def main():
parser = argparse.ArgumentParser(description="ESP-IDF app builder")
parser.add_argument(
"-v",
"--verbose",
action="count",
help=
"Increase the logging level of the script. Can be specified multiple times.",
)
parser.add_argument(
"--log-file",
type=argparse.FileType("w"),
help="Write the script log to the specified file, instead of stderr",
)
parser.add_argument(
"build_list",
type=argparse.FileType("r"),
nargs="?",
default=sys.stdin,
help=
"Name of the file to read the list of builds from. If not specified, read from stdin.",
)
args = parser.parse_args()
setup_logging(args)
build_items = [BuildItem.from_json(line) for line in args.build_list]
if not build_items:
logging.error("Empty build list!")
raise SystemExit(1)
found_warnings = 0
for build_item in build_items:
if not build_item.build_log_path:
logging.debug("No log file for {}".format(build_item.work_dir))
continue
with open(build_item.build_log_path, "r") as log_file:
for line_no, line in enumerate(log_file):
if line_has_warnings(line):
logging.error("Issue in app {}, config {}:".format(
build_item.app_dir, build_item.config_name))
logging.error(line.rstrip("\n"))
logging.error("See {}:{} for details".format(
os.path.basename(build_item.build_log_path),
line_no + 1))
found_warnings += 1
break
if found_warnings:
logging.error("Checked {} builds, found {} warnings".format(
len(build_items), found_warnings))
raise SystemExit(1)
logging.info("No warnings found")
def main(): # type: () -> None
parser = argparse.ArgumentParser(description='ESP-IDF app builder')
parser.add_argument(
'-v',
'--verbose',
action='count',
help=
'Increase the logging level of the script. Can be specified multiple times.',
)
parser.add_argument(
'--log-file',
type=argparse.FileType('w'),
help='Write the script log to the specified file, instead of stderr',
)
parser.add_argument(
'build_list',
type=argparse.FileType('r'),
nargs='?',
default=sys.stdin,
help=
'Name of the file to read the list of builds from. If not specified, read from stdin.',
)
args = parser.parse_args()
setup_logging(args)
build_items = [BuildItem.from_json(line) for line in args.build_list]
if not build_items:
logging.warning('Empty build list')
SystemExit(0)
found_warnings = 0
for build_item in build_items:
if not build_item.build_log_path:
logging.debug('No log file for {}'.format(build_item.work_dir))
continue
with open(build_item.build_log_path, 'r') as log_file:
for line_no, line in enumerate(log_file):
if line_has_warnings(line):
logging.error('Issue in app {}, config {}:'.format(
build_item.app_dir, build_item.config_name))
logging.error(line.rstrip('\n'))
logging.error('See {}:{} for details'.format(
os.path.basename(build_item.build_log_path),
line_no + 1))
found_warnings += 1
break
if found_warnings:
logging.error('Checked {} builds, found {} warnings'.format(
len(build_items), found_warnings))
raise SystemExit(1)
logging.info('No warnings found')
def main():
parser = argparse.ArgumentParser(
description="Tool to generate build steps for IDF apps")
parser.add_argument(
"-v",
"--verbose",
action="count",
help=
"Increase the logging level of the script. Can be specified multiple times.",
)
parser.add_argument(
"--log-file",
type=argparse.FileType("w"),
help="Write the script log to the specified file, instead of stderr",
)
parser.add_argument(
"--recursive",
action="store_true",
help="Look for apps in the specified directories recursively.",
)
parser.add_argument("--build-system", choices=BUILD_SYSTEMS.keys())
parser.add_argument(
"--work-dir",
help=
"If set, the app is first copied into the specified directory, and then built."
+ "If not set, the work directory is the directory of the app.",
)
parser.add_argument(
"--config",
action="append",
help=
"Adds configurations (sdkconfig file names) to build. This can either be "
+
"FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, "
+
"relative to the project directory, to be used. Optional NAME can be specified, "
+
"which can be used as a name of this configuration. FILEPATTERN is the name of "
+
"the sdkconfig file, relative to the project directory, with at most one wildcard. "
+
"The part captured by the wildcard is used as the name of the configuration.",
)
parser.add_argument(
"--build-dir",
help=
"If set, specifies the build directory name. Can expand placeholders. Can be either a "
+ "name relative to the work directory, or an absolute path.",
)
parser.add_argument(
"--build-log",
help=
"If specified, the build log will be written to this file. Can expand placeholders.",
)
parser.add_argument("--target", help="Build apps for given target.")
parser.add_argument(
"--format",
default="json",
choices=["json"],
help="Format to write the list of builds as",
)
parser.add_argument(
"--exclude",
action="append",
help=
"Ignore specified directory (if --recursive is given). Can be used multiple times.",
)
parser.add_argument(
"-o",
"--output",
type=argparse.FileType("w"),
help="Output the list of builds to the specified file",
)
parser.add_argument(
"--app-list",
default=None,
help=
"Scan tests results. Restrict the build/artifacts preservation behavior to apps need to be built. "
"If the file does not exist, will build all apps and upload all artifacts."
)
parser.add_argument("-p",
"--paths",
nargs="+",
help="One or more app paths.")
args = parser.parse_args()
setup_logging(args)
# Arguments Validation
if args.app_list:
conflict_args = [
args.recursive, args.build_system, args.target, args.exclude,
args.paths
]
if any(conflict_args):
raise ValueError(
'Conflict settings. "recursive", "build_system", "target", "exclude", "paths" should not '
'be specified with "app_list"')
if not os.path.exists(args.app_list):
raise OSError("File not found {}".format(args.app_list))
else:
# If the build target is not set explicitly, get it from the environment or use the default one (esp32)
if not args.target:
env_target = os.environ.get("IDF_TARGET")
if env_target:
logging.info(
"--target argument not set, using IDF_TARGET={} from the environment"
.format(env_target))
args.target = env_target
else:
logging.info(
"--target argument not set, using IDF_TARGET={} as the default"
.format(DEFAULT_TARGET))
args.target = DEFAULT_TARGET
if not args.build_system:
logging.info(
"--build-system argument not set, using {} as the default".
format(BUILD_SYSTEM_CMAKE))
args.build_system = BUILD_SYSTEM_CMAKE
required_args = [args.build_system, args.target, args.paths]
if not all(required_args):
raise ValueError(
'If app_list not set, arguments "build_system", "target", "paths" are required.'
)
# Prepare the list of app paths, try to read from the scan_tests result.
# If the file exists, then follow the file's app_dir and build/artifacts behavior, won't do find_apps() again.
# If the file not exists, will do find_apps() first, then build all apps and upload all artifacts.
if args.app_list:
apps = [json.loads(line) for line in open(args.app_list)]
else:
app_dirs = []
build_system_class = BUILD_SYSTEMS[args.build_system]
for path in args.paths:
app_dirs += find_apps(build_system_class, path, args.recursive,
args.exclude or [], args.target)
apps = [{
"app_dir": app_dir,
"build": True,
"preserve": True
} for app_dir in app_dirs]
if not apps:
logging.warning("No apps found")
SystemExit(0)
logging.info("Found {} apps".format(len(apps)))
apps.sort(key=lambda x: x["app_dir"])
# Find compatible configurations of each app, collect them as BuildItems
build_items = [] # type: typing.List[BuildItem]
config_rules = config_rules_from_str(args.config or [])
for app in apps:
build_items += find_builds_for_app(
app["app_dir"],
args.work_dir,
args.build_dir,
args.build_log,
args.target or app["target"],
args.build_system or app["build_system"],
config_rules,
app["preserve"],
)
logging.info("Found {} builds".format(len(build_items)))
# Write out the BuildItems. Only JSON supported now (will add YAML later).
if args.format != "json":
raise NotImplementedError()
out = args.output or sys.stdout
out.writelines([item.to_json() + "\n" for item in build_items])
action='store_true',
help="Don't actually build, only print the build commands",
)
parser.add_argument(
'--keep-going',
action='store_true',
help="Don't exit immediately when a build fails.",
)
parser.add_argument(
'--output-build-list',
type=argparse.FileType('w'),
help='If specified, the list of builds (with all the placeholders expanded) will be written to this file.',
)
parser.add_argument(
'--size-info',
type=argparse.FileType('a'),
help='If specified, the test case name and size info json will be written to this file'
)
parser.add_argument(
'build_list',
type=argparse.FileType('r'),
nargs='?',
default=sys.stdin,
help='Name of the file to read the list of builds from. If not specified, read from stdin.',
)
args = parser.parse_args()
setup_logging(args)
items = [BuildItem.from_json(line) for line in args.build_list]
build_apps(items, args.parallel_count, args.parallel_index, args.dry_run, args.build_verbose,
args.keep_going, args.output_build_list, args.size_info)
type=int,
help='Number of parallel build jobs.')
parser.add_argument(
'--parallel-index',
default=1,
type=int,
help=
'Index (1-based) of the job, out of the number specified by --parallel-count.',
)
parser.add_argument(
'--size-info',
type=argparse.FileType('a'),
help=
'If specified, the test case name and size info json will be written to this file'
)
parser.add_argument(
'-v',
'--verbose',
action='count',
help=
'Increase the logging level of the script. Can be specified multiple times.',
)
parser.add_argument(
'--build-verbose',
action='store_true',
help='Enable verbose output from build system.',
)
arguments = parser.parse_args()
setup_logging(arguments)
main(arguments)
def main():
parser = argparse.ArgumentParser(
description="Tool to generate build steps for IDF apps")
parser.add_argument(
"-v",
"--verbose",
action="count",
help=
"Increase the logging level of the script. Can be specified multiple times.",
)
parser.add_argument(
"--log-file",
type=argparse.FileType("w"),
help="Write the script log to the specified file, instead of stderr",
)
parser.add_argument(
"--recursive",
action="store_true",
help="Look for apps in the specified directories recursively.",
)
parser.add_argument("--build-system",
choices=BUILD_SYSTEMS.keys(),
default=BUILD_SYSTEM_CMAKE)
parser.add_argument(
"--work-dir",
help=
"If set, the app is first copied into the specified directory, and then built."
+ "If not set, the work directory is the directory of the app.",
)
parser.add_argument(
"--config",
action="append",
help=
"Adds configurations (sdkconfig file names) to build. This can either be "
+
"FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, "
+
"relative to the project directory, to be used. Optional NAME can be specified, "
+
"which can be used as a name of this configuration. FILEPATTERN is the name of "
+
"the sdkconfig file, relative to the project directory, with at most one wildcard. "
+
"The part captured by the wildcard is used as the name of the configuration.",
)
parser.add_argument(
"--build-dir",
help=
"If set, specifies the build directory name. Can expand placeholders. Can be either a "
+ "name relative to the work directory, or an absolute path.",
)
parser.add_argument(
"--build-log",
help=
"If specified, the build log will be written to this file. Can expand placeholders.",
)
parser.add_argument("--target", help="Build apps for given target.")
parser.add_argument(
"--format",
default="json",
choices=["json"],
help="Format to write the list of builds as",
)
parser.add_argument(
"--exclude",
action="append",
help=
"Ignore specified directory (if --recursive is given). Can be used multiple times.",
)
parser.add_argument(
"-o",
"--output",
type=argparse.FileType("w"),
help="Output the list of builds to the specified file",
)
parser.add_argument("paths", nargs="+", help="One or more app paths.")
args = parser.parse_args()
setup_logging(args)
build_system_class = BUILD_SYSTEMS[args.build_system]
# If the build target is not set explicitly, get it from the environment or use the default one (esp32)
if not args.target:
env_target = os.environ.get("IDF_TARGET")
if env_target:
logging.info(
"--target argument not set, using IDF_TARGET={} from the environment"
.format(env_target))
args.target = env_target
else:
logging.info(
"--target argument not set, using IDF_TARGET={} as the default"
.format(DEFAULT_TARGET))
args.target = DEFAULT_TARGET
# Prepare the list of app paths
app_paths = [] # type: typing.List[str]
for path in args.paths:
app_paths += find_apps(build_system_class, path, args.recursive,
args.exclude or [], args.target)
if not app_paths:
logging.critical("No {} apps found".format(build_system_class.NAME))
raise SystemExit(1)
logging.info("Found {} apps".format(len(app_paths)))
app_paths = sorted(app_paths)
# Find compatible configurations of each app, collect them as BuildItems
build_items = [] # type: typing.List[BuildItem]
config_rules = config_rules_from_str(args.config or [])
for app_path in app_paths:
build_items += find_builds_for_app(
app_path,
args.work_dir,
args.build_dir,
args.build_log,
args.target,
args.build_system,
config_rules,
)
logging.info("Found {} builds".format(len(build_items)))
# Write out the BuildItems. Only JSON supported now (will add YAML later).
if args.format != "json":
raise NotImplementedError()
out = args.output or sys.stdout
out.writelines([item.to_json() + "\n" for item in build_items])
def main():
parser = argparse.ArgumentParser(description="ESP-IDF app builder")
parser.add_argument(
"-v",
"--verbose",
action="count",
help=
"Increase the logging level of the script. Can be specified multiple times.",
)
parser.add_argument(
"--build-verbose",
action="store_true",
help="Enable verbose output from build system.",
)
parser.add_argument(
"--log-file",
type=argparse.FileType("w"),
help="Write the script log to the specified file, instead of stderr",
)
parser.add_argument(
"--parallel-count",
default=1,
type=int,
help=
"Number of parallel build jobs. Note that this script doesn't start the jobs, "
+
"it needs to be executed multiple times with same value of --parallel-count and "
+ "different values of --parallel-index.",
)
parser.add_argument(
"--parallel-index",
default=1,
type=int,
help=
"Index (1-based) of the job, out of the number specified by --parallel-count.",
)
parser.add_argument(
"--format",
default="json",
choices=["json"],
help="Format to read the list of builds",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Don't actually build, only print the build commands",
)
parser.add_argument(
"--keep-going",
action="store_true",
help="Don't exit immediately when a build fails.",
)
parser.add_argument(
"--output-build-list",
type=argparse.FileType("w"),
help=
"If specified, the list of builds (with all the placeholders expanded) will be written to this file.",
)
parser.add_argument(
"build_list",
type=argparse.FileType("r"),
nargs="?",
default=sys.stdin,
help=
"Name of the file to read the list of builds from. If not specified, read from stdin.",
)
args = parser.parse_args()
setup_logging(args)
build_items = [BuildItem.from_json(line) for line in args.build_list]
if not build_items:
logging.error("Empty build list!")
raise SystemExit(1)
num_builds = len(build_items)
num_jobs = args.parallel_count
job_index = args.parallel_index - 1 # convert to 0-based index
num_builds_per_job = (num_builds + num_jobs - 1) // num_jobs
min_job_index = num_builds_per_job * job_index
if min_job_index >= num_builds:
logging.warn(
"Nothing to do for job {} (build total: {}, per job: {})".format(
job_index + 1, num_builds, num_builds_per_job))
raise SystemExit(0)
max_job_index = min(num_builds_per_job * (job_index + 1) - 1,
num_builds - 1)
logging.info(
"Total {} builds, max. {} builds per job, running builds {}-{}".format(
num_builds, num_builds_per_job, min_job_index + 1,
max_job_index + 1))
builds_for_current_job = build_items[min_job_index:max_job_index + 1]
for i, build_info in enumerate(builds_for_current_job):
index = i + min_job_index + 1
build_info.index = index
build_info.dry_run = args.dry_run
build_info.verbose = args.build_verbose
build_info.keep_going = args.keep_going
logging.debug(" Build {}: {}".format(index, repr(build_info)))
if args.output_build_list:
args.output_build_list.write(build_info.to_json_expanded() + "\n")
failed_builds = []
for build_info in builds_for_current_job:
logging.info("Running build {}: {}".format(build_info.index,
repr(build_info)))
build_system_class = BUILD_SYSTEMS[build_info.build_system]
try:
build_system_class.build(build_info)
except BuildError as e:
logging.error(e.message)
if args.keep_going:
failed_builds.append(build_info)
else:
raise SystemExit(1)
if failed_builds:
logging.error("The following build have failed:")
for build in failed_builds:
logging.error(" {}".format(build))
raise SystemExit(1)
def main():
parser = argparse.ArgumentParser(description='ESP-IDF app builder')
parser.add_argument(
'-v',
'--verbose',
action='count',
help=
'Increase the logging level of the script. Can be specified multiple times.',
)
parser.add_argument(
'--build-verbose',
action='store_true',
help='Enable verbose output from build system.',
)
parser.add_argument(
'--log-file',
type=argparse.FileType('w'),
help='Write the script log to the specified file, instead of stderr',
)
parser.add_argument(
'--parallel-count',
default=1,
type=int,
help=
"Number of parallel build jobs. Note that this script doesn't start the jobs, "
+
'it needs to be executed multiple times with same value of --parallel-count and '
+ 'different values of --parallel-index.',
)
parser.add_argument(
'--parallel-index',
default=1,
type=int,
help=
'Index (1-based) of the job, out of the number specified by --parallel-count.',
)
parser.add_argument(
'--format',
default='json',
choices=['json'],
help='Format to read the list of builds',
)
parser.add_argument(
'--dry-run',
action='store_true',
help="Don't actually build, only print the build commands",
)
parser.add_argument(
'--keep-going',
action='store_true',
help="Don't exit immediately when a build fails.",
)
parser.add_argument(
'--output-build-list',
type=argparse.FileType('w'),
help=
'If specified, the list of builds (with all the placeholders expanded) will be written to this file.',
)
parser.add_argument(
'--size-info',
type=argparse.FileType('a'),
help=
'If specified, the test case name and size info json will be written to this file'
)
parser.add_argument(
'build_list',
type=argparse.FileType('r'),
nargs='?',
default=sys.stdin,
help=
'Name of the file to read the list of builds from. If not specified, read from stdin.',
)
args = parser.parse_args()
setup_logging(args)
build_items = [BuildItem.from_json(line) for line in args.build_list]
if not build_items:
logging.warning('Empty build list')
SystemExit(0)
num_builds = len(build_items)
num_jobs = args.parallel_count
job_index = args.parallel_index - 1 # convert to 0-based index
num_builds_per_job = (num_builds + num_jobs - 1) // num_jobs
min_job_index = num_builds_per_job * job_index
if min_job_index >= num_builds:
logging.warn(
'Nothing to do for job {} (build total: {}, per job: {})'.format(
job_index + 1, num_builds, num_builds_per_job))
raise SystemExit(0)
max_job_index = min(num_builds_per_job * (job_index + 1) - 1,
num_builds - 1)
logging.info(
'Total {} builds, max. {} builds per job, running builds {}-{}'.format(
num_builds, num_builds_per_job, min_job_index + 1,
max_job_index + 1))
builds_for_current_job = build_items[min_job_index:max_job_index + 1]
for i, build_info in enumerate(builds_for_current_job):
index = i + min_job_index + 1
build_info.index = index
build_info.dry_run = args.dry_run
build_info.verbose = args.build_verbose
build_info.keep_going = args.keep_going
logging.debug(' Build {}: {}'.format(index, repr(build_info)))
if args.output_build_list:
args.output_build_list.write(build_info.to_json_expanded() + '\n')
failed_builds = []
for build_info in builds_for_current_job:
logging.info('Running build {}: {}'.format(build_info.index,
repr(build_info)))
build_system_class = BUILD_SYSTEMS[build_info.build_system]
try:
build_system_class.build(build_info)
except BuildError as e:
logging.error(str(e))
if args.keep_going:
failed_builds.append(build_info)
else:
raise SystemExit(1)
else:
if args.size_info:
build_info.write_size_info(args.size_info)
if not build_info.preserve:
logging.info('Removing build directory {}'.format(
build_info.build_path))
# we only remove binaries here, log files are still needed by check_build_warnings.py
rmdir(build_info.build_path, exclude_file_pattern=SIZE_JSON_FN)
if failed_builds:
logging.error('The following build have failed:')
for build in failed_builds:
logging.error(' {}'.format(build))
raise SystemExit(1)
