def main(): parser = argparse.ArgumentParser(description="ESP-IDF app builder") parser.add_argument( "-v", "--verbose", action="count", help= "Increase the logging level of the script. Can be specified multiple times.", ) parser.add_argument( "--log-file", type=argparse.FileType("w"), help="Write the script log to the specified file, instead of stderr", ) parser.add_argument( "build_list", type=argparse.FileType("r"), nargs="?", default=sys.stdin, help= "Name of the file to read the list of builds from. If not specified, read from stdin.", ) args = parser.parse_args() setup_logging(args) build_items = [BuildItem.from_json(line) for line in args.build_list] if not build_items: logging.error("Empty build list!") raise SystemExit(1) found_warnings = 0 for build_item in build_items: if not build_item.build_log_path: logging.debug("No log file for {}".format(build_item.work_dir)) continue with open(build_item.build_log_path, "r") as log_file: for line_no, line in enumerate(log_file): if line_has_warnings(line): logging.error("Issue in app {}, config {}:".format( build_item.app_dir, build_item.config_name)) logging.error(line.rstrip("\n")) logging.error("See {}:{} for details".format( os.path.basename(build_item.build_log_path), line_no + 1)) found_warnings += 1 break if found_warnings: logging.error("Checked {} builds, found {} warnings".format( len(build_items), found_warnings)) raise SystemExit(1) logging.info("No warnings found")
def main(): # type: () -> None parser = argparse.ArgumentParser(description='ESP-IDF app builder') parser.add_argument( '-v', '--verbose', action='count', help= 'Increase the logging level of the script. Can be specified multiple times.', ) parser.add_argument( '--log-file', type=argparse.FileType('w'), help='Write the script log to the specified file, instead of stderr', ) parser.add_argument( 'build_list', type=argparse.FileType('r'), nargs='?', default=sys.stdin, help= 'Name of the file to read the list of builds from. If not specified, read from stdin.', ) args = parser.parse_args() setup_logging(args) build_items = [BuildItem.from_json(line) for line in args.build_list] if not build_items: logging.warning('Empty build list') SystemExit(0) found_warnings = 0 for build_item in build_items: if not build_item.build_log_path: logging.debug('No log file for {}'.format(build_item.work_dir)) continue with open(build_item.build_log_path, 'r') as log_file: for line_no, line in enumerate(log_file): if line_has_warnings(line): logging.error('Issue in app {}, config {}:'.format( build_item.app_dir, build_item.config_name)) logging.error(line.rstrip('\n')) logging.error('See {}:{} for details'.format( os.path.basename(build_item.build_log_path), line_no + 1)) found_warnings += 1 break if found_warnings: logging.error('Checked {} builds, found {} warnings'.format( len(build_items), found_warnings)) raise SystemExit(1) logging.info('No warnings found')
def main(): parser = argparse.ArgumentParser( description="Tool to generate build steps for IDF apps") parser.add_argument( "-v", "--verbose", action="count", help= "Increase the logging level of the script. Can be specified multiple times.", ) parser.add_argument( "--log-file", type=argparse.FileType("w"), help="Write the script log to the specified file, instead of stderr", ) parser.add_argument( "--recursive", action="store_true", help="Look for apps in the specified directories recursively.", ) parser.add_argument("--build-system", choices=BUILD_SYSTEMS.keys()) parser.add_argument( "--work-dir", help= "If set, the app is first copied into the specified directory, and then built." + "If not set, the work directory is the directory of the app.", ) parser.add_argument( "--config", action="append", help= "Adds configurations (sdkconfig file names) to build. This can either be " + "FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, " + "relative to the project directory, to be used. Optional NAME can be specified, " + "which can be used as a name of this configuration. FILEPATTERN is the name of " + "the sdkconfig file, relative to the project directory, with at most one wildcard. " + "The part captured by the wildcard is used as the name of the configuration.", ) parser.add_argument( "--build-dir", help= "If set, specifies the build directory name. Can expand placeholders. Can be either a " + "name relative to the work directory, or an absolute path.", ) parser.add_argument( "--build-log", help= "If specified, the build log will be written to this file. Can expand placeholders.", ) parser.add_argument("--target", help="Build apps for given target.") parser.add_argument( "--format", default="json", choices=["json"], help="Format to write the list of builds as", ) parser.add_argument( "--exclude", action="append", help= "Ignore specified directory (if --recursive is given). Can be used multiple times.", ) parser.add_argument( "-o", "--output", type=argparse.FileType("w"), help="Output the list of builds to the specified file", ) parser.add_argument( "--app-list", default=None, help= "Scan tests results. Restrict the build/artifacts preservation behavior to apps need to be built. " "If the file does not exist, will build all apps and upload all artifacts." ) parser.add_argument("-p", "--paths", nargs="+", help="One or more app paths.") args = parser.parse_args() setup_logging(args) # Arguments Validation if args.app_list: conflict_args = [ args.recursive, args.build_system, args.target, args.exclude, args.paths ] if any(conflict_args): raise ValueError( 'Conflict settings. "recursive", "build_system", "target", "exclude", "paths" should not ' 'be specified with "app_list"') if not os.path.exists(args.app_list): raise OSError("File not found {}".format(args.app_list)) else: # If the build target is not set explicitly, get it from the environment or use the default one (esp32) if not args.target: env_target = os.environ.get("IDF_TARGET") if env_target: logging.info( "--target argument not set, using IDF_TARGET={} from the environment" .format(env_target)) args.target = env_target else: logging.info( "--target argument not set, using IDF_TARGET={} as the default" .format(DEFAULT_TARGET)) args.target = DEFAULT_TARGET if not args.build_system: logging.info( "--build-system argument not set, using {} as the default". format(BUILD_SYSTEM_CMAKE)) args.build_system = BUILD_SYSTEM_CMAKE required_args = [args.build_system, args.target, args.paths] if not all(required_args): raise ValueError( 'If app_list not set, arguments "build_system", "target", "paths" are required.' ) # Prepare the list of app paths, try to read from the scan_tests result. # If the file exists, then follow the file's app_dir and build/artifacts behavior, won't do find_apps() again. # If the file not exists, will do find_apps() first, then build all apps and upload all artifacts. if args.app_list: apps = [json.loads(line) for line in open(args.app_list)] else: app_dirs = [] build_system_class = BUILD_SYSTEMS[args.build_system] for path in args.paths: app_dirs += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target) apps = [{ "app_dir": app_dir, "build": True, "preserve": True } for app_dir in app_dirs] if not apps: logging.warning("No apps found") SystemExit(0) logging.info("Found {} apps".format(len(apps))) apps.sort(key=lambda x: x["app_dir"]) # Find compatible configurations of each app, collect them as BuildItems build_items = [] # type: typing.List[BuildItem] config_rules = config_rules_from_str(args.config or []) for app in apps: build_items += find_builds_for_app( app["app_dir"], args.work_dir, args.build_dir, args.build_log, args.target or app["target"], args.build_system or app["build_system"], config_rules, app["preserve"], ) logging.info("Found {} builds".format(len(build_items))) # Write out the BuildItems. Only JSON supported now (will add YAML later). if args.format != "json": raise NotImplementedError() out = args.output or sys.stdout out.writelines([item.to_json() + "\n" for item in build_items])
action='store_true', help="Don't actually build, only print the build commands", ) parser.add_argument( '--keep-going', action='store_true', help="Don't exit immediately when a build fails.", ) parser.add_argument( '--output-build-list', type=argparse.FileType('w'), help='If specified, the list of builds (with all the placeholders expanded) will be written to this file.', ) parser.add_argument( '--size-info', type=argparse.FileType('a'), help='If specified, the test case name and size info json will be written to this file' ) parser.add_argument( 'build_list', type=argparse.FileType('r'), nargs='?', default=sys.stdin, help='Name of the file to read the list of builds from. If not specified, read from stdin.', ) args = parser.parse_args() setup_logging(args) items = [BuildItem.from_json(line) for line in args.build_list] build_apps(items, args.parallel_count, args.parallel_index, args.dry_run, args.build_verbose, args.keep_going, args.output_build_list, args.size_info)
type=int, help='Number of parallel build jobs.') parser.add_argument( '--parallel-index', default=1, type=int, help= 'Index (1-based) of the job, out of the number specified by --parallel-count.', ) parser.add_argument( '--size-info', type=argparse.FileType('a'), help= 'If specified, the test case name and size info json will be written to this file' ) parser.add_argument( '-v', '--verbose', action='count', help= 'Increase the logging level of the script. Can be specified multiple times.', ) parser.add_argument( '--build-verbose', action='store_true', help='Enable verbose output from build system.', ) arguments = parser.parse_args() setup_logging(arguments) main(arguments)
def main(): parser = argparse.ArgumentParser( description="Tool to generate build steps for IDF apps") parser.add_argument( "-v", "--verbose", action="count", help= "Increase the logging level of the script. Can be specified multiple times.", ) parser.add_argument( "--log-file", type=argparse.FileType("w"), help="Write the script log to the specified file, instead of stderr", ) parser.add_argument( "--recursive", action="store_true", help="Look for apps in the specified directories recursively.", ) parser.add_argument("--build-system", choices=BUILD_SYSTEMS.keys(), default=BUILD_SYSTEM_CMAKE) parser.add_argument( "--work-dir", help= "If set, the app is first copied into the specified directory, and then built." + "If not set, the work directory is the directory of the app.", ) parser.add_argument( "--config", action="append", help= "Adds configurations (sdkconfig file names) to build. This can either be " + "FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, " + "relative to the project directory, to be used. Optional NAME can be specified, " + "which can be used as a name of this configuration. FILEPATTERN is the name of " + "the sdkconfig file, relative to the project directory, with at most one wildcard. " + "The part captured by the wildcard is used as the name of the configuration.", ) parser.add_argument( "--build-dir", help= "If set, specifies the build directory name. Can expand placeholders. Can be either a " + "name relative to the work directory, or an absolute path.", ) parser.add_argument( "--build-log", help= "If specified, the build log will be written to this file. Can expand placeholders.", ) parser.add_argument("--target", help="Build apps for given target.") parser.add_argument( "--format", default="json", choices=["json"], help="Format to write the list of builds as", ) parser.add_argument( "--exclude", action="append", help= "Ignore specified directory (if --recursive is given). Can be used multiple times.", ) parser.add_argument( "-o", "--output", type=argparse.FileType("w"), help="Output the list of builds to the specified file", ) parser.add_argument("paths", nargs="+", help="One or more app paths.") args = parser.parse_args() setup_logging(args) build_system_class = BUILD_SYSTEMS[args.build_system] # If the build target is not set explicitly, get it from the environment or use the default one (esp32) if not args.target: env_target = os.environ.get("IDF_TARGET") if env_target: logging.info( "--target argument not set, using IDF_TARGET={} from the environment" .format(env_target)) args.target = env_target else: logging.info( "--target argument not set, using IDF_TARGET={} as the default" .format(DEFAULT_TARGET)) args.target = DEFAULT_TARGET # Prepare the list of app paths app_paths = [] # type: typing.List[str] for path in args.paths: app_paths += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target) if not app_paths: logging.critical("No {} apps found".format(build_system_class.NAME)) raise SystemExit(1) logging.info("Found {} apps".format(len(app_paths))) app_paths = sorted(app_paths) # Find compatible configurations of each app, collect them as BuildItems build_items = [] # type: typing.List[BuildItem] config_rules = config_rules_from_str(args.config or []) for app_path in app_paths: build_items += find_builds_for_app( app_path, args.work_dir, args.build_dir, args.build_log, args.target, args.build_system, config_rules, ) logging.info("Found {} builds".format(len(build_items))) # Write out the BuildItems. Only JSON supported now (will add YAML later). if args.format != "json": raise NotImplementedError() out = args.output or sys.stdout out.writelines([item.to_json() + "\n" for item in build_items])
def main(): parser = argparse.ArgumentParser(description="ESP-IDF app builder") parser.add_argument( "-v", "--verbose", action="count", help= "Increase the logging level of the script. Can be specified multiple times.", ) parser.add_argument( "--build-verbose", action="store_true", help="Enable verbose output from build system.", ) parser.add_argument( "--log-file", type=argparse.FileType("w"), help="Write the script log to the specified file, instead of stderr", ) parser.add_argument( "--parallel-count", default=1, type=int, help= "Number of parallel build jobs. Note that this script doesn't start the jobs, " + "it needs to be executed multiple times with same value of --parallel-count and " + "different values of --parallel-index.", ) parser.add_argument( "--parallel-index", default=1, type=int, help= "Index (1-based) of the job, out of the number specified by --parallel-count.", ) parser.add_argument( "--format", default="json", choices=["json"], help="Format to read the list of builds", ) parser.add_argument( "--dry-run", action="store_true", help="Don't actually build, only print the build commands", ) parser.add_argument( "--keep-going", action="store_true", help="Don't exit immediately when a build fails.", ) parser.add_argument( "--output-build-list", type=argparse.FileType("w"), help= "If specified, the list of builds (with all the placeholders expanded) will be written to this file.", ) parser.add_argument( "build_list", type=argparse.FileType("r"), nargs="?", default=sys.stdin, help= "Name of the file to read the list of builds from. If not specified, read from stdin.", ) args = parser.parse_args() setup_logging(args) build_items = [BuildItem.from_json(line) for line in args.build_list] if not build_items: logging.error("Empty build list!") raise SystemExit(1) num_builds = len(build_items) num_jobs = args.parallel_count job_index = args.parallel_index - 1 # convert to 0-based index num_builds_per_job = (num_builds + num_jobs - 1) // num_jobs min_job_index = num_builds_per_job * job_index if min_job_index >= num_builds: logging.warn( "Nothing to do for job {} (build total: {}, per job: {})".format( job_index + 1, num_builds, num_builds_per_job)) raise SystemExit(0) max_job_index = min(num_builds_per_job * (job_index + 1) - 1, num_builds - 1) logging.info( "Total {} builds, max. {} builds per job, running builds {}-{}".format( num_builds, num_builds_per_job, min_job_index + 1, max_job_index + 1)) builds_for_current_job = build_items[min_job_index:max_job_index + 1] for i, build_info in enumerate(builds_for_current_job): index = i + min_job_index + 1 build_info.index = index build_info.dry_run = args.dry_run build_info.verbose = args.build_verbose build_info.keep_going = args.keep_going logging.debug(" Build {}: {}".format(index, repr(build_info))) if args.output_build_list: args.output_build_list.write(build_info.to_json_expanded() + "\n") failed_builds = [] for build_info in builds_for_current_job: logging.info("Running build {}: {}".format(build_info.index, repr(build_info))) build_system_class = BUILD_SYSTEMS[build_info.build_system] try: build_system_class.build(build_info) except BuildError as e: logging.error(e.message) if args.keep_going: failed_builds.append(build_info) else: raise SystemExit(1) if failed_builds: logging.error("The following build have failed:") for build in failed_builds: logging.error(" {}".format(build)) raise SystemExit(1)
def main(): parser = argparse.ArgumentParser(description='ESP-IDF app builder') parser.add_argument( '-v', '--verbose', action='count', help= 'Increase the logging level of the script. Can be specified multiple times.', ) parser.add_argument( '--build-verbose', action='store_true', help='Enable verbose output from build system.', ) parser.add_argument( '--log-file', type=argparse.FileType('w'), help='Write the script log to the specified file, instead of stderr', ) parser.add_argument( '--parallel-count', default=1, type=int, help= "Number of parallel build jobs. Note that this script doesn't start the jobs, " + 'it needs to be executed multiple times with same value of --parallel-count and ' + 'different values of --parallel-index.', ) parser.add_argument( '--parallel-index', default=1, type=int, help= 'Index (1-based) of the job, out of the number specified by --parallel-count.', ) parser.add_argument( '--format', default='json', choices=['json'], help='Format to read the list of builds', ) parser.add_argument( '--dry-run', action='store_true', help="Don't actually build, only print the build commands", ) parser.add_argument( '--keep-going', action='store_true', help="Don't exit immediately when a build fails.", ) parser.add_argument( '--output-build-list', type=argparse.FileType('w'), help= 'If specified, the list of builds (with all the placeholders expanded) will be written to this file.', ) parser.add_argument( '--size-info', type=argparse.FileType('a'), help= 'If specified, the test case name and size info json will be written to this file' ) parser.add_argument( 'build_list', type=argparse.FileType('r'), nargs='?', default=sys.stdin, help= 'Name of the file to read the list of builds from. If not specified, read from stdin.', ) args = parser.parse_args() setup_logging(args) build_items = [BuildItem.from_json(line) for line in args.build_list] if not build_items: logging.warning('Empty build list') SystemExit(0) num_builds = len(build_items) num_jobs = args.parallel_count job_index = args.parallel_index - 1 # convert to 0-based index num_builds_per_job = (num_builds + num_jobs - 1) // num_jobs min_job_index = num_builds_per_job * job_index if min_job_index >= num_builds: logging.warn( 'Nothing to do for job {} (build total: {}, per job: {})'.format( job_index + 1, num_builds, num_builds_per_job)) raise SystemExit(0) max_job_index = min(num_builds_per_job * (job_index + 1) - 1, num_builds - 1) logging.info( 'Total {} builds, max. {} builds per job, running builds {}-{}'.format( num_builds, num_builds_per_job, min_job_index + 1, max_job_index + 1)) builds_for_current_job = build_items[min_job_index:max_job_index + 1] for i, build_info in enumerate(builds_for_current_job): index = i + min_job_index + 1 build_info.index = index build_info.dry_run = args.dry_run build_info.verbose = args.build_verbose build_info.keep_going = args.keep_going logging.debug(' Build {}: {}'.format(index, repr(build_info))) if args.output_build_list: args.output_build_list.write(build_info.to_json_expanded() + '\n') failed_builds = [] for build_info in builds_for_current_job: logging.info('Running build {}: {}'.format(build_info.index, repr(build_info))) build_system_class = BUILD_SYSTEMS[build_info.build_system] try: build_system_class.build(build_info) except BuildError as e: logging.error(str(e)) if args.keep_going: failed_builds.append(build_info) else: raise SystemExit(1) else: if args.size_info: build_info.write_size_info(args.size_info) if not build_info.preserve: logging.info('Removing build directory {}'.format( build_info.build_path)) # we only remove binaries here, log files are still needed by check_build_warnings.py rmdir(build_info.build_path, exclude_file_pattern=SIZE_JSON_FN) if failed_builds: logging.error('The following build have failed:') for build in failed_builds: logging.error(' {}'.format(build)) raise SystemExit(1)