def main(args: argparse.Namespace) -> None: pytest_cases = [] for path in args.paths: pytest_cases += get_pytest_cases(path, args.target) paths = set() app_configs = defaultdict(set) for case in pytest_cases: paths.add(case.app_path) app_configs[case.app_path].add(case.config) app_dirs = list(paths) if not app_dirs: raise RuntimeError('No apps found') logging.info(f'Found {len(app_dirs)} apps') app_dirs.sort() # Find compatible configurations of each app, collect them as BuildItems build_items: List[BuildItem] = [] config_rules = config_rules_from_str(args.config or []) for app_dir in app_dirs: app_dir = os.path.realpath(app_dir) build_items += find_builds_for_app( app_path=app_dir, work_dir=app_dir, build_dir='build_@t_@w', build_log=f'{app_dir}/build_@t_@w/build.log', target_arg=args.target, build_system='cmake', config_rules=config_rules, ) modified_build_items = [] # auto clean up the binaries if no flag --preserve-all for item in build_items: is_test_related = item.config_name in app_configs[item.app_dir] if args.test_only and not is_test_related: logging.info(f'Skipping non-test app: {item}') continue copied_item = copy.deepcopy(item) if not args.preserve_all and not is_test_related: copied_item.preserve = False modified_build_items.append(copied_item) logging.info(f'Found {len(modified_build_items)} builds') modified_build_items.sort(key=lambda x: x.build_path) # type: ignore build_apps( build_items=modified_build_items, parallel_count=args.parallel_count, parallel_index=args.parallel_index, dry_run=False, build_verbose=args.build_verbose, keep_going=True, output_build_list=None, size_info=args.size_info, )
def main(args: argparse.Namespace) -> None: if args.all_pytest_apps: paths = get_pytest_dirs(IDF_PATH, args.under_dir) args.recursive = True elif args.paths is None: paths = [os.getcwd()] else: paths = args.paths app_dirs = [] for path in paths: app_dirs += find_apps(CMakeBuildSystem, path, args.recursive, [], args.target) if not app_dirs: logging.error('No apps found') sys.exit(1) logging.info('Found {} apps'.format(len(app_dirs))) app_dirs.sort() # Find compatible configurations of each app, collect them as BuildItems build_items: List[BuildItem] = [] config_rules = config_rules_from_str(args.config or []) for app_dir in app_dirs: app_dir = os.path.realpath(app_dir) build_items += find_builds_for_app( app_dir, app_dir, 'build_@t_@w', f'{app_dir}/build_@t_@w/build.log', args.target, 'cmake', config_rules, True, ) logging.info('Found {} builds'.format(len(build_items))) build_items.sort(key=lambda x: x.build_path) # type: ignore build_apps(build_items, args.parallel_count, args.parallel_index, False, args.build_verbose, True, None, args.size_info)
def main(): parser = argparse.ArgumentParser( description="Tool to generate build steps for IDF apps") parser.add_argument( "-v", "--verbose", action="count", help= "Increase the logging level of the script. Can be specified multiple times.", ) parser.add_argument( "--log-file", type=argparse.FileType("w"), help="Write the script log to the specified file, instead of stderr", ) parser.add_argument( "--recursive", action="store_true", help="Look for apps in the specified directories recursively.", ) parser.add_argument("--build-system", choices=BUILD_SYSTEMS.keys()) parser.add_argument( "--work-dir", help= "If set, the app is first copied into the specified directory, and then built." + "If not set, the work directory is the directory of the app.", ) parser.add_argument( "--config", action="append", help= "Adds configurations (sdkconfig file names) to build. This can either be " + "FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, " + "relative to the project directory, to be used. Optional NAME can be specified, " + "which can be used as a name of this configuration. FILEPATTERN is the name of " + "the sdkconfig file, relative to the project directory, with at most one wildcard. " + "The part captured by the wildcard is used as the name of the configuration.", ) parser.add_argument( "--build-dir", help= "If set, specifies the build directory name. Can expand placeholders. Can be either a " + "name relative to the work directory, or an absolute path.", ) parser.add_argument( "--build-log", help= "If specified, the build log will be written to this file. Can expand placeholders.", ) parser.add_argument("--target", help="Build apps for given target.") parser.add_argument( "--format", default="json", choices=["json"], help="Format to write the list of builds as", ) parser.add_argument( "--exclude", action="append", help= "Ignore specified directory (if --recursive is given). Can be used multiple times.", ) parser.add_argument( "-o", "--output", type=argparse.FileType("w"), help="Output the list of builds to the specified file", ) parser.add_argument( "--app-list", default=None, help= "Scan tests results. Restrict the build/artifacts preservation behavior to apps need to be built. " "If the file does not exist, will build all apps and upload all artifacts." ) parser.add_argument("-p", "--paths", nargs="+", help="One or more app paths.") args = parser.parse_args() setup_logging(args) # Arguments Validation if args.app_list: conflict_args = [ args.recursive, args.build_system, args.target, args.exclude, args.paths ] if any(conflict_args): raise ValueError( 'Conflict settings. "recursive", "build_system", "target", "exclude", "paths" should not ' 'be specified with "app_list"') if not os.path.exists(args.app_list): raise OSError("File not found {}".format(args.app_list)) else: # If the build target is not set explicitly, get it from the environment or use the default one (esp32) if not args.target: env_target = os.environ.get("IDF_TARGET") if env_target: logging.info( "--target argument not set, using IDF_TARGET={} from the environment" .format(env_target)) args.target = env_target else: logging.info( "--target argument not set, using IDF_TARGET={} as the default" .format(DEFAULT_TARGET)) args.target = DEFAULT_TARGET if not args.build_system: logging.info( "--build-system argument not set, using {} as the default". format(BUILD_SYSTEM_CMAKE)) args.build_system = BUILD_SYSTEM_CMAKE required_args = [args.build_system, args.target, args.paths] if not all(required_args): raise ValueError( 'If app_list not set, arguments "build_system", "target", "paths" are required.' ) # Prepare the list of app paths, try to read from the scan_tests result. # If the file exists, then follow the file's app_dir and build/artifacts behavior, won't do find_apps() again. # If the file not exists, will do find_apps() first, then build all apps and upload all artifacts. if args.app_list: apps = [json.loads(line) for line in open(args.app_list)] else: app_dirs = [] build_system_class = BUILD_SYSTEMS[args.build_system] for path in args.paths: app_dirs += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target) apps = [{ "app_dir": app_dir, "build": True, "preserve": True } for app_dir in app_dirs] if not apps: logging.warning("No apps found") SystemExit(0) logging.info("Found {} apps".format(len(apps))) apps.sort(key=lambda x: x["app_dir"]) # Find compatible configurations of each app, collect them as BuildItems build_items = [] # type: typing.List[BuildItem] config_rules = config_rules_from_str(args.config or []) for app in apps: build_items += find_builds_for_app( app["app_dir"], args.work_dir, args.build_dir, args.build_log, args.target or app["target"], args.build_system or app["build_system"], config_rules, app["preserve"], ) logging.info("Found {} builds".format(len(build_items))) # Write out the BuildItems. Only JSON supported now (will add YAML later). if args.format != "json": raise NotImplementedError() out = args.output or sys.stdout out.writelines([item.to_json() + "\n" for item in build_items])
}) for app_dir in scan_info_dict[target]['standalone_apps']: if app_dir in pytest_app_paths: print(f'Skipping pytest app: {app_dir}') continue apps.append({ 'app_dir': app_dir, 'build_system': args.build_system, 'target': target, 'preserve': args.preserve_all }) output_path = os.path.join(args.output_path, 'scan_{}_{}.json'.format(target.lower(), build_system)) with open(output_path, 'w') as fw: if args.evaluate_parallel_count: build_items = [] config_rules = config_rules_from_str(args.config or []) for app in apps: build_items += find_builds_for_app( app['app_dir'], app['app_dir'], 'build', '', app['target'], app['build_system'], config_rules, app['preserve'], ) print('Found {} builds'.format(len(build_items))) if args.combine_all_targets: if (args.except_targets and target not in [t.lower() for t in args.except_targets]) \ or (not args.except_targets):
def main(): parser = argparse.ArgumentParser( description="Tool to generate build steps for IDF apps") parser.add_argument( "-v", "--verbose", action="count", help= "Increase the logging level of the script. Can be specified multiple times.", ) parser.add_argument( "--log-file", type=argparse.FileType("w"), help="Write the script log to the specified file, instead of stderr", ) parser.add_argument( "--recursive", action="store_true", help="Look for apps in the specified directories recursively.", ) parser.add_argument("--build-system", choices=BUILD_SYSTEMS.keys(), default=BUILD_SYSTEM_CMAKE) parser.add_argument( "--work-dir", help= "If set, the app is first copied into the specified directory, and then built." + "If not set, the work directory is the directory of the app.", ) parser.add_argument( "--config", action="append", help= "Adds configurations (sdkconfig file names) to build. This can either be " + "FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, " + "relative to the project directory, to be used. Optional NAME can be specified, " + "which can be used as a name of this configuration. FILEPATTERN is the name of " + "the sdkconfig file, relative to the project directory, with at most one wildcard. " + "The part captured by the wildcard is used as the name of the configuration.", ) parser.add_argument( "--build-dir", help= "If set, specifies the build directory name. Can expand placeholders. Can be either a " + "name relative to the work directory, or an absolute path.", ) parser.add_argument( "--build-log", help= "If specified, the build log will be written to this file. Can expand placeholders.", ) parser.add_argument("--target", help="Build apps for given target.") parser.add_argument( "--format", default="json", choices=["json"], help="Format to write the list of builds as", ) parser.add_argument( "--exclude", action="append", help= "Ignore specified directory (if --recursive is given). Can be used multiple times.", ) parser.add_argument( "-o", "--output", type=argparse.FileType("w"), help="Output the list of builds to the specified file", ) parser.add_argument("paths", nargs="+", help="One or more app paths.") args = parser.parse_args() setup_logging(args) build_system_class = BUILD_SYSTEMS[args.build_system] # If the build target is not set explicitly, get it from the environment or use the default one (esp32) if not args.target: env_target = os.environ.get("IDF_TARGET") if env_target: logging.info( "--target argument not set, using IDF_TARGET={} from the environment" .format(env_target)) args.target = env_target else: logging.info( "--target argument not set, using IDF_TARGET={} as the default" .format(DEFAULT_TARGET)) args.target = DEFAULT_TARGET # Prepare the list of app paths app_paths = [] # type: typing.List[str] for path in args.paths: app_paths += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target) if not app_paths: logging.critical("No {} apps found".format(build_system_class.NAME)) raise SystemExit(1) logging.info("Found {} apps".format(len(app_paths))) app_paths = sorted(app_paths) # Find compatible configurations of each app, collect them as BuildItems build_items = [] # type: typing.List[BuildItem] config_rules = config_rules_from_str(args.config or []) for app_path in app_paths: build_items += find_builds_for_app( app_path, args.work_dir, args.build_dir, args.build_log, args.target, args.build_system, config_rules, ) logging.info("Found {} builds".format(len(build_items))) # Write out the BuildItems. Only JSON supported now (will add YAML later). if args.format != "json": raise NotImplementedError() out = args.output or sys.stdout out.writelines([item.to_json() + "\n" for item in build_items])