def do(cls, ws, args): '''Executes the build subcmd.''' ws_config = get_ws_config(get_ws_dir(args.root, ws)) d = parse_manifest(args.root) # Validate. for project in args.projects: if project not in d: raise WSError('unknown project %s' % project) if len(args.projects) == 0: projects = d.keys() else: projects = args.projects # Build in reverse-dependency order. order = dependency_closure(d, projects) # Get all checksums; since this is a nop build bottle-neck, do it in # parallel. On my machine, this produces a ~20% speedup on a nop # "build-all". pool = multiprocessing.Pool(multiprocessing.cpu_count()) src_dirs = [get_source_dir(args.root, d, proj) for proj in order] checksums = pool.map(calculate_checksum, src_dirs) for i, proj in enumerate(order): log('building %s' % proj) checksum = checksums[i] success = _build(args.root, ws, proj, d, checksum, ws_config, args.force) if not success: raise WSError('%s build failed' % proj)
def do(cls, _, args): '''Executes the remove command.''' ws_dir = get_ws_dir(args.root, args.remove_ws) if not os.path.exists(ws_dir): raise WSError('workspace %s does not exist' % args.remove_ws) if args.default is not None: default_ws_dir = get_ws_dir(args.root, args.default) if not os.path.exists(default_ws_dir): raise WSError('workspace %s does not exist' % args.default) default_link = get_default_ws_link(args.root) is_default = (os.readlink(default_link) == args.remove_ws) if is_default: # If the deleted workspace is the default, force the user to choose # a new one. if args.default is None: raise WSError('trying to remove the default workspace; must ' 'specify a new default via -d/--default') elif args.default: raise WSError('-d/--default is not applicable unless you are ' 'removing the default workspace') # We are good to go. rmtree(ws_dir) if is_default: remove(default_link) symlink(args.default, default_link)
def do(cls, ws, args): '''Executes the test subcmd.''' d = parse_manifest(args.root) # Validate. for project in args.projects: if project not in d: raise WSError('unknown project %s' % project) if len(args.projects) == 0: projects = d.keys() else: projects = args.projects for proj in projects: build_dir = get_build_dir(ws, proj) if not os.path.isdir(build_dir): raise WSError('build directory for %s doesn\'t exist; have ' 'you built it yet?' % proj) if 'tests' not in d[proj]: raise WSError('no test configured for %s' % proj) for proj in projects: log('testing %s' % proj) build_env = get_build_env(ws, d, proj) cmds = d[proj]['tests'] _test(args.root, ws, proj, cmds, build_env)
def do(cls, ws, args): '''Executes the rename command.''' if args.old_ws == 'default': raise WSError('cannot rename the default workspace; please use ws ' 'default if you want to change it') old_ws_dir = get_ws_dir(args.root, args.old_ws) if not os.path.exists(old_ws_dir): raise WSError('workspace %s does not exist' % args.old_ws) d = parse_manifest(args.root) for proj in d: build_dir = get_build_dir(old_ws_dir, proj) if os.path.exists(build_dir): raise WSError('cannot rename a workspace that contains build ' 'artifacts, as some builds contain absolute ' 'paths and are thus not relocatable. Please ' 'force-clean this workspace first and then ' 'rename it.') new_ws_dir = get_ws_dir(args.root, args.new_ws) if os.path.exists(new_ws_dir): raise WSError('workspace %s already exists; please delete it ' 'first if you want to do this rename' % args.new_ws) rename(old_ws_dir, new_ws_dir) default_link = get_default_ws_link(args.root) if os.readlink(default_link) == args.old_ws: remove(default_link) symlink(args.new_ws, default_link)
def process(project): processed.add(project) for dep in d[project]['deps']: if dep not in order: if dep in processed: raise WSError('Projects %s and %s circularly depend on ' 'each other' % (project, dep)) process(dep) order.append(project)
def get_builder(d, proj): '''Returns the build properties for a given project. This function should be used instead of directly referencing _BUILD_TOOLS.''' build = d[proj]['build'] try: builder = _BUILD_TOOLS[build] except KeyError: raise WSError('unknown build tool %s for project %s' % (build, proj)) return builder
def include_paths(d, manifest): '''Return the manifest absolute paths included from the given parsed manifest.''' try: includes = d['include'] except KeyError: return () # The manifest's own directory is the default search path if none are # specified. search_paths = [os.path.realpath(os.path.join(manifest, os.pardir))] try: extra_search_paths = d['search-path'] except KeyError: pass else: for path in extra_search_paths: path = os.path.realpath(os.path.join(manifest, os.pardir, path)) search_paths.append(path) tweaked_includes = [] for i, path in enumerate(includes): if path[0] == '/': # This is an absolute path, so no tweaking is necessary. tweaked_includes.append(path) continue # Relative path, so try to match it using the search paths. found_match = False for search_path in search_paths: full_path = os.path.realpath(os.path.join(search_path, path)) if os.path.exists(full_path): found_match = True break if not found_match: raise WSError('cannot find manifest %s included by %s\n' 'search paths: %s' % (path, manifest, search_paths)) # For directories, we include every file in the directory. if os.path.isdir(full_path): for filename in os.listdir(full_path): if not (filename.endswith('.yaml') or filename.endswith('.yml')): # noqa: E501 continue filepath = os.path.join(full_path, filename) if not os.path.isfile(filepath): continue tweaked_includes.append(filepath) else: tweaked_includes.append(full_path) return tweaked_includes
def build(cls, proj, prefix, source_dir, build_dir, env, targets, args): '''Calls build using setuptools.''' if targets is not None and targets != DEFAULT_TARGETS: raise WSError('pip3 does not support alternate build targets but ' '"%s" was specified for targets' % targets) cmd = [ 'pip3', 'install', '--prefix=%s' % prefix, '--build=%s' % build_dir ] cmd.extend(args) cmd.append('.') return call_build(cmd, cwd=source_dir, env=env)
def _test(root, ws, proj, cmds, env): '''Tests a given project.''' for props in cmds: cwd = expand_vars(props['cwd'], ws, proj, env) cmds = props['cmds'] for cmd in cmds: cmd = expand_vars(cmd, ws, proj, env) success = call_test(cmd.split(), cwd=cwd, env=env) if not success: repro_cmd = '(cd %s && ws env %s %s)' % (cwd, proj, cmd) raise WSError(''' %s tests failed. You can reproduce this failure by running: %s''' % (proj, repro_cmd))
def do(cls, ws, args): '''Executes the clean command.''' # Validate. d = parse_manifest(args.root) for project in args.projects: if project not in d: raise WSError('unknown project %s' % project) if len(args.projects) == 0: projects = d.keys() else: projects = args.projects for project in projects: clean(args.root, ws, project, d, args.force)
def do(cls, _, args): '''Executes the default command.''' link = get_default_ws_link(args.root) if args.default_ws is None: # Just report what the default currently is. dest = os.path.basename(os.path.realpath(link)) print(dest) return ws_dir = get_ws_dir(args.root, args.default_ws) remove(link) if not os.path.exists(ws_dir): raise WSError('Cannot make non-existent workspace %s the default' % args.default_ws) symlink(args.default_ws, link)
def merge_manifest(parent, parent_path, child, child_path): '''Merges the keys from the child dictionary into the parent dictionary. If there are conflicts, bail out with an error.''' try: parent_projects = parent['projects'] except KeyError: parent_projects = {} parent['projects'] = parent_projects try: child_projects = child['projects'] except KeyError: child_projects = {} child['projects'] = child_projects intersection = set(parent_projects).intersection(set(child_projects)) if len(intersection) != 0: raise WSError('cannot include %s from %s, as the two share projects %s' % (child_path, parent_path, intersection)) parent_projects.update(child_projects)
def parse_bool_val(val): '''Parses a value meant to be interpreted as a bool. Accepts 0, 1, false, and true (with any casing). Raises an exception if none match.''' if val is None: return True if val == '0': return False if val == '1': return True val = val.lower() if val == 'false': return False elif val == 'true': return True raise WSError('value "%s" is not a valid boolean' % val)
def build(cls, proj, prefix, source_dir, build_dir, env, targets, builder_args, args): '''Calls build using setuptools.''' if targets is not None and targets != DEFAULT_TARGETS: raise WSError('pip3 does not support alternate build targets but ' '"%s" was specified for targets' % targets) python_exe = get_python_exe(builder_args) cmd = [ python_exe, '-m', 'pip', 'install', '--prefix=%s' % prefix, '--build=%s' % build_dir ] cmd.extend(args) path = '.' path += get_package_extras(builder_args) cmd.append(path) return call_build(cmd, cwd=source_dir, env=env)
def do(cls, ws, args): '''Executes the env command.''' build_dir = get_build_dir(ws, args.project) if not os.path.isdir(build_dir): raise WSError('build directory for %s doesn\'t exist; have you ' 'built it yet?' % args.project) d = parse_manifest(args.root) build_env = get_build_env(ws, d, args.project, True) # Add the build directory to the path for convenience of running # non-installed binaries, such as unit tests. merge_var(build_env, 'PATH', [build_dir]) if len(args.command) > 0: cmd = args.command else: cmd = [get_shell()] exe = os.path.basename(cmd[0]) if exe == 'bash': # Tweak the prompt to make it obvious we're in a special env. prompt = ( '\\[\033[1;32m\\][ws:%s env]\\[\033[m\\] \\u@\\h:\\w\\$ ' # noqa: E501 % args.project) build_env['PS1'] = prompt cmd.insert(1, '--norc') # Set an env var so the user can easily cd $WSBUILD and run tests or # similar inside the build directory. build_env['WSBUILD'] = build_dir log('execing with %s build environment: %s' % (args.project, cmd)) if args.build_dir: args.current_dir = build_dir if args.current_dir is not None: os.chdir(args.current_dir) os.execvpe(cmd[0], cmd, build_env)
def parse_manifest_file(root, manifest): '''Parses the given ws manifest file, returning a dictionary of the manifest data.''' d = parse_yaml(root, manifest) merge_includes(root, d, manifest) # Compute reverse-dependency list. projects = d['projects'] for proj, props in projects.items(): props['downstream'] = [] for proj, props in projects.items(): deps = props['deps'] for dep in deps: try: dep_props = projects[dep] except KeyError: raise WSError('project %s dependency %s not found in the ' 'manifest' % (proj, dep)) else: # Reverse-dependency list of downstream projects. dep_props['downstream'].append(proj) return projects
def parse_yaml(root, manifest): # noqa: E302 '''Parses the given manifest for YAML and syntax correctness, or bails if something went wrong.''' try: with open(manifest, 'r') as f: d = yaml.safe_load(f) except IOError: raise WSError('ws manifest %s not found' % manifest) try: includes = d['include'] except KeyError: includes = () d['include'] = includes else: if not isinstance(includes, list): raise WSError('"include" key %s in %s is not a list' % (includes, manifest)) if len(includes) == 0: raise WSError('"include" key in %s is an empty list!' % manifest) try: search_paths = d['search-path'] except KeyError: search_paths = () d['search-path'] = search_paths else: if not isinstance(search_paths, list): raise WSError('"search-path" key %s in %s is not a list' % (search_paths, manifest)) if len(search_paths) == 0: raise WSError('include in %s is an empty list!' % manifest) try: projects = d['projects'] except KeyError: if len(includes) == 0: raise WSError('"projects" key missing in manifest %s' % manifest) else: return d for proj, props in projects.items(): for prop in _REQUIRED_KEYS: if prop not in props: raise WSError('"%s" key missing from project %s in manifest' % (prop, proj)) for prop in props: if prop not in _ALL_KEYS: raise WSError('unknown key "%s" for project %s specified in ' 'manifest' % (prop, proj)) # Add computed keys. parent = os.path.realpath(os.path.join(root, os.pardir)) for proj, props in projects.items(): try: deps = props['deps'] except KeyError: props['deps'] = () else: if not isinstance(deps, list): raise WSError('"deps" key in project %s must be a list' % proj) if len(set(deps)) != len(deps): raise WSError('project %s has duplicate dependency' % proj) try: env = props['env'] except KeyError: props['env'] = {} else: if not isinstance(env, dict): raise WSError('"env" key in project %s must be a dictionary' % proj) for k, v in env.items(): if not isinstance(k, str): raise WSError('env key %s in project %s must be a string' % (k, proj)) if not isinstance(v, str): raise WSError('env value "%s" (key "%s") in project %s ' 'must be a string' % (v, k, proj)) try: args = props['args'] except KeyError: props['args'] = [] else: if not isinstance(args, list): raise WSError('"args" key in project %s must be a list' % proj) for opt in args: if not isinstance(opt, str): raise WSError('option "%s" in project %s must be a string' % (opt, proj)) args = [] for opt in props['args']: args.extend(opt.split()) props['args'] = args try: args = props['targets'] except KeyError: props['targets'] = DEFAULT_TARGETS else: if props['targets'] is None: props['targets'] = () else: if not isinstance(props['targets'], list): raise WSError('"targets" key in project %s must be a list' % proj) for target in props['targets']: if not isinstance(target, str): raise WSError('target "%s" in project %s must be a ' 'string' % (opt, proj)) try: builder_args = props['builder-args'] except KeyError: props['builder-args'] = {} builder_args = props['builder-args'] else: if not isinstance(builder_args, dict): raise WSError('"builder-args" key in project %s must be a ' 'list' % proj) for builder_arg in builder_args: if not isinstance(builder_arg, str): raise WSError('builder arg "%s" in project %s must be ' 'a string' % (builder_arg, proj)) try: tests = props['tests'] except KeyError: props['tests'] = [] tests = props['tests'] else: if not isinstance(tests, list): raise WSError('"tests" key in project %s must be a list' % proj) for i, test in enumerate(tests): if isinstance(test, str): cwd = '${BUILDDIR}' cmds = [test] elif isinstance(test, dict): try: cwd = test['cwd'] except KeyError: raise WSError('test "%s" in project %s is missing the ' '"cwd" key' % (test, proj)) try: cmds = test['cmds'] except KeyError: raise WSError('test "%s" in project %s is missing the ' '"cmds" key' % (test, proj)) if not isinstance(cmds, list): raise WSError('test "cmds" key "%s" in project %s is ' 'not a list' % (cmds, proj)) for cmd in cmds: if not isinstance(cmd, str): raise WSError('test command "%s" in project %s is ' 'not a string' % (cmd, proj)) else: raise WSError('test "%s" in project %s must be ' 'a string or dictionary' % (test, proj)) tests[i] = {'cwd': cwd, 'cmds': cmds} props['path'] = os.path.join(parent, proj) return d
def do(cls, ws, args): '''Executes the config command.''' config = get_ws_config(ws) if args.list: print(yaml.dump(config, default_flow_style=False), end='') return for arg in args.options: split = arg.split('=') split_len = len(split) key = split[0] if split_len == 1: val = None elif split_len == 2: val = split[1] else: val = '='.join(split[1:]) project = args.project if project is not None: # Project-specific option. d = parse_manifest(args.root) if project not in d: raise WSError('project "%s" not found' % project) can_taint = False if key == 'enable': val = parse_bool_val(val) can_taint = True elif key == 'args': val = parse_build_args(val) if val is None: raise WSError('build args are not in the right format ' '("args=key=val")') can_taint = True else: raise WSError('project key "%s" not found' % key) try: proj_config = config['projects'][project] except KeyError: proj_config = {} config['projects'][project] = proj_config if can_taint and proj_config[key] != val: log('tainting project %s' % project) proj_config['taint'] = True proj_config[key] = val else: # Global option. if key == 'type': if val is None or val not in BUILD_TYPES: raise WSError('"type" key must be one of %s' % str(BUILD_TYPES)) if config[key] != val: for proj_config in config['projects'].values(): proj_config['taint'] = True config[key] = val
def do(cls, _, args): '''Executes the init command.''' if args.root is None: root = '.ws' else: root = args.root if args.init_ws is None: ws = 'ws' else: reserved = (get_default_ws_name(), get_manifest_link_name()) for name in reserved: if args.init_ws == name: raise WSError('%s is a reserved name; please choose a ' 'different name' % name) ws = args.init_ws if '.' in ws or '/' in ws: raise WSError('Workspace name "%s" contains an illegal ' 'character (. or /). Please use a different ' 'name.' % ws) ws_dir = get_ws_dir(root, ws) if os.path.exists(ws_dir): raise WSError('Cannot initialize already existing workspace %s' % ws) if args.manifest_source == 'repo': parent = os.path.join(root, os.pardir) repo_manifest = os.path.realpath( os.path.join(parent, '.repo', 'manifest.xml')) base = os.path.dirname(repo_manifest) elif args.manifest_source == 'fs': base = '.' else: raise NotImplementedError('Manifest source %s should be ' 'implemented' % args.manifest_source) manifest = os.path.join(base, args.manifest) if os.path.isdir(manifest): # If -m points to a directory instead of a file, assume there is a # file with the default manifest name inside. manifest = os.path.join(manifest, get_default_manifest_name()) # Use a relative path for anything inside the parent of .ws and an # absolute path for anything outside. This is to maximize # relocatability for groups of git repos (e.g. using submodules, # repo-tool, etc.). manifest = os.path.abspath(manifest) parent = os.path.realpath(os.path.join(root, os.pardir)) if _is_subdir(manifest, parent): manifest = os.path.relpath(manifest, root) else: manifest = os.path.abspath(manifest) # Make sure the given manifest is sane. if os.path.isabs(manifest): abs_manifest = manifest else: abs_manifest = os.path.abspath(os.path.join(root, manifest)) d = parse_manifest_file(root, abs_manifest) try: mkdir(root) new_root = True except OSError as e: if e.errno != errno.EEXIST: raise new_root = False try: mkdir(ws_dir) new_ws = True except OSError as e: if e.errno != errno.EEXIST: raise new_ws = False if new_ws: # This is a brand-new workspace, so populate the initial workspace # directories. mkdir(get_toplevel_build_dir(ws_dir)) mkdir(get_checksum_dir(ws_dir)) proj_map = dict((proj, {}) for proj in d) for proj in proj_map: proj_map[proj] = get_new_config(proj) config = {'type': args.type, 'projects': proj_map} write_config(ws_dir, config) if new_root: # This is a brand new root .ws directory, so populate the initial # symlink defaults. symlink(ws, get_default_ws_link(root)) symlink(manifest, get_manifest_link(root))