def do(cls, ws, args):
'''Executes the build subcmd.'''
ws_config = get_ws_config(get_ws_dir(args.root, ws))
d = parse_manifest(args.root)
# Validate.
for project in args.projects:
if project not in d:
raise WSError('unknown project %s' % project)
if len(args.projects) == 0:
projects = d.keys()
else:
projects = args.projects
# Build in reverse-dependency order.
order = dependency_closure(d, projects)
# Get all checksums; since this is a nop build bottle-neck, do it in
# parallel. On my machine, this produces a ~20% speedup on a nop
# "build-all".
pool = multiprocessing.Pool(multiprocessing.cpu_count())
src_dirs = [get_source_dir(args.root, d, proj) for proj in order]
checksums = pool.map(calculate_checksum, src_dirs)
for i, proj in enumerate(order):
log('building %s' % proj)
checksum = checksums[i]
success = _build(args.root, ws, proj, d, checksum, ws_config,
args.force)
if not success:
raise WSError('%s build failed' % proj)
def do(cls, _, args):
'''Executes the remove command.'''
ws_dir = get_ws_dir(args.root, args.remove_ws)
if not os.path.exists(ws_dir):
raise WSError('workspace %s does not exist' % args.remove_ws)
if args.default is not None:
default_ws_dir = get_ws_dir(args.root, args.default)
if not os.path.exists(default_ws_dir):
raise WSError('workspace %s does not exist' % args.default)
default_link = get_default_ws_link(args.root)
is_default = (os.readlink(default_link) == args.remove_ws)
if is_default:
# If the deleted workspace is the default, force the user to choose
# a new one.
if args.default is None:
raise WSError('trying to remove the default workspace; must '
'specify a new default via -d/--default')
elif args.default:
raise WSError('-d/--default is not applicable unless you are '
'removing the default workspace')
# We are good to go.
rmtree(ws_dir)
if is_default:
remove(default_link)
symlink(args.default, default_link)
def do(cls, ws, args):
'''Executes the test subcmd.'''
d = parse_manifest(args.root)
# Validate.
for project in args.projects:
if project not in d:
raise WSError('unknown project %s' % project)
if len(args.projects) == 0:
projects = d.keys()
else:
projects = args.projects
for proj in projects:
build_dir = get_build_dir(ws, proj)
if not os.path.isdir(build_dir):
raise WSError('build directory for %s doesn\'t exist; have '
'you built it yet?' % proj)
if 'tests' not in d[proj]:
raise WSError('no test configured for %s' % proj)
for proj in projects:
log('testing %s' % proj)
build_env = get_build_env(ws, d, proj)
cmds = d[proj]['tests']
_test(args.root, ws, proj, cmds, build_env)
def do(cls, ws, args):
'''Executes the rename command.'''
if args.old_ws == 'default':
raise WSError('cannot rename the default workspace; please use ws '
'default if you want to change it')
old_ws_dir = get_ws_dir(args.root, args.old_ws)
if not os.path.exists(old_ws_dir):
raise WSError('workspace %s does not exist' % args.old_ws)
d = parse_manifest(args.root)
for proj in d:
build_dir = get_build_dir(old_ws_dir, proj)
if os.path.exists(build_dir):
raise WSError('cannot rename a workspace that contains build '
'artifacts, as some builds contain absolute '
'paths and are thus not relocatable. Please '
'force-clean this workspace first and then '
'rename it.')
new_ws_dir = get_ws_dir(args.root, args.new_ws)
if os.path.exists(new_ws_dir):
raise WSError('workspace %s already exists; please delete it '
'first if you want to do this rename' % args.new_ws)
rename(old_ws_dir, new_ws_dir)
default_link = get_default_ws_link(args.root)
if os.readlink(default_link) == args.old_ws:
remove(default_link)
symlink(args.new_ws, default_link)
def process(project):
processed.add(project)
for dep in d[project]['deps']:
if dep not in order:
if dep in processed:
raise WSError('Projects %s and %s circularly depend on '
'each other' % (project, dep))
process(dep)
order.append(project)
def get_builder(d, proj):
'''Returns the build properties for a given project. This function should
be used instead of directly referencing _BUILD_TOOLS.'''
build = d[proj]['build']
try:
builder = _BUILD_TOOLS[build]
except KeyError:
raise WSError('unknown build tool %s for project %s' % (build, proj))
return builder
def include_paths(d, manifest):
'''Return the manifest absolute paths included from the given parsed
manifest.'''
try:
includes = d['include']
except KeyError:
return ()
# The manifest's own directory is the default search path if none are
# specified.
search_paths = [os.path.realpath(os.path.join(manifest, os.pardir))]
try:
extra_search_paths = d['search-path']
except KeyError:
pass
else:
for path in extra_search_paths:
path = os.path.realpath(os.path.join(manifest, os.pardir, path))
search_paths.append(path)
tweaked_includes = []
for i, path in enumerate(includes):
if path[0] == '/':
# This is an absolute path, so no tweaking is necessary.
tweaked_includes.append(path)
continue
# Relative path, so try to match it using the search paths.
found_match = False
for search_path in search_paths:
full_path = os.path.realpath(os.path.join(search_path, path))
if os.path.exists(full_path):
found_match = True
break
if not found_match:
raise WSError('cannot find manifest %s included by %s\n'
'search paths: %s' % (path, manifest, search_paths))
# For directories, we include every file in the directory.
if os.path.isdir(full_path):
for filename in os.listdir(full_path):
if not (filename.endswith('.yaml')
or filename.endswith('.yml')): # noqa: E501
continue
filepath = os.path.join(full_path, filename)
if not os.path.isfile(filepath):
continue
tweaked_includes.append(filepath)
else:
tweaked_includes.append(full_path)
return tweaked_includes
def build(cls, proj, prefix, source_dir, build_dir, env, targets, args):
'''Calls build using setuptools.'''
if targets is not None and targets != DEFAULT_TARGETS:
raise WSError('pip3 does not support alternate build targets but '
'"%s" was specified for targets' % targets)
cmd = [
'pip3', 'install',
'--prefix=%s' % prefix,
'--build=%s' % build_dir
]
cmd.extend(args)
cmd.append('.')
return call_build(cmd, cwd=source_dir, env=env)
def _test(root, ws, proj, cmds, env):
'''Tests a given project.'''
for props in cmds:
cwd = expand_vars(props['cwd'], ws, proj, env)
cmds = props['cmds']
for cmd in cmds:
cmd = expand_vars(cmd, ws, proj, env)
success = call_test(cmd.split(), cwd=cwd, env=env)
if not success:
repro_cmd = '(cd %s && ws env %s %s)' % (cwd, proj, cmd)
raise WSError('''
%s tests failed. You can reproduce this failure by running:
%s''' % (proj, repro_cmd))
def do(cls, ws, args):
'''Executes the clean command.'''
# Validate.
d = parse_manifest(args.root)
for project in args.projects:
if project not in d:
raise WSError('unknown project %s' % project)
if len(args.projects) == 0:
projects = d.keys()
else:
projects = args.projects
for project in projects:
clean(args.root, ws, project, d, args.force)
def do(cls, _, args):
'''Executes the default command.'''
link = get_default_ws_link(args.root)
if args.default_ws is None:
# Just report what the default currently is.
dest = os.path.basename(os.path.realpath(link))
print(dest)
return
ws_dir = get_ws_dir(args.root, args.default_ws)
remove(link)
if not os.path.exists(ws_dir):
raise WSError('Cannot make non-existent workspace %s the default' %
args.default_ws)
symlink(args.default_ws, link)
def merge_manifest(parent, parent_path, child, child_path):
'''Merges the keys from the child dictionary into the parent dictionary. If
there are conflicts, bail out with an error.'''
try:
parent_projects = parent['projects']
except KeyError:
parent_projects = {}
parent['projects'] = parent_projects
try:
child_projects = child['projects']
except KeyError:
child_projects = {}
child['projects'] = child_projects
intersection = set(parent_projects).intersection(set(child_projects))
if len(intersection) != 0:
raise WSError('cannot include %s from %s, as the two share projects %s'
% (child_path, parent_path, intersection))
parent_projects.update(child_projects)
def parse_bool_val(val):
'''Parses a value meant to be interpreted as a bool. Accepts 0, 1, false,
and true (with any casing). Raises an exception if none match.'''
if val is None:
return True
if val == '0':
return False
if val == '1':
return True
val = val.lower()
if val == 'false':
return False
elif val == 'true':
return True
raise WSError('value "%s" is not a valid boolean' % val)
def build(cls, proj, prefix, source_dir, build_dir, env, targets,
builder_args, args):
'''Calls build using setuptools.'''
if targets is not None and targets != DEFAULT_TARGETS:
raise WSError('pip3 does not support alternate build targets but '
'"%s" was specified for targets' % targets)
python_exe = get_python_exe(builder_args)
cmd = [
python_exe, '-m', 'pip', 'install',
'--prefix=%s' % prefix,
'--build=%s' % build_dir
]
cmd.extend(args)
path = '.'
path += get_package_extras(builder_args)
cmd.append(path)
return call_build(cmd, cwd=source_dir, env=env)
def do(cls, ws, args):
'''Executes the env command.'''
build_dir = get_build_dir(ws, args.project)
if not os.path.isdir(build_dir):
raise WSError('build directory for %s doesn\'t exist; have you '
'built it yet?' % args.project)
d = parse_manifest(args.root)
build_env = get_build_env(ws, d, args.project, True)
# Add the build directory to the path for convenience of running
# non-installed binaries, such as unit tests.
merge_var(build_env, 'PATH', [build_dir])
if len(args.command) > 0:
cmd = args.command
else:
cmd = [get_shell()]
exe = os.path.basename(cmd[0])
if exe == 'bash':
# Tweak the prompt to make it obvious we're in a special env.
prompt = (
'\\[\033[1;32m\\][ws:%s env]\\[\033[m\\] \\u@\\h:\\w\\$ ' # noqa: E501
% args.project)
build_env['PS1'] = prompt
cmd.insert(1, '--norc')
# Set an env var so the user can easily cd $WSBUILD and run tests or
# similar inside the build directory.
build_env['WSBUILD'] = build_dir
log('execing with %s build environment: %s' % (args.project, cmd))
if args.build_dir:
args.current_dir = build_dir
if args.current_dir is not None:
os.chdir(args.current_dir)
os.execvpe(cmd[0], cmd, build_env)
def parse_manifest_file(root, manifest):
'''Parses the given ws manifest file, returning a dictionary of the
manifest data.'''
d = parse_yaml(root, manifest)
merge_includes(root, d, manifest)
# Compute reverse-dependency list.
projects = d['projects']
for proj, props in projects.items():
props['downstream'] = []
for proj, props in projects.items():
deps = props['deps']
for dep in deps:
try:
dep_props = projects[dep]
except KeyError:
raise WSError('project %s dependency %s not found in the '
'manifest' % (proj, dep))
else:
# Reverse-dependency list of downstream projects.
dep_props['downstream'].append(proj)
return projects
def parse_yaml(root, manifest): # noqa: E302
'''Parses the given manifest for YAML and syntax correctness, or bails if
something went wrong.'''
try:
with open(manifest, 'r') as f:
d = yaml.safe_load(f)
except IOError:
raise WSError('ws manifest %s not found' % manifest)
try:
includes = d['include']
except KeyError:
includes = ()
d['include'] = includes
else:
if not isinstance(includes, list):
raise WSError('"include" key %s in %s is not a list'
% (includes, manifest))
if len(includes) == 0:
raise WSError('"include" key in %s is an empty list!' % manifest)
try:
search_paths = d['search-path']
except KeyError:
search_paths = ()
d['search-path'] = search_paths
else:
if not isinstance(search_paths, list):
raise WSError('"search-path" key %s in %s is not a list'
% (search_paths, manifest))
if len(search_paths) == 0:
raise WSError('include in %s is an empty list!' % manifest)
try:
projects = d['projects']
except KeyError:
if len(includes) == 0:
raise WSError('"projects" key missing in manifest %s' % manifest)
else:
return d
for proj, props in projects.items():
for prop in _REQUIRED_KEYS:
if prop not in props:
raise WSError('"%s" key missing from project %s in manifest'
% (prop, proj))
for prop in props:
if prop not in _ALL_KEYS:
raise WSError('unknown key "%s" for project %s specified in '
'manifest' % (prop, proj))
# Add computed keys.
parent = os.path.realpath(os.path.join(root, os.pardir))
for proj, props in projects.items():
try:
deps = props['deps']
except KeyError:
props['deps'] = ()
else:
if not isinstance(deps, list):
raise WSError('"deps" key in project %s must be a list' % proj)
if len(set(deps)) != len(deps):
raise WSError('project %s has duplicate dependency' % proj)
try:
env = props['env']
except KeyError:
props['env'] = {}
else:
if not isinstance(env, dict):
raise WSError('"env" key in project %s must be a dictionary'
% proj)
for k, v in env.items():
if not isinstance(k, str):
raise WSError('env key %s in project %s must be a string' %
(k, proj))
if not isinstance(v, str):
raise WSError('env value "%s" (key "%s") in project %s '
'must be a string' % (v, k, proj))
try:
args = props['args']
except KeyError:
props['args'] = []
else:
if not isinstance(args, list):
raise WSError('"args" key in project %s must be a list' % proj)
for opt in args:
if not isinstance(opt, str):
raise WSError('option "%s" in project %s must be a string'
% (opt, proj))
args = []
for opt in props['args']:
args.extend(opt.split())
props['args'] = args
try:
args = props['targets']
except KeyError:
props['targets'] = DEFAULT_TARGETS
else:
if props['targets'] is None:
props['targets'] = ()
else:
if not isinstance(props['targets'], list):
raise WSError('"targets" key in project %s must be a list'
% proj)
for target in props['targets']:
if not isinstance(target, str):
raise WSError('target "%s" in project %s must be a '
'string' % (opt, proj))
try:
builder_args = props['builder-args']
except KeyError:
props['builder-args'] = {}
builder_args = props['builder-args']
else:
if not isinstance(builder_args, dict):
raise WSError('"builder-args" key in project %s must be a '
'list' % proj)
for builder_arg in builder_args:
if not isinstance(builder_arg, str):
raise WSError('builder arg "%s" in project %s must be '
'a string' % (builder_arg, proj))
try:
tests = props['tests']
except KeyError:
props['tests'] = []
tests = props['tests']
else:
if not isinstance(tests, list):
raise WSError('"tests" key in project %s must be a list'
% proj)
for i, test in enumerate(tests):
if isinstance(test, str):
cwd = '${BUILDDIR}'
cmds = [test]
elif isinstance(test, dict):
try:
cwd = test['cwd']
except KeyError:
raise WSError('test "%s" in project %s is missing the '
'"cwd" key' % (test, proj))
try:
cmds = test['cmds']
except KeyError:
raise WSError('test "%s" in project %s is missing the '
'"cmds" key' % (test, proj))
if not isinstance(cmds, list):
raise WSError('test "cmds" key "%s" in project %s is '
'not a list' % (cmds, proj))
for cmd in cmds:
if not isinstance(cmd, str):
raise WSError('test command "%s" in project %s is '
'not a string' % (cmd, proj))
else:
raise WSError('test "%s" in project %s must be '
'a string or dictionary' % (test, proj))
tests[i] = {'cwd': cwd, 'cmds': cmds}
props['path'] = os.path.join(parent, proj)
return d
def do(cls, ws, args):
'''Executes the config command.'''
config = get_ws_config(ws)
if args.list:
print(yaml.dump(config, default_flow_style=False), end='')
return
for arg in args.options:
split = arg.split('=')
split_len = len(split)
key = split[0]
if split_len == 1:
val = None
elif split_len == 2:
val = split[1]
else:
val = '='.join(split[1:])
project = args.project
if project is not None:
# Project-specific option.
d = parse_manifest(args.root)
if project not in d:
raise WSError('project "%s" not found' % project)
can_taint = False
if key == 'enable':
val = parse_bool_val(val)
can_taint = True
elif key == 'args':
val = parse_build_args(val)
if val is None:
raise WSError('build args are not in the right format '
'("args=key=val")')
can_taint = True
else:
raise WSError('project key "%s" not found' % key)
try:
proj_config = config['projects'][project]
except KeyError:
proj_config = {}
config['projects'][project] = proj_config
if can_taint and proj_config[key] != val:
log('tainting project %s' % project)
proj_config['taint'] = True
proj_config[key] = val
else:
# Global option.
if key == 'type':
if val is None or val not in BUILD_TYPES:
raise WSError('"type" key must be one of %s' %
str(BUILD_TYPES))
if config[key] != val:
for proj_config in config['projects'].values():
proj_config['taint'] = True
config[key] = val
def do(cls, _, args):
'''Executes the init command.'''
if args.root is None:
root = '.ws'
else:
root = args.root
if args.init_ws is None:
ws = 'ws'
else:
reserved = (get_default_ws_name(), get_manifest_link_name())
for name in reserved:
if args.init_ws == name:
raise WSError('%s is a reserved name; please choose a '
'different name' % name)
ws = args.init_ws
if '.' in ws or '/' in ws:
raise WSError('Workspace name "%s" contains an illegal '
'character (. or /). Please use a different '
'name.' % ws)
ws_dir = get_ws_dir(root, ws)
if os.path.exists(ws_dir):
raise WSError('Cannot initialize already existing workspace %s' %
ws)
if args.manifest_source == 'repo':
parent = os.path.join(root, os.pardir)
repo_manifest = os.path.realpath(
os.path.join(parent, '.repo', 'manifest.xml'))
base = os.path.dirname(repo_manifest)
elif args.manifest_source == 'fs':
base = '.'
else:
raise NotImplementedError('Manifest source %s should be '
'implemented' % args.manifest_source)
manifest = os.path.join(base, args.manifest)
if os.path.isdir(manifest):
# If -m points to a directory instead of a file, assume there is a
# file with the default manifest name inside.
manifest = os.path.join(manifest, get_default_manifest_name())
# Use a relative path for anything inside the parent of .ws and an
# absolute path for anything outside. This is to maximize
# relocatability for groups of git repos (e.g. using submodules,
# repo-tool, etc.).
manifest = os.path.abspath(manifest)
parent = os.path.realpath(os.path.join(root, os.pardir))
if _is_subdir(manifest, parent):
manifest = os.path.relpath(manifest, root)
else:
manifest = os.path.abspath(manifest)
# Make sure the given manifest is sane.
if os.path.isabs(manifest):
abs_manifest = manifest
else:
abs_manifest = os.path.abspath(os.path.join(root, manifest))
d = parse_manifest_file(root, abs_manifest)
try:
mkdir(root)
new_root = True
except OSError as e:
if e.errno != errno.EEXIST:
raise
new_root = False
try:
mkdir(ws_dir)
new_ws = True
except OSError as e:
if e.errno != errno.EEXIST:
raise
new_ws = False
if new_ws:
# This is a brand-new workspace, so populate the initial workspace
# directories.
mkdir(get_toplevel_build_dir(ws_dir))
mkdir(get_checksum_dir(ws_dir))
proj_map = dict((proj, {}) for proj in d)
for proj in proj_map:
proj_map[proj] = get_new_config(proj)
config = {'type': args.type, 'projects': proj_map}
write_config(ws_dir, config)
if new_root:
# This is a brand new root .ws directory, so populate the initial
# symlink defaults.
symlink(ws, get_default_ws_link(root))
symlink(manifest, get_manifest_link(root))
