def do(cls, ws, args):
'''Executes the build subcmd.'''
ws_config = get_ws_config(get_ws_dir(args.root, ws))
d = parse_manifest(args.root)
# Validate.
for project in args.projects:
if project not in d:
raise WSError('unknown project %s' % project)
if len(args.projects) == 0:
projects = d.keys()
else:
projects = args.projects
# Build in reverse-dependency order.
order = dependency_closure(d, projects)
# Get all checksums; since this is a nop build bottle-neck, do it in
# parallel. On my machine, this produces a ~20% speedup on a nop
# "build-all".
pool = multiprocessing.Pool(multiprocessing.cpu_count())
src_dirs = [get_source_dir(args.root, d, proj) for proj in order]
checksums = pool.map(calculate_checksum, src_dirs)
for i, proj in enumerate(order):
log('building %s' % proj)
checksum = checksums[i]
success = _build(args.root, ws, proj, d, checksum, ws_config,
args.force)
if not success:
raise WSError('%s build failed' % proj)
def do(cls, ws, args):
'''Executes the test subcmd.'''
d = parse_manifest(args.root)
# Validate.
for project in args.projects:
if project not in d:
raise WSError('unknown project %s' % project)
if len(args.projects) == 0:
projects = d.keys()
else:
projects = args.projects
for proj in projects:
build_dir = get_build_dir(ws, proj)
if not os.path.isdir(build_dir):
raise WSError('build directory for %s doesn\'t exist; have '
'you built it yet?' % proj)
if 'tests' not in d[proj]:
raise WSError('no test configured for %s' % proj)
for proj in projects:
log('testing %s' % proj)
build_env = get_build_env(ws, d, proj)
cmds = d[proj]['tests']
_test(args.root, ws, proj, cmds, build_env)
def rmtree(path, fail_ok=False):
'''Removes the given file or directory, recursively.'''
log('recursively removing %s' % path)
if not dry_run():
try:
shutil.rmtree(path)
except FileNotFoundError:
if not fail_ok:
raise
def remove(path, fail_ok=False):
'''Removes the given path.'''
log('removing %s' % path)
if not dry_run():
try:
os.unlink(path)
except FileNotFoundError:
if not fail_ok:
raise
def invalidate_checksum(ws, proj):
'''Invalidates the current project checksum. This can be used to force a
project to rebuild, for example if one of its dependencies rebuilds.'''
log('invalidating checksum for %s' % proj)
if dry_run():
return
try:
remove(get_checksum_file(ws, proj))
except OSError as e:
if e.errno != errno.ENOENT:
raise
def sync_config(ws):
'''Writes out the config if and only if it changed since we first read
it.'''
if _WS_CONFIG == _ORIG_WS_CONFIG:
log('ws config did not change, so not updating')
return
log('updating config at %s' % ws)
if dry_run():
return
write_config(ws, _WS_CONFIG)
def get_ws_config(ws): # noqa: E302
'''Parses the current workspace config, returning a dictionary of the
state.'''
global _WS_CONFIG
if _WS_CONFIG is None:
config_path = get_ws_config_path(ws)
with open(config_path, 'r') as f:
_WS_CONFIG = yaml.safe_load(f)
global _ORIG_WS_CONFIG
# Save a copy of the config so we know later whether or not to
# write it out when someone asks to sync the config.
_ORIG_WS_CONFIG = copy.deepcopy(_WS_CONFIG)
# Check if projects were added or removed from the manifest. If so, the
# config needs to be updated accordingly.
d = parse_manifest(get_ws_root(ws))
deletions = []
for proj in _WS_CONFIG['projects']:
if proj not in d:
deletions.append(proj)
for proj in deletions:
del _WS_CONFIG['projects'][proj]
checksum_file = get_checksum_file(ws, proj)
proj_dir = get_proj_dir(ws, proj)
log('removing project %s, which is not in the manifest' % proj,
logging.INFO)
remove(checksum_file, True)
rmtree(proj_dir, True)
for proj in d:
if proj in _WS_CONFIG['projects']:
continue
# Project is not in the config, so add it in.
_WS_CONFIG['projects'][proj] = get_new_config(proj)
# Split all project arguments by spaces so that args like '-D something'
# turn into ['-D', 'something'], which is what exec requires. We do this
# at parse time rather than in the "config" command to allow the user to
# hand-edit in a natural way and have the config still work properly. The
# next time the config is saved, it will be split by spaces.
for proj, proj_map in _WS_CONFIG['projects'].items():
parsed_args = []
args = proj_map['args']
for arg in args:
parsed_args.extend(arg.split())
proj_map['args'] = parsed_args
return _WS_CONFIG
def merge_includes(root, d, parent_manifest):
'''Recursively merge all the include lines from the manifest into the given
dictionary. Include paths are relative to the including manifest's parent
directory.'''
included = set(include_paths(d, parent_manifest))
queue = collections.deque(included)
while len(queue) > 0:
manifest = queue.popleft()
d_include = parse_yaml(root, manifest)
log('merging manifest %s into %s' % (manifest, parent_manifest))
merge_manifest(d, parent_manifest, d_include, manifest)
for path in include_paths(d_include, manifest):
# Prevent double-inclusion.
if path in included:
continue
queue.append(path)
included.add(path)
def _force_clean(ws, proj):
'''Performs a force-clean of a project, removing all files instead of
politely calling the clean function of the underlying build system.'''
build_dir = get_build_dir(ws, proj)
log('removing %s' % build_dir)
if dry_run():
return
try:
rmtree(build_dir)
except OSError as e:
if e.errno == errno.ENOENT:
log('%s already removed' % build_dir)
else:
raise
config = get_ws_config(ws)
config['projects'][proj]['taint'] = False
def do(cls, ws, args):
'''Executes the env command.'''
build_dir = get_build_dir(ws, args.project)
if not os.path.isdir(build_dir):
raise WSError('build directory for %s doesn\'t exist; have you '
'built it yet?' % args.project)
d = parse_manifest(args.root)
build_env = get_build_env(ws, d, args.project, True)
# Add the build directory to the path for convenience of running
# non-installed binaries, such as unit tests.
merge_var(build_env, 'PATH', [build_dir])
if len(args.command) > 0:
cmd = args.command
else:
cmd = [get_shell()]
exe = os.path.basename(cmd[0])
if exe == 'bash':
# Tweak the prompt to make it obvious we're in a special env.
prompt = (
'\\[\033[1;32m\\][ws:%s env]\\[\033[m\\] \\u@\\h:\\w\\$ ' # noqa: E501
% args.project)
build_env['PS1'] = prompt
cmd.insert(1, '--norc')
# Set an env var so the user can easily cd $WSBUILD and run tests or
# similar inside the build directory.
build_env['WSBUILD'] = build_dir
log('execing with %s build environment: %s' % (args.project, cmd))
if args.build_dir:
args.current_dir = build_dir
if args.current_dir is not None:
os.chdir(args.current_dir)
os.execvpe(cmd[0], cmd, build_env)
def rename(old, new):
'''Renames the given path.'''
log('renaming %s --> %s' % (old, new))
if not dry_run():
os.rename(old, new)
def symlink(dest, src):
'''Makes a symlink from src to dest.'''
log('making symlink %s --> %s' % (src, dest))
if not dry_run():
os.symlink(dest, src)
def mkdir(path):
'''Makes a directory.'''
log('making directory %s' % path)
if not dry_run():
os.mkdir(path)
def _build(root, ws, proj, d, current, ws_config, force):
'''Builds a given project.'''
if not ws_config['projects'][proj]['enable']:
log('not building manually disabled project %s' % proj,
logging.WARNING)
return True
if ws_config['projects'][proj]['taint']:
log('force-cleaning tainted project %s' % proj, logging.WARNING)
clean(root, ws, proj, d, True)
source_dir = get_source_dir(root, d, proj)
if not force:
stored = get_stored_checksum(ws, proj)
if current == stored:
log('checksum for %s is current; skipping' % proj)
return True
else:
log('forcing a build of %s' % proj)
# Make the project directory if needed.
proj_dir = get_proj_dir(ws, proj)
try:
mkdir(proj_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
# Make the build directory if needed.
build_dir = get_build_dir(ws, proj)
try:
mkdir(build_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
needs_configure = False
else:
needs_configure = True
# Populate the convenience source link.
source_link = get_source_link(ws, proj)
if not os.path.exists(source_link):
source_dir = get_source_dir(root, d, proj)
symlink(source_dir, source_link)
# Invalidate the checksums for any downstream projects.
for downstream_dep in d[proj]['downstream']:
invalidate_checksum(ws, downstream_dep)
# Add envs to find all projects on which this project is dependent.
build_env = get_build_env(ws, d, proj)
# Configure.
builder = get_builder(d, proj)
prefix = get_install_dir(ws, proj)
extra_args = d[proj]['args'] + ws_config['projects'][proj]['args']
if needs_configure:
try:
success = builder.conf(proj, prefix, source_dir, build_dir,
build_env, ws_config['type'],
d[proj]['builder-args'], extra_args)
except Exception as _e:
success = False
e = _e
else:
e = None
if not success:
# Remove the build directory if we failed so that we are forced to
# re-run configure next time.
rmtree(build_dir)
if e is not None:
raise e
else:
return False
# Build.
success = builder.build(proj, prefix, source_dir, build_dir, build_env,
d[proj]['targets'], d[proj]['builder-args'],
d[proj]['args'])
if success:
set_stored_checksum(ws, proj, current)
return success
def do(cls, ws, args):
'''Executes the config command.'''
config = get_ws_config(ws)
if args.list:
print(yaml.dump(config, default_flow_style=False), end='')
return
for arg in args.options:
split = arg.split('=')
split_len = len(split)
key = split[0]
if split_len == 1:
val = None
elif split_len == 2:
val = split[1]
else:
val = '='.join(split[1:])
project = args.project
if project is not None:
# Project-specific option.
d = parse_manifest(args.root)
if project not in d:
raise WSError('project "%s" not found' % project)
can_taint = False
if key == 'enable':
val = parse_bool_val(val)
can_taint = True
elif key == 'args':
val = parse_build_args(val)
if val is None:
raise WSError('build args are not in the right format '
'("args=key=val")')
can_taint = True
else:
raise WSError('project key "%s" not found' % key)
try:
proj_config = config['projects'][project]
except KeyError:
proj_config = {}
config['projects'][project] = proj_config
if can_taint and proj_config[key] != val:
log('tainting project %s' % project)
proj_config['taint'] = True
proj_config[key] = val
else:
# Global option.
if key == 'type':
if val is None or val not in BUILD_TYPES:
raise WSError('"type" key must be one of %s' %
str(BUILD_TYPES))
if config[key] != val:
for proj_config in config['projects'].values():
proj_config['taint'] = True
config[key] = val
