def load(target_path, ext=None, **kwargs):
'''
Select an appropriate loader based on the file extension, or `ext` if specified.
If a loader is found, then `open_dep` is called with the default `open_dep_kwargs` registered by `add_loader`,
except updated by any values with matching keys in `kwargs`.
The remaining `kwargs` are passed to the loader function registered by `add_loader`.
Thus, keyword arguments passed to `load` get divvied up between `open_dep` and the custom load function.
If no loader is found, raise an error.
Muck's static analysis looks specifically for this function to infer dependencies;
`target_path` must be a string literal.
'''
if ext is None:
ext = path_ext(target_path)
elif not isinstance(ext, str): raise TypeError(ext)
try: load_fn, std_open_args = _loaders[ext]
except KeyError:
errFL('ERROR: No loader found for target: {!r}', target_path)
errFL('NOTE: extension: {!r}', ext)
raise
open_args = std_open_args.copy()
# transfer all matching kwargs to open_args.
for k in _open_deps_parameters:
try: v = kwargs[k]
except KeyError: continue
open_args[k] = v
del kwargs[k] # only pass this arg to open_deps; del is safe because kwargs has local lifetime.
file = open_dep(target_path, **open_args)
return load_fn(file, **kwargs)
def muck_patch(ctx, args):
if not len(args) in (1, 2):
failF('''\
muck patch error: patch command takes one or two arguments. usage:
muck patch [original_target] [target]
creates a new target by copying either the source or product of the original to _build/[target],
and then creates an empty [target].pat.
muck patch [target.pat]
update the patch file with the diff of the previously specified original and target.
''')
if len(args) == 2: # create new patch.
orig_target_path, target_path = args
if orig_target_path.endswith('.pat'):
errFL('muck patch error: original should not be a patch file: {}', orig_target_path)
if target_path.endswith('.pat'):
errFL('muck patch error: {} {}: target should not be a patch file: {}', target_path)
patch_path = target_path + '.pat'
if path_exists(patch_path):
failF('muck patch error: {}: patch already exists.', patch_path)
update_dependency(ctx, orig_target_path, dependent=None)
orig_path = actual_path_for_target(orig_target_path)
prod_path = product_path_for_target(target_path)
if path_exists(prod_path):
errFL('muck patch note: product already exists: {}', prod_path)
else:
errFL('muck patch note: copying original to product: {} -> {}', orig_path, prod_path)
copy_file(orig_path, prod_path)
else: # update existing patch.
patch_path = args[0]
if path_ext(patch_path) != '.pat':
failF('muck patch error: argument does not specify a .pat file: {!r}', patch_path)
deps = pat_dependencies(patch_path, open(patch_path), {})
orig_target_path = deps[0]
update_dependency(ctx, orig_target_path, dependent=None)
orig_path = actual_path_for_target(orig_target_path)
target_path = path_stem(patch_path)
prod_path = product_path_for_target(target_path)
# update patch (both cases).
patch_path_tmp = patch_path + tmp_ext
cmd = ['pat', 'diff', orig_path, prod_path]
errFL('muck patch note: diffing: `{}`', ' '.join(shlex.quote(w) for w in cmd))
with open(patch_path_tmp, 'wb') as f:
code = runC(cmd, out=f)
move_file(patch_path_tmp, patch_path, overwrite=True)
if len(args) == 1: # updated existing patch.
# need to remove or update the target record to avoid the 'did you mean to patch?' safeguard.
# for now, just delete it to be safe; this makes the target look stale.
try:
ctx.db.delete_record(target_path=target_path)
except KeyError: pass
def calc_dependencies(path, dir_names):
'''
Infer the dependencies for the file at `path`.
'''
ext = path_ext(path)
try:
dep_fn = dependency_fns[ext]
except KeyError:
return []
with open(path) as f:
return sorted(dep_fn(path, f, dir_names))
def list_dir_filtered(src_dir, cache):
'''
Given src_dir, Cache and return the list of names that might be source files.
TODO: eventually this should be replaced by using os.scandir.
'''
try: return cache[src_dir]
except KeyError: pass
names = [n for n in list_dir(src_dir, hidden=False)
if n not in reserved_names and path_ext(n) not in ignored_exts]
if cache is not None:
cache[dir] = names
return names
def update_dependency(ctx: Ctx, target_path: str, dependent: Optional[str], force=False) -> bool:
'''
returns is_changed.
'''
target_ext = path_ext(target_path)
if not target_path.strip():
failF(repr(target_path), 'invalid target name.')
if target_path in reserved_names:
failF(target_path, 'target name is reserved; please rename the target.')
if target_ext in reserved_exts:
failF(target_path, 'target name has reserved extension; please rename the target.')
if dependent is not None:
ctx.dependents[target_path].add(dependent)
try: # if in ctx.statuses, this path has already been visited on this run.
status = ctx.statuses[target_path]
if status is Ellipsis: # recursion sentinal.
involved_paths = sorted(path for path, status in ctx.statuses.items() if status is Ellipsis)
failF(target_path, 'target has circular dependency; involved paths:\n {}',
'\n '.join(involved_paths))
return status
except KeyError: pass
ctx.statuses[target_path] = Ellipsis # recursion sentinal is replaced before return.
ctx.dbgF(target_path, 'examining... (dependent={})', dependent)
is_product = not path_exists(target_path)
actual_path = product_path_for_target(target_path) if is_product else target_path
size, mtime, old = calc_size_mtime_old(ctx, target_path, actual_path)
has_old_file = (mtime > 0)
has_old_record = not is_empty_record(old)
is_changed = force or (not has_old_file) or (not has_old_record)
if has_old_record:
old_is_product = (old.src is not None)
if is_product != old_is_product: # nature of the target changed.
noteF(target_path, 'target is {} a product.', 'now' if is_product else 'no longer')
is_changed = True
if not has_old_file and target_ext: # product was deleted and not a phony target.
noteF(target_path, 'old product was deleted.')
if is_product:
if has_old_file and has_old_record:
check_product_not_modified(ctx, target_path, actual_path, size=size, mtime=mtime, old=old)
return update_product(ctx, target_path, actual_path, is_changed=is_changed, size=size, mtime=mtime, old=old)
else:
return update_non_product(ctx, target_path, is_changed=is_changed, size=size, mtime=mtime, old=old)
def main():
parser = ArgumentParser(description='Count lines of source code.')
parser.add_argument('paths', nargs='+', help='Directories to explore.')
args = parser.parse_args()
ext_counts = defaultdict(int)
for path in walk_files(*args.paths):
ext = path_ext(path)
ext_counts[ext] += 1
empty_name = '[none]'
max_len = max(len(empty_name), max(len(k) for k in ext_counts))
for k in sorted(ext_counts.keys()):
v = ext_counts[k]
print('{0:{1}}: {2}'.format(k or empty_name, max_len, v))
def build_product(ctx, target_path: str, src_path: str, prod_path: str) -> bool:
'''
Run a source file, producing zero or more products.
Return a list of produced product paths.
'''
src_ext = path_ext(src_path)
try:
build_tool = build_tools[src_ext]
except KeyError:
# TODO: fall back to generic .deps file.
failF(target_path, 'unsupported source file extension: `{}`', src_ext)
prod_path_out = prod_path + out_ext
prod_path_tmp = prod_path + tmp_ext
remove_file_if_exists(prod_path_out)
remove_file_if_exists(prod_path_tmp)
if not build_tool:
noteF(target_path, 'no op.')
return False # no product.
prod_dir = path_dir(prod_path)
make_dirs(prod_dir)
# Extract args from the combination of wilds in the source and the matching target.
m = match_wilds(target_path_for_source(src_path), target_path)
if m is None:
failF(target_path, 'internal error: match failed; src_path: {!r}', src_path)
argv = [src_path] + list(m.groups())
cmd = build_tool + argv
try: env_fn = build_tool_env_fns[src_ext]
except KeyError: env = None
else:
env = os.environ.copy()
custom_env = env_fn()
env.update(custom_env)
noteF(target_path, 'building: `{}`', ' '.join(shlex.quote(w) for w in cmd))
out_file = open(prod_path_out, 'wb')
time_start = time.time()
code = runC(cmd, env=env, out=out_file)
time_elapsed = time.time() - time_start
out_file.close()
if code != 0:
failF(target_path, 'build failed with code: {}', code)
def cleanup_out():
if file_size(prod_path_out) == 0:
remove_file(prod_path_out)
else:
warnF(target_path, 'wrote data directly to `{}`;\n ignoring output captured in `{}`', prod_path_tmp, prod_path_out)
manif_path = manifest_path(argv)
try: f = open(manif_path)
except FileNotFoundError: # no list.
if not path_exists(prod_path_tmp):
via = 'stdout'
tmp_paths = [prod_path_out]
else:
via = 'tmp'
tmp_paths = [prod_path_tmp]
cleanup_out()
else:
via = 'manifest'
tmp_paths = list(line[:-1] for line in f) # strip newlines.
cleanup_out()
if ('%' not in prod_path_tmp) and prod_path_tmp not in tmp_paths:
failF(target_path, 'product does not appear in manifest ({} records): {}',
len(tmp_paths), manif_path)
remove_file(manif_path)
time_msg = '{:0.2f} seconds '.format(time_elapsed) if ctx.report_times else ''
noteF(target_path, 'finished: {}(via {}).', time_msg, via)
return tmp_paths
def mush_dependencies(src_path, src_file, dir_names):
'Calculate dependencies for .mush files.'
for line in src_file:
for token in shlex.split(line):
if path_ext(token):
yield token