def process_features(self, features, attrib=None):
with timing('features'):
for name, feat in list(features.items()):
if not attrib or attrib in getattr(feat, 'feature_attribs', []):
with timing(name):
try:
feat(self)
except PermissionError as e:
error('Feature "{}" tried to change stored file "{}"'
.format(name, e.filename))
del features[name]
def build(caf, dry: '--dry', do_init: 'init'):
"""
Prepare tasks and targets defined in cscript.
Usage:
caf [init] build [--dry]
Options:
-n, --dry Dry run (do not write to disk).
Tasks are created in .caf/db/Brewery/Latest and if their preparation does
not depened on unfinished tasks, they are prepared and stored in
.caf/db/Cellar based on their SHA1 hash. Targets (collections of symlinks to
tasks) are created in ./build.
"""
if not hasattr(caf.cscript, 'build'):
error('cscript has to contain function build(ctx)')
if do_init:
init(['caf', 'init'], caf)
ctx = Context(caf.cache/cellar, caf.top, caf.libpath)
with timing('dependency tree'):
caf.cscript.build(ctx)
if not dry:
timestamp = get_timestamp()
mkdir(caf.brewery/timestamp)
relink(timestamp, caf.brewery/latest, relative=False)
with timing('build'):
ctx.build(caf.brewery/latest)
if caf.out.is_dir():
shutil.rmtree(str(caf.out))
mkdir(caf.out)
with timing('targets'):
ctx.make_targets(caf.out, caf.cache)
if hasattr(caf.cscript, 'json'):
warn('Make sure json is not printing dictionaries in features')
with open(os.devnull, 'w') as null:
sp.call(['git', 'add', '--all', 'build'], stdout=null)
sp.call(['git', 'commit', '-a', '-m', '#build'], stdout=null)
def build(self, batch):
try:
batch = batch.resolve()
except FileNotFoundError:
error('Batch does not exist, maybe `caf build new` first?')
with timing('task sorting'):
self.sort_tasks()
ntskdigit = ceil(log10(len(self.tasks)+1))
with ProgressBar(maxval=len(self.tasks), redirect_stdout=True) as progress:
for i, task in enumerate(self.tasks):
task.build(batch/'{:0{n}d}'.format(i, n=ntskdigit))
progress.update(i)
for report in _reports:
report()
def build(self, path):
"""Prepare, lock and store the task.
Check if not already locked. Touch (link in children, save chilren to
.caf/children). Check if needed children are already sealed. Check if
children are already locked. Prepare task. Get hashes. Lock task with
hashes. Check if a task has been already stored previously and if not,
store it and relink children.
"""
with timing('task init'):
if not path.is_dir():
mkdir(path)
self.path = Path(path).resolve()
if self.is_locked():
warn('{} already locked'.format(self))
return
if not self.is_touched():
self.touch()
for linkname, link in self.links.items():
if link.needed and not link.task.is_sealed():
warn('{}: dependency "{}" not sealed'.format(self, linkname))
return
if not all(child.is_locked() for child in self.children):
return
with timing('prepare'):
self.prepare()
with timing('hash'):
hashes = self.get_hashes()
with timing('lock'):
self.lock(hashes)
myhash = get_file_hash(self.path/'.caf/lock')
with timing('storing'):
cellarpath = self.ctx.cellar/str_to_path(myhash)
if cellarpath.is_dir():
env_file = Path(self.path/'.caf/env')
if env_file.is_file():
env_file.rename(cellarpath/'.caf/env')
shutil.rmtree(str(self.path))
else:
info('Stored new task {}'.format(self))
mkdir(cellarpath.parent, parents=True, exist_ok=True)
self.path.rename(cellarpath)
relink(cellarpath, self.path)
self.path = cellarpath
with timing('linking deps'):
self.link_deps()
def __init__(self, libpath):
super().__init__('caf')
self.conf = Configuration('.caf/conf.yaml')
self.conf.set_global(Configuration('{}/.config/caf/conf.yaml'
.format(os.environ['HOME'])))
for cscriptname in ['cscript', 'cscript.py']:
if Path(cscriptname).is_file():
break
else:
cscriptname = None
with timing('reading cscript'):
try:
self.cscript = load_module(cscriptname, self.commands[('unpack',)]._func) \
if cscriptname else object()
except RuntimeError:
error('There was an error while reading cscript.')
self.out = Path(getattr(self.cscript, 'out', 'build'))
self.cache = Path(getattr(self.cscript, 'cache', '.caf/db'))
self.top = Path(getattr(self.cscript, 'top', '.'))
self.cellar = self.cache/cellar
self.brewery = self.cache/brewery
self.remotes = {name: Remote(r['host'], r['path'], self.top)
for name, r in self.conf.get('remotes', {}).items()}
self.libpath = libpath
def prepare(self):
"""Prepare a task.
Pull in files and templates, link in files from children, execute
features and save the command. Check that all attributes have been
consumed.
"""
try:
features = OrderedDict((feat, _features[feat])
if isinstance(feat, str)
else (feat.__name__, feat)
for feat in listify(self.consume('features')))
except KeyError as e:
error('Feature {} is not registered'.format(e.args[0]))
self.process_features(features, 'before_files')
with cd(self.ctx.top):
with timing('files'):
for filename in listify(self.consume('files')):
if isinstance(filename, tuple):
self.store_link_file(filename[0], filename[1])
else:
if isinstance(filename, str) \
and ('*' in filename or '?' in filename):
for member in glob(filename):
self.store_link_file(member)
else:
self.store_link_file(filename)
with timing('hooks'):
hooks = {filename: process_hook(filename)
for filename in listify(self.consume('hooks'))}
with timing('templates'):
templates = {}
for filename in listify(self.consume('templates')):
if isinstance(filename, tuple):
source, target = filename
elif isinstance(filename, str):
source = target = filename
else:
error("Don't know how to store {!r}".format(filename))
templates[target] = Template(source)
with cd(self.path):
self.process_features(features, 'before_templates')
with timing('templates'):
for target, template in templates.items():
processed, used = template.substitute(self.attrs)
self.store_link_text(processed, target, template.name)
for attr in used:
self.consume(attr)
with timing('linking'):
for linkname, link in self.links.items():
for symlink in link.links:
if isinstance(symlink, tuple):
target, symlink = symlink
else:
target = symlink
relink('{}/{}'.format(linkname, target), symlink)
self.process_features(features)
commands = []
env = defaultdict(list)
for var, val in (self.consume('_env') or {}).items():
env[var].append(str(val))
for hook_path, (hook_src, hook_cmd, hook_env) in hooks.items():
commands.append(hook_cmd)
for var, vals in hook_env.items():
env[var].extend(vals)
self.store_link_text(hook_src, hook_path, label=True)
command = self.consume('command')
if command:
commands.append(command)
if commands:
with open('command', 'w') as f:
f.write('\n'.join(commands))
if env:
with open('.caf/env', 'w') as f:
for var, vals in env.items():
f.write('export {}={}\n'
.format(var, ':'.join(map(str, vals))))
if self.attrs:
error('Task {} has non-consumed attributs: {}'
.format(self, list(self.attrs)))
