def process_features(self, features, attrib=None): with timing('features'): for name, feat in list(features.items()): if not attrib or attrib in getattr(feat, 'feature_attribs', []): with timing(name): try: feat(self) except PermissionError as e: error('Feature "{}" tried to change stored file "{}"' .format(name, e.filename)) del features[name]
def build(caf, dry: '--dry', do_init: 'init'): """ Prepare tasks and targets defined in cscript. Usage: caf [init] build [--dry] Options: -n, --dry Dry run (do not write to disk). Tasks are created in .caf/db/Brewery/Latest and if their preparation does not depened on unfinished tasks, they are prepared and stored in .caf/db/Cellar based on their SHA1 hash. Targets (collections of symlinks to tasks) are created in ./build. """ if not hasattr(caf.cscript, 'build'): error('cscript has to contain function build(ctx)') if do_init: init(['caf', 'init'], caf) ctx = Context(caf.cache/cellar, caf.top, caf.libpath) with timing('dependency tree'): caf.cscript.build(ctx) if not dry: timestamp = get_timestamp() mkdir(caf.brewery/timestamp) relink(timestamp, caf.brewery/latest, relative=False) with timing('build'): ctx.build(caf.brewery/latest) if caf.out.is_dir(): shutil.rmtree(str(caf.out)) mkdir(caf.out) with timing('targets'): ctx.make_targets(caf.out, caf.cache) if hasattr(caf.cscript, 'json'): warn('Make sure json is not printing dictionaries in features') with open(os.devnull, 'w') as null: sp.call(['git', 'add', '--all', 'build'], stdout=null) sp.call(['git', 'commit', '-a', '-m', '#build'], stdout=null)
def build(self, batch): try: batch = batch.resolve() except FileNotFoundError: error('Batch does not exist, maybe `caf build new` first?') with timing('task sorting'): self.sort_tasks() ntskdigit = ceil(log10(len(self.tasks)+1)) with ProgressBar(maxval=len(self.tasks), redirect_stdout=True) as progress: for i, task in enumerate(self.tasks): task.build(batch/'{:0{n}d}'.format(i, n=ntskdigit)) progress.update(i) for report in _reports: report()
def build(self, path): """Prepare, lock and store the task. Check if not already locked. Touch (link in children, save chilren to .caf/children). Check if needed children are already sealed. Check if children are already locked. Prepare task. Get hashes. Lock task with hashes. Check if a task has been already stored previously and if not, store it and relink children. """ with timing('task init'): if not path.is_dir(): mkdir(path) self.path = Path(path).resolve() if self.is_locked(): warn('{} already locked'.format(self)) return if not self.is_touched(): self.touch() for linkname, link in self.links.items(): if link.needed and not link.task.is_sealed(): warn('{}: dependency "{}" not sealed'.format(self, linkname)) return if not all(child.is_locked() for child in self.children): return with timing('prepare'): self.prepare() with timing('hash'): hashes = self.get_hashes() with timing('lock'): self.lock(hashes) myhash = get_file_hash(self.path/'.caf/lock') with timing('storing'): cellarpath = self.ctx.cellar/str_to_path(myhash) if cellarpath.is_dir(): env_file = Path(self.path/'.caf/env') if env_file.is_file(): env_file.rename(cellarpath/'.caf/env') shutil.rmtree(str(self.path)) else: info('Stored new task {}'.format(self)) mkdir(cellarpath.parent, parents=True, exist_ok=True) self.path.rename(cellarpath) relink(cellarpath, self.path) self.path = cellarpath with timing('linking deps'): self.link_deps()
def __init__(self, libpath): super().__init__('caf') self.conf = Configuration('.caf/conf.yaml') self.conf.set_global(Configuration('{}/.config/caf/conf.yaml' .format(os.environ['HOME']))) for cscriptname in ['cscript', 'cscript.py']: if Path(cscriptname).is_file(): break else: cscriptname = None with timing('reading cscript'): try: self.cscript = load_module(cscriptname, self.commands[('unpack',)]._func) \ if cscriptname else object() except RuntimeError: error('There was an error while reading cscript.') self.out = Path(getattr(self.cscript, 'out', 'build')) self.cache = Path(getattr(self.cscript, 'cache', '.caf/db')) self.top = Path(getattr(self.cscript, 'top', '.')) self.cellar = self.cache/cellar self.brewery = self.cache/brewery self.remotes = {name: Remote(r['host'], r['path'], self.top) for name, r in self.conf.get('remotes', {}).items()} self.libpath = libpath
def prepare(self): """Prepare a task. Pull in files and templates, link in files from children, execute features and save the command. Check that all attributes have been consumed. """ try: features = OrderedDict((feat, _features[feat]) if isinstance(feat, str) else (feat.__name__, feat) for feat in listify(self.consume('features'))) except KeyError as e: error('Feature {} is not registered'.format(e.args[0])) self.process_features(features, 'before_files') with cd(self.ctx.top): with timing('files'): for filename in listify(self.consume('files')): if isinstance(filename, tuple): self.store_link_file(filename[0], filename[1]) else: if isinstance(filename, str) \ and ('*' in filename or '?' in filename): for member in glob(filename): self.store_link_file(member) else: self.store_link_file(filename) with timing('hooks'): hooks = {filename: process_hook(filename) for filename in listify(self.consume('hooks'))} with timing('templates'): templates = {} for filename in listify(self.consume('templates')): if isinstance(filename, tuple): source, target = filename elif isinstance(filename, str): source = target = filename else: error("Don't know how to store {!r}".format(filename)) templates[target] = Template(source) with cd(self.path): self.process_features(features, 'before_templates') with timing('templates'): for target, template in templates.items(): processed, used = template.substitute(self.attrs) self.store_link_text(processed, target, template.name) for attr in used: self.consume(attr) with timing('linking'): for linkname, link in self.links.items(): for symlink in link.links: if isinstance(symlink, tuple): target, symlink = symlink else: target = symlink relink('{}/{}'.format(linkname, target), symlink) self.process_features(features) commands = [] env = defaultdict(list) for var, val in (self.consume('_env') or {}).items(): env[var].append(str(val)) for hook_path, (hook_src, hook_cmd, hook_env) in hooks.items(): commands.append(hook_cmd) for var, vals in hook_env.items(): env[var].extend(vals) self.store_link_text(hook_src, hook_path, label=True) command = self.consume('command') if command: commands.append(command) if commands: with open('command', 'w') as f: f.write('\n'.join(commands)) if env: with open('.caf/env', 'w') as f: for var, vals in env.items(): f.write('export {}={}\n' .format(var, ':'.join(map(str, vals)))) if self.attrs: error('Task {} has non-consumed attributs: {}' .format(self, list(self.attrs)))