def init(caf): """ Initialize the Caf repository. Usage: caf init By default create directory in .caf/db. If 'cache' is defined in ~/.config/caf/conf.yaml, the repository is created there and symlinked to .caf/db, otherwise it is created locally. """ if 'cache' in caf.conf: timestamp = get_timestamp() cache_path = Path(caf.conf['cache'])/'{}_{}'.format(Path().resolve().name, timestamp) mkdir(cache_path) relink(cache_path, caf.cache, relative=False) else: cache_path = caf.cache if cache_path.exists(): error('{} exists, cannot overwrite'.format(cache_path)) mkdir(cache_path) info('Initializing an empty repository at {}.'.format(cache_path)) mkdir(caf.cellar) mkdir(caf.brewery) with open('.gitignore', 'w') as f: f.write('\n'.join(['.caf'])) with open(os.devnull, 'w') as null: sp.call(['git', 'init'], stdout=null) sp.call(['git', 'add', 'caf', 'cscript.py', '.gitignore'], stdout=null) sp.call(['git', 'commit', '-m', 'initial commit'], stdout=null)
def make_targets(self, out, cache): for target, tasks in self.targets.items(): if len(tasks) == 1 and None in tasks: relink(Path('../.caf/db')/'/'.join(tasks[None].path.parts[-4:]), out/target, relative=False) else: if not (out/target).is_dir(): mkdir(out/target) for name, task in tasks.items(): relink(Path('../../.caf/db')/'/'.join(task.path.parts[-4:]), out/target/name, relative=False)
def build(self, path): """Prepare, lock and store the task. Check if not already locked. Touch (link in children, save chilren to .caf/children). Check if needed children are already sealed. Check if children are already locked. Prepare task. Get hashes. Lock task with hashes. Check if a task has been already stored previously and if not, store it and relink children. """ with timing('task init'): if not path.is_dir(): mkdir(path) self.path = Path(path).resolve() if self.is_locked(): warn('{} already locked'.format(self)) return if not self.is_touched(): self.touch() for linkname, link in self.links.items(): if link.needed and not link.task.is_sealed(): warn('{}: dependency "{}" not sealed'.format(self, linkname)) return if not all(child.is_locked() for child in self.children): return with timing('prepare'): self.prepare() with timing('hash'): hashes = self.get_hashes() with timing('lock'): self.lock(hashes) myhash = get_file_hash(self.path/'.caf/lock') with timing('storing'): cellarpath = self.ctx.cellar/str_to_path(myhash) if cellarpath.is_dir(): env_file = Path(self.path/'.caf/env') if env_file.is_file(): env_file.rename(cellarpath/'.caf/env') shutil.rmtree(str(self.path)) else: info('Stored new task {}'.format(self)) mkdir(cellarpath.parent, parents=True, exist_ok=True) self.path.rename(cellarpath) relink(cellarpath, self.path) self.path = cellarpath with timing('linking deps'): self.link_deps()
def store_link_text(self, text, target, label=None): h = hashlib.new(hashf) h.update(text.encode()) texthash = h.hexdigest() cellarpath = self.ctx.cellar/str_to_path(texthash) if not cellarpath.is_file(): if label is True: info('Stored new file "{}"'.format(target)) elif label: info('Stored new text labeled "{}"'.format(label)) else: info('Stored new text') mkdir(cellarpath.parent, parents=True, exist_ok=True) with cellarpath.open('w') as f: f.write(text) make_nonwritable(cellarpath) with cd(self.path): relink(os.path.relpath(str(cellarpath)), target) self.files[target] = cellarpath
def build(caf, dry: '--dry', do_init: 'init'): """ Prepare tasks and targets defined in cscript. Usage: caf [init] build [--dry] Options: -n, --dry Dry run (do not write to disk). Tasks are created in .caf/db/Brewery/Latest and if their preparation does not depened on unfinished tasks, they are prepared and stored in .caf/db/Cellar based on their SHA1 hash. Targets (collections of symlinks to tasks) are created in ./build. """ if not hasattr(caf.cscript, 'build'): error('cscript has to contain function build(ctx)') if do_init: init(['caf', 'init'], caf) ctx = Context(caf.cache/cellar, caf.top, caf.libpath) with timing('dependency tree'): caf.cscript.build(ctx) if not dry: timestamp = get_timestamp() mkdir(caf.brewery/timestamp) relink(timestamp, caf.brewery/latest, relative=False) with timing('build'): ctx.build(caf.brewery/latest) if caf.out.is_dir(): shutil.rmtree(str(caf.out)) mkdir(caf.out) with timing('targets'): ctx.make_targets(caf.out, caf.cache) if hasattr(caf.cscript, 'json'): warn('Make sure json is not printing dictionaries in features') with open(os.devnull, 'w') as null: sp.call(['git', 'add', '--all', 'build'], stdout=null) sp.call(['git', 'commit', '-a', '-m', '#build'], stdout=null)
def store_link_file(self, source, target=None): try: text = source.getvalue() except AttributeError: pass else: assert target self.store_link_text(text, target) return if not target: target = source if Path(source).is_dir(): (self.path/target).mkdir() return filehash = get_file_hash(Path(source)) cellarpath = self.ctx.cellar/str_to_path(filehash) if not cellarpath.is_file(): info('Stored new file "{}"'.format(source)) mkdir(cellarpath.parent, parents=True, exist_ok=True) shutil.copy(str(source), str(cellarpath)) make_nonwritable(cellarpath) with cd(self.path): relink(os.path.relpath(str(cellarpath)), target) self.files[str(target)] = cellarpath
def link_deps(self): with cd(self.path): for linkname, link in self.links.items(): relink(os.path.relpath(str(link.task.path)), linkname) for filename, path in self.files.items(): try: relink(os.path.relpath(str(path)), filename) except FileExistsError: if 'RELINK' in os.environ: Path(filename).unlink() relink(os.path.relpath(str(path)), filename) else: error('Something replaced a linked file "{}" with a real file in {}' .format(filename, self))
def prepare(self): """Prepare a task. Pull in files and templates, link in files from children, execute features and save the command. Check that all attributes have been consumed. """ try: features = OrderedDict((feat, _features[feat]) if isinstance(feat, str) else (feat.__name__, feat) for feat in listify(self.consume('features'))) except KeyError as e: error('Feature {} is not registered'.format(e.args[0])) self.process_features(features, 'before_files') with cd(self.ctx.top): with timing('files'): for filename in listify(self.consume('files')): if isinstance(filename, tuple): self.store_link_file(filename[0], filename[1]) else: if isinstance(filename, str) \ and ('*' in filename or '?' in filename): for member in glob(filename): self.store_link_file(member) else: self.store_link_file(filename) with timing('hooks'): hooks = {filename: process_hook(filename) for filename in listify(self.consume('hooks'))} with timing('templates'): templates = {} for filename in listify(self.consume('templates')): if isinstance(filename, tuple): source, target = filename elif isinstance(filename, str): source = target = filename else: error("Don't know how to store {!r}".format(filename)) templates[target] = Template(source) with cd(self.path): self.process_features(features, 'before_templates') with timing('templates'): for target, template in templates.items(): processed, used = template.substitute(self.attrs) self.store_link_text(processed, target, template.name) for attr in used: self.consume(attr) with timing('linking'): for linkname, link in self.links.items(): for symlink in link.links: if isinstance(symlink, tuple): target, symlink = symlink else: target = symlink relink('{}/{}'.format(linkname, target), symlink) self.process_features(features) commands = [] env = defaultdict(list) for var, val in (self.consume('_env') or {}).items(): env[var].append(str(val)) for hook_path, (hook_src, hook_cmd, hook_env) in hooks.items(): commands.append(hook_cmd) for var, vals in hook_env.items(): env[var].extend(vals) self.store_link_text(hook_src, hook_path, label=True) command = self.consume('command') if command: commands.append(command) if commands: with open('command', 'w') as f: f.write('\n'.join(commands)) if env: with open('.caf/env', 'w') as f: for var, vals in env.items(): f.write('export {}={}\n' .format(var, ':'.join(map(str, vals)))) if self.attrs: error('Task {} has non-consumed attributs: {}' .format(self, list(self.attrs)))