def get_hashes(self): """Get hashes of task's dependencies. Dependencies consist of all files and on locks of children. """ with cd(self.path): hashes = {} for dirpath, dirnames, filenames in os.walk('.'): if dirpath == '.': dirnames[:] = [name for name in dirnames if name not in ['.caf'] + list(self.links)] for name in filenames: filepath = Path(dirpath)/name if filepath.is_symlink(): target = os.readlink(str(filepath)) if Path(target).is_absolute(): error('Cannot link to absolute paths in tasks') if str(filepath) in self.files: with cd(filepath.parent): hashes[str(filepath)] = get_file_hash(Path(target)) else: hashes[str(filepath)] = target else: make_nonwritable(filepath) hashes[str(filepath)] = get_file_hash(filepath) for linkname in self.links: hashes[linkname] = get_file_hash(Path(linkname)/'.caf/lock') return hashes
def run_command(self, taskid): with cd(self.root/taskid): if Path('command').is_file(): with open('command') as f: command = f.read() else: command = '' if Path('.caf/env').is_file(): command = 'source .caf/env\n' + command with open('run.out', 'w') as stdout, open('run.err', 'w') as stderr: try: subprocess.check_call( command, shell=True, stdout=stdout, stderr=stderr ) except subprocess.CalledProcessError as e: print(e) self.info( 'error: There was an error when working on {}'.format(taskid) ) with Path('.caf/error').open('w') as f: f.write(self.myid + '\n') self.task_error(taskid) else: if 'CAFWAIT' in os.environ: from time import sleep sleep(int(os.environ['CAFWAIT'])) with Path('.caf/seal').open('w') as f: f.write(self.myid + '\n') if Path('.caf/error').is_file(): Path('.caf/error').unlink() self.task_done(taskid)
def link_deps(self): with cd(self.path): for linkname, link in self.links.items(): relink(os.path.relpath(str(link.task.path)), linkname) for filename, path in self.files.items(): try: relink(os.path.relpath(str(path)), filename) except FileExistsError: if 'RELINK' in os.environ: Path(filename).unlink() relink(os.path.relpath(str(path)), filename) else: error('Something replaced a linked file "{}" with a real file in {}' .format(filename, self))
def store_link_text(self, text, target, label=None): h = hashlib.new(hashf) h.update(text.encode()) texthash = h.hexdigest() cellarpath = self.ctx.cellar/str_to_path(texthash) if not cellarpath.is_file(): if label is True: info('Stored new file "{}"'.format(target)) elif label: info('Stored new text labeled "{}"'.format(label)) else: info('Stored new text') mkdir(cellarpath.parent, parents=True, exist_ok=True) with cellarpath.open('w') as f: f.write(text) make_nonwritable(cellarpath) with cd(self.path): relink(os.path.relpath(str(cellarpath)), target) self.files[target] = cellarpath
def store_link_file(self, source, target=None): try: text = source.getvalue() except AttributeError: pass else: assert target self.store_link_text(text, target) return if not target: target = source if Path(source).is_dir(): (self.path/target).mkdir() return filehash = get_file_hash(Path(source)) cellarpath = self.ctx.cellar/str_to_path(filehash) if not cellarpath.is_file(): info('Stored new file "{}"'.format(source)) mkdir(cellarpath.parent, parents=True, exist_ok=True) shutil.copy(str(source), str(cellarpath)) make_nonwritable(cellarpath) with cd(self.path): relink(os.path.relpath(str(cellarpath)), target) self.files[str(target)] = cellarpath
def prepare(self): """Prepare a task. Pull in files and templates, link in files from children, execute features and save the command. Check that all attributes have been consumed. """ try: features = OrderedDict((feat, _features[feat]) if isinstance(feat, str) else (feat.__name__, feat) for feat in listify(self.consume('features'))) except KeyError as e: error('Feature {} is not registered'.format(e.args[0])) self.process_features(features, 'before_files') with cd(self.ctx.top): with timing('files'): for filename in listify(self.consume('files')): if isinstance(filename, tuple): self.store_link_file(filename[0], filename[1]) else: if isinstance(filename, str) \ and ('*' in filename or '?' in filename): for member in glob(filename): self.store_link_file(member) else: self.store_link_file(filename) with timing('hooks'): hooks = {filename: process_hook(filename) for filename in listify(self.consume('hooks'))} with timing('templates'): templates = {} for filename in listify(self.consume('templates')): if isinstance(filename, tuple): source, target = filename elif isinstance(filename, str): source = target = filename else: error("Don't know how to store {!r}".format(filename)) templates[target] = Template(source) with cd(self.path): self.process_features(features, 'before_templates') with timing('templates'): for target, template in templates.items(): processed, used = template.substitute(self.attrs) self.store_link_text(processed, target, template.name) for attr in used: self.consume(attr) with timing('linking'): for linkname, link in self.links.items(): for symlink in link.links: if isinstance(symlink, tuple): target, symlink = symlink else: target = symlink relink('{}/{}'.format(linkname, target), symlink) self.process_features(features) commands = [] env = defaultdict(list) for var, val in (self.consume('_env') or {}).items(): env[var].append(str(val)) for hook_path, (hook_src, hook_cmd, hook_env) in hooks.items(): commands.append(hook_cmd) for var, vals in hook_env.items(): env[var].extend(vals) self.store_link_text(hook_src, hook_path, label=True) command = self.consume('command') if command: commands.append(command) if commands: with open('command', 'w') as f: f.write('\n'.join(commands)) if env: with open('.caf/env', 'w') as f: for var, vals in env.items(): f.write('export {}={}\n' .format(var, ':'.join(map(str, vals)))) if self.attrs: error('Task {} has non-consumed attributs: {}' .format(self, list(self.attrs)))