def get_hashes(self):
"""Get hashes of task's dependencies.
Dependencies consist of all files and on locks of children.
"""
with cd(self.path):
hashes = {}
for dirpath, dirnames, filenames in os.walk('.'):
if dirpath == '.':
dirnames[:] = [name for name in dirnames
if name not in ['.caf'] + list(self.links)]
for name in filenames:
filepath = Path(dirpath)/name
if filepath.is_symlink():
target = os.readlink(str(filepath))
if Path(target).is_absolute():
error('Cannot link to absolute paths in tasks')
if str(filepath) in self.files:
with cd(filepath.parent):
hashes[str(filepath)] = get_file_hash(Path(target))
else:
hashes[str(filepath)] = target
else:
make_nonwritable(filepath)
hashes[str(filepath)] = get_file_hash(filepath)
for linkname in self.links:
hashes[linkname] = get_file_hash(Path(linkname)/'.caf/lock')
return hashes
def run_command(self, taskid):
with cd(self.root/taskid):
if Path('command').is_file():
with open('command') as f:
command = f.read()
else:
command = ''
if Path('.caf/env').is_file():
command = 'source .caf/env\n' + command
with open('run.out', 'w') as stdout, open('run.err', 'w') as stderr:
try:
subprocess.check_call(
command, shell=True, stdout=stdout, stderr=stderr
)
except subprocess.CalledProcessError as e:
print(e)
self.info(
'error: There was an error when working on {}'.format(taskid)
)
with Path('.caf/error').open('w') as f:
f.write(self.myid + '\n')
self.task_error(taskid)
else:
if 'CAFWAIT' in os.environ:
from time import sleep
sleep(int(os.environ['CAFWAIT']))
with Path('.caf/seal').open('w') as f:
f.write(self.myid + '\n')
if Path('.caf/error').is_file():
Path('.caf/error').unlink()
self.task_done(taskid)
def link_deps(self):
with cd(self.path):
for linkname, link in self.links.items():
relink(os.path.relpath(str(link.task.path)), linkname)
for filename, path in self.files.items():
try:
relink(os.path.relpath(str(path)), filename)
except FileExistsError:
if 'RELINK' in os.environ:
Path(filename).unlink()
relink(os.path.relpath(str(path)), filename)
else:
error('Something replaced a linked file "{}" with a real file in {}'
.format(filename, self))
def store_link_text(self, text, target, label=None):
h = hashlib.new(hashf)
h.update(text.encode())
texthash = h.hexdigest()
cellarpath = self.ctx.cellar/str_to_path(texthash)
if not cellarpath.is_file():
if label is True:
info('Stored new file "{}"'.format(target))
elif label:
info('Stored new text labeled "{}"'.format(label))
else:
info('Stored new text')
mkdir(cellarpath.parent, parents=True, exist_ok=True)
with cellarpath.open('w') as f:
f.write(text)
make_nonwritable(cellarpath)
with cd(self.path):
relink(os.path.relpath(str(cellarpath)), target)
self.files[target] = cellarpath
def store_link_file(self, source, target=None):
try:
text = source.getvalue()
except AttributeError:
pass
else:
assert target
self.store_link_text(text, target)
return
if not target:
target = source
if Path(source).is_dir():
(self.path/target).mkdir()
return
filehash = get_file_hash(Path(source))
cellarpath = self.ctx.cellar/str_to_path(filehash)
if not cellarpath.is_file():
info('Stored new file "{}"'.format(source))
mkdir(cellarpath.parent, parents=True, exist_ok=True)
shutil.copy(str(source), str(cellarpath))
make_nonwritable(cellarpath)
with cd(self.path):
relink(os.path.relpath(str(cellarpath)), target)
self.files[str(target)] = cellarpath
def prepare(self):
"""Prepare a task.
Pull in files and templates, link in files from children, execute
features and save the command. Check that all attributes have been
consumed.
"""
try:
features = OrderedDict((feat, _features[feat])
if isinstance(feat, str)
else (feat.__name__, feat)
for feat in listify(self.consume('features')))
except KeyError as e:
error('Feature {} is not registered'.format(e.args[0]))
self.process_features(features, 'before_files')
with cd(self.ctx.top):
with timing('files'):
for filename in listify(self.consume('files')):
if isinstance(filename, tuple):
self.store_link_file(filename[0], filename[1])
else:
if isinstance(filename, str) \
and ('*' in filename or '?' in filename):
for member in glob(filename):
self.store_link_file(member)
else:
self.store_link_file(filename)
with timing('hooks'):
hooks = {filename: process_hook(filename)
for filename in listify(self.consume('hooks'))}
with timing('templates'):
templates = {}
for filename in listify(self.consume('templates')):
if isinstance(filename, tuple):
source, target = filename
elif isinstance(filename, str):
source = target = filename
else:
error("Don't know how to store {!r}".format(filename))
templates[target] = Template(source)
with cd(self.path):
self.process_features(features, 'before_templates')
with timing('templates'):
for target, template in templates.items():
processed, used = template.substitute(self.attrs)
self.store_link_text(processed, target, template.name)
for attr in used:
self.consume(attr)
with timing('linking'):
for linkname, link in self.links.items():
for symlink in link.links:
if isinstance(symlink, tuple):
target, symlink = symlink
else:
target = symlink
relink('{}/{}'.format(linkname, target), symlink)
self.process_features(features)
commands = []
env = defaultdict(list)
for var, val in (self.consume('_env') or {}).items():
env[var].append(str(val))
for hook_path, (hook_src, hook_cmd, hook_env) in hooks.items():
commands.append(hook_cmd)
for var, vals in hook_env.items():
env[var].extend(vals)
self.store_link_text(hook_src, hook_path, label=True)
command = self.consume('command')
if command:
commands.append(command)
if commands:
with open('command', 'w') as f:
f.write('\n'.join(commands))
if env:
with open('.caf/env', 'w') as f:
for var, vals in env.items():
f.write('export {}={}\n'
.format(var, ':'.join(map(str, vals))))
if self.attrs:
error('Task {} has non-consumed attributs: {}'
.format(self, list(self.attrs)))
