def effective_task_id(self):
""" Replace date in task id with closest date. """
task_name, task_params = id_to_name_and_params(self.task_id)
if 'date' in task_params and is_closest_date_parameter(self, 'date'):
task_params['date'] = self.closest()
task_id_parts = ['%s=%s' % (k, v) for k, v in task_params.iteritems()]
return '%s(%s)' % (self.task_family, ', '.join(task_id_parts))
else:
return self.task_id
def effective_task_id(self):
""" Replace date in task id with closest date. """
task_name, task_params = id_to_name_and_params(self.task_id)
if 'date' in task_params and is_closest_date_parameter(self, 'date'):
task_params['date'] = self.closest()
task_id_parts = [
'%s=%s' % (k, v) for k, v in task_params.iteritems()
]
return '%s(%s)' % (self.task_family, ', '.join(task_id_parts))
else:
return self.task_id
def path(self, filename=None, ext='tsv', digest=False, shard=False):
"""
Return the path for this class with a certain set of parameters.
`ext` sets the extension of the file.
If `hash` is true, the filename (w/o extenstion) will be hashed.
If `shard` is true, the files are placed in shards, based on the first
two chars of the filename (hashed).
"""
if self.TAG is NotImplemented or self.BASE is NotImplemented:
raise RuntimeError('TAG and BASE must be set.')
task_name, task_params = id_to_name_and_params(self.task_id)
if filename is None:
if 'date' in task_params and is_closest_date_parameter(
self, 'date'):
task_params['date'] = self.closest()
parts = ('{k}-{v}'.format(k=k, v=delistify(v))
for k, v in task_params.iteritems())
name = '-'.join(sorted(parts))
if len(name) == 0:
name = 'output'
if digest:
name = hashlib.sha1(name).hexdigest()
if not ext:
filename = '{fn}'.format(ext=ext, fn=name)
else:
filename = '{fn}.{ext}'.format(ext=ext, fn=name)
if shard:
prefix = hashlib.sha1(filename).hexdigest()[:2]
return os.path.join(unicode(self.BASE), unicode(self.TAG),
task_name, prefix, filename)
return os.path.join(unicode(self.BASE), unicode(self.TAG), task_name,
filename)
def path(self, filename=None, ext='tsv', digest=False, shard=False):
"""
Return the path for this class with a certain set of parameters.
`ext` sets the extension of the file.
If `hash` is true, the filename (w/o extenstion) will be hashed.
If `shard` is true, the files are placed in shards, based on the first
two chars of the filename (hashed).
"""
if self.TAG is NotImplemented or self.BASE is NotImplemented:
raise RuntimeError('TAG and BASE must be set.')
task_name, task_params = id_to_name_and_params(self.task_id)
if filename is None:
if 'date' in task_params and is_closest_date_parameter(self, 'date'):
task_params['date'] = self.closest()
parts = ('{k}-{v}'.format(k=k, v=delistify(v))
for k, v in task_params.iteritems())
name = '-'.join(sorted(parts))
if len(name) == 0:
name = 'output'
if digest:
name = hashlib.sha1(name).hexdigest()
if not ext:
filename = '{fn}'.format(ext=ext, fn=name)
else:
filename = '{fn}.{ext}'.format(ext=ext, fn=name)
if shard:
prefix = hashlib.sha1(filename).hexdigest()[:2]
return os.path.join(unicode(self.BASE), unicode(self.TAG),
task_name, prefix, filename)
return os.path.join(unicode(self.BASE), unicode(self.TAG), task_name,
filename)
def taskdir(self):
""" Return the directory under which all artefacts are stored. """
task_name, task_params = id_to_name_and_params(self.task_id)
return os.path.join(unicode(self.BASE), unicode(self.TAG), task_name)
def taskdir(self):
""" Return the directory under which all artefacts are stored. """
task_name, task_params = id_to_name_and_params(self.task_id)
return os.path.join(unicode(self.BASE), unicode(self.TAG), task_name)
