def effective_task_id(self): """ Replace date in task id with closest date. """ task_name, task_params = id_to_name_and_params(self.task_id) if 'date' in task_params and is_closest_date_parameter(self, 'date'): task_params['date'] = self.closest() task_id_parts = ['%s=%s' % (k, v) for k, v in task_params.iteritems()] return '%s(%s)' % (self.task_family, ', '.join(task_id_parts)) else: return self.task_id
def effective_task_id(self): """ Replace date in task id with closest date. """ task_name, task_params = id_to_name_and_params(self.task_id) if 'date' in task_params and is_closest_date_parameter(self, 'date'): task_params['date'] = self.closest() task_id_parts = [ '%s=%s' % (k, v) for k, v in task_params.iteritems() ] return '%s(%s)' % (self.task_family, ', '.join(task_id_parts)) else: return self.task_id
def path(self, filename=None, ext='tsv', digest=False, shard=False): """ Return the path for this class with a certain set of parameters. `ext` sets the extension of the file. If `hash` is true, the filename (w/o extenstion) will be hashed. If `shard` is true, the files are placed in shards, based on the first two chars of the filename (hashed). """ if self.TAG is NotImplemented or self.BASE is NotImplemented: raise RuntimeError('TAG and BASE must be set.') task_name, task_params = id_to_name_and_params(self.task_id) if filename is None: if 'date' in task_params and is_closest_date_parameter( self, 'date'): task_params['date'] = self.closest() parts = ('{k}-{v}'.format(k=k, v=delistify(v)) for k, v in task_params.iteritems()) name = '-'.join(sorted(parts)) if len(name) == 0: name = 'output' if digest: name = hashlib.sha1(name).hexdigest() if not ext: filename = '{fn}'.format(ext=ext, fn=name) else: filename = '{fn}.{ext}'.format(ext=ext, fn=name) if shard: prefix = hashlib.sha1(filename).hexdigest()[:2] return os.path.join(unicode(self.BASE), unicode(self.TAG), task_name, prefix, filename) return os.path.join(unicode(self.BASE), unicode(self.TAG), task_name, filename)
def path(self, filename=None, ext='tsv', digest=False, shard=False): """ Return the path for this class with a certain set of parameters. `ext` sets the extension of the file. If `hash` is true, the filename (w/o extenstion) will be hashed. If `shard` is true, the files are placed in shards, based on the first two chars of the filename (hashed). """ if self.TAG is NotImplemented or self.BASE is NotImplemented: raise RuntimeError('TAG and BASE must be set.') task_name, task_params = id_to_name_and_params(self.task_id) if filename is None: if 'date' in task_params and is_closest_date_parameter(self, 'date'): task_params['date'] = self.closest() parts = ('{k}-{v}'.format(k=k, v=delistify(v)) for k, v in task_params.iteritems()) name = '-'.join(sorted(parts)) if len(name) == 0: name = 'output' if digest: name = hashlib.sha1(name).hexdigest() if not ext: filename = '{fn}'.format(ext=ext, fn=name) else: filename = '{fn}.{ext}'.format(ext=ext, fn=name) if shard: prefix = hashlib.sha1(filename).hexdigest()[:2] return os.path.join(unicode(self.BASE), unicode(self.TAG), task_name, prefix, filename) return os.path.join(unicode(self.BASE), unicode(self.TAG), task_name, filename)
def taskdir(self): """ Return the directory under which all artefacts are stored. """ task_name, task_params = id_to_name_and_params(self.task_id) return os.path.join(unicode(self.BASE), unicode(self.TAG), task_name)