def data_derived_post(self):
todo = [[['meta', 'modality'], 'TODO should come from sheets maybe?']]
derives = [[[['meta', 'award_number']], lambda an:
(self.lifters.organ(an), ), [['meta', 'organ']]]]
data = super().data_derived_post
DictTransformer.derive(data, derives)
return data
def added(self):
""" bind referencing data such as ids or uris regardless
or anything else that should happen regardless of whether
pipelines failed or not """
data = self.pipeline_end
adds = [[path, function(self.lifters)] for path, function in self.adds]
DictTransformer.add(data, adds)
return data
def cleaned(self):
data = self.moved
removed = list(
DictTransformer.pop(data, self.cleans, source_key_optional=True))
#log.debug(f'cleaned the following values from {self}' + lj(removed))
log.debug(f'cleaned {len(removed)} values from {self}')
return data
def subpipelined(self):
data = self.pipeline_start
errors_subpipelines = list(DictTransformer.subpipeline(data,
self.runtime_context,
self.subpipelines,
lifters=self.lifters))
errors = tuple(es for es in errors_subpipelines if isinstance(es, tuple))
self.subpipeline_errors(errors)
self.subpipeline_instances = tuple(es for es in errors_subpipelines if isinstance(es, Pipeline))
return data
def augmented(self):
data = self.updated
DictTransformer.derive(data, self.derives, source_key_optional=True)
return data
def updated(self):
data = self.cleaned
DictTransformer.update(data, self.updates, source_key_optional=True)
return data
def moved(self):
data = self.copied
DictTransformer.move(data, self.moves, source_key_optional=True)
return data
def copied(self):
data = self.subpipelined
DictTransformer.copy(data, self.copies, source_key_optional=True)
return data