async def _run_job(self, job):
# Get stage from pipeline
pf = PipelineStageFactory()
config = PipelineStageConfig(job['stage_name'], job['stage_config'])
stage = pf.create_pipeline_stage(config)
# Load input artifact payloads from cache
loaded_artifacts = []
for artifact in job['artifacts']:
art_obj = Artifact(stage._config)
art_obj.meta_from_dict(artifact)
loaded = self._backend.load_artifact(art_obj)
if loaded is None:
self._log("Could not find payload for artifact")
raise Exception("Could not find payload for artifact")
loaded_artifacts.append(loaded)
# Execute the task
exec_task = self._executor.create_task(stage, loaded_artifacts)
result = await exec_task.generate_artifacts()
for art in result:
art._creation_time = float(time.time())
art._dependency_hash = Artifact.dependency_hash(loaded_artifacts)
self._backend.save_artifact(art)
self._backend.log_pipeline_stage_run_complete(
config, Artifact.dependency_hash(loaded_artifacts))
return result
def test_metadata_from_dict(self):
stage_a = PipelineStageConfig('some_name', {
"A": 1,
"B": 2,
"type": "ExecutorPipelineStage"
})
art_a = Artifact(stage_a)
d = {
"antecedents": {},
"creation_time": 124566722.3,
"definition_hash": "dac9630aec642a428cd73f4be0a03569",
"specific_hash": "bc1687bbb3b97214d46b7c30ab307cc1",
"dependency_hash": "ecad5fc98abf66565e009155f5e57dda",
"pipeline_stage": "some_stage",
"item": {
"meta": {
"loss": 0.2
},
"tags": ["my_pipeline_run"],
"type": "my_item_type"
}
}
art_a.meta_from_dict(d)
for prop in d:
if prop == "item":
for iprop in d['item']:
value = getattr(art_a.item, iprop)
self.assertEqual(d['item'][iprop], value)
else:
value = getattr(art_a, "_" + prop)
self.assertEqual(d[prop], value)
def test_metadata_from_bad_dict(self):
stage_a = PipelineStageConfig('some_name', {
"A": 1,
"B": 2,
"type": "ExecutorPipelineStage"
})
art_a = Artifact(stage_a)
try:
art_a.meta_from_dict({})
self.fail()
except InvalidArtifactMetadataError:
pass
def _sorted_artifacts(self, artifact):
"""
Returns a sorted list of artifacts, based upon pruning ordering
"""
item_meta = self._load_item_meta(artifact._pipeline_stage,
artifact.item.type)
result = []
for k in item_meta:
result.append(item_meta[k])
sorted_metadata = sorted(result, key=lambda x: x["creation_time"])
sorted_artifacts = []
for x in sorted_metadata:
a = Artifact(artifact._config, artifact.item.type)
a.meta_from_dict(x)
sorted_artifacts.append(a)
return sorted_artifacts
async def _run_job(self, job):
# Get stage from pipeline
pf = PipelineStageFactory()
config = PipelineStageConfig(job['stage_name'], job['stage_config'])
stage = pf.create_pipeline_stage(config)
# Load artifact payloads from cache
loaded_artifacts = []
for artifact in job['artifacts']:
art_obj = Artifact(stage._config)
art_obj.meta_from_dict(artifact)
print(art_obj._pipeline_stage)
loaded = self._backend.load_artifact(art_obj)
if loaded is None:
raise Exception("Could not find payload for artifact")
loaded_artifacts.append(loaded)
# Execute the task
exec_task = self._executor.create_task(stage, loaded_artifacts)
result = await exec_task.generate_artifacts()
return result