def testio(request):
full_name = f"tests/input_execution_order/{request.param}"
with open(full_name, "r") as f:
description = json.load(f)
pipeline = Pipeline.from_json(description)
return IO(pipeline)
def run_partial(self,
pipeline_description: PipelineDescription,
run_config: Dict[str, Union[str, Dict[str, str]]],
task_id: Optional[str] = None) -> str:
"""Runs a pipeline partially.
A partial run is described by the pipeline description The
call-order of the steps is always preserved, e.g. a --> b then a
will always be run before b.
Args:
pipeline_description: a json description of the pipeline.
run_config: configuration of the run for the compute backend.
Returns:
Status of the pipeline run. "FAILURE" or "SUCCESS".
"""
# Get the pipeline to run.
pipeline = Pipeline.from_json(pipeline_description)
# TODO: don't think this task_id is needed anymore. It was
# introduced as part of the scheduled runs which we don't use
# anymore.
# Run the subgraph in parallel. And pass the id of the AsyncResult
# object.
# TODO: The commented line below is once we can introduce sessions.
# session = run_partial.session
task_id = task_id if task_id is not None else self.request.id
return asyncio.run(pipeline.run(task_id, run_config=run_config))
def testio(request):
full_name = f'tests/input_execution_order/{request.param}'
with open(full_name, 'r') as f:
description = json.load(f)
pipeline = Pipeline.from_json(description)
correct_execution_order = description['correct_execution_order']
return IO(pipeline, correct_execution_order)
def run_pipeline(
self,
pipeline_definition: PipelineDefinition,
project_uuid: str,
run_config: Dict[str, Union[str, Dict[str, str]]],
task_id: Optional[str] = None,
) -> str:
"""Runs a pipeline partially.
A partial run is described by the pipeline definition The
call-order of the steps is always preserved, e.g. a --> b then a
will always be run before b.
Args:
pipeline_definition: a json description of the pipeline.
run_config: configuration of the run for the compute backend.
Example: {
'run_endpoint': 'runs',
'project_dir': '/home/../pipelines/uuid',
'env_uuid_docker_id_mappings': {
'b6527b0b-bfcc-4aff-91d1-37f9dfd5d8e8':
'sha256:61f82126945bb25dd85d6a5b122a1815df1c0c5f91621089cde0938be4f698d4'
}
}
Returns:
Status of the pipeline run. "FAILURE" or "SUCCESS".
"""
run_config["pipeline_uuid"] = pipeline_definition["uuid"]
run_config["project_uuid"] = project_uuid
# Get the pipeline to run.
pipeline = Pipeline.from_json(pipeline_definition)
# TODO: don't think this task_id is needed anymore. It was
# introduced as part of the scheduled runs which we don't use
# anymore.
# Run the subgraph in parallel. And pass the id of the AsyncResult
# object.
# TODO: The commented line below is once we can introduce sessions.
# session = run_pipeline.session
task_id = task_id if task_id is not None else self.request.id
# TODO: could make the celery task fail in case the pipeline run
# failed. Although the run did complete successfully from a task
# scheduler perspective.
# https://stackoverflow.com/questions/7672327/how-to-make-a-celery-task-fail-from-within-the-task
return asyncio.run(run_pipeline_async(run_config, pipeline, task_id))
def run_pipeline(
self,
pipeline_definition: PipelineDefinition,
run_config: RunConfig,
session_uuid: str,
task_id: Optional[str] = None,
) -> str:
"""Runs a pipeline partially.
A partial run is described by the pipeline definition The
call-order of the steps is always preserved, e.g. a --> b then a
will always be run before b.
Args:
pipeline_definition: a json description of the pipeline.
run_config: configuration of the run for the compute backend.
Returns:
Status of the pipeline run. "FAILURE" or "SUCCESS".
"""
# Get the pipeline to run.
pipeline = Pipeline.from_json(pipeline_definition)
# TODO: don't think this task_id is needed anymore. It was
# introduced as part of the scheduled runs which we don't use
# anymore.
# Run the subgraph in parallel. And pass the id of the AsyncResult
# object.
# TODO: The commented line below is once we can introduce sessions.
# session = run_pipeline.session
task_id = task_id if task_id is not None else self.request.id
# TODO: could make the celery task fail in case the pipeline run
# failed. Although the run did complete successfully from a task
# scheduler perspective.
# https://stackoverflow.com/questions/7672327/how-to-make-a-celery-task-fail-from-within-the-task
return asyncio.run(run_pipeline_async(session_uuid, run_config, pipeline, task_id))
def pipeline():
with open('tests/input_operations/pipeline.json', 'r') as f:
description = json.load(f)
pipeline = Pipeline.from_json(description)
return pipeline
def test_serialization():
with open("tests/input_operations/pipeline.json", "r") as f:
description = json.load(f)
pipeline = Pipeline.from_json(description)
assert pipeline.to_dict() == description