def get_config_type(info, pipeline_name, type_name):
pipeline_or_error = _pipeline_or_error_from_container(
info, info.context.repository_container,
ExecutionSelector(pipeline_name))
return pipeline_or_error.chain(lambda pipeline: _config_type_or_error(
info, pipeline, type_name)).value()
def test_execution_crash():
run_id = 'run-1'
repository_container = RepositoryContainer(
RepositoryTargetInfo(
repository_yaml=None,
python_file=__file__,
fn_name='define_crashy_pipeline',
module_name=None,
)
)
pipeline = define_crashy_pipeline()
env_config = {
'solids': {
'sum_solid': {'inputs': {'num': {'csv': {'path': script_relative_path('num.csv')}}}}
}
}
selector = ExecutionSelector('pandas_hello_world')
pipeline_run = InMemoryPipelineRun(
run_id, selector, env_config, create_execution_plan(pipeline, env_config)
)
execution_manager = MultiprocessingExecutionManager()
execution_manager.execute_pipeline(repository_container, pipeline, pipeline_run)
execution_manager.join()
assert pipeline_run.status == PipelineRunStatus.FAILURE
last_log = pipeline_run.all_logs()[-1]
assert last_log.message == (
'Exception: Pipeline execution process for {run_id} unexpectedly exited\n'
).format(run_id=run_id)
def test_failing():
run_id = 'run-1'
repository_container = RepositoryContainer(
RepositoryTargetInfo(
repository_yaml=None,
python_file=__file__,
fn_name='define_failing_pipeline',
module_name=None,
)
)
pipeline = define_failing_pipeline()
env_config = {
'solids': {
'sum_solid': {'inputs': {'num': {'csv': {'path': script_relative_path('num.csv')}}}}
}
}
selector = ExecutionSelector('pandas_hello_world')
pipeline_run = InMemoryPipelineRun(
run_id, selector, env_config, create_execution_plan(pipeline, env_config)
)
execution_manager = MultiprocessingExecutionManager()
execution_manager.execute_pipeline(repository_container, pipeline, pipeline_run)
execution_manager.join()
assert pipeline_run.status == PipelineRunStatus.FAILURE
assert pipeline_run.all_logs()
def test_running():
run_id = 'run-1'
repository_container = RepositoryContainer(
RepositoryTargetInfo(
repository_yaml=None,
python_file=__file__,
fn_name='define_passing_pipeline',
module_name=None,
)
)
pipeline = define_passing_pipeline()
env_config = {
'solids': {
'sum_solid': {'inputs': {'num': {'csv': {'path': script_relative_path('num.csv')}}}}
}
}
selector = ExecutionSelector('pandas_hello_world')
pipeline_run = InMemoryPipelineRun(
run_id, selector, env_config, create_execution_plan(pipeline, env_config)
)
execution_manager = MultiprocessingExecutionManager()
execution_manager.execute_pipeline(repository_container, pipeline, pipeline_run)
execution_manager.join()
assert pipeline_run.status == PipelineRunStatus.SUCCESS
events = pipeline_run.all_logs()
assert events
process_start_events = get_events_of_type(events, EventType.PIPELINE_PROCESS_START)
assert len(process_start_events) == 1
process_started_events = get_events_of_type(events, EventType.PIPELINE_PROCESS_STARTED)
assert len(process_started_events) == 1
def get_runtime_type(graphene_info, pipeline_name, type_name):
pipeline_or_error = _pipeline_or_error_from_container(
graphene_info, graphene_info.context.repository_container,
ExecutionSelector(pipeline_name))
return pipeline_or_error.chain(lambda pipeline: _runtime_type_or_error(
graphene_info, pipeline, type_name)).value()
def get_pipeline_type(info, pipelineName, typeName):
check.inst_param(info, 'info', ResolveInfo)
check.str_param(pipelineName, 'pipelineName')
check.str_param(typeName, 'typeName')
pipeline_or_error = _pipeline_or_error_from_container(
info, info.context.repository_container,
ExecutionSelector(pipelineName))
return pipeline_or_error.chain(
lambda pip: pip.get_type(info, typeName)).value_or_raise()
def start_subplan_execution(args):
check.inst_param(args, 'args', SubplanExecutionArgs)
info = args.info
# this is a sequence of validations to valiadate inputs:
# validate_pipeline => validate_config => validate_execution_plan => execute_execution_plan
return (_pipeline_or_error_from_container(
info, info.context.repository_container,
ExecutionSelector(args.pipeline_name)).chain(
lambda dauphin_pipeline: _chain_config_or_error_from_pipeline(
args, dauphin_pipeline)).value())
def do_execute_plan(graphene_info, pipeline_name, environment_dict,
execution_metadata, step_keys):
execute_plan_args = ExecutePlanArgs(
graphene_info=graphene_info,
pipeline_name=pipeline_name,
environment_dict=environment_dict,
execution_metadata=execution_metadata,
step_keys=step_keys,
)
return (_pipeline_or_error_from_container(
graphene_info, ExecutionSelector(pipeline_name)).chain(
lambda dauphin_pipeline: _execute_plan_resolve_config(
execute_plan_args, dauphin_pipeline)).value())
def to_selector(self):
return ExecutionSelector(self.name, self.solidSubset)
def get_config_type(graphene_info, pipeline_name, type_name):
pipeline_or_error = _pipeline_or_error_from_container(
graphene_info, ExecutionSelector(pipeline_name))
return pipeline_or_error.chain(lambda pipeline: _config_type_or_error(
graphene_info, pipeline, type_name)).value()