def _pipeline_or_error_from_repository(info, repository, selector):
if not repository.has_pipeline(selector.name):
return EitherError(
info.schema.type_named('PipelineNotFoundError')(
pipeline_name=selector.name))
else:
orig_pipeline = repository.get_pipeline(selector.name)
if selector.solid_subset:
for solid_name in selector.solid_subset:
if not orig_pipeline.has_solid(solid_name):
return EitherError(
info.schema.type_named('SolidNotFoundError')(
solid_name=solid_name))
pipeline = get_subset_pipeline(orig_pipeline, selector.solid_subset)
return EitherValue(info.schema.type_named('Pipeline')(pipeline))
def execute_pipeline_through_queue(
repository_info, pipeline_name, pipeline_solid_subset, config, run_id, message_queue
):
"""
Execute pipeline using message queue as a transport
"""
message_queue.put(ProcessStartedSentinel(os.getpid()))
execution_metadata = ExecutionMetadata(run_id, event_callback=message_queue.put)
from .app import RepositoryContainer
repository_container = RepositoryContainer(repository_info)
if repository_container.repo_error:
message_queue.put(
MultiprocessingError(
serializable_error_info_from_exc_info(repository_container.repo_error)
)
)
return
pipeline = repository_container.repository.get_pipeline(pipeline_name)
pipeline = get_subset_pipeline(pipeline, pipeline_solid_subset)
typed_environment = construct_environment_config(
evaluate_config_value(pipeline.environment_type, config).value
)
try:
result = execute_pipeline_with_metadata(
pipeline,
typed_environment,
execution_metadata=execution_metadata,
throw_on_user_error=False,
)
return result
except: # pylint: disable=W0702
message_queue.put(
MultiprocessingError(serializable_error_info_from_exc_info(sys.exc_info()))
)
finally:
message_queue.put(MultiprocessingDone())
def test_pipeline_disjoint_subset():
disjoint_pipeline = get_subset_pipeline(define_three_part_pipeline(), ['add_one', 'add_three'])
assert len(disjoint_pipeline.solids) == 2
def define_created_disjoint_three_part_pipeline():
return get_subset_pipeline(define_three_part_pipeline(), ['add_one', 'add_three'])