def execute_pipeline(self, _, pipeline, pipeline_run, instance, raise_on_error):
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
run_config = RunConfig(
pipeline_run.run_id,
mode=pipeline_run.mode,
event_sink=CallbackEventSink(instance.handle_new_event),
reexecution_config=pipeline_run.reexecution_config,
step_keys_to_execute=pipeline_run.step_keys_to_execute,
)
# We do this instead of just using execute_pipeline to avoid spurious pipeline start and
# pipeline success or pipeline failure events.
try:
event_list = []
for event in execute_pipeline_iterator(
pipeline, pipeline_run.environment_dict, run_config, instance
):
event_list.append(event)
return PipelineExecutionResult(pipeline, run_config.run_id, event_list, lambda: None)
except Exception: # pylint: disable=broad-except
if raise_on_error:
six.reraise(*sys.exc_info())
instance.handle_new_event(
build_synthetic_pipeline_error_record(
pipeline_run.run_id,
serializable_error_info_from_exc_info(sys.exc_info()),
pipeline.name,
)
)
def execute_pipeline_through_queue(
handle,
pipeline_name,
solid_subset,
environment_dict,
mode,
run_id,
message_queue,
reexecution_config,
step_keys_to_execute,
instance_dir,
):
"""
Execute pipeline using message queue as a transport
"""
check.opt_str_param(mode, 'mode')
message_queue.put(build_process_started_event(run_id, pipeline_name, os.getpid()))
run_config = RunConfig(
run_id,
mode=mode,
event_sink=CallbackEventSink(message_queue.put),
reexecution_config=reexecution_config,
step_keys_to_execute=step_keys_to_execute,
)
if 'execution' not in environment_dict or not environment_dict['execution']:
environment_dict['execution'] = {'in_process': {'config': {'raise_on_error': False}}}
try:
handle.build_repository_definition()
pipeline_def = handle.with_pipeline_name(pipeline_name).build_pipeline_definition()
except Exception: # pylint: disable=broad-except
repo_error = sys.exc_info()
message_queue.put(
build_synthetic_pipeline_error_record(
run_id, serializable_error_info_from_exc_info(repo_error), pipeline_name
)
)
return
try:
event_list = []
for event in execute_pipeline_iterator(
pipeline_def.build_sub_pipeline(solid_subset),
environment_dict,
run_config=run_config,
instance=DagsterInstance.ephemeral(instance_dir),
):
event_list.append(event)
return PipelineExecutionResult(pipeline_def, run_config.run_id, event_list, lambda: None)
except Exception: # pylint: disable=broad-except
error_info = serializable_error_info_from_exc_info(sys.exc_info())
message_queue.put(build_synthetic_pipeline_error_record(run_id, error_info, pipeline_name))
finally:
message_queue.put(build_process_exited_event(run_id, pipeline_name, os.getpid()))
message_queue.close()
def execute_pipeline(self, _, pipeline, pipeline_run, instance):
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
event_list = []
for event in execute_run_iterator(pipeline, pipeline_run, instance):
event_list.append(event)
return PipelineExecutionResult(pipeline, pipeline_run.run_id,
event_list, lambda: None)
def _in_mp_process(handle, pipeline_run, instance_ref, term_event):
"""
Execute pipeline using message queue as a transport
"""
run_id = pipeline_run.run_id
pipeline_name = pipeline_run.pipeline_name
instance = DagsterInstance.from_ref(instance_ref)
instance.handle_new_event(
build_process_started_event(run_id, pipeline_name, os.getpid()))
start_termination_thread(term_event)
try:
handle.build_repository_definition()
pipeline_def = handle.with_pipeline_name(
pipeline_name).build_pipeline_definition()
except Exception: # pylint: disable=broad-except
repo_error = sys.exc_info()
instance.handle_new_event(
build_synthetic_pipeline_error_record(
run_id, serializable_error_info_from_exc_info(repo_error),
pipeline_name))
return
try:
event_list = []
for event in execute_run_iterator(
pipeline_def.build_sub_pipeline(
pipeline_run.selector.solid_subset),
pipeline_run,
instance,
):
event_list.append(event)
return PipelineExecutionResult(pipeline_def, run_id, event_list,
lambda: None)
# Add a DagsterEvent for unexpected exceptions
# Explicitly ignore KeyboardInterrupts since they are used for termination
except DagsterSubprocessError as err:
if not all([
err_info.cls_name == 'KeyboardInterrupt'
for err_info in err.subprocess_error_infos
]):
error_info = serializable_error_info_from_exc_info(sys.exc_info())
instance.handle_new_event(
build_synthetic_pipeline_error_record(run_id, error_info,
pipeline_name))
except Exception: # pylint: disable=broad-except
error_info = serializable_error_info_from_exc_info(sys.exc_info())
instance.handle_new_event(
build_synthetic_pipeline_error_record(run_id, error_info,
pipeline_name))
finally:
instance.handle_new_event(
build_process_exited_event(run_id, pipeline_name, os.getpid()))
def execute_pipeline(self, _, pipeline, pipeline_run, instance):
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
check.inst_param(instance, 'instance', DagsterInstance)
event_list = []
self._active.add(pipeline_run.run_id)
for event in execute_run_iterator(pipeline, pipeline_run, instance):
event_list.append(event)
self._active.remove(pipeline_run.run_id)
return PipelineExecutionResult(pipeline, pipeline_run.run_id,
event_list, lambda: None)
def execute_pipeline_through_queue(handle, pipeline_run, message_queue,
instance_ref):
"""
Execute pipeline using message queue as a transport
"""
run_id = pipeline_run.run_id
pipeline_name = pipeline_run.pipeline_name
message_queue.put(
build_process_started_event(run_id, pipeline_name, os.getpid()))
try:
handle.build_repository_definition()
pipeline_def = handle.with_pipeline_name(
pipeline_name).build_pipeline_definition()
except Exception: # pylint: disable=broad-except
repo_error = sys.exc_info()
message_queue.put(
build_synthetic_pipeline_error_record(
run_id, serializable_error_info_from_exc_info(repo_error),
pipeline_name))
return
try:
event_list = []
for event in execute_run_iterator(
pipeline_def.build_sub_pipeline(
pipeline_run.selector.solid_subset),
pipeline_run,
DagsterInstance.from_ref(instance_ref),
):
event_list.append(event)
return PipelineExecutionResult(pipeline_def, run_id, event_list,
lambda: None)
except Exception: # pylint: disable=broad-except
error_info = serializable_error_info_from_exc_info(sys.exc_info())
message_queue.put(
build_synthetic_pipeline_error_record(run_id, error_info,
pipeline_name))
finally:
message_queue.put(
build_process_exited_event(run_id, pipeline_name, os.getpid()))
message_queue.close()
def execute_pipeline(self, _, pipeline, pipeline_run, instance,
raise_on_error):
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
try:
event_list = []
for event in execute_run_iterator(pipeline, pipeline_run,
instance):
event_list.append(event)
return PipelineExecutionResult(pipeline, pipeline_run.run_id,
event_list, lambda: None)
except Exception: # pylint: disable=broad-except
if raise_on_error:
six.reraise(*sys.exc_info())
instance.handle_new_event(
build_synthetic_pipeline_error_record(
pipeline_run.run_id,
serializable_error_info_from_exc_info(sys.exc_info()),
pipeline.name,
))
def in_mp_process(cls, handle, pipeline_run, instance_ref, term_event):
"""
Execute pipeline using message queue as a transport
"""
run_id = pipeline_run.run_id
pipeline_name = pipeline_run.pipeline_name
instance = DagsterInstance.from_ref(instance_ref)
pid = os.getpid()
instance.report_engine_event(
'Started process for pipeline (pid: {pid}).'.format(pid=pid),
pipeline_run,
EngineEventData.in_process(pid,
marker_end='dagit_subprocess_init'),
cls,
)
start_termination_thread(term_event)
try:
handle.build_repository_definition()
pipeline_def = handle.with_pipeline_name(
pipeline_name).build_pipeline_definition()
except Exception: # pylint: disable=broad-except
instance.report_engine_event(
'Failed attempting to load pipeline "{}"'.format(
pipeline_name),
pipeline_run,
EngineEventData.engine_error(
serializable_error_info_from_exc_info(sys.exc_info())),
cls,
)
return
try:
event_list = []
for event in execute_run_iterator(
pipeline_def.build_sub_pipeline(
pipeline_run.selector.solid_subset),
pipeline_run,
instance,
):
event_list.append(event)
return PipelineExecutionResult(pipeline_def, run_id, event_list,
lambda: None)
# Add a DagsterEvent for unexpected exceptions
# Explicitly ignore KeyboardInterrupts since they are used for termination
except DagsterSubprocessError as err:
if not all([
err_info.cls_name == 'KeyboardInterrupt'
for err_info in err.subprocess_error_infos
]):
instance.report_engine_event(
'An exception was thrown during execution that is likely a framework error, '
'rather than an error in user code.',
pipeline_run,
EngineEventData.engine_error(
serializable_error_info_from_exc_info(sys.exc_info())),
cls,
)
except Exception: # pylint: disable=broad-except
instance.report_engine_event(
'An exception was thrown during execution that is likely a framework error, '
'rather than an error in user code.',
pipeline_run,
EngineEventData.engine_error(
serializable_error_info_from_exc_info(sys.exc_info())),
cls,
)
finally:
instance.report_engine_event(
'Process for pipeline exited (pid: {pid}).'.format(pid=pid),
pipeline_run,
cls=cls,
)
def execute_pipeline_through_queue(
handle,
pipeline_name,
solid_subset,
environment_dict,
mode,
run_id,
message_queue,
reexecution_config,
step_keys_to_execute,
):
"""
Execute pipeline using message queue as a transport
"""
check.opt_str_param(mode, 'mode')
message_queue.put(ProcessStartedSentinel(os.getpid()))
run_config = RunConfig(
run_id,
mode=mode,
event_callback=message_queue.put,
reexecution_config=reexecution_config,
step_keys_to_execute=step_keys_to_execute,
)
if 'execution' not in environment_dict or not environment_dict['execution']:
environment_dict['execution'] = {
'in_process': {
'config': {
'raise_on_error': False
}
}
}
try:
handle.build_repository_definition()
pipeline_def = handle.with_pipeline_name(
pipeline_name).build_pipeline_definition()
except Exception: # pylint: disable=broad-except
repo_error = sys.exc_info()
message_queue.put(
MultiprocessingError(
serializable_error_info_from_exc_info(repo_error)))
return
try:
event_list = []
for event in execute_pipeline_iterator(
pipeline_def.build_sub_pipeline(solid_subset),
environment_dict,
run_config=run_config):
# message_queue.put(event)
event_list.append(event)
return PipelineExecutionResult(pipeline_def, run_config.run_id,
event_list, lambda: None)
except Exception: # pylint: disable=broad-except
error_info = serializable_error_info_from_exc_info(sys.exc_info())
message_queue.put(MultiprocessingError(error_info))
finally:
message_queue.put(MultiprocessingDone())
message_queue.close()