def execute(self, pipeline_context, execution_plan): check.inst_param(pipeline_context, 'pipeline_context', SystemPipelineExecutionContext) check.inst_param(execution_plan, 'execution_plan', ExecutionPlan) step_keys_to_execute = execution_plan.step_keys_to_execute yield DagsterEvent.engine_event( pipeline_context, 'Executing steps in process (pid: {pid})'.format(pid=os.getpid()), event_specific_data=EngineEventData.in_process(os.getpid(), step_keys_to_execute), ) with time_execution_scope() as timer_result: for event in inner_plan_execution_iterator( pipeline_context, execution_plan, self.retries ): yield event yield DagsterEvent.engine_event( pipeline_context, 'Finished steps in process (pid: {pid}) in {duration_ms}'.format( pid=os.getpid(), duration_ms=format_duration(timer_result.millis) ), event_specific_data=EngineEventData.in_process(os.getpid(), step_keys_to_execute), )
def execute(self, pipeline_context, execution_plan): check.inst_param(pipeline_context, "pipeline_context", SystemPipelineExecutionContext) check.inst_param(execution_plan, "execution_plan", ExecutionPlan) step_keys_to_execute = execution_plan.step_keys_to_execute yield DagsterEvent.engine_event( pipeline_context, "Executing steps in process (pid: {pid})".format(pid=os.getpid()), event_specific_data=EngineEventData.in_process( os.getpid(), step_keys_to_execute), ) with time_execution_scope() as timer_result: yield from inner_plan_execution_iterator(pipeline_context, execution_plan) yield DagsterEvent.engine_event( pipeline_context, "Finished steps in process (pid: {pid}) in {duration_ms}".format( pid=os.getpid(), duration_ms=format_duration(timer_result.millis)), event_specific_data=EngineEventData.in_process( os.getpid(), step_keys_to_execute), )
def _execute_run_command_body(recon_pipeline, pipeline_run_id, instance, write_stream_fn): if instance.should_start_background_run_thread: cancellation_thread, cancellation_thread_shutdown_event = start_run_cancellation_thread( instance, pipeline_run_id) pipeline_run = instance.get_run_by_id(pipeline_run_id) pid = os.getpid() instance.report_engine_event( "Started process for run (pid: {pid}).".format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"), ) try: for event in core_execute_run(recon_pipeline, pipeline_run, instance): write_stream_fn(event) finally: if instance.should_start_background_run_thread: cancellation_thread_shutdown_event.set() if cancellation_thread.is_alive(): cancellation_thread.join(timeout=15) if cancellation_thread.is_alive(): instance.report_engine_event( "Cancellation thread did not shutdown gracefully", pipeline_run, ) instance.report_engine_event( "Process for run exited (pid: {pid}).".format(pid=pid), pipeline_run, )
def test_correct_timezone(conn_string): event_log_storage = PostgresEventLogStorage.create_clean_storage( conn_string) curr_time = time.time() event = DagsterEventRecord( None, "Message2", "debug", "", "foo", curr_time, dagster_event=DagsterEvent( DagsterEventType.PIPELINE_START.value, "nonce", event_specific_data=EngineEventData.in_process(999), ), ) event_log_storage.store_event(event) logs = event_log_storage.get_logs_for_run("foo") assert len(logs) == 1 log = logs[0] stats = event_log_storage.get_stats_for_run("foo") assert int(log.timestamp) == int(stats.start_time) assert int(log.timestamp) == int(curr_time)
def _execute_run_command_body(recon_pipeline, pipeline_run_id, instance, write_stream_fn): # we need to send but the fact that we have loaded the args so the calling # process knows it is safe to clean up the temp input file write_stream_fn(ExecuteRunArgsLoadComplete()) pipeline_run = instance.get_run_by_id(pipeline_run_id) pid = os.getpid() instance.report_engine_event( "Started process for pipeline (pid: {pid}).".format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"), ) # Perform setup so that termination of the execution will unwind and report to the # instance correctly setup_windows_interrupt_support() try: for event in core_execute_run(recon_pipeline, pipeline_run, instance): write_stream_fn(event) finally: instance.report_engine_event( "Process for pipeline exited (pid: {pid}).".format(pid=pid), pipeline_run, )
def test_correct_timezone(self, storage): curr_time = time.time() event = EventRecord( None, "Message2", "debug", "", "foo", curr_time, dagster_event=DagsterEvent( DagsterEventType.PIPELINE_START.value, "nonce", event_specific_data=EngineEventData.in_process(999), ), ) storage.store_event(event) logs = storage.get_logs_for_run("foo") assert len(logs) == 1 log = logs[0] stats = storage.get_stats_for_run("foo") assert int(log.timestamp) == int(stats.start_time) assert int(log.timestamp) == int(curr_time)
def _resume_run_command_body( recon_pipeline: ReconstructablePipeline, pipeline_run_id: Optional[str], instance: DagsterInstance, write_stream_fn: Callable[[DagsterEvent], Any], set_exit_code_on_failure: bool, ): if instance.should_start_background_run_thread: cancellation_thread, cancellation_thread_shutdown_event = start_run_cancellation_thread( instance, pipeline_run_id) pipeline_run = instance.get_run_by_id(pipeline_run_id) check.inst( pipeline_run, PipelineRun, "Pipeline run with id '{}' not found for run execution.".format( pipeline_run_id), ) pid = os.getpid() instance.report_engine_event( "Started process for resuming pipeline (pid: {pid}).".format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"), ) run_worker_failed = False try: for event in core_execute_run( recon_pipeline, pipeline_run, instance, resume_from_failure=True, ): write_stream_fn(event) if event.event_type == DagsterEventType.PIPELINE_FAILURE: run_worker_failed = True except: # relies on core_execute_run writing failures to the event log before raising run_worker_failed = True finally: if instance.should_start_background_run_thread: cancellation_thread_shutdown_event.set() if cancellation_thread.is_alive(): cancellation_thread.join(timeout=15) if cancellation_thread.is_alive(): instance.report_engine_event( "Cancellation thread did not shutdown gracefully", pipeline_run, ) instance.report_engine_event( "Process for pipeline exited (pid: {pid}).".format(pid=pid), pipeline_run, ) return 1 if (run_worker_failed and set_exit_code_on_failure) else 0
def _execute_run_command_body( output_file, recon_repo, pipeline_run_id, instance_ref_json, ): with ipc_write_stream(output_file) as stream: instance = _get_instance(stream, instance_ref_json) if not instance: return pipeline_run = instance.get_run_by_id(pipeline_run_id) pid = os.getpid() instance.report_engine_event( 'Started process for pipeline (pid: {pid}).'.format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end='cli_api_subprocess_init'), ) recon_pipeline = _recon_pipeline(stream, recon_repo, pipeline_run) # Perform setup so that termination of the execution will unwind and report to the # instance correctly setup_interrupt_support() try: for event in execute_run_iterator(recon_pipeline, pipeline_run, instance): stream.send(event) except DagsterSubprocessError as err: if not all([ err_info.cls_name == 'KeyboardInterrupt' for err_info in err.subprocess_error_infos ]): instance.report_engine_event( 'An exception was thrown during execution that is likely a framework error, ' 'rather than an error in user code.', pipeline_run, EngineEventData.engine_error( serializable_error_info_from_exc_info(sys.exc_info())), ) except Exception: # pylint: disable=broad-except instance.report_engine_event( 'An exception was thrown during execution that is likely a framework error, ' 'rather than an error in user code.', pipeline_run, EngineEventData.engine_error( serializable_error_info_from_exc_info(sys.exc_info())), ) finally: instance.report_engine_event( 'Process for pipeline exited (pid: {pid}).'.format(pid=pid), pipeline_run, )
def _execute_run_command_body(recon_pipeline, pipeline_run_id, instance, write_stream_fn): # we need to send but the fact that we have loaded the args so the calling # process knows it is safe to clean up the temp input file write_stream_fn(ExecuteRunArgsLoadComplete()) pipeline_run = instance.get_run_by_id(pipeline_run_id) pid = os.getpid() instance.report_engine_event( "Started process for pipeline (pid: {pid}).".format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"), ) # Perform setup so that termination of the execution will unwind and report to the # instance correctly setup_windows_interrupt_support() try: for event in execute_run_iterator(recon_pipeline, pipeline_run, instance): write_stream_fn(event) except KeyboardInterrupt: instance.report_engine_event( message="Pipeline execution terminated by interrupt", pipeline_run=pipeline_run, ) except DagsterSubprocessError as err: if not all([ err_info.cls_name == "KeyboardInterrupt" for err_info in err.subprocess_error_infos ]): instance.report_engine_event( "An exception was thrown during execution that is likely a framework error, " "rather than an error in user code.", pipeline_run, EngineEventData.engine_error( serializable_error_info_from_exc_info(sys.exc_info())), ) except Exception: # pylint: disable=broad-except instance.report_engine_event( "An exception was thrown during execution that is likely a framework error, " "rather than an error in user code.", pipeline_run, EngineEventData.engine_error( serializable_error_info_from_exc_info(sys.exc_info())), ) finally: instance.report_engine_event( "Process for pipeline exited (pid: {pid}).".format(pid=pid), pipeline_run, )
def _execute_run_command_body(output_file, recon_pipeline, pipeline_run_id, instance_ref): with ipc_write_stream(output_file) as stream: # we need to send but the fact that we have loaded the args so the calling # process knows it is safe to clean up the temp input file stream.send(ExecuteRunArgsLoadComplete()) instance = DagsterInstance.from_ref(instance_ref) pipeline_run = instance.get_run_by_id(pipeline_run_id) pid = os.getpid() instance.report_engine_event( 'Started process for pipeline (pid: {pid}).'.format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end='cli_api_subprocess_init'), ) # Perform setup so that termination of the execution will unwind and report to the # instance correctly setup_interrupt_support() try: for event in execute_run_iterator(recon_pipeline, pipeline_run, instance): stream.send(event) except DagsterSubprocessError as err: if not all([ err_info.cls_name == 'KeyboardInterrupt' for err_info in err.subprocess_error_infos ]): instance.report_engine_event( 'An exception was thrown during execution that is likely a framework error, ' 'rather than an error in user code.', pipeline_run, EngineEventData.engine_error( serializable_error_info_from_exc_info(sys.exc_info())), ) except Exception: # pylint: disable=broad-except instance.report_engine_event( 'An exception was thrown during execution that is likely a framework error, ' 'rather than an error in user code.', pipeline_run, EngineEventData.engine_error( serializable_error_info_from_exc_info(sys.exc_info())), ) finally: instance.report_engine_event( 'Process for pipeline exited (pid: {pid}).'.format(pid=pid), pipeline_run, )
def execute(self, pipeline_context, execution_plan): check.inst_param(pipeline_context, "pipeline_context", PlanOrchestrationContext) check.inst_param(execution_plan, "execution_plan", ExecutionPlan) step_keys_to_execute = execution_plan.step_keys_to_execute yield DagsterEvent.engine_event( pipeline_context, "Executing steps in process (pid: {pid})".format(pid=os.getpid()), event_specific_data=EngineEventData.in_process( os.getpid(), step_keys_to_execute), ) with time_execution_scope() as timer_result: yield from iter( ExecuteRunWithPlanIterable( execution_plan=pipeline_context.execution_plan, iterator=inner_plan_execution_iterator, execution_context_manager=PlanExecutionContextManager( pipeline=pipeline_context.pipeline, retry_mode=pipeline_context.retry_mode, execution_plan=pipeline_context.execution_plan, run_config=pipeline_context.run_config, pipeline_run=pipeline_context.pipeline_run, instance=pipeline_context.instance, raise_on_error=pipeline_context.raise_on_error, output_capture=pipeline_context.output_capture, ), )) yield DagsterEvent.engine_event( pipeline_context, "Finished steps in process (pid: {pid}) in {duration_ms}".format( pid=os.getpid(), duration_ms=format_duration(timer_result.millis)), event_specific_data=EngineEventData.in_process( os.getpid(), step_keys_to_execute), )
def create_event(count: int, run_id: str = RUN_ID): return EventLogEntry( error_info=None, user_message=str(count), level="debug", run_id=run_id, timestamp=time.time(), dagster_event=DagsterEvent( DagsterEventType.ENGINE_EVENT.value, "nonce", event_specific_data=EngineEventData.in_process(999), ), )
def create_test_event_log_record(message: str, run_id: str = DEFAULT_RUN_ID): return EventRecord( None, message, "debug", "", run_id, time.time(), dagster_event=DagsterEvent( DagsterEventType.ENGINE_EVENT.value, "nonce", event_specific_data=EngineEventData.in_process(999), ), )
def create_event(count: int, run_id: str = RUN_ID): return EventLogEntry( None, str(count), "debug", "", run_id, time.time(), dagster_event=DagsterEvent( DagsterEventType.ENGINE_EVENT.value, "nonce", event_specific_data=EngineEventData.in_process(999), ), )
def evt(name): return DagsterEventRecord( None, name, 'debug', '', 'foo', time.time(), dagster_event=DagsterEvent( DagsterEventType.ENGINE_EVENT.value, 'nonce', event_specific_data=EngineEventData.in_process(999), ), )
def evt(name): return EventRecord( None, name, "debug", "", "foo", time.time(), dagster_event=DagsterEvent( DagsterEventType.ENGINE_EVENT.value, "nonce", event_specific_data=EngineEventData.in_process(999), ), )
def _execute_run_command_body(recon_pipeline, pipeline_run_id, instance, write_stream_fn, set_exit_code_on_failure): if instance.should_start_background_run_thread: cancellation_thread, cancellation_thread_shutdown_event = start_run_cancellation_thread( instance, pipeline_run_id) pipeline_run = instance.get_run_by_id(pipeline_run_id) pid = os.getpid() instance.report_engine_event( "Started process for run (pid: {pid}).".format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"), ) run_worker_failed = 0 try: for event in core_execute_run( recon_pipeline, pipeline_run, instance, ): write_stream_fn(event) if event.event_type == DagsterEventType.PIPELINE_FAILURE: run_worker_failed = True except: # relies on core_execute_run writing failures to the event log before raising run_worker_failed = True finally: if instance.should_start_background_run_thread: cancellation_thread_shutdown_event.set() if cancellation_thread.is_alive(): cancellation_thread.join(timeout=15) if cancellation_thread.is_alive(): instance.report_engine_event( "Cancellation thread did not shutdown gracefully", pipeline_run, ) instance.report_engine_event( "Process for run exited (pid: {pid}).".format(pid=pid), pipeline_run, ) return 1 if (run_worker_failed and set_exit_code_on_failure) else 0
def _execute_run(request): try: execute_run_args = deserialize_json_to_dagster_namedtuple( request.serialized_execute_run_args) check.inst_param(execute_run_args, 'execute_run_args', ExecuteRunArgs) recon_pipeline = recon_pipeline_from_origin( execute_run_args.pipeline_origin) instance = DagsterInstance.from_ref(execute_run_args.instance_ref) pipeline_run = instance.get_run_by_id(execute_run_args.pipeline_run_id) pid = os.getpid() except: # pylint: disable=bare-except yield IPCErrorMessage( serializable_error_info=serializable_error_info_from_exc_info( sys.exc_info()), message='Error during RPC setup for ExecuteRun', ) return yield instance.report_engine_event( 'Started process for pipeline (pid: {pid}).'.format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end='cli_api_subprocess_init'), ) # This is so nasty but seemingly unavoidable # https://amir.rachum.com/blog/2017/03/03/generator-cleanup/ closed = False try: for event in _core_execute_run(recon_pipeline, pipeline_run, instance): yield event except GeneratorExit: closed = True raise finally: if not closed: yield instance.report_engine_event( 'Process for pipeline exited (pid: {pid}).'.format(pid=pid), pipeline_run, )
def test_event_log_storage_store_events_and_wipe(self, storage): assert len(storage.get_logs_for_run(DEFAULT_RUN_ID)) == 0 storage.store_event( EventRecord( None, "Message2", "debug", "", DEFAULT_RUN_ID, time.time(), dagster_event=DagsterEvent( DagsterEventType.ENGINE_EVENT.value, "nonce", event_specific_data=EngineEventData.in_process(999), ), )) assert len(storage.get_logs_for_run(DEFAULT_RUN_ID)) == 1 assert storage.get_stats_for_run(DEFAULT_RUN_ID) storage.wipe() assert len(storage.get_logs_for_run(DEFAULT_RUN_ID)) == 0
def test_in_memory_event_log_storage_store_events_and_wipe(): storage = InMemoryEventLogStorage() assert len(storage.get_logs_for_run('foo')) == 0 storage.store_event( DagsterEventRecord( None, 'Message2', 'debug', '', 'foo', time.time(), dagster_event=DagsterEvent( DagsterEventType.ENGINE_EVENT.value, 'nonce', event_specific_data=EngineEventData.in_process(999), ), )) assert len(storage.get_logs_for_run('foo')) == 1 storage.wipe() assert len(storage.get_logs_for_run('foo')) == 0
def test_event_log_storage_store_events_and_wipe(event_storage_factory_cm_fn): with event_storage_factory_cm_fn() as storage: assert len(storage.get_logs_for_run('foo')) == 0 storage.store_event( DagsterEventRecord( None, 'Message2', 'debug', '', 'foo', time.time(), dagster_event=DagsterEvent( DagsterEventType.ENGINE_EVENT.value, 'nonce', event_specific_data=EngineEventData.in_process(999), ), )) assert len(storage.get_logs_for_run('foo')) == 1 assert storage.get_stats_for_run('foo') storage.wipe() assert len(storage.get_logs_for_run('foo')) == 0
def test_filesystem_event_log_storage_store_events_and_wipe(): with seven.TemporaryDirectory() as tmpdir_path: storage = SqliteEventLogStorage(tmpdir_path) assert len(storage.get_logs_for_run('foo')) == 0 storage.store_event( DagsterEventRecord( None, 'Message2', 'debug', '', 'foo', time.time(), dagster_event=DagsterEvent( DagsterEventType.ENGINE_EVENT.value, 'nonce', event_specific_data=EngineEventData.in_process(999), ), )) assert len(storage.get_logs_for_run('foo')) == 1 storage.wipe() assert len(storage.get_logs_for_run('foo')) == 0
def test_event_log_delete(event_storage_factory_cm_fn): with event_storage_factory_cm_fn() as storage: assert len(storage.get_logs_for_run("foo")) == 0 storage.store_event( EventRecord( None, "Message2", "debug", "", "foo", time.time(), dagster_event=DagsterEvent( DagsterEventType.ENGINE_EVENT.value, "nonce", event_specific_data=EngineEventData.in_process(999), ), )) assert len(storage.get_logs_for_run("foo")) == 1 assert storage.get_stats_for_run("foo") storage.delete_events("foo") assert len(storage.get_logs_for_run("foo")) == 0
def execute(pipeline_context, execution_plan): check.inst_param(pipeline_context, 'pipeline_context', SystemPipelineExecutionContext) check.inst_param(execution_plan, 'execution_plan', ExecutionPlan) yield DagsterEvent.engine_event( pipeline_context, 'Executing steps in process (pid: {pid})'.format(pid=os.getpid()), event_specific_data=EngineEventData.in_process( os.getpid(), execution_plan.step_keys_to_execute), ) with time_execution_scope() as timer_result: check.param_invariant( isinstance(pipeline_context.executor_config, ExecutorConfig), 'pipeline_context', 'Expected executor_config to be ExecutorConfig got {}'.format( pipeline_context.executor_config), ) for event in copy_required_intermediates_for_execution( pipeline_context, execution_plan): yield event # It would be good to implement a reference tracking algorithm here to # garbage collect results that are no longer needed by any steps # https://github.com/dagster-io/dagster/issues/811 active_execution = execution_plan.start() while not active_execution.is_complete: steps = active_execution.get_steps_to_execute(limit=1) check.invariant( len(steps) == 1, 'Invariant Violation: expected step to be available to execute' ) step = steps[0] step_context = pipeline_context.for_step(step) check.invariant( all( hasattr(step_context.resources, resource_key) for resource_key in step_context.required_resource_keys), 'expected step context to have all required resources', ) with mirror_step_io(step_context): # capture all of the logs for this step uncovered_inputs = pipeline_context.intermediates_manager.uncovered_inputs( step_context, step) if uncovered_inputs: # In partial pipeline execution, we may end up here without having validated the # missing dependent outputs were optional _assert_missing_inputs_optional( uncovered_inputs, execution_plan, step.key) step_context.log.info(( 'Not all inputs covered for {step}. Not executing. Output missing for ' 'inputs: {uncovered_inputs}').format( uncovered_inputs=uncovered_inputs, step=step.key)) yield DagsterEvent.step_skipped_event(step_context) active_execution.mark_skipped(step.key) continue for step_event in check.generator( dagster_event_sequence_for_step(step_context)): check.inst(step_event, DagsterEvent) yield step_event active_execution.handle_event(step_event) active_execution.verify_complete(pipeline_context, step.key) # process skips from failures or uncovered inputs for event in active_execution.skipped_step_events_iterator( pipeline_context): yield event yield DagsterEvent.engine_event( pipeline_context, 'Finished steps in process (pid: {pid}) in {duration_ms}'.format( pid=os.getpid(), duration_ms=format_duration(timer_result.millis)), event_specific_data=EngineEventData.in_process( os.getpid(), execution_plan.step_keys_to_execute), )
def execute(self, pipeline_context, execution_plan): check.inst_param(pipeline_context, "pipeline_context", SystemPipelineExecutionContext) check.inst_param(execution_plan, "execution_plan", ExecutionPlan) limit = self.max_concurrent yield DagsterEvent.engine_event( pipeline_context, "Executing steps using multithread executor (pid: {pid})".format(pid=os.getpid()), event_specific_data=EngineEventData.in_process(os.getpid(), execution_plan.step_keys_to_execute), ) with time_execution_scope() as timer_result: with execution_plan.start(retries=self.retries) as active_execution: active_iters = {} errors = {} while not active_execution.is_complete or active_iters: # start iterators while len(active_iters) < limit: steps = active_execution.get_steps_to_execute(limit=(limit - len(active_iters))) if not steps: break for step in steps: step_context = pipeline_context.for_step(step) active_iters[step.key] = self.execute_step_in_thread(step.key, step_context, errors) # process active iterators empty_iters = [] for key, step_iter in active_iters.items(): try: event_or_none = next(step_iter) if event_or_none is None: continue yield event_or_none active_execution.handle_event(event_or_none) except ThreadCrashException: serializable_error = serializable_error_info_from_exc_info(sys.exc_info()) yield DagsterEvent.engine_event( pipeline_context, f"Multithread executor: thread for step {key} exited unexpectedly", EngineEventData.engine_error(serializable_error), ) step_failure_event = DagsterEvent.step_failure_event( step_context=pipeline_context.for_step(active_execution.get_step_by_key(key)), step_failure_data=StepFailureData(error=serializable_error, user_failure_data=None), ) active_execution.handle_event(step_failure_event) yield step_failure_event empty_iters.append(key) except StopIteration: empty_iters.append(key) # clear and mark complete finished iterators for key in empty_iters: del active_iters[key] active_execution.verify_complete(pipeline_context, key) # process skipped and abandoned steps for event in active_execution.plan_events_iterator(pipeline_context): yield event errs = {tid: err for tid, err in errors.items() if err} if errs: raise DagsterThreadError( "During multithread execution errors occurred in threads:\n{error_list}".format( error_list="\n".join( [ "In thread {tid}: {err}".format(tid=tid, err=err.to_string()) for tid, err in errs.items() ] ) ), thread_error_infos=list(errs.values()), ) yield DagsterEvent.engine_event( pipeline_context, "Multithread executor: parent process exiting after {duration} (pid: {pid})".format( duration=format_duration(timer_result.millis), pid=os.getpid() ), event_specific_data=EngineEventData.multiprocess(os.getpid()), )
def in_mp_process(cls, handle, pipeline_run, instance_ref, term_event): """ Execute pipeline using message queue as a transport """ run_id = pipeline_run.run_id pipeline_name = pipeline_run.pipeline_name instance = DagsterInstance.from_ref(instance_ref) pid = os.getpid() instance.report_engine_event( 'Started process for pipeline (pid: {pid}).'.format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end='dagit_subprocess_init'), cls, ) start_termination_thread(term_event) try: handle.build_repository_definition() pipeline_def = handle.with_pipeline_name( pipeline_name).build_pipeline_definition() except Exception: # pylint: disable=broad-except instance.report_engine_event( 'Failed attempting to load pipeline "{}"'.format( pipeline_name), pipeline_run, EngineEventData.engine_error( serializable_error_info_from_exc_info(sys.exc_info())), cls, ) return try: event_list = [] for event in execute_run_iterator( pipeline_def.build_sub_pipeline( pipeline_run.selector.solid_subset), pipeline_run, instance, ): event_list.append(event) return PipelineExecutionResult(pipeline_def, run_id, event_list, lambda: None) # Add a DagsterEvent for unexpected exceptions # Explicitly ignore KeyboardInterrupts since they are used for termination except DagsterSubprocessError as err: if not all([ err_info.cls_name == 'KeyboardInterrupt' for err_info in err.subprocess_error_infos ]): instance.report_engine_event( 'An exception was thrown during execution that is likely a framework error, ' 'rather than an error in user code.', pipeline_run, EngineEventData.engine_error( serializable_error_info_from_exc_info(sys.exc_info())), cls, ) except Exception: # pylint: disable=broad-except instance.report_engine_event( 'An exception was thrown during execution that is likely a framework error, ' 'rather than an error in user code.', pipeline_run, EngineEventData.engine_error( serializable_error_info_from_exc_info(sys.exc_info())), cls, ) finally: instance.report_engine_event( 'Process for pipeline exited (pid: {pid}).'.format(pid=pid), pipeline_run, cls=cls, )
def _execute_run(request): try: execute_run_args = deserialize_json_to_dagster_namedtuple( request.serialized_execute_run_args) check.inst_param(execute_run_args, 'execute_run_args', ExecuteRunArgs) recon_pipeline = recon_pipeline_from_origin( execute_run_args.pipeline_origin) instance = DagsterInstance.from_ref(execute_run_args.instance_ref) pipeline_run = instance.get_run_by_id(execute_run_args.pipeline_run_id) pid = os.getpid() except: # pylint: disable=bare-except yield IPCErrorMessage( serializable_error_info=serializable_error_info_from_exc_info( sys.exc_info()), message='Error during RPC setup for ExecuteRun', ) return yield instance.report_engine_event( 'Started process for pipeline (pid: {pid}).'.format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end='cli_api_subprocess_init'), ) # This is so nasty but seemingly unavoidable # https://amir.rachum.com/blog/2017/03/03/generator-cleanup/ closed = False try: for event in execute_run_iterator(recon_pipeline, pipeline_run, instance): yield event except DagsterSubprocessError as err: if not all([ err_info.cls_name == 'KeyboardInterrupt' for err_info in err.subprocess_error_infos ]): yield instance.report_engine_event( 'An exception was thrown during execution that is likely a framework error, ' 'rather than an error in user code.', pipeline_run, EngineEventData.engine_error( serializable_error_info_from_exc_info(sys.exc_info())), ) instance.report_run_failed(pipeline_run) except GeneratorExit: closed = True raise except Exception: # pylint: disable=broad-except yield instance.report_engine_event( 'An exception was thrown during execution that is likely a framework error, ' 'rather than an error in user code.', pipeline_run, EngineEventData.engine_error( serializable_error_info_from_exc_info(sys.exc_info())), ) instance.report_run_failed(pipeline_run) finally: if not closed: yield instance.report_engine_event( 'Process for pipeline exited (pid: {pid}).'.format(pid=pid), pipeline_run, )
def execute(pipeline_context, execution_plan): check.inst_param(pipeline_context, 'pipeline_context', SystemPipelineExecutionContext) check.inst_param(execution_plan, 'execution_plan', ExecutionPlan) step_levels = execution_plan.execution_step_levels() step_key_set = set(step.key for step_level in step_levels for step in step_level) yield DagsterEvent.engine_event( pipeline_context, 'Executing steps in process (pid: {pid})'.format(pid=os.getpid()), event_specific_data=EngineEventData.in_process( os.getpid(), step_key_set), ) with time_execution_scope() as timer_result: check.param_invariant( isinstance(pipeline_context.executor_config, ExecutorConfig), 'pipeline_context', 'Expected executor_config to be ExecutorConfig got {}'.format( pipeline_context.executor_config), ) for event in copy_required_intermediates_for_execution( pipeline_context, execution_plan): yield event failed_or_skipped_steps = set() # It would be good to implement a reference tracking algorithm here to # garbage collect results that are no longer needed by any steps # https://github.com/dagster-io/dagster/issues/811 for step_level in step_levels: for step in step_level: step_context = pipeline_context.for_step(step) with mirror_step_io(step_context): # capture all of the logs for this step failed_inputs = [] for step_input in step.step_inputs: failed_inputs.extend( failed_or_skipped_steps.intersection( step_input.dependency_keys)) if failed_inputs: step_context.log.info(( 'Dependencies for step {step} failed: {failed_inputs}. Not executing.' ).format(step=step.key, failed_inputs=failed_inputs)) failed_or_skipped_steps.add(step.key) yield DagsterEvent.step_skipped_event(step_context) continue uncovered_inputs = pipeline_context.intermediates_manager.uncovered_inputs( step_context, step) if uncovered_inputs: # In partial pipeline execution, we may end up here without having validated the # missing dependent outputs were optional _assert_missing_inputs_optional( uncovered_inputs, execution_plan, step.key) step_context.log.info(( 'Not all inputs covered for {step}. Not executing. Output missing for ' 'inputs: {uncovered_inputs}').format( uncovered_inputs=uncovered_inputs, step=step.key)) failed_or_skipped_steps.add(step.key) yield DagsterEvent.step_skipped_event(step_context) continue for step_event in check.generator( dagster_event_sequence_for_step(step_context)): check.inst(step_event, DagsterEvent) if step_event.is_step_failure: failed_or_skipped_steps.add(step.key) yield step_event yield DagsterEvent.engine_event( pipeline_context, 'Finished steps in process (pid: {pid}) in {duration_ms}'.format( pid=os.getpid(), duration_ms=format_duration(timer_result.millis)), event_specific_data=EngineEventData.in_process( os.getpid(), step_key_set), )
def _run_in_subprocess( serialized_execute_run_args, recon_pipeline, termination_event, subprocess_status_handler, run_event_handler, ): start_termination_thread(termination_event) try: execute_run_args = deserialize_json_to_dagster_namedtuple( serialized_execute_run_args) check.inst_param(execute_run_args, 'execute_run_args', ExecuteRunArgs) instance = DagsterInstance.from_ref(execute_run_args.instance_ref) pipeline_run = instance.get_run_by_id(execute_run_args.pipeline_run_id) pid = os.getpid() except: # pylint: disable=bare-except event = IPCErrorMessage( serializable_error_info=serializable_error_info_from_exc_info( sys.exc_info()), message='Error during RPC setup for ExecuteRun', ) subprocess_status_handler(event) subprocess_status_handler(RunInSubprocessComplete()) return subprocess_status_handler(StartRunInSubprocessSuccessful()) run_event_handler( instance.report_engine_event( 'Started process for pipeline (pid: {pid}).'.format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end='cli_api_subprocess_init'), )) # This is so nasty but seemingly unavoidable # https://amir.rachum.com/blog/2017/03/03/generator-cleanup/ closed = False try: for event in _core_execute_run(recon_pipeline, pipeline_run, instance): run_event_handler(event) except KeyboardInterrupt: run_event_handler( instance.report_engine_event( message='Pipeline execution terminated by interrupt', pipeline_run=pipeline_run, )) raise except GeneratorExit: closed = True raise finally: if not closed: run_event_handler( instance.report_engine_event( 'Process for pipeline exited (pid: {pid}).'.format( pid=pid), pipeline_run, )) subprocess_status_handler(RunInSubprocessComplete())
def _run_in_subprocess( serialized_execute_run_args, recon_pipeline, termination_event, subprocess_status_handler, run_event_handler, ): start_termination_thread(termination_event) try: execute_run_args = deserialize_json_to_dagster_namedtuple( serialized_execute_run_args) check.inst_param(execute_run_args, "execute_run_args", ExecuteExternalPipelineArgs) instance = DagsterInstance.from_ref(execute_run_args.instance_ref) pipeline_run = instance.get_run_by_id(execute_run_args.pipeline_run_id) if not pipeline_run: raise DagsterRunNotFoundError( "gRPC server could not load run {run_id} in order to execute it. Make sure that the gRPC server has access to your run storage." .format(run_id=execute_run_args.pipeline_run_id), invalid_run_id=execute_run_args.pipeline_run_id, ) pid = os.getpid() except: # pylint: disable=bare-except serializable_error_info = serializable_error_info_from_exc_info( sys.exc_info()) event = IPCErrorMessage( serializable_error_info=serializable_error_info, message="Error during RPC setup for executing run: {message}". format(message=serializable_error_info.message), ) subprocess_status_handler(event) subprocess_status_handler(RunInSubprocessComplete()) if instance: instance.dispose() return subprocess_status_handler(StartRunInSubprocessSuccessful()) run_event_handler( instance.report_engine_event( "Started process for pipeline (pid: {pid}).".format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"), )) # This is so nasty but seemingly unavoidable # https://amir.rachum.com/blog/2017/03/03/generator-cleanup/ closed = False try: for event in _core_execute_run(recon_pipeline, pipeline_run, instance): run_event_handler(event) except GeneratorExit: closed = True raise finally: if not closed: run_event_handler( instance.report_engine_event( "Process for pipeline exited (pid: {pid}).".format( pid=pid), pipeline_run, )) subprocess_status_handler(RunInSubprocessComplete()) instance.dispose()