def get_external_executable_params(recon_repo, external_executable_args): check.inst_param(recon_repo, "recon_repo", ReconstructableRepository) check.inst_param( external_executable_args, "external_executable_args", ExternalExecutableArgs, ) definition = recon_repo.get_definition() executable_def = definition.get_executable_def( external_executable_args.name) with DagsterInstance.from_ref( external_executable_args.instance_ref) as instance: context = ExecutableContext(instance) try: with user_code_error_boundary( ExecutableError, lambda: "Error occured during the execution of run_config_fn for triggered " "execution {name}".format(name=executable_def.name), ): run_config = executable_def.get_run_config(context) with user_code_error_boundary( ExecutableError, lambda: "Error occured during the execution of tags_fn for triggered " "execution {name}".format(name=executable_def.name), ): tags = executable_def.get_tags(context) return ExternalExecutionParamsData(run_config=run_config, tags=tags) except ExecutableError: return ExternalExecutionParamsErrorData( serializable_error_info_from_exc_info(sys.exc_info()))
def start_run(self, execute_run_args): check.inst_param(execute_run_args, 'execute_run_args', ExecuteRunArgs) try: instance = DagsterInstance.from_ref(execute_run_args.instance_ref) pipeline_run = instance.get_run_by_id( execute_run_args.pipeline_run_id) res = self._query( 'StartRun', api_pb2.StartRunRequest, serialized_execute_run_args=serialize_dagster_namedtuple( execute_run_args), ) return deserialize_json_to_dagster_namedtuple( res.serialized_start_run_result) except Exception: # pylint: disable=bare-except instance.report_engine_event( message='Unexpected error in IPC client', pipeline_run=pipeline_run, engine_event_data=EngineEventData.engine_error( serializable_error_info_from_exc_info(sys.exc_info())), ) raise
def test_event_log_step_key_migration(): src_dir = file_relative_path( __file__, "snapshot_0_7_6_pre_event_log_migration/sqlite") with copy_directory(src_dir) as test_dir: instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) # Make sure the schema is migrated instance.upgrade() runs = instance.get_runs() assert len(runs) == 1 run_ids = instance._event_storage.get_all_run_ids() assert run_ids == ["6405c4a0-3ccc-4600-af81-b5ee197f8528"] assert isinstance(instance._event_storage, SqlEventLogStorage) events_by_id = instance._event_storage.get_logs_for_run_by_log_id( "6405c4a0-3ccc-4600-af81-b5ee197f8528") assert len(events_by_id) == 40 step_key_records = [] for record_id, _event in events_by_id.items(): row_data = instance._event_storage.get_event_log_table_data( "6405c4a0-3ccc-4600-af81-b5ee197f8528", record_id) if row_data.step_key is not None: step_key_records.append(row_data) assert len(step_key_records) == 0 # run the event_log backfill migration migrate_event_log_data(instance=instance) step_key_records = [] for record_id, _event in events_by_id.items(): row_data = instance._event_storage.get_event_log_table_data( "6405c4a0-3ccc-4600-af81-b5ee197f8528", record_id) if row_data.step_key is not None: step_key_records.append(row_data) assert len(step_key_records) > 0
def test_0_6_6_sqlite_exc(): test_dir = file_relative_path(__file__, 'snapshot_0_6_6/sqlite') with restore_directory(test_dir): instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) runs = instance.get_runs() # Note that this is a deliberate choice -- old runs are simply invisible, and their # presence won't raise DagsterInstanceMigrationRequired. This is a reasonable choice since # the runs.db has moved and otherwise we would have to do a check for the existence of an # old runs.db every time we accessed the runs. Instead, we'll do this only in the upgrade # method. assert len(runs) == 0 run_ids = instance._event_storage.get_all_run_ids() assert run_ids == ['89296095-892d-4a15-aa0d-9018d1580945'] with pytest.raises( DagsterInstanceMigrationRequired, match=re.escape( 'Instance is out of date and must be migrated (SqliteEventLogStorage for run ' '89296095-892d-4a15-aa0d-9018d1580945). Database is at revision None, head is ' '3b1e175a2be3. Please run `dagster instance migrate`.'), ): instance._event_storage.get_logs_for_run( '89296095-892d-4a15-aa0d-9018d1580945')
def get_external_schedule_execution(external_schedule_execution_args): check.inst_param( external_schedule_execution_args, 'external_schedule_execution_args', ExternalScheduleExecutionArgs, ) recon_repo = recon_repository_from_origin(external_schedule_execution_args.repository_origin) definition = recon_repo.get_definition() schedule_def = definition.get_schedule_def(external_schedule_execution_args.schedule_name) instance = DagsterInstance.from_ref(external_schedule_execution_args.instance_ref) schedule_context = ScheduleExecutionContext(instance) try: with user_code_error_boundary( ScheduleExecutionError, lambda: 'Error occurred during the execution of run_config_fn for schedule ' '{schedule_name}'.format(schedule_name=schedule_def.name), ): run_config = schedule_def.get_run_config(schedule_context) return ExternalScheduleExecutionData(run_config=run_config) except ScheduleExecutionError: return ExternalScheduleExecutionErrorData( serializable_error_info_from_exc_info(sys.exc_info()) )
def execute(self): check.inst(self.executor_config, MultiprocessExecutorConfig) pipeline = self.executor_config.pipeline instance = DagsterInstance.from_ref(self.instance_ref) start_termination_thread(self.term_event) execution_plan = create_execution_plan( pipeline=pipeline, environment_dict=self.environment_dict, mode=self.pipeline_run.mode, step_keys_to_execute=self.pipeline_run.step_keys_to_execute, ).build_subset_plan([self.step_key]) yield instance.report_engine_event( 'Executing step {} in subprocess'.format(self.step_key), self.pipeline_run, EngineEventData( [ EventMetadataEntry.text(str(os.getpid()), 'pid'), EventMetadataEntry.text(self.step_key, 'step_key'), ], marker_end=DELEGATE_MARKER, ), MultiprocessEngine, self.step_key, ) for step_event in execute_plan_iterator( execution_plan, self.pipeline_run, environment_dict=self.environment_dict, retries=self.executor_config.retries.for_inner_plan(), instance=instance, ): yield step_event
def reconstitute_pipeline_context( self, output_log_path=None, marshal_dir=None, run_config=None, executable_dict=None, pipeline_run_dict=None, solid_handle_kwargs=None, instance_ref_dict=None, ): """Reconstitutes a context for dagstermill-managed execution. You'll see this function called to reconstruct a pipeline context within the ``injected parameters`` cell of a dagstermill output notebook. Users should not call this function interactively except when debugging output notebooks. Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a context for interactive exploration and development. This call will be replaced by one to :func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by dagstermill. """ check.opt_str_param(output_log_path, "output_log_path") check.opt_str_param(marshal_dir, "marshal_dir") run_config = check.opt_dict_param(run_config, "run_config", key_type=str) check.dict_param(pipeline_run_dict, "pipeline_run_dict") check.dict_param(executable_dict, "executable_dict") check.dict_param(solid_handle_kwargs, "solid_handle_kwargs") check.dict_param(instance_ref_dict, "instance_ref_dict") pipeline = ReconstructablePipeline.from_dict(executable_dict) pipeline_def = pipeline.get_definition() try: instance_ref = unpack_value(instance_ref_dict) instance = DagsterInstance.from_ref(instance_ref) except Exception as err: # pylint: disable=broad-except raise DagstermillError( "Error when attempting to resolve DagsterInstance from serialized InstanceRef" ) from err pipeline_run = unpack_value(pipeline_run_dict) solid_handle = SolidHandle.from_dict(solid_handle_kwargs) solid_def = pipeline_def.get_solid(solid_handle).definition self.marshal_dir = marshal_dir self.in_pipeline = True self.solid_def = solid_def self.pipeline = pipeline environment_config = EnvironmentConfig.build(pipeline_def, run_config, mode=pipeline_run.mode) execution_plan = ExecutionPlan.build( self.pipeline, environment_config, step_keys_to_execute=pipeline_run.step_keys_to_execute, ) with scoped_pipeline_context( execution_plan, pipeline, run_config, pipeline_run, instance, scoped_resources_builder_cm=self._setup_resources, # Set this flag even though we're not in test for clearer error reporting raise_on_error=True, ) as pipeline_context: self.context = DagstermillRuntimeExecutionContext( pipeline_context=pipeline_context, pipeline_def=pipeline_def, solid_config=run_config.get("solids", {}).get(solid_def.name, {}).get("config"), resource_keys_to_init=get_required_resource_keys_to_init( execution_plan, pipeline_def, environment_config, pipeline_context.intermediate_storage_def, ), solid_name=solid_def.name, ) return self.context
def __init__( self, task_id, environment_dict=None, pipeline_name=None, mode=None, step_keys=None, dag=None, instance_ref=None, *args, **kwargs ): check.str_param(pipeline_name, 'pipeline_name') step_keys = check.opt_list_param(step_keys, 'step_keys', of_type=str) environment_dict = check.opt_dict_param(environment_dict, 'environment_dict', key_type=str) check.opt_inst_param(instance_ref, 'instance_ref', InstanceRef) tmp_dir = kwargs.pop('tmp_dir', DOCKER_TEMPDIR) host_tmp_dir = kwargs.pop('host_tmp_dir', seven.get_system_temp_directory()) if not environment_dict.get('storage'): raise AirflowException( 'No storage config found -- must configure storage for ' 'the DagsterDockerOperator. Ex.: \n' 'storage:\n' ' filesystem:\n' ' config:' ' base_dir: \'/some/shared/volume/mount/special_place\'' '\n\n --or--\n\n' 'storage:\n' ' s3:\n' ' s3_bucket: \'my-s3-bucket\'\n' '\n\n --or--\n\n' 'storage:\n' ' gcs:\n' ' gcs_bucket: \'my-gcs-bucket\'\n' ) if 'filesystem' in environment_dict['storage']: if ( 'config' in (environment_dict['storage'].get('filesystem', {}) or {}) and 'base_dir' in ( (environment_dict['storage'].get('filesystem', {}) or {}).get('config', {}) or {} ) and environment_dict['storage']['filesystem']['config']['base_dir'] != tmp_dir ): warnings.warn( 'Found base_dir \'{base_dir}\' set in filesystem storage config, which was not ' 'the tmp_dir we expected (\'{tmp_dir}\', mounting host_tmp_dir ' '\'{host_tmp_dir}\' from the host). We assume you know what you are doing, but ' 'if you are having trouble executing containerized workloads, this may be the ' 'issue'.format( base_dir=environment_dict['storage']['filesystem']['config']['base_dir'], tmp_dir=tmp_dir, host_tmp_dir=host_tmp_dir, ) ) else: environment_dict['storage']['filesystem'] = dict( environment_dict['storage']['filesystem'] or {}, **{ 'config': dict( ( (environment_dict['storage'].get('filesystem', {}) or {}).get( 'config', {} ) or {} ), **{'base_dir': tmp_dir} ) } ) self.docker_conn_id_set = kwargs.get('docker_conn_id') is not None self.environment_dict = environment_dict self.pipeline_name = pipeline_name self.mode = mode self.step_keys = step_keys self._run_id = None # self.instance might be None in, for instance, a unit test setting where the operator # was being directly instantiated without passing through make_airflow_dag self.instance = DagsterInstance.from_ref(instance_ref) if instance_ref else None # These shenanigans are so we can override DockerOperator.get_hook in order to configure # a docker client using docker.from_env, rather than messing with the logic of # DockerOperator.execute if not self.docker_conn_id_set: try: from_env().version() except Exception: # pylint: disable=broad-except pass else: kwargs['docker_conn_id'] = True # We do this because log lines won't necessarily be emitted in order (!) -- so we can't # just check the last log line to see if it's JSON. kwargs['xcom_all'] = True # Store Airflow DAG run timestamp so that we can pass along via execution metadata self.airflow_ts = kwargs.get('ts') if 'environment' not in kwargs: kwargs['environment'] = get_aws_environment() super(DagsterDockerOperator, self).__init__( task_id=task_id, dag=dag, tmp_dir=tmp_dir, host_tmp_dir=host_tmp_dir, *args, **kwargs )
def execute_inner(step_key, pipeline_run, instance_ref): instance = DagsterInstance.from_ref(instance_ref) inner_step(instance, pipeline_run, step_key)
def _run_in_subprocess( serialized_execute_run_args, recon_pipeline, termination_event, subprocess_status_handler, run_event_handler, ): start_termination_thread(termination_event) try: execute_run_args = deserialize_json_to_dagster_namedtuple( serialized_execute_run_args) check.inst_param(execute_run_args, "execute_run_args", ExecuteExternalPipelineArgs) instance = DagsterInstance.from_ref(execute_run_args.instance_ref) pipeline_run = instance.get_run_by_id(execute_run_args.pipeline_run_id) if not pipeline_run: raise DagsterRunNotFoundError( "gRPC server could not load run {run_id} in order to execute it. Make sure that the gRPC server has access to your run storage." .format(run_id=execute_run_args.pipeline_run_id), invalid_run_id=execute_run_args.pipeline_run_id, ) pid = os.getpid() except: # pylint: disable=bare-except serializable_error_info = serializable_error_info_from_exc_info( sys.exc_info()) event = IPCErrorMessage( serializable_error_info=serializable_error_info, message="Error during RPC setup for executing run: {message}". format(message=serializable_error_info.message), ) subprocess_status_handler(event) subprocess_status_handler(RunInSubprocessComplete()) if instance: instance.dispose() return subprocess_status_handler(StartRunInSubprocessSuccessful()) run_event_handler( instance.report_engine_event( "Started process for pipeline (pid: {pid}).".format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"), )) # This is so nasty but seemingly unavoidable # https://amir.rachum.com/blog/2017/03/03/generator-cleanup/ closed = False try: for event in _core_execute_run(recon_pipeline, pipeline_run, instance): run_event_handler(event) except GeneratorExit: closed = True raise finally: if not closed: run_event_handler( instance.report_engine_event( "Process for pipeline exited (pid: {pid}).".format( pid=pid), pipeline_run, )) subprocess_status_handler(RunInSubprocessComplete()) instance.dispose()
def execute_run(self, execute_run_args): check.inst_param(execute_run_args, "execute_run_args", ExecuteRunArgs) with DagsterInstance.from_ref( execute_run_args.instance_ref) as instance: try: pipeline_run = instance.get_run_by_id( execute_run_args.pipeline_run_id) event_iterator = self._streaming_query( "ExecuteRun", api_pb2.ExecuteRunRequest, serialized_execute_run_args=serialize_dagster_namedtuple( execute_run_args), ) except Exception as exc: # pylint: disable=bare-except yield instance.report_engine_event( message="Unexpected error in IPC client", pipeline_run=pipeline_run, engine_event_data=EngineEventData.engine_error( serializable_error_info_from_exc_info(sys.exc_info())), ) raise exc try: for event in event_iterator: yield deserialize_json_to_dagster_namedtuple( event.serialized_dagster_event_or_ipc_error_message) except KeyboardInterrupt: self.cancel_execution( CancelExecutionRequest( run_id=execute_run_args.pipeline_run_id)) raise except grpc.RpcError as rpc_error: if ( # posix "Socket closed" in rpc_error.debug_error_string() # pylint: disable=no-member # windows or "Stream removed" in rpc_error.debug_error_string() # pylint: disable=no-member ): yield instance.report_engine_event( message= "User process: GRPC server for {run_id} terminated unexpectedly" .format(run_id=pipeline_run.run_id), pipeline_run=pipeline_run, engine_event_data=EngineEventData.engine_error( serializable_error_info_from_exc_info( sys.exc_info())), ) yield instance.report_run_failed(pipeline_run) else: yield instance.report_engine_event( message="Unexpected error in IPC client", pipeline_run=pipeline_run, engine_event_data=EngineEventData.engine_error( serializable_error_info_from_exc_info( sys.exc_info())), ) raise rpc_error except Exception as exc: # pylint: disable=bare-except yield instance.report_engine_event( message="Unexpected error in IPC client", pipeline_run=pipeline_run, engine_event_data=EngineEventData.engine_error( serializable_error_info_from_exc_info(sys.exc_info())), ) raise exc
def test_init_compute_log_with_bad_config_override(): with seven.TemporaryDirectory() as tmpdir_path: with pytest.raises(DagsterInvalidConfigError, match='Undefined field "garbage"'): DagsterInstance.from_ref( InstanceRef.from_dir(tmpdir_path, overrides={'compute_logs': {'garbage': 'flargh'}}) )
def test_backcompat_asset_materializations(): src_dir = file_relative_path( __file__, "compat_tests/snapshot_0_11_0_asset_materialization") # should contain materialization events for asset keys a, b, c, d, e, f # events a and b have been wiped, but b has been rematerialized @op def materialize(): yield AssetMaterialization(AssetKey("c"), tags={"foo": "bar"}) yield Output(None) @job def my_job(): materialize() def _validate_materialization(asset_key, event, expected_tags): assert isinstance(event, EventLogEntry) assert event.dagster_event assert event.dagster_event.is_step_materialization assert event.dagster_event.step_materialization_data.materialization.asset_key == asset_key assert event.dagster_event.step_materialization_data.materialization.tags == expected_tags a = AssetKey("a") b = AssetKey("b") c = AssetKey("c") with copy_directory(src_dir) as test_dir: with DagsterInstance.from_ref( InstanceRef.from_dir(test_dir)) as instance: storage = instance.event_log_storage a_mat = storage.get_latest_materialization_events([a]).get(a) assert a_mat is None b_mat = storage.get_latest_materialization_events([b]).get(b) _validate_materialization(b, b_mat, expected_tags={}) c_mat = storage.get_latest_materialization_events([c]).get(c) _validate_materialization(c, c_mat, expected_tags={}) mat_by_key = storage.get_latest_materialization_events([a, b, c]) assert mat_by_key.get(a) is None _validate_materialization(b, mat_by_key.get(b), expected_tags={}) _validate_materialization(c, mat_by_key.get(c), expected_tags={}) # materialize c with tags my_job.execute_in_process(instance=instance) a_mat = storage.get_latest_materialization_events([a]).get(a) assert a_mat is None b_mat = storage.get_latest_materialization_events([b]).get(b) _validate_materialization(b, b_mat, expected_tags={}) c_mat = storage.get_latest_materialization_events([c]).get(c) _validate_materialization(c, c_mat, expected_tags={"foo": "bar"}) mat_by_key = storage.get_latest_materialization_events([a, b, c]) assert mat_by_key.get(a) is None _validate_materialization(b, mat_by_key.get(b), expected_tags={}) _validate_materialization(c, c_mat, expected_tags={"foo": "bar"})
def reconstitute_pipeline_context( self, output_log_path=None, marshal_dir=None, environment_dict=None, executable_dict=None, pipeline_run_dict=None, solid_handle_kwargs=None, instance_ref_dict=None, ): '''Reconstitutes a context for dagstermill-managed execution. You'll see this function called to reconstruct a pipeline context within the ``injected parameters`` cell of a dagstermill output notebook. Users should not call this function interactively except when debugging output notebooks. Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a context for interactive exploration and development. This call will be replaced by one to :func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by dagstermill. ''' check.opt_str_param(output_log_path, 'output_log_path') check.opt_str_param(marshal_dir, 'marshal_dir') environment_dict = check.opt_dict_param(environment_dict, 'environment_dict', key_type=str) check.dict_param(pipeline_run_dict, 'pipeline_run_dict') check.dict_param(executable_dict, 'executable_dict') check.dict_param(solid_handle_kwargs, 'solid_handle_kwargs') check.dict_param(instance_ref_dict, 'instance_ref_dict') pipeline = InterProcessExecutablePipeline.from_dict(executable_dict) pipeline_def = pipeline.get_definition() try: instance_ref = unpack_value(instance_ref_dict) instance = DagsterInstance.from_ref(instance_ref) except Exception as err: # pylint: disable=broad-except six.raise_from( DagstermillError( 'Error when attempting to resolve DagsterInstance from serialized InstanceRef' ), err, ) pipeline_run = unpack_value(pipeline_run_dict) solid_handle = SolidHandle.from_dict(solid_handle_kwargs) solid_def = pipeline_def.get_solid(solid_handle).definition self.marshal_dir = marshal_dir self.in_pipeline = True self.solid_def = solid_def self.pipeline = pipeline execution_plan = create_execution_plan( self.pipeline, environment_dict, mode=pipeline_run.mode, step_keys_to_execute=pipeline_run.step_keys_to_execute, ) with scoped_pipeline_context( execution_plan, environment_dict, pipeline_run, instance, scoped_resources_builder_cm=self._setup_resources, # Set this flag even though we're not in test for clearer error reporting raise_on_error=True, ) as pipeline_context: self.context = DagstermillRuntimeExecutionContext( pipeline_context=pipeline_context, solid_config=None, resource_keys_to_init=get_required_resource_keys_to_init( execution_plan, pipeline_context.system_storage_def), ) return self.context
def in_mp_process(cls, handle, pipeline_run, instance_ref, term_event): """ Execute pipeline using message queue as a transport """ run_id = pipeline_run.run_id pipeline_name = pipeline_run.pipeline_name instance = DagsterInstance.from_ref(instance_ref) pid = os.getpid() instance.report_engine_event( 'Started process for pipeline (pid: {pid}).'.format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end='dagit_subprocess_init'), cls, ) start_termination_thread(term_event) try: handle.build_repository_definition() pipeline_def = handle.with_pipeline_name(pipeline_name).build_pipeline_definition() except Exception: # pylint: disable=broad-except instance.report_engine_event( 'Failed attempting to load pipeline "{}"'.format(pipeline_name), pipeline_run, EngineEventData.engine_error(serializable_error_info_from_exc_info(sys.exc_info())), cls, ) return try: event_list = [] for event in execute_run_iterator( pipeline_def.build_sub_pipeline(pipeline_run.selector.solid_subset), pipeline_run, instance, ): event_list.append(event) return PipelineExecutionResult(pipeline_def, run_id, event_list, lambda: None) # Add a DagsterEvent for unexpected exceptions # Explicitly ignore KeyboardInterrupts since they are used for termination except DagsterSubprocessError as err: if not all( [ err_info.cls_name == 'KeyboardInterrupt' for err_info in err.subprocess_error_infos ] ): instance.report_engine_event( 'An exception was thrown during execution that is likely a framework error, ' 'rather than an error in user code.', pipeline_run, EngineEventData.engine_error( serializable_error_info_from_exc_info(sys.exc_info()) ), cls, ) except Exception: # pylint: disable=broad-except instance.report_engine_event( 'An exception was thrown during execution that is likely a framework error, ' 'rather than an error in user code.', pipeline_run, EngineEventData.engine_error(serializable_error_info_from_exc_info(sys.exc_info())), cls, ) finally: instance.report_engine_event( 'Process for pipeline exited (pid: {pid}).'.format(pid=pid), pipeline_run, cls=cls, )
def StartRun(self, request, _context): execute_run_args = check.inst( deserialize_json_to_dagster_namedtuple( request.serialized_execute_run_args), ExecuteRunArgs, ) try: execute_run_args = check.inst( deserialize_json_to_dagster_namedtuple( request.serialized_execute_run_args), ExecuteRunArgs, ) run_id = execute_run_args.pipeline_run_id recon_pipeline = self._recon_pipeline_from_origin( execute_run_args.pipeline_origin) except: # pylint: disable=bare-except return api_pb2.StartRunReply( serialized_start_run_result=serialize_dagster_namedtuple( StartRunResult( success=False, message=None, serializable_error_info= serializable_error_info_from_exc_info(sys.exc_info()), ))) event_queue = multiprocessing.Queue() termination_event = multiprocessing.Event() execution_process = multiprocessing.Process( target=start_run_in_subprocess, args=[ request.serialized_execute_run_args, recon_pipeline, event_queue, termination_event, ], ) with self._execution_lock: execution_process.start() self._executions[run_id] = ( execution_process, DagsterInstance.from_ref(execute_run_args.instance_ref), ) self._termination_events[run_id] = termination_event success = None message = None serializable_error_info = None while success is None: time.sleep(EVENT_QUEUE_POLL_INTERVAL) # We use `get_nowait()` instead of `get()` so that we can handle the case where the # execution process has died unexpectedly -- `get()` would hang forever in that case try: dagster_event_or_ipc_error_message_or_done = event_queue.get_nowait( ) except queue.Empty: if not execution_process.is_alive(): # subprocess died unexpectedly success = False message = ( "GRPC server: Subprocess for {run_id} terminated unexpectedly with " "exit code {exit_code}".format( run_id=run_id, exit_code=execution_process.exitcode, )) serializable_error_info = serializable_error_info_from_exc_info( sys.exc_info()) else: if isinstance(dagster_event_or_ipc_error_message_or_done, StartRunInSubprocessSuccessful): success = True elif isinstance(dagster_event_or_ipc_error_message_or_done, RunInSubprocessComplete): continue if isinstance(dagster_event_or_ipc_error_message_or_done, IPCErrorMessage): success = False message = dagster_event_or_ipc_error_message_or_done.message serializable_error_info = ( dagster_event_or_ipc_error_message_or_done. serializable_error_info) # Ensure that if the run failed, we remove it from the executions map before # returning so that CanCancel will never return True if not success: self._check_for_orphaned_runs() return api_pb2.StartRunReply( serialized_start_run_result=serialize_dagster_namedtuple( StartRunResult( success=success, message=message, serializable_error_info=serializable_error_info, )))
def test_init_compute_log_with_bad_config(): with seven.TemporaryDirectory() as tmpdir_path: with open(os.path.join(tmpdir_path, 'dagster.yaml'), 'w') as fd: yaml.dump({'compute_logs': {'garbage': 'flargh'}}, fd, default_flow_style=False) with pytest.raises(DagsterInvalidConfigError, match='Undefined field "garbage"'): DagsterInstance.from_ref(InstanceRef.from_dir(tmpdir_path))
def _start_pipeline_execution(self, job_args): handle = job_args['handle'] pipeline_run = job_args['pipeline_run'] pipeline = handle.build_repository_definition().get_pipeline(pipeline_run.pipeline_name) instance = DagsterInstance.from_ref(job_args['instance_ref']) self._delegate.execute_pipeline(handle, pipeline, pipeline_run, instance)
def __init__(self, dagster_operator_parameters, *args): kwargs = dagster_operator_parameters.op_kwargs tmp_dir = kwargs.pop("tmp_dir", DOCKER_TEMPDIR) host_tmp_dir = kwargs.pop("host_tmp_dir", seven.get_system_temp_directory()) self.host_tmp_dir = host_tmp_dir run_config = dagster_operator_parameters.run_config if "filesystem" in run_config["storage"]: if ("config" in (run_config["storage"].get("filesystem", {}) or {}) and "base_dir" in ((run_config["storage"].get( "filesystem", {}) or {}).get("config", {}) or {}) and run_config["storage"]["filesystem"]["config"]["base_dir"] != tmp_dir): warnings.warn( "Found base_dir '{base_dir}' set in filesystem storage config, which was not " "the tmp_dir we expected ('{tmp_dir}', mounting host_tmp_dir " "'{host_tmp_dir}' from the host). We assume you know what you are doing, but " "if you are having trouble executing containerized workloads, this may be the " "issue".format( base_dir=run_config["storage"]["filesystem"]["config"] ["base_dir"], tmp_dir=tmp_dir, host_tmp_dir=host_tmp_dir, )) else: run_config["storage"]["filesystem"] = dict( run_config["storage"]["filesystem"] or {}, **{ "config": dict(((run_config["storage"].get("filesystem", {}) or {}).get("config", {}) or {}), **{"base_dir": tmp_dir}) }) self.docker_conn_id_set = kwargs.get("docker_conn_id") is not None self.run_config = run_config self.pipeline_name = dagster_operator_parameters.pipeline_name self.pipeline_snapshot = dagster_operator_parameters.pipeline_snapshot self.execution_plan_snapshot = dagster_operator_parameters.execution_plan_snapshot self.parent_pipeline_snapshot = dagster_operator_parameters.parent_pipeline_snapshot self.mode = dagster_operator_parameters.mode self.step_keys = dagster_operator_parameters.step_keys self.recon_repo = dagster_operator_parameters.recon_repo self._run_id = None self.instance_ref = dagster_operator_parameters.instance_ref check.invariant(self.instance_ref) self.instance = DagsterInstance.from_ref(self.instance_ref) # These shenanigans are so we can override DockerOperator.get_hook in order to configure # a docker client using docker.from_env, rather than messing with the logic of # DockerOperator.execute if not self.docker_conn_id_set: try: from_env().version() except Exception: # pylint: disable=broad-except pass else: kwargs["docker_conn_id"] = True if "environment" not in kwargs: kwargs["environment"] = get_aws_environment() super(DagsterDockerOperator, self).__init__( task_id=dagster_operator_parameters.task_id, dag=dagster_operator_parameters.dag, tmp_dir=tmp_dir, host_tmp_dir=host_tmp_dir, xcom_push=True, # We do this because log lines won't necessarily be emitted in order (!) -- so we can't # just check the last log line to see if it's JSON. xcom_all=True, *args, **kwargs)
'''This file is used by the EphemeralGrpcRunLauncher to execute runs in a subprocess.''' import sys from dagster import check from dagster.api.execute_run import cli_api_execute_run_grpc from dagster.core.instance import DagsterInstance, InstanceRef from dagster.core.origin import PipelinePythonOrigin from dagster.serdes import deserialize_json_to_dagster_namedtuple from dagster.serdes.ipc import setup_interrupt_support from dagster.seven import json if __name__ == '__main__': setup_interrupt_support() kwargs = json.loads(sys.argv[1]) instance_ref = check.inst( deserialize_json_to_dagster_namedtuple(kwargs['instance_ref']), InstanceRef) pipeline_origin = check.inst( deserialize_json_to_dagster_namedtuple(kwargs['pipeline_origin']), PipelinePythonOrigin) pipeline_run_id = kwargs['pipeline_run_id'] instance = DagsterInstance.from_ref(instance_ref) pipeline_run = instance.get_run_by_id(pipeline_run_id) events = [ evt for evt in cli_api_execute_run_grpc(instance_ref=instance_ref, pipeline_origin=pipeline_origin, pipeline_run=pipeline_run) ] print(len(events)) # pylint: disable=print-call
def get_external_schedule_execution( recon_repo, instance_ref, schedule_name, schedule_execution_data_mode, scheduled_execution_timestamp, scheduled_execution_timezone, ): check.inst_param( recon_repo, "recon_repo", ReconstructableRepository, ) definition = recon_repo.get_definition() schedule_def = definition.get_schedule_def(schedule_name) with DagsterInstance.from_ref(instance_ref) as instance: scheduled_execution_time = (pendulum.from_timestamp( scheduled_execution_timestamp, tz=scheduled_execution_timezone, ) if scheduled_execution_timestamp else None) schedule_context = ScheduleExecutionContext(instance, scheduled_execution_time) try: with user_code_error_boundary( ScheduleExecutionError, lambda: "Error occurred during the execution of should_execute for schedule " "{schedule_name}".format(schedule_name=schedule_def.name), ): should_execute = None if (schedule_execution_data_mode == ScheduleExecutionDataMode.LAUNCH_SCHEDULED_EXECUTION): should_execute = schedule_def.should_execute( schedule_context) if not should_execute: return ExternalScheduleExecutionData( should_execute=False, run_config=None, tags=None) with user_code_error_boundary( ScheduleExecutionError, lambda: "Error occurred during the execution of run_config_fn for schedule " "{schedule_name}".format(schedule_name=schedule_def.name), ): run_config = schedule_def.get_run_config(schedule_context) with user_code_error_boundary( ScheduleExecutionError, lambda: "Error occurred during the execution of tags_fn for schedule " "{schedule_name}".format(schedule_name=schedule_def.name), ): tags = schedule_def.get_tags(schedule_context) return ExternalScheduleExecutionData(run_config=run_config, tags=tags, should_execute=should_execute) except ScheduleExecutionError: return ExternalScheduleExecutionErrorData( serializable_error_info_from_exc_info(sys.exc_info()))
def instance(self): if not self._instance: self._instance = self._exit_stack.enter_context( DagsterInstance.from_ref(self._instance_ref) ) return self._instance
def reconstitute_pipeline_context( self, output_log_path=None, marshal_dir=None, environment_dict=None, handle_kwargs=None, pipeline_run_dict=None, solid_subset=None, solid_handle_kwargs=None, instance_ref_dict=None, ): '''Reconstitutes a context for dagstermill-managed execution. You'll see this function called to reconstruct a pipeline context within the ``injected parameters`` cell of a dagstermill output notebook. Users should not call this function interactively except when debugging output notebooks. Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a context for interactive exploration and development. This call will be replaced by one to :func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by dagstermill. ''' check.opt_str_param(output_log_path, 'output_log_path') check.opt_str_param(marshal_dir, 'marshal_dir') environment_dict = check.opt_dict_param(environment_dict, 'environment_dict', key_type=str) check.dict_param(pipeline_run_dict, 'pipeline_run_dict') check.dict_param(handle_kwargs, 'handle_kwargs') check.opt_list_param(solid_subset, 'solid_subset', of_type=str) check.dict_param(solid_handle_kwargs, 'solid_handle_kwargs') check.dict_param(instance_ref_dict, 'instance_ref_dict') try: handle = load_handle.handle_for_pipeline_cli_args( handle_kwargs, use_default_repository_yaml=False) except (check.CheckError, load_handle.UsageError) as err: six.raise_from( DagstermillError( 'Cannot invoke a dagstermill solid from an in-memory pipeline that was not loaded ' 'from an ExecutionTargetHandle. Run this pipeline using dagit, the dagster CLI, ' 'through dagster-graphql, or in-memory after loading it through an ' 'ExecutionTargetHandle.'), err, ) try: instance_ref = unpack_value(instance_ref_dict) instance = DagsterInstance.from_ref(instance_ref) except Exception as err: # pylint: disable=broad-except six.raise_from( DagstermillError( 'Error when attempting to resolve DagsterInstance from serialized InstanceRef' ), err, ) pipeline_def = check.inst_param( handle.build_pipeline_definition(), 'pipeline_def (from handle {handle_dict})'.format( handle_dict=handle.data._asdict()), PipelineDefinition, ).build_sub_pipeline(solid_subset) solid_handle = SolidHandle.from_dict(solid_handle_kwargs) solid_def = pipeline_def.get_solid(solid_handle) pipeline_run = unpack_value(pipeline_run_dict) self.marshal_dir = marshal_dir self.in_pipeline = True self.solid_def = solid_def self.pipeline_def = pipeline_def with scoped_pipeline_context( self.pipeline_def, environment_dict, pipeline_run, instance=instance, scoped_resources_builder_cm=self._setup_resources, ) as pipeline_context: self.context = DagstermillExecutionContext(pipeline_context) return self.context
def test_0_12_0_extract_asset_index_cols(): src_dir = file_relative_path(__file__, "snapshot_0_12_0_pre_asset_index_cols/sqlite") @solid def asset_solid(_): yield AssetMaterialization( asset_key=AssetKey(["a"]), partition="partition_1", tags={"foo": "FOO"} ) yield AssetMaterialization(asset_key=AssetKey(["b"]), tags={"bar": "BAR"}) yield Output(1) @pipeline def asset_pipeline(): asset_solid() with copy_directory(src_dir) as test_dir: db_path = os.path.join(test_dir, "history", "runs", "index.db") assert get_current_alembic_version(db_path) == "3b529ad30626" assert "last_materialization_timestamp" not in set( get_sqlite3_columns(db_path, "asset_keys") ) assert "wipe_timestamp" not in set(get_sqlite3_columns(db_path, "asset_keys")) assert "tags" not in set(get_sqlite3_columns(db_path, "asset_keys")) with DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) as instance: storage = instance._event_storage # make sure that executing the pipeline works execute_pipeline(asset_pipeline, instance=instance) assert storage.has_asset_key(AssetKey(["a"])) assert storage.has_asset_key(AssetKey(["b"])) # make sure that wiping works storage.wipe_asset(AssetKey(["a"])) assert not storage.has_asset_key(AssetKey(["a"])) assert storage.has_asset_key(AssetKey(["b"])) execute_pipeline(asset_pipeline, instance=instance) assert storage.has_asset_key(AssetKey(["a"])) # wipe and leave asset wiped storage.wipe_asset(AssetKey(["b"])) assert not storage.has_asset_key(AssetKey(["b"])) old_keys = storage.all_asset_keys() instance.upgrade() assert "last_materialization_timestamp" in set( get_sqlite3_columns(db_path, "asset_keys") ) assert "wipe_timestamp" in set(get_sqlite3_columns(db_path, "asset_keys")) assert "tags" in set(get_sqlite3_columns(db_path, "asset_keys")) assert storage.has_asset_key(AssetKey(["a"])) assert not storage.has_asset_key(AssetKey(["b"])) new_keys = storage.all_asset_keys() assert set(old_keys) == set(new_keys) # make sure that storing assets still works execute_pipeline(asset_pipeline, instance=instance) # make sure that wiping still works storage.wipe_asset(AssetKey(["a"])) assert not storage.has_asset_key(AssetKey(["a"]))
def _run_in_subprocess( serialized_execute_run_args, recon_pipeline, termination_event, subprocess_status_handler, run_event_handler, ): start_termination_thread(termination_event) try: execute_run_args = deserialize_json_to_dagster_namedtuple( serialized_execute_run_args) check.inst_param(execute_run_args, 'execute_run_args', ExecuteRunArgs) instance = DagsterInstance.from_ref(execute_run_args.instance_ref) pipeline_run = instance.get_run_by_id(execute_run_args.pipeline_run_id) pid = os.getpid() except: # pylint: disable=bare-except event = IPCErrorMessage( serializable_error_info=serializable_error_info_from_exc_info( sys.exc_info()), message='Error during RPC setup for ExecuteRun', ) subprocess_status_handler(event) subprocess_status_handler(RunInSubprocessComplete()) return subprocess_status_handler(StartRunInSubprocessSuccessful()) run_event_handler( instance.report_engine_event( 'Started process for pipeline (pid: {pid}).'.format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end='cli_api_subprocess_init'), )) # This is so nasty but seemingly unavoidable # https://amir.rachum.com/blog/2017/03/03/generator-cleanup/ closed = False try: for event in _core_execute_run(recon_pipeline, pipeline_run, instance): run_event_handler(event) except KeyboardInterrupt: run_event_handler( instance.report_engine_event( message='Pipeline execution terminated by interrupt', pipeline_run=pipeline_run, )) raise except GeneratorExit: closed = True raise finally: if not closed: run_event_handler( instance.report_engine_event( 'Process for pipeline exited (pid: {pid}).'.format( pid=pid), pipeline_run, )) subprocess_status_handler(RunInSubprocessComplete())
def __init__(self, task_id, environment_dict=None, pipeline_name=None, mode=None, step_keys=None, dag=None, instance_ref=None, *args, **kwargs): check.str_param(pipeline_name, 'pipeline_name') step_keys = check.opt_list_param(step_keys, 'step_keys', of_type=str) environment_dict = check.opt_dict_param(environment_dict, 'environment_dict', key_type=str) check.opt_inst_param(instance_ref, 'instance_ref', InstanceRef) kwargs['name'] = 'dagster.{pipeline_name}.{task_id}'.format( pipeline_name=pipeline_name, task_id=task_id).replace( '_', '-' # underscores are not permissible DNS names ) if 'storage' not in environment_dict: raise AirflowException( 'No storage config found -- must configure either filesystem or s3 storage for ' 'the DagsterKubernetesPodOperator. Ex.: \n' 'storage:\n' ' filesystem:\n' ' base_dir: \'/some/shared/volume/mount/special_place\'' '\n\n --or--\n\n' 'storage:\n' ' s3:\n' ' s3_bucket: \'my-s3-bucket\'\n') check.invariant( 'in_memory' not in environment_dict.get('storage', {}), 'Cannot use in-memory storage with Airflow, must use S3', ) self.environment_dict = environment_dict self.pipeline_name = pipeline_name self.mode = mode self.step_keys = step_keys self._run_id = None # self.instance might be None in, for instance, a unit test setting where the operator # was being directly instantiated without passing through make_airflow_dag self.instance = DagsterInstance.from_ref( instance_ref) if instance_ref else None # Store Airflow DAG run timestamp so that we can pass along via execution metadata self.airflow_ts = kwargs.get('ts') # Add AWS creds self.env_vars = kwargs.get('env_vars', {}) for k, v in get_aws_environment().items(): self.env_vars.setdefault(k, v) kwargs.setdefault('labels', {}) kwargs['labels'].setdefault('dagster_pipeline', self.pipeline_name) kwargs['labels'].setdefault('app.kubernetes.io/name', 'dagster') kwargs['labels'].setdefault('app.kubernetes.io/instance', self.pipeline_name) kwargs['labels'].setdefault('app.kubernetes.io/version', dagster_version) kwargs['labels'].setdefault('app.kubernetes.io/component', 'pipeline-execution') kwargs['labels'].setdefault('app.kubernetes.io/part-of', 'dagster-airflow') kwargs['labels'].setdefault('app.kubernetes.io/managed-by', 'dagster-airflow') # The xcom mechanism for the pod operator is very unlike that of the Docker operator, so # we disable it if 'xcom_push' in kwargs: self.log.warning( 'xcom_push cannot be enabled with the DagsterKubernetesPodOperator, disabling' ) kwargs['xcom_push'] = False super(DagsterKubernetesPodOperator, self).__init__(task_id=task_id, dag=dag, *args, **kwargs)
def _execute_run(request): try: execute_run_args = deserialize_json_to_dagster_namedtuple( request.serialized_execute_run_args) check.inst_param(execute_run_args, 'execute_run_args', ExecuteRunArgs) recon_pipeline = recon_pipeline_from_origin( execute_run_args.pipeline_origin) instance = DagsterInstance.from_ref(execute_run_args.instance_ref) pipeline_run = instance.get_run_by_id(execute_run_args.pipeline_run_id) pid = os.getpid() except: # pylint: disable=bare-except yield IPCErrorMessage( serializable_error_info=serializable_error_info_from_exc_info( sys.exc_info()), message='Error during RPC setup for ExecuteRun', ) return yield instance.report_engine_event( 'Started process for pipeline (pid: {pid}).'.format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end='cli_api_subprocess_init'), ) # This is so nasty but seemingly unavoidable # https://amir.rachum.com/blog/2017/03/03/generator-cleanup/ closed = False try: for event in execute_run_iterator(recon_pipeline, pipeline_run, instance): yield event except DagsterSubprocessError as err: if not all([ err_info.cls_name == 'KeyboardInterrupt' for err_info in err.subprocess_error_infos ]): yield instance.report_engine_event( 'An exception was thrown during execution that is likely a framework error, ' 'rather than an error in user code.', pipeline_run, EngineEventData.engine_error( serializable_error_info_from_exc_info(sys.exc_info())), ) instance.report_run_failed(pipeline_run) except GeneratorExit: closed = True raise except Exception: # pylint: disable=broad-except yield instance.report_engine_event( 'An exception was thrown during execution that is likely a framework error, ' 'rather than an error in user code.', pipeline_run, EngineEventData.engine_error( serializable_error_info_from_exc_info(sys.exc_info())), ) instance.report_run_failed(pipeline_run) finally: if not closed: yield instance.report_engine_event( 'Process for pipeline exited (pid: {pid}).'.format(pid=pid), pipeline_run, )
def __init__(self, dagster_operator_parameters, *args): kwargs = dagster_operator_parameters.op_kwargs tmp_dir = kwargs.pop('tmp_dir', DOCKER_TEMPDIR) host_tmp_dir = kwargs.pop('host_tmp_dir', seven.get_system_temp_directory()) environment_dict = dagster_operator_parameters.environment_dict if 'filesystem' in environment_dict['storage']: if ('config' in (environment_dict['storage'].get('filesystem', {}) or {}) and 'base_dir' in ((environment_dict['storage'].get( 'filesystem', {}) or {}).get('config', {}) or {}) and environment_dict['storage']['filesystem']['config'] ['base_dir'] != tmp_dir): warnings.warn( 'Found base_dir \'{base_dir}\' set in filesystem storage config, which was not ' 'the tmp_dir we expected (\'{tmp_dir}\', mounting host_tmp_dir ' '\'{host_tmp_dir}\' from the host). We assume you know what you are doing, but ' 'if you are having trouble executing containerized workloads, this may be the ' 'issue'.format( base_dir=environment_dict['storage']['filesystem'] ['config']['base_dir'], tmp_dir=tmp_dir, host_tmp_dir=host_tmp_dir, )) else: environment_dict['storage']['filesystem'] = dict( environment_dict['storage']['filesystem'] or {}, **{ 'config': dict(((environment_dict['storage'].get( 'filesystem', {}) or {}).get('config', {}) or {}), **{'base_dir': tmp_dir}) }) self.docker_conn_id_set = kwargs.get('docker_conn_id') is not None self.environment_dict = environment_dict self.pipeline_name = dagster_operator_parameters.pipeline_name self.pipeline_snapshot = dagster_operator_parameters.pipeline_snapshot self.execution_plan_snapshot = dagster_operator_parameters.execution_plan_snapshot self.parent_pipeline_snapshot = dagster_operator_parameters.parent_pipeline_snapshot self.mode = dagster_operator_parameters.mode self.step_keys = dagster_operator_parameters.step_keys self._run_id = None # self.instance might be None in, for instance, a unit test setting where the operator # was being directly instantiated without passing through make_airflow_dag self.instance = (DagsterInstance.from_ref( dagster_operator_parameters.instance_ref) if dagster_operator_parameters.instance_ref else None) # These shenanigans are so we can override DockerOperator.get_hook in order to configure # a docker client using docker.from_env, rather than messing with the logic of # DockerOperator.execute if not self.docker_conn_id_set: try: from_env().version() except Exception: # pylint: disable=broad-except pass else: kwargs['docker_conn_id'] = True # We do this because log lines won't necessarily be emitted in order (!) -- so we can't # just check the last log line to see if it's JSON. kwargs['xcom_all'] = True # Store Airflow DAG run timestamp so that we can pass along via execution metadata self.airflow_ts = kwargs.get('ts') if 'environment' not in kwargs: kwargs['environment'] = get_aws_environment() super(DagsterDockerOperator, self).__init__(task_id=dagster_operator_parameters.task_id, dag=dagster_operator_parameters.dag, tmp_dir=tmp_dir, host_tmp_dir=host_tmp_dir, *args, **kwargs)
def get_instance(self): with DagsterInstance.from_ref(self._instance_ref) as instance: yield instance
def ExecuteRun(self, request, _context): try: execute_run_args = deserialize_json_to_dagster_namedtuple( request.serialized_execute_run_args) check.inst_param(execute_run_args, "execute_run_args", ExecuteRunArgs) run_id = execute_run_args.pipeline_run_id recon_pipeline = self._recon_pipeline_from_origin( execute_run_args.pipeline_origin) except: # pylint: disable=bare-except yield api_pb2.ExecuteRunEvent( serialized_dagster_event_or_ipc_error_message= serialize_dagster_namedtuple( IPCErrorMessage( serializable_error_info= serializable_error_info_from_exc_info(sys.exc_info()), message="Error during RPC setup for ExecuteRun", ))) return event_queue = multiprocessing.Queue() termination_event = multiprocessing.Event() execution_process = multiprocessing.Process( target=execute_run_in_subprocess, args=[ request.serialized_execute_run_args, recon_pipeline, event_queue, termination_event, ], ) with self._execution_lock: execution_process.start() self._executions[run_id] = ( execution_process, DagsterInstance.from_ref(execute_run_args.instance_ref), ) self._termination_events[run_id] = termination_event done = False while not done: try: # We use `get_nowait()` instead of `get()` so that we can handle the case where the # execution process has died unexpectedly -- `get()` would hang forever in that case dagster_event_or_ipc_error_message_or_done = event_queue.get_nowait( ) except queue.Empty: if not execution_process.is_alive(): # subprocess died unexpectedly yield api_pb2.ExecuteRunEvent( serialized_dagster_event_or_ipc_error_message= serialize_dagster_namedtuple( IPCErrorMessage( serializable_error_info= serializable_error_info_from_exc_info( sys.exc_info()), message= ("GRPC server: Subprocess for {run_id} terminated unexpectedly" ).format(run_id=run_id), ))) done = True time.sleep(EVENT_QUEUE_POLL_INTERVAL) else: if isinstance(dagster_event_or_ipc_error_message_or_done, RunInSubprocessComplete): done = True elif isinstance(dagster_event_or_ipc_error_message_or_done, StartRunInSubprocessSuccessful): continue else: yield api_pb2.ExecuteRunEvent( serialized_dagster_event_or_ipc_error_message= serialize_dagster_namedtuple( dagster_event_or_ipc_error_message_or_done)) with self._execution_lock: if run_id in self._executions: del self._executions[run_id] if run_id in self._termination_events: del self._termination_events[run_id]