def launch_run(self, context: LaunchRunContext) -> None: run = context.pipeline_run check.inst_param(run, "run", PipelineRun) if not context.workspace: raise DagsterInvariantViolationError( "DefaultRunLauncher requires a workspace to be included in its LaunchRunContext" ) external_pipeline_origin = check.not_none(run.external_pipeline_origin) repository_location = context.workspace.get_location( external_pipeline_origin.external_repository_origin.repository_location_origin.location_name ) check.inst( repository_location, GrpcServerRepositoryLocation, "DefaultRunLauncher: Can't launch runs for pipeline not loaded from a GRPC server", ) self._instance.add_run_tags( run.run_id, { GRPC_INFO_TAG: seven.json.dumps( merge_dicts( {"host": repository_location.host}, ( {"port": repository_location.port} if repository_location.port else {"socket": repository_location.socket} ), ({"use_ssl": True} if repository_location.use_ssl else {}), ) ) }, ) res = deserialize_as( repository_location.client.start_run( ExecuteExternalPipelineArgs( pipeline_origin=external_pipeline_origin, pipeline_run_id=run.run_id, instance_ref=self._instance.get_ref(), ) ), StartRunResult, ) if not res.success: raise ( DagsterLaunchFailedError( res.message, serializable_error_info=res.serializable_error_info ) ) self._run_ids.add(run.run_id) if self._wait_for_processes: self._locations_to_wait_for.append(repository_location)
def launch_run(self, run, external_pipeline): check.inst_param(run, "run", PipelineRun) check.inst_param(external_pipeline, "external_pipeline", ExternalPipeline) repository_location_handle = external_pipeline.repository_handle.repository_location_handle check.inst( repository_location_handle, GRPC_REPOSITORY_LOCATION_HANDLE_TYPES, "DefaultRunLauncher: Can't launch runs for pipeline not loaded from a GRPC server", ) self._instance.add_run_tags( run.run_id, { GRPC_INFO_TAG: seven.json.dumps( merge_dicts( {"host": repository_location_handle.host}, ({ "port": repository_location_handle.port } if repository_location_handle.port else { "socket": repository_location_handle.socket }), ({ "use_ssl": True } if repository_location_handle.use_ssl else {}), )) }, ) res = repository_location_handle.client.start_run( ExecuteExternalPipelineArgs( pipeline_origin=external_pipeline.get_external_origin(), pipeline_run_id=run.run_id, instance_ref=self._instance.get_ref(), )) if not res.success: raise (DagsterLaunchFailedError( res.message, serializable_error_info=res.serializable_error_info)) self._run_ids.add(run.run_id) if self._wait_for_processes and isinstance( repository_location_handle, ManagedGrpcPythonEnvRepositoryLocationHandle): self._processes_to_wait_for.append( repository_location_handle.grpc_server_process) return run
def launch_run(self, instance, run, external_pipeline): """Subclasses must implement this method.""" check.inst_param(run, "run", PipelineRun) check.inst_param(external_pipeline, "external_pipeline", ExternalPipeline) repository_location_handle = external_pipeline.repository_handle.repository_location_handle check.inst( repository_location_handle, GRPC_REPOSITORY_LOCATION_HANDLE_TYPES, "DefaultRunLauncher: Can't launch runs for pipeline not loaded from a GRPC server", ) self._instance.add_run_tags( run.run_id, { GRPC_INFO_TAG: seven.json.dumps( merge_dicts( {"host": repository_location_handle.host}, {"port": repository_location_handle.port} if repository_location_handle.port else {"socket": repository_location_handle.socket}, )) }, ) res = repository_location_handle.client.start_run( ExecuteExternalPipelineArgs( pipeline_origin=external_pipeline.get_external_origin(), pipeline_run_id=run.run_id, instance_ref=self._instance.get_ref(), )) if not res.success: raise (DagsterLaunchFailedError( res.message, serializable_error_info=res.serializable_error_info)) self._run_id_to_repository_location_handle_cache[ run.run_id] = repository_location_handle return run
def test_cancel_run(): with instance_for_test() as instance: loadable_target_origin = LoadableTargetOrigin( executable_path=sys.executable, python_file=__file__, working_directory=None, ) server_process = GrpcServerProcess(loadable_target_origin, max_workers=10) with server_process.create_ephemeral_client() as api_client: streaming_results = [] pipeline_run = instance.create_run_for_pipeline( streaming_pipeline, run_config={ "solids": { "streamer": { "config": { "length": 20 } } } }, ) execute_run_args = ExecuteExternalPipelineArgs( pipeline_origin=ExternalPipelineOrigin( ExternalRepositoryOrigin( repository_location_origin= GrpcServerRepositoryLocationOrigin( host="localhost", socket=api_client.socket, port=api_client.port, ), repository_name="test_repository", ), pipeline_name="streaming_pipeline", ), pipeline_run_id=pipeline_run.run_id, instance_ref=instance.get_ref(), ) stream_events_result_thread = threading.Thread( target=_stream_events_target, args=[streaming_results, api_client, execute_run_args]) stream_events_result_thread.daemon = True stream_events_result_thread.start() poll_for_step_start(instance, pipeline_run.run_id) res = api_client.cancel_execution( cancel_execution_request=CancelExecutionRequest( run_id=pipeline_run.run_id)) assert res.success is True poll_for_finished_run(instance, pipeline_run.run_id) logs = instance.all_logs(pipeline_run.run_id) assert (len([ ev for ev in logs if ev.dagster_event.event_type_value == "STEP_MATERIALIZATION" ]) < 20) # soft termination assert [ ev for ev in logs if ev.dagster_event.event_type_value == "STEP_FAILURE" ] server_process.wait()