def launch_run(self, context: LaunchRunContext) -> None:
run = context.pipeline_run
check.inst_param(run, "run", PipelineRun)
if not context.workspace:
raise DagsterInvariantViolationError(
"DefaultRunLauncher requires a workspace to be included in its LaunchRunContext"
)
external_pipeline_origin = check.not_none(run.external_pipeline_origin)
repository_location = context.workspace.get_location(
external_pipeline_origin.external_repository_origin.repository_location_origin.location_name
)
check.inst(
repository_location,
GrpcServerRepositoryLocation,
"DefaultRunLauncher: Can't launch runs for pipeline not loaded from a GRPC server",
)
self._instance.add_run_tags(
run.run_id,
{
GRPC_INFO_TAG: seven.json.dumps(
merge_dicts(
{"host": repository_location.host},
(
{"port": repository_location.port}
if repository_location.port
else {"socket": repository_location.socket}
),
({"use_ssl": True} if repository_location.use_ssl else {}),
)
)
},
)
res = deserialize_as(
repository_location.client.start_run(
ExecuteExternalPipelineArgs(
pipeline_origin=external_pipeline_origin,
pipeline_run_id=run.run_id,
instance_ref=self._instance.get_ref(),
)
),
StartRunResult,
)
if not res.success:
raise (
DagsterLaunchFailedError(
res.message, serializable_error_info=res.serializable_error_info
)
)
self._run_ids.add(run.run_id)
if self._wait_for_processes:
self._locations_to_wait_for.append(repository_location)
def launch_run(self, run, external_pipeline):
check.inst_param(run, "run", PipelineRun)
check.inst_param(external_pipeline, "external_pipeline",
ExternalPipeline)
repository_location_handle = external_pipeline.repository_handle.repository_location_handle
check.inst(
repository_location_handle,
GRPC_REPOSITORY_LOCATION_HANDLE_TYPES,
"DefaultRunLauncher: Can't launch runs for pipeline not loaded from a GRPC server",
)
self._instance.add_run_tags(
run.run_id,
{
GRPC_INFO_TAG:
seven.json.dumps(
merge_dicts(
{"host": repository_location_handle.host},
({
"port": repository_location_handle.port
} if repository_location_handle.port else {
"socket": repository_location_handle.socket
}),
({
"use_ssl": True
} if repository_location_handle.use_ssl else {}),
))
},
)
res = repository_location_handle.client.start_run(
ExecuteExternalPipelineArgs(
pipeline_origin=external_pipeline.get_external_origin(),
pipeline_run_id=run.run_id,
instance_ref=self._instance.get_ref(),
))
if not res.success:
raise (DagsterLaunchFailedError(
res.message,
serializable_error_info=res.serializable_error_info))
self._run_ids.add(run.run_id)
if self._wait_for_processes and isinstance(
repository_location_handle,
ManagedGrpcPythonEnvRepositoryLocationHandle):
self._processes_to_wait_for.append(
repository_location_handle.grpc_server_process)
return run
def launch_run(self, instance, run, external_pipeline):
"""Subclasses must implement this method."""
check.inst_param(run, "run", PipelineRun)
check.inst_param(external_pipeline, "external_pipeline",
ExternalPipeline)
repository_location_handle = external_pipeline.repository_handle.repository_location_handle
check.inst(
repository_location_handle,
GRPC_REPOSITORY_LOCATION_HANDLE_TYPES,
"DefaultRunLauncher: Can't launch runs for pipeline not loaded from a GRPC server",
)
self._instance.add_run_tags(
run.run_id,
{
GRPC_INFO_TAG:
seven.json.dumps(
merge_dicts(
{"host": repository_location_handle.host},
{"port": repository_location_handle.port}
if repository_location_handle.port else
{"socket": repository_location_handle.socket},
))
},
)
res = repository_location_handle.client.start_run(
ExecuteExternalPipelineArgs(
pipeline_origin=external_pipeline.get_external_origin(),
pipeline_run_id=run.run_id,
instance_ref=self._instance.get_ref(),
))
if not res.success:
raise (DagsterLaunchFailedError(
res.message,
serializable_error_info=res.serializable_error_info))
self._run_id_to_repository_location_handle_cache[
run.run_id] = repository_location_handle
return run
def test_cancel_run():
with instance_for_test() as instance:
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable,
python_file=__file__,
working_directory=None,
)
server_process = GrpcServerProcess(loadable_target_origin,
max_workers=10)
with server_process.create_ephemeral_client() as api_client:
streaming_results = []
pipeline_run = instance.create_run_for_pipeline(
streaming_pipeline,
run_config={
"solids": {
"streamer": {
"config": {
"length": 20
}
}
}
},
)
execute_run_args = ExecuteExternalPipelineArgs(
pipeline_origin=ExternalPipelineOrigin(
ExternalRepositoryOrigin(
repository_location_origin=
GrpcServerRepositoryLocationOrigin(
host="localhost",
socket=api_client.socket,
port=api_client.port,
),
repository_name="test_repository",
),
pipeline_name="streaming_pipeline",
),
pipeline_run_id=pipeline_run.run_id,
instance_ref=instance.get_ref(),
)
stream_events_result_thread = threading.Thread(
target=_stream_events_target,
args=[streaming_results, api_client, execute_run_args])
stream_events_result_thread.daemon = True
stream_events_result_thread.start()
poll_for_step_start(instance, pipeline_run.run_id)
res = api_client.cancel_execution(
cancel_execution_request=CancelExecutionRequest(
run_id=pipeline_run.run_id))
assert res.success is True
poll_for_finished_run(instance, pipeline_run.run_id)
logs = instance.all_logs(pipeline_run.run_id)
assert (len([
ev for ev in logs
if ev.dagster_event.event_type_value == "STEP_MATERIALIZATION"
]) < 20)
# soft termination
assert [
ev for ev in logs
if ev.dagster_event.event_type_value == "STEP_FAILURE"
]
server_process.wait()