def cli_api_execute_run_grpc(instance_ref, pipeline_origin, pipeline_run):
check.inst_param(instance_ref, 'instance_ref', InstanceRef)
check.inst_param(pipeline_origin, 'pipeline_origin', PipelinePythonOrigin)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
instance = DagsterInstance.from_ref(instance_ref)
yield instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'.format(
pipeline_name=pipeline_run.pipeline_name, run_id=pipeline_run.run_id
),
pipeline_run,
engine_event_data=EngineEventData(marker_start='cli_api_subprocess_init'),
)
with ephemeral_grpc_api_client() as api_client:
execute_run_args = ExecuteRunArgs(
pipeline_origin=pipeline_origin,
pipeline_run_id=pipeline_run.run_id,
instance_ref=instance_ref,
)
for event in api_client.execute_run(execute_run_args=execute_run_args):
if isinstance(event, IPCErrorMessage):
instance.report_engine_event(
event.message,
pipeline_run=pipeline_run,
engine_event_data=EngineEventData(
marker_end='cli_api_subprocess_init', error=event.serializable_error_info
),
)
instance.report_run_failed(pipeline_run)
return
yield event
def _launch_k8s_job_with_args(self, job_name, args, run, pipeline_origin):
pod_name = job_name
user_defined_k8s_config = get_user_defined_k8s_config(frozentags(run.tags))
repository_origin = pipeline_origin.repository_origin
job_config = (
self._get_grpc_job_config(repository_origin.container_image)
if repository_origin.container_image
else self.get_static_job_config()
)
self._instance.add_run_tags(
run.run_id,
{DOCKER_IMAGE_TAG: job_config.job_image},
)
job = construct_dagster_k8s_job(
job_config=job_config,
args=args,
job_name=job_name,
pod_name=pod_name,
component="run_worker",
user_defined_k8s_config=user_defined_k8s_config,
labels={
"dagster/job": pipeline_origin.pipeline_name,
},
)
self._instance.report_engine_event(
"Creating Kubernetes run worker job",
run,
EngineEventData(
[
EventMetadataEntry.text(job_name, "Kubernetes Job name"),
EventMetadataEntry.text(self.job_namespace, "Kubernetes Namespace"),
EventMetadataEntry.text(run.run_id, "Run ID"),
]
),
cls=self.__class__,
)
self._batch_api.create_namespaced_job(body=job, namespace=self.job_namespace)
self._instance.report_engine_event(
"Kubernetes run worker job created",
run,
EngineEventData(
[
EventMetadataEntry.text(job_name, "Kubernetes Job name"),
EventMetadataEntry.text(self.job_namespace, "Kubernetes Namespace"),
EventMetadataEntry.text(run.run_id, "Run ID"),
]
),
cls=self.__class__,
)
def cli_api_execute_run(output_file, instance, repo_cli_args, pipeline_run):
check.str_param(output_file, 'output_file')
check.inst_param(instance, 'instance', DagsterInstance)
check.str_param(repo_cli_args, 'repo_cli_args')
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
parts = ([
'dagster',
'api',
'execute_run',
output_file,
] + xplat_shlex_split(repo_cli_args) + [
'--instance-ref',
'{instance_ref}'.format(
instance_ref=serialize_dagster_namedtuple(instance.get_ref())),
'--pipeline-run',
'{pipeline_run}'.format(
pipeline_run=serialize_dagster_namedtuple(pipeline_run)),
])
instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'
.format(pipeline_name=pipeline_run.pipeline_name,
run_id=pipeline_run.run_id),
pipeline_run,
engine_event_data=EngineEventData(
marker_start='cli_api_subprocess_init'),
)
return open_ipc_subprocess(parts)
def execute(self):
pipeline = self.recon_pipeline
with DagsterInstance.from_ref(self.instance_ref) as instance:
start_termination_thread(self.term_event)
execution_plan = create_execution_plan(
pipeline=pipeline,
run_config=self.run_config,
mode=self.pipeline_run.mode,
step_keys_to_execute=[self.step_key],
known_state=self.known_state,
)
yield instance.report_engine_event(
"Executing step {} in subprocess".format(self.step_key),
self.pipeline_run,
EngineEventData(
[
MetadataEntry("pid", value=str(os.getpid())),
MetadataEntry("step_key", value=self.step_key),
],
marker_end=DELEGATE_MARKER,
),
MultiprocessExecutor,
self.step_key,
)
yield from execute_plan_iterator(
execution_plan,
pipeline,
self.pipeline_run,
run_config=self.run_config,
retry_mode=self.retry_mode.for_inner_plan(),
instance=instance,
)
def execute(self):
pipeline = self.recon_pipeline
with DagsterInstance.from_ref(self.instance_ref) as instance:
start_termination_thread(self.term_event)
execution_plan = create_execution_plan(
pipeline=pipeline,
run_config=self.run_config,
mode=self.pipeline_run.mode,
step_keys_to_execute=self.pipeline_run.step_keys_to_execute,
).build_subset_plan([self.step_key])
yield instance.report_engine_event(
"Executing step {} in subprocess".format(self.step_key),
self.pipeline_run,
EngineEventData(
[
EventMetadataEntry.text(str(os.getpid()), "pid"),
EventMetadataEntry.text(self.step_key, "step_key"),
],
marker_end=DELEGATE_MARKER,
),
MultiprocessExecutor,
self.step_key,
)
for step_event in execute_plan_iterator(
execution_plan,
self.pipeline_run,
run_config=self.run_config,
retries=self.retries.for_inner_plan(),
instance=instance,
):
yield step_event
def execute_step_out_of_process(self, step_context, step, errors,
term_events, known_state):
command = InProcessExecutorChildProcessCommand(
run_config=step_context.run_config,
pipeline_run=step_context.pipeline_run,
step_key=step.key,
instance_ref=step_context.instance.get_ref(),
term_event=term_events[step.key],
recon_pipeline=self.pipeline,
retry_mode=self.retries,
known_state=known_state,
)
yield DagsterEvent.engine_event(
step_context,
"Launching subprocess for {}".format(step.key),
EngineEventData(marker_start=DELEGATE_MARKER),
step_handle=step.handle,
)
for ret in execute_child_process_command(command):
if ret is None or isinstance(ret, DagsterEvent):
yield ret
elif isinstance(ret, ChildProcessEvent):
if isinstance(ret, ChildProcessSystemErrorEvent):
errors[ret.pid] = ret.error_info
else:
check.failed(
"Unexpected return value from child process {}".format(
type(ret)))
def _execute_plan(self, execute_step_args_packed, executable_dict):
execute_step_args = unpack_value(
check.dict_param(
execute_step_args_packed,
"execute_step_args_packed",
))
check.inst_param(execute_step_args, "execute_step_args",
ExecuteStepArgs)
check.dict_param(executable_dict, "executable_dict")
instance = DagsterInstance.from_ref(execute_step_args.instance_ref)
pipeline = ReconstructablePipeline.from_dict(executable_dict)
retries = Retries.from_config(execute_step_args.retries_dict)
pipeline_run = instance.get_run_by_id(
execute_step_args.pipeline_run_id)
check.invariant(
pipeline_run,
"Could not load run {}".format(execute_step_args.pipeline_run_id))
step_keys_str = ", ".join(execute_step_args.step_keys_to_execute)
execution_plan = create_execution_plan(
pipeline,
pipeline_run.run_config,
mode=pipeline_run.mode,
step_keys_to_execute=execute_step_args.step_keys_to_execute,
)
engine_event = instance.report_engine_event(
"Executing steps {} in celery worker".format(step_keys_str),
pipeline_run,
EngineEventData(
[
EventMetadataEntry.text(step_keys_str, "step_keys"),
EventMetadataEntry.text(self.request.hostname,
"Celery worker"),
],
marker_end=DELEGATE_MARKER,
),
CeleryExecutor,
step_key=execution_plan.step_key_for_single_step_plans(),
)
events = [engine_event]
for step_event in execute_plan_iterator(
execution_plan,
pipeline_run=pipeline_run,
run_config=pipeline_run.run_config,
instance=instance,
retries=retries,
):
events.append(step_event)
serialized_events = [
serialize_dagster_namedtuple(event) for event in events
]
return serialized_events
def execute_step_out_of_process(step_context, step, errors, term_events):
command = InProcessExecutorChildProcessCommand(
step_context.run_config,
step_context.pipeline_run,
step_context.executor_config,
step.key,
step_context.instance.get_ref(),
term_events[step.key],
)
yield DagsterEvent.engine_event(
step_context,
'Launching subprocess for {}'.format(step.key),
EngineEventData(marker_start=DELEGATE_MARKER),
step_key=step.key,
)
for ret in execute_child_process_command(command):
if ret is None or isinstance(ret, DagsterEvent):
yield ret
elif isinstance(ret, ChildProcessEvent):
if isinstance(ret, ChildProcessSystemErrorEvent):
errors[ret.pid] = ret.error_info
elif isinstance(ret, KeyboardInterrupt):
yield DagsterEvent.engine_event(
step_context,
'Multiprocess engine: received KeyboardInterrupt - forwarding to active child processes',
EngineEventData.interrupted(list(term_events.keys())),
)
for term_event in term_events.values():
term_event.set()
else:
check.failed(
'Unexpected return value from child process {}'.format(
type(ret)))
def execute(self):
check.inst(self.executor_config, MultiprocessExecutorConfig)
pipeline_def = self.executor_config.load_pipeline(self.pipeline_run)
instance = DagsterInstance.from_ref(self.instance_ref)
start_termination_thread(self.term_event)
execution_plan = create_execution_plan(
pipeline_def, self.environment_dict,
self.pipeline_run).build_subset_plan([self.step_key])
yield instance.report_engine_event(
MultiprocessEngine,
'Executing step {} in subprocess'.format(self.step_key),
self.pipeline_run,
EngineEventData(
[
EventMetadataEntry.text(str(os.getpid()), 'pid'),
EventMetadataEntry.text(self.step_key, 'step_key'),
],
marker_end=DELEGATE_MARKER,
),
self.step_key,
)
for step_event in execute_plan_iterator(
execution_plan,
self.pipeline_run,
environment_dict=self.environment_dict,
retries=self.executor_config.retries.for_inner_plan(),
instance=instance,
):
yield step_event
def make_event(event_id):
return DagsterEvent(
event_type_value="ENGINE_EVENT",
pipeline_name="some_pipeline",
event_specific_data=EngineEventData(),
message=str(event_id),
)
def execute_pipeline(self, handle, pipeline, pipeline_run, instance):
'''Subclasses must implement this method.'''
check.inst_param(handle, 'handle', ExecutionTargetHandle)
term_event = self._multiprocessing_context.Event()
mp_process = self._multiprocessing_context.Process(
target=self.__class__.in_mp_process,
kwargs={
'handle': handle,
'pipeline_run': pipeline_run,
'instance_ref': instance.get_ref(),
'term_event': term_event,
},
)
instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'
.format(pipeline_name=pipeline_run.pipeline_name,
run_id=pipeline_run.run_id),
pipeline_run,
engine_event_data=EngineEventData(
marker_start='dagit_subprocess_init'),
cls=self.__class__,
)
mp_process.start()
with self._processes_lock:
self._living_process_by_run_id[pipeline_run.run_id] = mp_process
self._term_events[pipeline_run.run_id] = term_event
def launch_step(self, step_handler_context: StepHandlerContext):
events = []
assert (len(
step_handler_context.execute_step_args.step_keys_to_execute) == 1
), "Launching multiple steps is not currently supported"
step_key = step_handler_context.execute_step_args.step_keys_to_execute[
0]
k8s_name_key = get_k8s_job_name(
step_handler_context.execute_step_args.pipeline_run_id,
step_key,
)
job_name = "dagster-job-%s" % (k8s_name_key)
pod_name = "dagster-job-%s" % (k8s_name_key)
input_json = serialize_dagster_namedtuple(
step_handler_context.execute_step_args)
args = ["dagster", "api", "execute_step", input_json]
job_config = self._job_config
if not job_config.job_image:
job_config = job_config.with_image(
step_handler_context.execute_step_args.pipeline_origin.
repository_origin.container_image)
if not job_config.job_image:
raise Exception(
"No image included in either executor config or the pipeline")
user_defined_k8s_config = get_user_defined_k8s_config(
frozentags(step_handler_context.step_tags[step_key]))
job = construct_dagster_k8s_job(
job_config=job_config,
args=args,
job_name=job_name,
pod_name=pod_name,
component="step_worker",
user_defined_k8s_config=user_defined_k8s_config,
)
events.append(
DagsterEvent(
event_type_value=DagsterEventType.ENGINE_EVENT.value,
pipeline_name=step_handler_context.execute_step_args.
pipeline_origin.pipeline_name,
step_key=step_key,
message=
f"Executing step {step_key} in Kubernetes job {job_name}",
event_specific_data=EngineEventData([
EventMetadataEntry.text(step_key, "Step key"),
EventMetadataEntry.text(job_name, "Kubernetes Job name"),
], ),
))
self._batch_api.create_namespaced_job(body=job,
namespace=self._job_namespace)
return events
def terminate_pipeline_execution(instance, run_id, terminate_policy):
from ...schema.errors import GrapheneRunNotFoundError
from ...schema.pipelines.pipeline import GrapheneRun
from ...schema.roots.mutation import (
GrapheneTerminateRunFailure,
GrapheneTerminateRunPolicy,
GrapheneTerminateRunSuccess,
)
check.inst_param(instance, "instance", DagsterInstance)
check.str_param(run_id, "run_id")
records = instance.get_run_records(RunsFilter(run_ids=[run_id]))
force_mark_as_canceled = (terminate_policy == GrapheneTerminateRunPolicy.
MARK_AS_CANCELED_IMMEDIATELY)
if not records:
return GrapheneRunNotFoundError(run_id)
record = records[0]
run = record.pipeline_run
graphene_run = GrapheneRun(record)
valid_status = not run.is_finished and (
force_mark_as_canceled or (run.status == PipelineRunStatus.STARTED
or run.status == PipelineRunStatus.QUEUED))
if not valid_status:
return GrapheneTerminateRunFailure(
run=graphene_run,
message=
"Run {run_id} could not be terminated due to having status {status}."
.format(run_id=run.run_id, status=run.status.value),
)
if force_mark_as_canceled:
try:
if instance.run_coordinator and instance.run_coordinator.can_cancel_run(
run_id):
instance.run_coordinator.cancel_run(run_id)
except:
instance.report_engine_event(
"Exception while attempting to force-terminate run. Run will still be marked as canceled.",
pipeline_name=run.pipeline_name,
run_id=run.run_id,
engine_event_data=EngineEventData(
error=serializable_error_info_from_exc_info(
sys.exc_info()), ),
)
return _force_mark_as_canceled(instance, run_id)
if (instance.run_coordinator
and instance.run_coordinator.can_cancel_run(run_id)
and instance.run_coordinator.cancel_run(run_id)):
return GrapheneTerminateRunSuccess(graphene_run)
return GrapheneTerminateRunFailure(
run=graphene_run,
message="Unable to terminate run {run_id}".format(run_id=run.run_id))
def _cli_api_execute_run_process(input_file, output_file, instance, pipeline_origin, pipeline_run):
write_unary_input(
input_file,
ExecuteRunArgs(
pipeline_origin=pipeline_origin,
pipeline_run_id=pipeline_run.run_id,
instance_ref=instance.get_ref(),
),
)
parts = [
pipeline_origin.executable_path,
'-m',
'dagster',
'api',
'execute_run',
input_file,
output_file,
]
instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'.format(
pipeline_name=pipeline_run.pipeline_name, run_id=pipeline_run.run_id
),
pipeline_run,
engine_event_data=EngineEventData(marker_start='cli_api_subprocess_init'),
)
return open_ipc_subprocess(parts)
def launch_run(self, instance, run, external_pipeline):
check.inst_param(instance, 'instance', DagsterInstance)
check.inst_param(run, 'run', PipelineRun)
job_name = 'dagster-run-{}'.format(run.run_id)
pod_name = job_name
exc_config = _get_validated_celery_k8s_executor_config(run.environment_dict)
job_config = DagsterK8sJobConfig(
dagster_home=self.dagster_home,
instance_config_map=self.instance_config_map,
postgres_password_secret=self.postgres_password_secret,
job_image=exc_config.get('job_image'),
image_pull_policy=exc_config.get('image_pull_policy'),
image_pull_secrets=exc_config.get('image_pull_secrets'),
service_account_name=exc_config.get('service_account_name'),
env_config_maps=exc_config.get('env_config_maps'),
env_secrets=exc_config.get('env_secrets'),
)
job = construct_dagster_graphql_k8s_job(
job_config,
args=[
'-p',
'executeRunInProcess',
'-v',
seven.json.dumps(
{
'runId': run.run_id,
'repositoryName': external_pipeline.handle.repository_name,
'repositoryLocationName': external_pipeline.handle.location_name,
}
),
],
job_name=job_name,
pod_name=pod_name,
component='runmaster',
)
job_namespace = exc_config.get('job_namespace')
api = kubernetes.client.BatchV1Api()
api.create_namespaced_job(body=job, namespace=job_namespace)
instance.report_engine_event(
'Kubernetes runmaster job launched',
run,
EngineEventData(
[
EventMetadataEntry.text(job_name, 'Kubernetes Job name'),
EventMetadataEntry.text(pod_name, 'Kubernetes Pod name'),
EventMetadataEntry.text(job_namespace, 'Kubernetes Namespace'),
EventMetadataEntry.text(run.run_id, 'Run ID'),
]
),
cls=CeleryK8sRunLauncher,
)
return run
def terminate_step(
self,
step_handler_context: StepHandlerContext) -> List[DagsterEvent]:
container_context = self._get_docker_container_context(
step_handler_context)
assert (len(
step_handler_context.execute_step_args.step_keys_to_execute) == 1
), "Launching multiple steps is not currently supported"
step_key = step_handler_context.execute_step_args.step_keys_to_execute[
0]
events = [
DagsterEvent(
event_type_value=DagsterEventType.ENGINE_EVENT.value,
pipeline_name=step_handler_context.execute_step_args.
pipeline_origin.pipeline_name,
step_key=step_key,
message="Stopping Docker container for step",
event_specific_data=EngineEventData(),
)
]
client = self._get_client(container_context)
try:
container = client.containers.get(
self._get_container_name(
step_handler_context.execute_step_args.pipeline_run_id,
step_handler_context.execute_step_args.
step_keys_to_execute[0],
))
container.stop()
except Exception as e:
events.append(
DagsterEvent(
event_type_value=DagsterEventType.ENGINE_EVENT.value,
pipeline_name=step_handler_context.execute_step_args.
pipeline_origin.pipeline_name,
step_key=step_key,
message=
f"Hit error while terminating Docker container:\n{e}",
event_specific_data=EngineEventData(),
))
return events
def launch_step(
self,
step_handler_context: StepHandlerContext) -> List[DagsterEvent]:
client = self._get_client()
step_image = (step_handler_context.execute_step_args.pipeline_origin.
repository_origin.container_image)
if not step_image:
step_image = self._image
if not step_image:
raise Exception(
"No docker image specified by the executor config or repository"
)
validate_docker_image(step_image)
try:
step_container = self._create_step_container(
client, step_image, step_handler_context.execute_step_args)
except docker.errors.ImageNotFound:
client.images.pull(step_image)
step_container = self._create_step_container(
client, step_image, step_handler_context.execute_step_args)
if len(self._networks) > 1:
for network_name in self._networks[1:]:
network = client.networks.get(network_name)
network.connect(step_container)
assert (len(
step_handler_context.execute_step_args.step_keys_to_execute) == 1
), "Launching multiple steps is not currently supported"
step_key = step_handler_context.execute_step_args.step_keys_to_execute[
0]
events = [
DagsterEvent(
event_type_value=DagsterEventType.ENGINE_EVENT.value,
pipeline_name=step_handler_context.execute_step_args.
pipeline_origin.pipeline_name,
step_key=step_key,
message="Launching step in Docker container",
event_specific_data=EngineEventData([
EventMetadataEntry.text(step_key, "Step key"),
EventMetadataEntry.text(step_container.id,
"Docker container id"),
], ),
)
]
step_container.start()
return events
def _execute_plan(_self, instance_ref_dict, handle_dict, run_id, step_keys,
retries_dict):
check.dict_param(instance_ref_dict, 'instance_ref_dict')
check.dict_param(handle_dict, 'handle_dict')
check.str_param(run_id, 'run_id')
check.list_param(step_keys, 'step_keys', of_type=str)
check.dict_param(retries_dict, 'retries_dict')
instance_ref = InstanceRef.from_dict(instance_ref_dict)
instance = DagsterInstance.from_ref(instance_ref)
handle = ExecutionTargetHandle.from_dict(handle_dict)
retries = Retries.from_config(retries_dict)
pipeline_run = instance.get_run_by_id(run_id)
check.invariant(pipeline_run, 'Could not load run {}'.format(run_id))
pipeline_def = handle.build_pipeline_definition().build_sub_pipeline(
pipeline_run.selector.solid_subset)
step_keys_str = ", ".join(step_keys)
execution_plan = create_execution_plan(
pipeline_def,
pipeline_run.environment_dict,
mode=pipeline_run.mode,
step_keys_to_execute=pipeline_run.step_keys_to_execute,
).build_subset_plan(step_keys)
engine_event = instance.report_engine_event(
'Executing steps {} in celery worker'.format(step_keys_str),
pipeline_run,
EngineEventData(
[
EventMetadataEntry.text(step_keys_str, 'step_keys'),
],
marker_end=DELEGATE_MARKER,
),
CeleryEngine,
step_key=execution_plan.step_key_for_single_step_plans(),
)
events = [engine_event]
for step_event in execute_plan_iterator(
execution_plan,
pipeline_run=pipeline_run,
environment_dict=pipeline_run.environment_dict,
instance=instance,
retries=retries,
):
events.append(step_event)
serialized_events = [
serialize_dagster_namedtuple(event) for event in events
]
return serialized_events
def launch_run(self, run, external_pipeline):
check.inst_param(run, "run", PipelineRun)
check.inst_param(external_pipeline, "external_pipeline",
ExternalPipeline)
job_name = "dagster-run-{}".format(run.run_id)
pod_name = job_name
user_defined_k8s_config = get_user_defined_k8s_config(
frozentags(run.tags))
pipeline_origin = external_pipeline.get_python_origin()
repository_origin = pipeline_origin.repository_origin
job_config = (self._get_grpc_job_config(
repository_origin.container_image)
if repository_origin.container_image else
self.get_static_job_config())
self._instance.add_run_tags(
run.run_id,
{DOCKER_IMAGE_TAG: job_config.job_image},
)
input_json = serialize_dagster_namedtuple(
ExecuteRunArgs(
pipeline_origin=pipeline_origin,
pipeline_run_id=run.run_id,
instance_ref=None,
))
job = construct_dagster_k8s_job(
job_config=job_config,
args=["dagster", "api", "execute_run", input_json],
job_name=job_name,
pod_name=pod_name,
component="run_coordinator",
user_defined_k8s_config=user_defined_k8s_config,
)
self._batch_api.create_namespaced_job(body=job,
namespace=self.job_namespace)
self._instance.report_engine_event(
"Kubernetes run worker job launched",
run,
EngineEventData([
EventMetadataEntry.text(job_name, "Kubernetes Job name"),
EventMetadataEntry.text(self.job_namespace,
"Kubernetes Namespace"),
EventMetadataEntry.text(run.run_id, "Run ID"),
]),
cls=self.__class__,
)
return run
def execute_run_grpc(api_client, instance_ref, pipeline_origin, pipeline_run):
'''Asynchronously execute a run over GRPC.'''
check.inst_param(api_client, 'api_client', DagsterGrpcClient)
check.inst_param(instance_ref, 'instance_ref', InstanceRef)
check.inst_param(pipeline_origin, 'pipeline_origin', PipelineOrigin)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
instance = DagsterInstance.from_ref(instance_ref)
yield instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'
.format(pipeline_name=pipeline_run.pipeline_name,
run_id=pipeline_run.run_id),
pipeline_run,
engine_event_data=EngineEventData(
marker_start='cli_api_subprocess_init'),
)
run_did_fail = False
execute_run_args = ExecuteRunArgs(
pipeline_origin=pipeline_origin,
pipeline_run_id=pipeline_run.run_id,
instance_ref=instance_ref,
)
for event in api_client.execute_run(execute_run_args=execute_run_args):
if isinstance(event, IPCErrorMessage):
yield instance.report_engine_event(
event.message,
pipeline_run=pipeline_run,
engine_event_data=EngineEventData(
marker_end='cli_api_subprocess_init',
error=event.serializable_error_info),
)
if not run_did_fail:
run_did_fail = True
yield instance.report_run_failed(pipeline_run)
else:
yield event
def _execute_plan(_self, instance_ref_dict, executable_dict, run_id,
step_keys, retries_dict):
check.dict_param(instance_ref_dict, "instance_ref_dict")
check.dict_param(executable_dict, "executable_dict")
check.str_param(run_id, "run_id")
check.list_param(step_keys, "step_keys", of_type=str)
check.dict_param(retries_dict, "retries_dict")
instance_ref = InstanceRef.from_dict(instance_ref_dict)
instance = DagsterInstance.from_ref(instance_ref)
pipeline = ReconstructablePipeline.from_dict(executable_dict)
retries = Retries.from_config(retries_dict)
pipeline_run = instance.get_run_by_id(run_id)
check.invariant(pipeline_run, "Could not load run {}".format(run_id))
step_keys_str = ", ".join(step_keys)
execution_plan = create_execution_plan(
pipeline,
pipeline_run.run_config,
mode=pipeline_run.mode,
step_keys_to_execute=pipeline_run.step_keys_to_execute,
).build_subset_plan(step_keys)
engine_event = instance.report_engine_event(
"Executing steps {} in celery worker".format(step_keys_str),
pipeline_run,
EngineEventData(
[
EventMetadataEntry.text(step_keys_str, "step_keys"),
],
marker_end=DELEGATE_MARKER,
),
CeleryExecutor,
step_key=execution_plan.step_key_for_single_step_plans(),
)
events = [engine_event]
for step_event in execute_plan_iterator(
execution_plan,
pipeline_run=pipeline_run,
run_config=pipeline_run.run_config,
instance=instance,
retries=retries,
):
events.append(step_event)
serialized_events = [
serialize_dagster_namedtuple(event) for event in events
]
return serialized_events
def execute_step_in_thread(self, step_key, step_context, errors):
yield DagsterEvent.engine_event(
step_context, "Spawning thread for {}".format(step_key), EngineEventData(marker_start=DELEGATE_MARKER)
)
for ret in execute_thread_step(step_context, self.retries):
if ret is None or isinstance(ret, DagsterEvent):
yield ret
elif isinstance(ret, ThreadEvent):
if isinstance(ret, ThreadSystemErrorEvent):
errors[ret.tid] = ret.error_info
else:
check.failed("Unexpected return value from thread {}".format(type(ret)))
def execute_run_grpc(api_client, instance_ref, pipeline_origin, pipeline_run):
"""Asynchronously execute a run over GRPC."""
from dagster.grpc.client import DagsterGrpcClient
check.inst_param(api_client, "api_client", DagsterGrpcClient)
check.inst_param(instance_ref, "instance_ref", InstanceRef)
check.inst_param(pipeline_origin, "pipeline_origin", ExternalPipelineOrigin)
check.inst_param(pipeline_run, "pipeline_run", PipelineRun)
with DagsterInstance.from_ref(instance_ref) as instance:
yield instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'.format(
pipeline_name=pipeline_run.pipeline_name, run_id=pipeline_run.run_id
),
pipeline_run,
engine_event_data=EngineEventData(marker_start="cli_api_subprocess_init"),
)
run_did_fail = False
execute_run_args = ExecuteExternalPipelineArgs(
pipeline_origin=pipeline_origin,
pipeline_run_id=pipeline_run.run_id,
instance_ref=instance_ref,
)
for event in api_client.execute_run(execute_run_args=execute_run_args):
if isinstance(event, IPCErrorMessage):
yield instance.report_engine_event(
event.message,
pipeline_run=pipeline_run,
engine_event_data=EngineEventData(
marker_end="cli_api_subprocess_init", error=event.serializable_error_info
),
)
if not run_did_fail:
run_did_fail = True
yield instance.report_run_failed(pipeline_run)
else:
yield event
def report_engine_event(
self,
message,
pipeline_run,
engine_event_data=None,
cls=None,
step_key=None,
):
'''
Report a EngineEvent that occurred outside of a pipeline execution context.
'''
from dagster.core.events import EngineEventData, DagsterEvent, DagsterEventType
from dagster.core.events.log import DagsterEventRecord
check.class_param(cls, 'cls')
check.str_param(message, 'message')
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
engine_event_data = check.opt_inst_param(
engine_event_data,
'engine_event_data',
EngineEventData,
EngineEventData([]),
)
if cls:
message = "[{}] {}".format(cls.__name__, message)
log_level = logging.INFO
if engine_event_data and engine_event_data.error:
log_level = logging.ERROR
dagster_event = DagsterEvent(
event_type_value=DagsterEventType.ENGINE_EVENT.value,
pipeline_name=pipeline_run.pipeline_name,
message=message,
event_specific_data=engine_event_data,
)
event_record = DagsterEventRecord(
message=message,
user_message=message,
level=log_level,
pipeline_name=pipeline_run.pipeline_name,
run_id=pipeline_run.run_id,
error_info=None,
timestamp=time.time(),
step_key=step_key,
dagster_event=dagster_event,
)
self.handle_new_event(event_record)
return dagster_event
def _launch_k8s_job_with_args(self, job_name, args, run):
container_context = self.get_container_context_for_run(run)
pod_name = job_name
pipeline_origin = run.pipeline_code_origin
user_defined_k8s_config = get_user_defined_k8s_config(
frozentags(run.tags))
repository_origin = pipeline_origin.repository_origin
job_config = container_context.get_k8s_job_config(
job_image=repository_origin.container_image, run_launcher=self)
self._instance.add_run_tags(
run.run_id,
{DOCKER_IMAGE_TAG: job_config.job_image},
)
job = construct_dagster_k8s_job(
job_config=job_config,
args=args,
job_name=job_name,
pod_name=pod_name,
component="run_worker",
user_defined_k8s_config=user_defined_k8s_config,
labels={
"dagster/job": pipeline_origin.pipeline_name,
"dagster/run-id": run.run_id,
},
)
self._instance.report_engine_event(
"Creating Kubernetes run worker job",
run,
EngineEventData([
MetadataEntry("Kubernetes Job name", value=job_name),
MetadataEntry("Kubernetes Namespace",
value=container_context.namespace),
MetadataEntry("Run ID", value=run.run_id),
]),
cls=self.__class__,
)
self._batch_api.create_namespaced_job(
body=job, namespace=container_context.namespace)
self._instance.report_engine_event(
"Kubernetes run worker job created",
run,
cls=self.__class__,
)
def launch_run(self, instance, run, external_pipeline):
check.inst_param(run, 'run', PipelineRun)
check.inst_param(external_pipeline, 'external_pipeline',
ExternalPipeline)
job_name = 'dagster-run-{}'.format(run.run_id)
pod_name = job_name
resources = get_k8s_resource_requirements(
frozentags(external_pipeline.tags))
job = construct_dagster_k8s_job(
job_config=self.job_config,
command=['dagster-graphql'],
args=[
'-p',
'executeRunInProcess',
'-v',
seven.json.dumps({
'runId':
run.run_id,
'repositoryName':
external_pipeline.handle.repository_name,
'repositoryLocationName':
external_pipeline.handle.location_name,
}),
'--remap-sigterm',
],
job_name=job_name,
pod_name=pod_name,
component='runmaster',
resources=resources,
)
self._batch_api.create_namespaced_job(body=job,
namespace=self.job_namespace)
self._instance.report_engine_event(
'Kubernetes runmaster job launched',
run,
EngineEventData([
EventMetadataEntry.text(job_name, 'Kubernetes Job name'),
EventMetadataEntry.text(pod_name, 'Kubernetes Pod name'),
EventMetadataEntry.text(self.job_namespace,
'Kubernetes Namespace'),
EventMetadataEntry.text(run.run_id, 'Run ID'),
]),
cls=K8sRunLauncher,
)
return run
def launch_run(self, instance, run, external_pipeline):
if isinstance(
external_pipeline.get_external_origin().
external_repository_origin.repository_location_origin,
GrpcServerRepositoryLocationOrigin,
):
repository_location_handle = (
external_pipeline.repository_handle.repository_location_handle)
if not isinstance(repository_location_handle,
GrpcServerRepositoryLocationHandle):
raise DagsterInvariantViolationError(
"Expected RepositoryLocationHandle to be of type "
"GrpcServerRepositoryLocationHandle but found type {}".
format(type(repository_location_handle)))
repository_name = external_pipeline.repository_handle.repository_name
location_name = external_pipeline.repository_handle.repository_location_handle.location_name
pipeline_origin = PipelinePythonOrigin(
pipeline_name=external_pipeline.name,
repository_origin=repository_location_handle.
get_repository_python_origin(repository_name),
)
else:
location_name = 'local'
pipeline_origin = external_pipeline.get_python_origin()
input_json = serialize_dagster_namedtuple(
ExecuteRunArgs(
pipeline_origin=pipeline_origin,
pipeline_run_id=run.run_id,
instance_ref=None,
))
app = self._get_app(location_name)
sig = app.signature('launch_run',
args=(input_json, ),
queue=f"{location_name}-pipelines")
result = sig.delay()
instance.report_engine_event(
"Started Celery task for pipeline (task id: {result.id}).".format(
result=result),
run,
EngineEventData(metadata_entries=[
EventMetadataEntry.text(result.id, "task_id"),
]),
)
return run
def cli_api_execute_run(output_file, instance, repository_handle,
pipeline_run):
check.str_param(output_file, 'output_file')
check.inst_param(instance, 'instance', DagsterInstance)
check.inst_param(repository_handle, 'repository_handle', RepositoryHandle)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
pointer = repository_handle.get_pointer()
location_handle = repository_handle.repository_location_handle
if isinstance(location_handle, PythonEnvRepositoryLocationHandle):
executable_path = location_handle.executable_path
elif isinstance(location_handle, InProcessRepositoryLocationHandle):
# [legacy] default to using sys.executable for the in process
# location handle
executable_path = sys.executable
else:
raise DagsterInvariantViolationError(
"Unable to resolve executable_path for repository location handle of type {}"
.format(location_handle.__class__))
executable_path = (location_handle.executable_path if isinstance(
location_handle, PythonEnvRepositoryLocationHandle) else
sys.executable)
parts = (
[executable_path, '-m', 'dagster', 'api', 'execute_run', output_file] +
xplat_shlex_split(pointer.get_cli_args()) + [
'--instance-ref',
'{instance_ref}'.format(
instance_ref=serialize_dagster_namedtuple(instance.get_ref())),
'--pipeline-run-id',
'{pipeline_run_id}'.format(pipeline_run_id=pipeline_run.run_id),
])
instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'
.format(pipeline_name=pipeline_run.pipeline_name,
run_id=pipeline_run.run_id),
pipeline_run,
engine_event_data=EngineEventData(
marker_start='cli_api_subprocess_init'),
)
return open_ipc_subprocess(parts)
def cli_api_execute_run(output_file, instance, pipeline_origin, pipeline_run):
check.str_param(output_file, 'output_file')
check.inst_param(instance, 'instance', DagsterInstance)
check.inst_param(pipeline_origin, 'pipeline_origin', PipelinePythonOrigin)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
from dagster.cli.api import ExecuteRunArgsLoadComplete
with safe_tempfile_path() as input_file:
write_unary_input(
input_file,
ExecuteRunArgs(
pipeline_origin=pipeline_origin,
pipeline_run_id=pipeline_run.run_id,
instance_ref=instance.get_ref(),
),
)
parts = [
pipeline_origin.executable_path,
'-m',
'dagster',
'api',
'execute_run',
input_file,
output_file,
]
instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'
.format(pipeline_name=pipeline_run.pipeline_name,
run_id=pipeline_run.run_id),
pipeline_run,
engine_event_data=EngineEventData(
marker_start='cli_api_subprocess_init'),
)
process = open_ipc_subprocess(parts)
# we need to process this event in order to ensure that the called process loads the input
event = next(ipc_read_event_stream(output_file))
check.inst(event, ExecuteRunArgsLoadComplete)
return process
def launch_run(self, instance, run, external_pipeline):
check.inst_param(instance, 'instance', DagsterInstance)
check.inst_param(run, 'run', PipelineRun)
check.inst_param(external_pipeline, 'external_pipeline', ExternalPipeline)
job_name = 'dagster-run-{}'.format(run.run_id)
pod_name = job_name
job = construct_dagster_graphql_k8s_job(
self.job_config,
args=[
'-p',
'executeRunInProcess',
'-v',
seven.json.dumps(
{
'runId': run.run_id,
'repositoryName': external_pipeline.handle.repository_name,
'repositoryLocationName': external_pipeline.handle.location_name,
}
),
],
job_name=job_name,
pod_name=pod_name,
component='runmaster',
)
api = kubernetes.client.BatchV1Api()
api.create_namespaced_job(body=job, namespace=self.job_namespace)
instance.report_engine_event(
'Kubernetes runmaster job launched',
run,
EngineEventData(
[
EventMetadataEntry.text(job_name, 'Kubernetes Job name'),
EventMetadataEntry.text(pod_name, 'Kubernetes Pod name'),
EventMetadataEntry.text(self.job_namespace, 'Kubernetes Namespace'),
EventMetadataEntry.text(run.run_id, 'Run ID'),
]
),
cls=K8sRunLauncher,
)
return run
