def cli_api_execute_run_grpc(instance_ref, pipeline_origin, pipeline_run): check.inst_param(instance_ref, 'instance_ref', InstanceRef) check.inst_param(pipeline_origin, 'pipeline_origin', PipelinePythonOrigin) check.inst_param(pipeline_run, 'pipeline_run', PipelineRun) instance = DagsterInstance.from_ref(instance_ref) yield instance.report_engine_event( 'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'.format( pipeline_name=pipeline_run.pipeline_name, run_id=pipeline_run.run_id ), pipeline_run, engine_event_data=EngineEventData(marker_start='cli_api_subprocess_init'), ) with ephemeral_grpc_api_client() as api_client: execute_run_args = ExecuteRunArgs( pipeline_origin=pipeline_origin, pipeline_run_id=pipeline_run.run_id, instance_ref=instance_ref, ) for event in api_client.execute_run(execute_run_args=execute_run_args): if isinstance(event, IPCErrorMessage): instance.report_engine_event( event.message, pipeline_run=pipeline_run, engine_event_data=EngineEventData( marker_end='cli_api_subprocess_init', error=event.serializable_error_info ), ) instance.report_run_failed(pipeline_run) return yield event
def _launch_k8s_job_with_args(self, job_name, args, run, pipeline_origin): pod_name = job_name user_defined_k8s_config = get_user_defined_k8s_config(frozentags(run.tags)) repository_origin = pipeline_origin.repository_origin job_config = ( self._get_grpc_job_config(repository_origin.container_image) if repository_origin.container_image else self.get_static_job_config() ) self._instance.add_run_tags( run.run_id, {DOCKER_IMAGE_TAG: job_config.job_image}, ) job = construct_dagster_k8s_job( job_config=job_config, args=args, job_name=job_name, pod_name=pod_name, component="run_worker", user_defined_k8s_config=user_defined_k8s_config, labels={ "dagster/job": pipeline_origin.pipeline_name, }, ) self._instance.report_engine_event( "Creating Kubernetes run worker job", run, EngineEventData( [ EventMetadataEntry.text(job_name, "Kubernetes Job name"), EventMetadataEntry.text(self.job_namespace, "Kubernetes Namespace"), EventMetadataEntry.text(run.run_id, "Run ID"), ] ), cls=self.__class__, ) self._batch_api.create_namespaced_job(body=job, namespace=self.job_namespace) self._instance.report_engine_event( "Kubernetes run worker job created", run, EngineEventData( [ EventMetadataEntry.text(job_name, "Kubernetes Job name"), EventMetadataEntry.text(self.job_namespace, "Kubernetes Namespace"), EventMetadataEntry.text(run.run_id, "Run ID"), ] ), cls=self.__class__, )
def cli_api_execute_run(output_file, instance, repo_cli_args, pipeline_run): check.str_param(output_file, 'output_file') check.inst_param(instance, 'instance', DagsterInstance) check.str_param(repo_cli_args, 'repo_cli_args') check.inst_param(pipeline_run, 'pipeline_run', PipelineRun) parts = ([ 'dagster', 'api', 'execute_run', output_file, ] + xplat_shlex_split(repo_cli_args) + [ '--instance-ref', '{instance_ref}'.format( instance_ref=serialize_dagster_namedtuple(instance.get_ref())), '--pipeline-run', '{pipeline_run}'.format( pipeline_run=serialize_dagster_namedtuple(pipeline_run)), ]) instance.report_engine_event( 'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).' .format(pipeline_name=pipeline_run.pipeline_name, run_id=pipeline_run.run_id), pipeline_run, engine_event_data=EngineEventData( marker_start='cli_api_subprocess_init'), ) return open_ipc_subprocess(parts)
def execute(self): pipeline = self.recon_pipeline with DagsterInstance.from_ref(self.instance_ref) as instance: start_termination_thread(self.term_event) execution_plan = create_execution_plan( pipeline=pipeline, run_config=self.run_config, mode=self.pipeline_run.mode, step_keys_to_execute=[self.step_key], known_state=self.known_state, ) yield instance.report_engine_event( "Executing step {} in subprocess".format(self.step_key), self.pipeline_run, EngineEventData( [ MetadataEntry("pid", value=str(os.getpid())), MetadataEntry("step_key", value=self.step_key), ], marker_end=DELEGATE_MARKER, ), MultiprocessExecutor, self.step_key, ) yield from execute_plan_iterator( execution_plan, pipeline, self.pipeline_run, run_config=self.run_config, retry_mode=self.retry_mode.for_inner_plan(), instance=instance, )
def execute(self): pipeline = self.recon_pipeline with DagsterInstance.from_ref(self.instance_ref) as instance: start_termination_thread(self.term_event) execution_plan = create_execution_plan( pipeline=pipeline, run_config=self.run_config, mode=self.pipeline_run.mode, step_keys_to_execute=self.pipeline_run.step_keys_to_execute, ).build_subset_plan([self.step_key]) yield instance.report_engine_event( "Executing step {} in subprocess".format(self.step_key), self.pipeline_run, EngineEventData( [ EventMetadataEntry.text(str(os.getpid()), "pid"), EventMetadataEntry.text(self.step_key, "step_key"), ], marker_end=DELEGATE_MARKER, ), MultiprocessExecutor, self.step_key, ) for step_event in execute_plan_iterator( execution_plan, self.pipeline_run, run_config=self.run_config, retries=self.retries.for_inner_plan(), instance=instance, ): yield step_event
def execute_step_out_of_process(self, step_context, step, errors, term_events, known_state): command = InProcessExecutorChildProcessCommand( run_config=step_context.run_config, pipeline_run=step_context.pipeline_run, step_key=step.key, instance_ref=step_context.instance.get_ref(), term_event=term_events[step.key], recon_pipeline=self.pipeline, retry_mode=self.retries, known_state=known_state, ) yield DagsterEvent.engine_event( step_context, "Launching subprocess for {}".format(step.key), EngineEventData(marker_start=DELEGATE_MARKER), step_handle=step.handle, ) for ret in execute_child_process_command(command): if ret is None or isinstance(ret, DagsterEvent): yield ret elif isinstance(ret, ChildProcessEvent): if isinstance(ret, ChildProcessSystemErrorEvent): errors[ret.pid] = ret.error_info else: check.failed( "Unexpected return value from child process {}".format( type(ret)))
def _execute_plan(self, execute_step_args_packed, executable_dict): execute_step_args = unpack_value( check.dict_param( execute_step_args_packed, "execute_step_args_packed", )) check.inst_param(execute_step_args, "execute_step_args", ExecuteStepArgs) check.dict_param(executable_dict, "executable_dict") instance = DagsterInstance.from_ref(execute_step_args.instance_ref) pipeline = ReconstructablePipeline.from_dict(executable_dict) retries = Retries.from_config(execute_step_args.retries_dict) pipeline_run = instance.get_run_by_id( execute_step_args.pipeline_run_id) check.invariant( pipeline_run, "Could not load run {}".format(execute_step_args.pipeline_run_id)) step_keys_str = ", ".join(execute_step_args.step_keys_to_execute) execution_plan = create_execution_plan( pipeline, pipeline_run.run_config, mode=pipeline_run.mode, step_keys_to_execute=execute_step_args.step_keys_to_execute, ) engine_event = instance.report_engine_event( "Executing steps {} in celery worker".format(step_keys_str), pipeline_run, EngineEventData( [ EventMetadataEntry.text(step_keys_str, "step_keys"), EventMetadataEntry.text(self.request.hostname, "Celery worker"), ], marker_end=DELEGATE_MARKER, ), CeleryExecutor, step_key=execution_plan.step_key_for_single_step_plans(), ) events = [engine_event] for step_event in execute_plan_iterator( execution_plan, pipeline_run=pipeline_run, run_config=pipeline_run.run_config, instance=instance, retries=retries, ): events.append(step_event) serialized_events = [ serialize_dagster_namedtuple(event) for event in events ] return serialized_events
def execute_step_out_of_process(step_context, step, errors, term_events): command = InProcessExecutorChildProcessCommand( step_context.run_config, step_context.pipeline_run, step_context.executor_config, step.key, step_context.instance.get_ref(), term_events[step.key], ) yield DagsterEvent.engine_event( step_context, 'Launching subprocess for {}'.format(step.key), EngineEventData(marker_start=DELEGATE_MARKER), step_key=step.key, ) for ret in execute_child_process_command(command): if ret is None or isinstance(ret, DagsterEvent): yield ret elif isinstance(ret, ChildProcessEvent): if isinstance(ret, ChildProcessSystemErrorEvent): errors[ret.pid] = ret.error_info elif isinstance(ret, KeyboardInterrupt): yield DagsterEvent.engine_event( step_context, 'Multiprocess engine: received KeyboardInterrupt - forwarding to active child processes', EngineEventData.interrupted(list(term_events.keys())), ) for term_event in term_events.values(): term_event.set() else: check.failed( 'Unexpected return value from child process {}'.format( type(ret)))
def execute(self): check.inst(self.executor_config, MultiprocessExecutorConfig) pipeline_def = self.executor_config.load_pipeline(self.pipeline_run) instance = DagsterInstance.from_ref(self.instance_ref) start_termination_thread(self.term_event) execution_plan = create_execution_plan( pipeline_def, self.environment_dict, self.pipeline_run).build_subset_plan([self.step_key]) yield instance.report_engine_event( MultiprocessEngine, 'Executing step {} in subprocess'.format(self.step_key), self.pipeline_run, EngineEventData( [ EventMetadataEntry.text(str(os.getpid()), 'pid'), EventMetadataEntry.text(self.step_key, 'step_key'), ], marker_end=DELEGATE_MARKER, ), self.step_key, ) for step_event in execute_plan_iterator( execution_plan, self.pipeline_run, environment_dict=self.environment_dict, retries=self.executor_config.retries.for_inner_plan(), instance=instance, ): yield step_event
def make_event(event_id): return DagsterEvent( event_type_value="ENGINE_EVENT", pipeline_name="some_pipeline", event_specific_data=EngineEventData(), message=str(event_id), )
def execute_pipeline(self, handle, pipeline, pipeline_run, instance): '''Subclasses must implement this method.''' check.inst_param(handle, 'handle', ExecutionTargetHandle) term_event = self._multiprocessing_context.Event() mp_process = self._multiprocessing_context.Process( target=self.__class__.in_mp_process, kwargs={ 'handle': handle, 'pipeline_run': pipeline_run, 'instance_ref': instance.get_ref(), 'term_event': term_event, }, ) instance.report_engine_event( 'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).' .format(pipeline_name=pipeline_run.pipeline_name, run_id=pipeline_run.run_id), pipeline_run, engine_event_data=EngineEventData( marker_start='dagit_subprocess_init'), cls=self.__class__, ) mp_process.start() with self._processes_lock: self._living_process_by_run_id[pipeline_run.run_id] = mp_process self._term_events[pipeline_run.run_id] = term_event
def launch_step(self, step_handler_context: StepHandlerContext): events = [] assert (len( step_handler_context.execute_step_args.step_keys_to_execute) == 1 ), "Launching multiple steps is not currently supported" step_key = step_handler_context.execute_step_args.step_keys_to_execute[ 0] k8s_name_key = get_k8s_job_name( step_handler_context.execute_step_args.pipeline_run_id, step_key, ) job_name = "dagster-job-%s" % (k8s_name_key) pod_name = "dagster-job-%s" % (k8s_name_key) input_json = serialize_dagster_namedtuple( step_handler_context.execute_step_args) args = ["dagster", "api", "execute_step", input_json] job_config = self._job_config if not job_config.job_image: job_config = job_config.with_image( step_handler_context.execute_step_args.pipeline_origin. repository_origin.container_image) if not job_config.job_image: raise Exception( "No image included in either executor config or the pipeline") user_defined_k8s_config = get_user_defined_k8s_config( frozentags(step_handler_context.step_tags[step_key])) job = construct_dagster_k8s_job( job_config=job_config, args=args, job_name=job_name, pod_name=pod_name, component="step_worker", user_defined_k8s_config=user_defined_k8s_config, ) events.append( DagsterEvent( event_type_value=DagsterEventType.ENGINE_EVENT.value, pipeline_name=step_handler_context.execute_step_args. pipeline_origin.pipeline_name, step_key=step_key, message= f"Executing step {step_key} in Kubernetes job {job_name}", event_specific_data=EngineEventData([ EventMetadataEntry.text(step_key, "Step key"), EventMetadataEntry.text(job_name, "Kubernetes Job name"), ], ), )) self._batch_api.create_namespaced_job(body=job, namespace=self._job_namespace) return events
def terminate_pipeline_execution(instance, run_id, terminate_policy): from ...schema.errors import GrapheneRunNotFoundError from ...schema.pipelines.pipeline import GrapheneRun from ...schema.roots.mutation import ( GrapheneTerminateRunFailure, GrapheneTerminateRunPolicy, GrapheneTerminateRunSuccess, ) check.inst_param(instance, "instance", DagsterInstance) check.str_param(run_id, "run_id") records = instance.get_run_records(RunsFilter(run_ids=[run_id])) force_mark_as_canceled = (terminate_policy == GrapheneTerminateRunPolicy. MARK_AS_CANCELED_IMMEDIATELY) if not records: return GrapheneRunNotFoundError(run_id) record = records[0] run = record.pipeline_run graphene_run = GrapheneRun(record) valid_status = not run.is_finished and ( force_mark_as_canceled or (run.status == PipelineRunStatus.STARTED or run.status == PipelineRunStatus.QUEUED)) if not valid_status: return GrapheneTerminateRunFailure( run=graphene_run, message= "Run {run_id} could not be terminated due to having status {status}." .format(run_id=run.run_id, status=run.status.value), ) if force_mark_as_canceled: try: if instance.run_coordinator and instance.run_coordinator.can_cancel_run( run_id): instance.run_coordinator.cancel_run(run_id) except: instance.report_engine_event( "Exception while attempting to force-terminate run. Run will still be marked as canceled.", pipeline_name=run.pipeline_name, run_id=run.run_id, engine_event_data=EngineEventData( error=serializable_error_info_from_exc_info( sys.exc_info()), ), ) return _force_mark_as_canceled(instance, run_id) if (instance.run_coordinator and instance.run_coordinator.can_cancel_run(run_id) and instance.run_coordinator.cancel_run(run_id)): return GrapheneTerminateRunSuccess(graphene_run) return GrapheneTerminateRunFailure( run=graphene_run, message="Unable to terminate run {run_id}".format(run_id=run.run_id))
def _cli_api_execute_run_process(input_file, output_file, instance, pipeline_origin, pipeline_run): write_unary_input( input_file, ExecuteRunArgs( pipeline_origin=pipeline_origin, pipeline_run_id=pipeline_run.run_id, instance_ref=instance.get_ref(), ), ) parts = [ pipeline_origin.executable_path, '-m', 'dagster', 'api', 'execute_run', input_file, output_file, ] instance.report_engine_event( 'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'.format( pipeline_name=pipeline_run.pipeline_name, run_id=pipeline_run.run_id ), pipeline_run, engine_event_data=EngineEventData(marker_start='cli_api_subprocess_init'), ) return open_ipc_subprocess(parts)
def launch_run(self, instance, run, external_pipeline): check.inst_param(instance, 'instance', DagsterInstance) check.inst_param(run, 'run', PipelineRun) job_name = 'dagster-run-{}'.format(run.run_id) pod_name = job_name exc_config = _get_validated_celery_k8s_executor_config(run.environment_dict) job_config = DagsterK8sJobConfig( dagster_home=self.dagster_home, instance_config_map=self.instance_config_map, postgres_password_secret=self.postgres_password_secret, job_image=exc_config.get('job_image'), image_pull_policy=exc_config.get('image_pull_policy'), image_pull_secrets=exc_config.get('image_pull_secrets'), service_account_name=exc_config.get('service_account_name'), env_config_maps=exc_config.get('env_config_maps'), env_secrets=exc_config.get('env_secrets'), ) job = construct_dagster_graphql_k8s_job( job_config, args=[ '-p', 'executeRunInProcess', '-v', seven.json.dumps( { 'runId': run.run_id, 'repositoryName': external_pipeline.handle.repository_name, 'repositoryLocationName': external_pipeline.handle.location_name, } ), ], job_name=job_name, pod_name=pod_name, component='runmaster', ) job_namespace = exc_config.get('job_namespace') api = kubernetes.client.BatchV1Api() api.create_namespaced_job(body=job, namespace=job_namespace) instance.report_engine_event( 'Kubernetes runmaster job launched', run, EngineEventData( [ EventMetadataEntry.text(job_name, 'Kubernetes Job name'), EventMetadataEntry.text(pod_name, 'Kubernetes Pod name'), EventMetadataEntry.text(job_namespace, 'Kubernetes Namespace'), EventMetadataEntry.text(run.run_id, 'Run ID'), ] ), cls=CeleryK8sRunLauncher, ) return run
def terminate_step( self, step_handler_context: StepHandlerContext) -> List[DagsterEvent]: container_context = self._get_docker_container_context( step_handler_context) assert (len( step_handler_context.execute_step_args.step_keys_to_execute) == 1 ), "Launching multiple steps is not currently supported" step_key = step_handler_context.execute_step_args.step_keys_to_execute[ 0] events = [ DagsterEvent( event_type_value=DagsterEventType.ENGINE_EVENT.value, pipeline_name=step_handler_context.execute_step_args. pipeline_origin.pipeline_name, step_key=step_key, message="Stopping Docker container for step", event_specific_data=EngineEventData(), ) ] client = self._get_client(container_context) try: container = client.containers.get( self._get_container_name( step_handler_context.execute_step_args.pipeline_run_id, step_handler_context.execute_step_args. step_keys_to_execute[0], )) container.stop() except Exception as e: events.append( DagsterEvent( event_type_value=DagsterEventType.ENGINE_EVENT.value, pipeline_name=step_handler_context.execute_step_args. pipeline_origin.pipeline_name, step_key=step_key, message= f"Hit error while terminating Docker container:\n{e}", event_specific_data=EngineEventData(), )) return events
def launch_step( self, step_handler_context: StepHandlerContext) -> List[DagsterEvent]: client = self._get_client() step_image = (step_handler_context.execute_step_args.pipeline_origin. repository_origin.container_image) if not step_image: step_image = self._image if not step_image: raise Exception( "No docker image specified by the executor config or repository" ) validate_docker_image(step_image) try: step_container = self._create_step_container( client, step_image, step_handler_context.execute_step_args) except docker.errors.ImageNotFound: client.images.pull(step_image) step_container = self._create_step_container( client, step_image, step_handler_context.execute_step_args) if len(self._networks) > 1: for network_name in self._networks[1:]: network = client.networks.get(network_name) network.connect(step_container) assert (len( step_handler_context.execute_step_args.step_keys_to_execute) == 1 ), "Launching multiple steps is not currently supported" step_key = step_handler_context.execute_step_args.step_keys_to_execute[ 0] events = [ DagsterEvent( event_type_value=DagsterEventType.ENGINE_EVENT.value, pipeline_name=step_handler_context.execute_step_args. pipeline_origin.pipeline_name, step_key=step_key, message="Launching step in Docker container", event_specific_data=EngineEventData([ EventMetadataEntry.text(step_key, "Step key"), EventMetadataEntry.text(step_container.id, "Docker container id"), ], ), ) ] step_container.start() return events
def _execute_plan(_self, instance_ref_dict, handle_dict, run_id, step_keys, retries_dict): check.dict_param(instance_ref_dict, 'instance_ref_dict') check.dict_param(handle_dict, 'handle_dict') check.str_param(run_id, 'run_id') check.list_param(step_keys, 'step_keys', of_type=str) check.dict_param(retries_dict, 'retries_dict') instance_ref = InstanceRef.from_dict(instance_ref_dict) instance = DagsterInstance.from_ref(instance_ref) handle = ExecutionTargetHandle.from_dict(handle_dict) retries = Retries.from_config(retries_dict) pipeline_run = instance.get_run_by_id(run_id) check.invariant(pipeline_run, 'Could not load run {}'.format(run_id)) pipeline_def = handle.build_pipeline_definition().build_sub_pipeline( pipeline_run.selector.solid_subset) step_keys_str = ", ".join(step_keys) execution_plan = create_execution_plan( pipeline_def, pipeline_run.environment_dict, mode=pipeline_run.mode, step_keys_to_execute=pipeline_run.step_keys_to_execute, ).build_subset_plan(step_keys) engine_event = instance.report_engine_event( 'Executing steps {} in celery worker'.format(step_keys_str), pipeline_run, EngineEventData( [ EventMetadataEntry.text(step_keys_str, 'step_keys'), ], marker_end=DELEGATE_MARKER, ), CeleryEngine, step_key=execution_plan.step_key_for_single_step_plans(), ) events = [engine_event] for step_event in execute_plan_iterator( execution_plan, pipeline_run=pipeline_run, environment_dict=pipeline_run.environment_dict, instance=instance, retries=retries, ): events.append(step_event) serialized_events = [ serialize_dagster_namedtuple(event) for event in events ] return serialized_events
def launch_run(self, run, external_pipeline): check.inst_param(run, "run", PipelineRun) check.inst_param(external_pipeline, "external_pipeline", ExternalPipeline) job_name = "dagster-run-{}".format(run.run_id) pod_name = job_name user_defined_k8s_config = get_user_defined_k8s_config( frozentags(run.tags)) pipeline_origin = external_pipeline.get_python_origin() repository_origin = pipeline_origin.repository_origin job_config = (self._get_grpc_job_config( repository_origin.container_image) if repository_origin.container_image else self.get_static_job_config()) self._instance.add_run_tags( run.run_id, {DOCKER_IMAGE_TAG: job_config.job_image}, ) input_json = serialize_dagster_namedtuple( ExecuteRunArgs( pipeline_origin=pipeline_origin, pipeline_run_id=run.run_id, instance_ref=None, )) job = construct_dagster_k8s_job( job_config=job_config, args=["dagster", "api", "execute_run", input_json], job_name=job_name, pod_name=pod_name, component="run_coordinator", user_defined_k8s_config=user_defined_k8s_config, ) self._batch_api.create_namespaced_job(body=job, namespace=self.job_namespace) self._instance.report_engine_event( "Kubernetes run worker job launched", run, EngineEventData([ EventMetadataEntry.text(job_name, "Kubernetes Job name"), EventMetadataEntry.text(self.job_namespace, "Kubernetes Namespace"), EventMetadataEntry.text(run.run_id, "Run ID"), ]), cls=self.__class__, ) return run
def execute_run_grpc(api_client, instance_ref, pipeline_origin, pipeline_run): '''Asynchronously execute a run over GRPC.''' check.inst_param(api_client, 'api_client', DagsterGrpcClient) check.inst_param(instance_ref, 'instance_ref', InstanceRef) check.inst_param(pipeline_origin, 'pipeline_origin', PipelineOrigin) check.inst_param(pipeline_run, 'pipeline_run', PipelineRun) instance = DagsterInstance.from_ref(instance_ref) yield instance.report_engine_event( 'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).' .format(pipeline_name=pipeline_run.pipeline_name, run_id=pipeline_run.run_id), pipeline_run, engine_event_data=EngineEventData( marker_start='cli_api_subprocess_init'), ) run_did_fail = False execute_run_args = ExecuteRunArgs( pipeline_origin=pipeline_origin, pipeline_run_id=pipeline_run.run_id, instance_ref=instance_ref, ) for event in api_client.execute_run(execute_run_args=execute_run_args): if isinstance(event, IPCErrorMessage): yield instance.report_engine_event( event.message, pipeline_run=pipeline_run, engine_event_data=EngineEventData( marker_end='cli_api_subprocess_init', error=event.serializable_error_info), ) if not run_did_fail: run_did_fail = True yield instance.report_run_failed(pipeline_run) else: yield event
def _execute_plan(_self, instance_ref_dict, executable_dict, run_id, step_keys, retries_dict): check.dict_param(instance_ref_dict, "instance_ref_dict") check.dict_param(executable_dict, "executable_dict") check.str_param(run_id, "run_id") check.list_param(step_keys, "step_keys", of_type=str) check.dict_param(retries_dict, "retries_dict") instance_ref = InstanceRef.from_dict(instance_ref_dict) instance = DagsterInstance.from_ref(instance_ref) pipeline = ReconstructablePipeline.from_dict(executable_dict) retries = Retries.from_config(retries_dict) pipeline_run = instance.get_run_by_id(run_id) check.invariant(pipeline_run, "Could not load run {}".format(run_id)) step_keys_str = ", ".join(step_keys) execution_plan = create_execution_plan( pipeline, pipeline_run.run_config, mode=pipeline_run.mode, step_keys_to_execute=pipeline_run.step_keys_to_execute, ).build_subset_plan(step_keys) engine_event = instance.report_engine_event( "Executing steps {} in celery worker".format(step_keys_str), pipeline_run, EngineEventData( [ EventMetadataEntry.text(step_keys_str, "step_keys"), ], marker_end=DELEGATE_MARKER, ), CeleryExecutor, step_key=execution_plan.step_key_for_single_step_plans(), ) events = [engine_event] for step_event in execute_plan_iterator( execution_plan, pipeline_run=pipeline_run, run_config=pipeline_run.run_config, instance=instance, retries=retries, ): events.append(step_event) serialized_events = [ serialize_dagster_namedtuple(event) for event in events ] return serialized_events
def execute_step_in_thread(self, step_key, step_context, errors): yield DagsterEvent.engine_event( step_context, "Spawning thread for {}".format(step_key), EngineEventData(marker_start=DELEGATE_MARKER) ) for ret in execute_thread_step(step_context, self.retries): if ret is None or isinstance(ret, DagsterEvent): yield ret elif isinstance(ret, ThreadEvent): if isinstance(ret, ThreadSystemErrorEvent): errors[ret.tid] = ret.error_info else: check.failed("Unexpected return value from thread {}".format(type(ret)))
def execute_run_grpc(api_client, instance_ref, pipeline_origin, pipeline_run): """Asynchronously execute a run over GRPC.""" from dagster.grpc.client import DagsterGrpcClient check.inst_param(api_client, "api_client", DagsterGrpcClient) check.inst_param(instance_ref, "instance_ref", InstanceRef) check.inst_param(pipeline_origin, "pipeline_origin", ExternalPipelineOrigin) check.inst_param(pipeline_run, "pipeline_run", PipelineRun) with DagsterInstance.from_ref(instance_ref) as instance: yield instance.report_engine_event( 'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'.format( pipeline_name=pipeline_run.pipeline_name, run_id=pipeline_run.run_id ), pipeline_run, engine_event_data=EngineEventData(marker_start="cli_api_subprocess_init"), ) run_did_fail = False execute_run_args = ExecuteExternalPipelineArgs( pipeline_origin=pipeline_origin, pipeline_run_id=pipeline_run.run_id, instance_ref=instance_ref, ) for event in api_client.execute_run(execute_run_args=execute_run_args): if isinstance(event, IPCErrorMessage): yield instance.report_engine_event( event.message, pipeline_run=pipeline_run, engine_event_data=EngineEventData( marker_end="cli_api_subprocess_init", error=event.serializable_error_info ), ) if not run_did_fail: run_did_fail = True yield instance.report_run_failed(pipeline_run) else: yield event
def report_engine_event( self, message, pipeline_run, engine_event_data=None, cls=None, step_key=None, ): ''' Report a EngineEvent that occurred outside of a pipeline execution context. ''' from dagster.core.events import EngineEventData, DagsterEvent, DagsterEventType from dagster.core.events.log import DagsterEventRecord check.class_param(cls, 'cls') check.str_param(message, 'message') check.inst_param(pipeline_run, 'pipeline_run', PipelineRun) engine_event_data = check.opt_inst_param( engine_event_data, 'engine_event_data', EngineEventData, EngineEventData([]), ) if cls: message = "[{}] {}".format(cls.__name__, message) log_level = logging.INFO if engine_event_data and engine_event_data.error: log_level = logging.ERROR dagster_event = DagsterEvent( event_type_value=DagsterEventType.ENGINE_EVENT.value, pipeline_name=pipeline_run.pipeline_name, message=message, event_specific_data=engine_event_data, ) event_record = DagsterEventRecord( message=message, user_message=message, level=log_level, pipeline_name=pipeline_run.pipeline_name, run_id=pipeline_run.run_id, error_info=None, timestamp=time.time(), step_key=step_key, dagster_event=dagster_event, ) self.handle_new_event(event_record) return dagster_event
def _launch_k8s_job_with_args(self, job_name, args, run): container_context = self.get_container_context_for_run(run) pod_name = job_name pipeline_origin = run.pipeline_code_origin user_defined_k8s_config = get_user_defined_k8s_config( frozentags(run.tags)) repository_origin = pipeline_origin.repository_origin job_config = container_context.get_k8s_job_config( job_image=repository_origin.container_image, run_launcher=self) self._instance.add_run_tags( run.run_id, {DOCKER_IMAGE_TAG: job_config.job_image}, ) job = construct_dagster_k8s_job( job_config=job_config, args=args, job_name=job_name, pod_name=pod_name, component="run_worker", user_defined_k8s_config=user_defined_k8s_config, labels={ "dagster/job": pipeline_origin.pipeline_name, "dagster/run-id": run.run_id, }, ) self._instance.report_engine_event( "Creating Kubernetes run worker job", run, EngineEventData([ MetadataEntry("Kubernetes Job name", value=job_name), MetadataEntry("Kubernetes Namespace", value=container_context.namespace), MetadataEntry("Run ID", value=run.run_id), ]), cls=self.__class__, ) self._batch_api.create_namespaced_job( body=job, namespace=container_context.namespace) self._instance.report_engine_event( "Kubernetes run worker job created", run, cls=self.__class__, )
def launch_run(self, instance, run, external_pipeline): check.inst_param(run, 'run', PipelineRun) check.inst_param(external_pipeline, 'external_pipeline', ExternalPipeline) job_name = 'dagster-run-{}'.format(run.run_id) pod_name = job_name resources = get_k8s_resource_requirements( frozentags(external_pipeline.tags)) job = construct_dagster_k8s_job( job_config=self.job_config, command=['dagster-graphql'], args=[ '-p', 'executeRunInProcess', '-v', seven.json.dumps({ 'runId': run.run_id, 'repositoryName': external_pipeline.handle.repository_name, 'repositoryLocationName': external_pipeline.handle.location_name, }), '--remap-sigterm', ], job_name=job_name, pod_name=pod_name, component='runmaster', resources=resources, ) self._batch_api.create_namespaced_job(body=job, namespace=self.job_namespace) self._instance.report_engine_event( 'Kubernetes runmaster job launched', run, EngineEventData([ EventMetadataEntry.text(job_name, 'Kubernetes Job name'), EventMetadataEntry.text(pod_name, 'Kubernetes Pod name'), EventMetadataEntry.text(self.job_namespace, 'Kubernetes Namespace'), EventMetadataEntry.text(run.run_id, 'Run ID'), ]), cls=K8sRunLauncher, ) return run
def launch_run(self, instance, run, external_pipeline): if isinstance( external_pipeline.get_external_origin(). external_repository_origin.repository_location_origin, GrpcServerRepositoryLocationOrigin, ): repository_location_handle = ( external_pipeline.repository_handle.repository_location_handle) if not isinstance(repository_location_handle, GrpcServerRepositoryLocationHandle): raise DagsterInvariantViolationError( "Expected RepositoryLocationHandle to be of type " "GrpcServerRepositoryLocationHandle but found type {}". format(type(repository_location_handle))) repository_name = external_pipeline.repository_handle.repository_name location_name = external_pipeline.repository_handle.repository_location_handle.location_name pipeline_origin = PipelinePythonOrigin( pipeline_name=external_pipeline.name, repository_origin=repository_location_handle. get_repository_python_origin(repository_name), ) else: location_name = 'local' pipeline_origin = external_pipeline.get_python_origin() input_json = serialize_dagster_namedtuple( ExecuteRunArgs( pipeline_origin=pipeline_origin, pipeline_run_id=run.run_id, instance_ref=None, )) app = self._get_app(location_name) sig = app.signature('launch_run', args=(input_json, ), queue=f"{location_name}-pipelines") result = sig.delay() instance.report_engine_event( "Started Celery task for pipeline (task id: {result.id}).".format( result=result), run, EngineEventData(metadata_entries=[ EventMetadataEntry.text(result.id, "task_id"), ]), ) return run
def cli_api_execute_run(output_file, instance, repository_handle, pipeline_run): check.str_param(output_file, 'output_file') check.inst_param(instance, 'instance', DagsterInstance) check.inst_param(repository_handle, 'repository_handle', RepositoryHandle) check.inst_param(pipeline_run, 'pipeline_run', PipelineRun) pointer = repository_handle.get_pointer() location_handle = repository_handle.repository_location_handle if isinstance(location_handle, PythonEnvRepositoryLocationHandle): executable_path = location_handle.executable_path elif isinstance(location_handle, InProcessRepositoryLocationHandle): # [legacy] default to using sys.executable for the in process # location handle executable_path = sys.executable else: raise DagsterInvariantViolationError( "Unable to resolve executable_path for repository location handle of type {}" .format(location_handle.__class__)) executable_path = (location_handle.executable_path if isinstance( location_handle, PythonEnvRepositoryLocationHandle) else sys.executable) parts = ( [executable_path, '-m', 'dagster', 'api', 'execute_run', output_file] + xplat_shlex_split(pointer.get_cli_args()) + [ '--instance-ref', '{instance_ref}'.format( instance_ref=serialize_dagster_namedtuple(instance.get_ref())), '--pipeline-run-id', '{pipeline_run_id}'.format(pipeline_run_id=pipeline_run.run_id), ]) instance.report_engine_event( 'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).' .format(pipeline_name=pipeline_run.pipeline_name, run_id=pipeline_run.run_id), pipeline_run, engine_event_data=EngineEventData( marker_start='cli_api_subprocess_init'), ) return open_ipc_subprocess(parts)
def cli_api_execute_run(output_file, instance, pipeline_origin, pipeline_run): check.str_param(output_file, 'output_file') check.inst_param(instance, 'instance', DagsterInstance) check.inst_param(pipeline_origin, 'pipeline_origin', PipelinePythonOrigin) check.inst_param(pipeline_run, 'pipeline_run', PipelineRun) from dagster.cli.api import ExecuteRunArgsLoadComplete with safe_tempfile_path() as input_file: write_unary_input( input_file, ExecuteRunArgs( pipeline_origin=pipeline_origin, pipeline_run_id=pipeline_run.run_id, instance_ref=instance.get_ref(), ), ) parts = [ pipeline_origin.executable_path, '-m', 'dagster', 'api', 'execute_run', input_file, output_file, ] instance.report_engine_event( 'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).' .format(pipeline_name=pipeline_run.pipeline_name, run_id=pipeline_run.run_id), pipeline_run, engine_event_data=EngineEventData( marker_start='cli_api_subprocess_init'), ) process = open_ipc_subprocess(parts) # we need to process this event in order to ensure that the called process loads the input event = next(ipc_read_event_stream(output_file)) check.inst(event, ExecuteRunArgsLoadComplete) return process
def launch_run(self, instance, run, external_pipeline): check.inst_param(instance, 'instance', DagsterInstance) check.inst_param(run, 'run', PipelineRun) check.inst_param(external_pipeline, 'external_pipeline', ExternalPipeline) job_name = 'dagster-run-{}'.format(run.run_id) pod_name = job_name job = construct_dagster_graphql_k8s_job( self.job_config, args=[ '-p', 'executeRunInProcess', '-v', seven.json.dumps( { 'runId': run.run_id, 'repositoryName': external_pipeline.handle.repository_name, 'repositoryLocationName': external_pipeline.handle.location_name, } ), ], job_name=job_name, pod_name=pod_name, component='runmaster', ) api = kubernetes.client.BatchV1Api() api.create_namespaced_job(body=job, namespace=self.job_namespace) instance.report_engine_event( 'Kubernetes runmaster job launched', run, EngineEventData( [ EventMetadataEntry.text(job_name, 'Kubernetes Job name'), EventMetadataEntry.text(pod_name, 'Kubernetes Pod name'), EventMetadataEntry.text(self.job_namespace, 'Kubernetes Namespace'), EventMetadataEntry.text(run.run_id, 'Run ID'), ] ), cls=K8sRunLauncher, ) return run