def _check_execute_pipeline_args(pipeline, environment_dict, mode, preset, tags, solid_subset, instance): pipeline, pipeline_def = _check_pipeline(pipeline) environment_dict = check.opt_dict_param(environment_dict, 'environment_dict') check.opt_str_param(mode, 'mode') check.opt_str_param(preset, 'preset') check.invariant( not (mode is not None and preset is not None), 'You may set only one of `mode` (got {mode}) or `preset` (got {preset}).' .format(mode=mode, preset=preset), ) tags = check.opt_dict_param(tags, 'tags', key_type=str) check.opt_list_param(solid_subset, 'solid_subset', of_type=str) if preset is not None: pipeline_preset = pipeline_def.get_preset(preset) if pipeline_preset.environment_dict is not None: check.invariant( (not environment_dict) or (pipeline_preset.environment_dict == environment_dict), 'The environment set in preset \'{preset}\' does not agree with the environment ' 'passed in the `environment_dict` argument.'.format( preset=preset), ) environment_dict = pipeline_preset.environment_dict if pipeline_preset.solid_subset is not None: check.invariant( solid_subset is None or solid_subset == pipeline_preset.solid_subset, 'The solid_subset set in preset \'{preset}\', {preset_subset}, does not agree with ' 'the `solid_subset` argument: {solid_subset}'.format( preset=preset, preset_subset=pipeline_preset.solid_subset, solid_subset=solid_subset, ), ) solid_subset = pipeline_preset.solid_subset check.invariant( mode is None or mode == pipeline_preset.mode, 'Mode {mode} does not agree with the mode set in preset \'{preset}\': ' '(\'{preset_mode}\')'.format(preset=preset, preset_mode=pipeline_preset.mode, mode=mode), ) mode = pipeline_preset.mode if mode is not None: if not pipeline_def.has_mode_definition(mode): raise DagsterInvariantViolationError(( 'You have attempted to execute pipeline {name} with mode {mode}. ' 'Available modes: {modes}').format( name=pipeline_def.name, mode=mode, modes=pipeline_def.available_modes, )) else: if pipeline_def.is_multi_mode: raise DagsterInvariantViolationError(( 'Pipeline {name} has multiple modes (Available modes: {modes}) and you have ' 'attempted to execute it without specifying a mode. Set ' 'mode property on the PipelineRun object.').format( name=pipeline_def.name, modes=pipeline_def.available_modes)) mode = pipeline_def.get_default_mode_name() tags = merge_dicts(pipeline_def.tags, tags) check.opt_inst_param(instance, 'instance', DagsterInstance) instance = instance or DagsterInstance.ephemeral() if solid_subset: pipeline = pipeline.subset_for_execution(solid_subset) pipeline_def = pipeline.get_definition() else: solid_subset = pipeline_def.solid_subset return (pipeline, pipeline_def, environment_dict, instance, mode, tags, solid_subset)
def define_context_for_repository_yaml(path, instance=None): return DagsterGraphQLContext( handle=ExecutionTargetHandle.for_repo_yaml(path), instance=instance or DagsterInstance.ephemeral(), execution_manager=SynchronousExecutionManager(), )
def StartRun(self, request, _context): execute_run_args = check.inst( deserialize_json_to_dagster_namedtuple( request.serialized_execute_run_args), ExecuteRunArgs, ) try: execute_run_args = check.inst( deserialize_json_to_dagster_namedtuple( request.serialized_execute_run_args), ExecuteRunArgs, ) run_id = execute_run_args.pipeline_run_id recon_pipeline = self._recon_pipeline_from_origin( execute_run_args.pipeline_origin) except: # pylint: disable=bare-except return api_pb2.StartRunReply( serialized_start_run_result=serialize_dagster_namedtuple( StartRunResult( success=False, message=None, serializable_error_info= serializable_error_info_from_exc_info(sys.exc_info()), ))) event_queue = multiprocessing.Queue() termination_event = multiprocessing.Event() execution_process = multiprocessing.Process( target=start_run_in_subprocess, args=[ request.serialized_execute_run_args, recon_pipeline, event_queue, termination_event, ], ) with self._execution_lock: execution_process.start() self._executions[run_id] = ( execution_process, DagsterInstance.from_ref(execute_run_args.instance_ref), ) self._termination_events[run_id] = termination_event success = None message = None serializable_error_info = None while success is None: time.sleep(EVENT_QUEUE_POLL_INTERVAL) # We use `get_nowait()` instead of `get()` so that we can handle the case where the # execution process has died unexpectedly -- `get()` would hang forever in that case try: dagster_event_or_ipc_error_message_or_done = event_queue.get_nowait( ) except queue.Empty: if not execution_process.is_alive(): # subprocess died unexpectedly success = False message = ( "GRPC server: Subprocess for {run_id} terminated unexpectedly with " "exit code {exit_code}".format( run_id=run_id, exit_code=execution_process.exitcode, )) serializable_error_info = serializable_error_info_from_exc_info( sys.exc_info()) else: if isinstance(dagster_event_or_ipc_error_message_or_done, StartRunInSubprocessSuccessful): success = True elif isinstance(dagster_event_or_ipc_error_message_or_done, RunInSubprocessComplete): continue if isinstance(dagster_event_or_ipc_error_message_or_done, IPCErrorMessage): success = False message = dagster_event_or_ipc_error_message_or_done.message serializable_error_info = ( dagster_event_or_ipc_error_message_or_done. serializable_error_info) # Ensure that if the run failed, we remove it from the executions map before # returning so that CanCancel will never return True if not success: self._check_for_orphaned_runs() return api_pb2.StartRunReply( serialized_start_run_result=serialize_dagster_namedtuple( StartRunResult( success=success, message=message, serializable_error_info=serializable_error_info, )))
def execute_backfill_command(cli_args, print_fn, instance=None): pipeline_name = cli_args.pop('pipeline_name') repo_args = {k: v for k, v in cli_args.items() if k in REPO_ARG_NAMES} if pipeline_name and not isinstance(pipeline_name, six.string_types): if len(pipeline_name) == 1: pipeline_name = pipeline_name[0] instance = instance or DagsterInstance.get() handle = handle_for_repo_cli_args(repo_args) repository = handle.build_repository_definition() noprompt = cli_args.get('noprompt') # check run launcher if not instance.run_launcher: raise click.UsageError( 'A run launcher must be configured before running a backfill. You can configure a run ' 'launcher (e.g. dagster_graphql.launcher.RemoteDagitRunLauncher) in your instance ' '`dagster.yaml` settings. See ' 'https://docs.dagster.io/latest/deploying/instance/ for more' 'information.') # Resolve pipeline if not pipeline_name and noprompt: raise click.UsageError('No pipeline specified') if not pipeline_name: pipeline_name = click.prompt( 'Select a pipeline to backfill: {}'.format(', '.join( repository.pipeline_names))) repository = handle.build_repository_definition() if not repository.has_pipeline(pipeline_name): raise click.UsageError( 'No pipeline found named `{}`'.format(pipeline_name)) pipeline = repository.get_pipeline(pipeline_name) # Resolve partition set all_partition_sets = get_partition_sets_for_handle(handle) pipeline_partition_sets = [ x for x in all_partition_sets if x.pipeline_name == pipeline.name ] if not pipeline_partition_sets: raise click.UsageError( 'No partition sets found for pipeline `{}`'.format(pipeline.name)) partition_set_name = cli_args.get('partition_set') if not partition_set_name: if len(pipeline_partition_sets) == 1: partition_set_name = pipeline_partition_sets[0].name elif noprompt: raise click.UsageError( 'No partition set specified (see option `--partition-set`)') else: partition_set_name = click.prompt( 'Select a partition set to use for backfill: {}'.format( ', '.join(x.name for x in pipeline_partition_sets))) partition_set = next( (x for x in pipeline_partition_sets if x.name == partition_set_name), None) if not partition_set: raise click.UsageError( 'No partition set found named `{}`'.format(partition_set_name)) # Resolve partitions to backfill partitions = gen_partitions_from_args(partition_set, cli_args) # Resolve priority celery_priority = get_backfill_priority_from_args(cli_args) # Print backfill info print_fn('\n Pipeline: {}'.format(pipeline.name)) print_fn('Partition set: {}'.format(partition_set.name)) print_fn(' Partitions: {}\n'.format( print_partition_format(partitions, indent_level=15))) # Confirm and launch if noprompt or click.confirm( 'Do you want to proceed with the backfill ({} partitions)?'.format( len(partitions))): print_fn('Launching runs... ') backfill_id = make_new_backfill_id() run_tags = merge_dicts( PipelineRun.tags_for_backfill_id(backfill_id), get_tags_from_args(cli_args), ) # for backwards compatibility - remove once prezi switched over to using tags argument if celery_priority is not None: run_tags['dagster-celery/run_priority'] = celery_priority for partition in partitions: # execution_plan = create_execution_plan() run = instance.create_run_for_pipeline( pipeline=pipeline, # execution_plan=execution_plan, environment_dict=partition_set.environment_dict_for_partition( partition), mode=cli_args.get('mode') or 'default', tags=merge_dicts(partition_set.tags_for_partition(partition), run_tags), ) instance.launch_run(run) # Remove once we can handle synchronous execution... currently limited by sqlite time.sleep(0.1) print_fn('Launched backfill job `{}`'.format(backfill_id)) else: print_fn(' Aborted!')
def test_interrupt_multiproc(): with seven.TemporaryDirectory() as tempdir: file_1 = os.path.join(tempdir, "file_1") file_2 = os.path.join(tempdir, "file_2") file_3 = os.path.join(tempdir, "file_3") file_4 = os.path.join(tempdir, "file_4") # launch a thread that waits until the file is written to launch an interrupt Thread(target=_send_kbd_int, args=([file_1, file_2, file_3, file_4], )).start() results = [] try: # launch a pipeline that writes a file and loops infinitely # next time the launched thread wakes up it will send a keyboard # interrupt for result in execute_pipeline_iterator( reconstructable(write_files_pipeline), run_config={ "solids": { "write_1": { "config": { "tempfile": file_1 } }, "write_2": { "config": { "tempfile": file_2 } }, "write_3": { "config": { "tempfile": file_3 } }, "write_4": { "config": { "tempfile": file_4 } }, }, "execution": { "multiprocess": { "config": { "max_concurrent": 4 } } }, "storage": { "filesystem": {} }, }, instance=DagsterInstance.local_temp(tempdir=tempdir), ): results.append(result) assert False # should never reach except (DagsterSubprocessError, KeyboardInterrupt): pass assert [result.event_type for result in results ].count(DagsterEventType.STEP_FAILURE) == 4 assert DagsterEventType.PIPELINE_FAILURE in [ result.event_type for result in results ]
def pipeline_execute_command(**kwargs): with delay_interrupts(): with DagsterInstance.get() as instance: execute_execute_command(instance, kwargs)
def pipeline_backfill_command(**kwargs): with DagsterInstance.get() as instance: execute_backfill_command(kwargs, click.echo, instance)
def get_context(self, solid_config=None, mode_def=None, run_config=None): """Get a dagstermill execution context for interactive exploration and development. Args: solid_config (Optional[Any]): If specified, this value will be made available on the context as its ``solid_config`` property. mode_def (Optional[:class:`dagster.ModeDefinition`]): If specified, defines the mode to use to construct the context. Specify this if you would like a context constructed with specific ``resource_defs`` or ``logger_defs``. By default, an ephemeral mode with a console logger will be constructed. run_config(Optional[dict]): The environment config dict with which to construct the context. Returns: :py:class:`~dagstermill.DagstermillExecutionContext` """ check.opt_inst_param(mode_def, "mode_def", ModeDefinition) run_config = check.opt_dict_param(run_config, "run_config", key_type=str) # If we are running non-interactively, and there is already a context reconstituted, return # that context rather than overwriting it. if self.context is not None and isinstance( self.context, DagstermillRuntimeExecutionContext ): return self.context if not mode_def: mode_def = ModeDefinition(logger_defs={"dagstermill": colored_console_logger}) run_config["loggers"] = {"dagstermill": {}} solid_def = SolidDefinition( name="this_solid", input_defs=[], compute_fn=lambda *args, **kwargs: None, output_defs=[], description="Ephemeral solid constructed by dagstermill.get_context()", required_resource_keys=mode_def.resource_key_set, ) pipeline_def = PipelineDefinition( [solid_def], mode_defs=[mode_def], name="ephemeral_dagstermill_pipeline" ) run_id = make_new_run_id() # construct stubbed PipelineRun for notebook exploration... # The actual pipeline run during pipeline execution will be serialized and reconstituted # in the `reconstitute_pipeline_context` call pipeline_run = PipelineRun( pipeline_name=pipeline_def.name, run_id=run_id, run_config=run_config, mode=mode_def.name, step_keys_to_execute=None, status=PipelineRunStatus.NOT_STARTED, tags=None, ) self.in_pipeline = False self.solid_def = solid_def self.pipeline = pipeline_def execution_plan = create_execution_plan(self.pipeline, run_config, mode=mode_def.name) with scoped_pipeline_context( execution_plan, run_config, pipeline_run, DagsterInstance.ephemeral(), scoped_resources_builder_cm=self._setup_resources, ) as pipeline_context: self.context = DagstermillExecutionContext( pipeline_context=pipeline_context, solid_config=solid_config, resource_keys_to_init=get_required_resource_keys_to_init( execution_plan, pipeline_context.system_storage_def, pipeline_context.intermediate_storage_def, ), ) return self.context
def reconstitute_pipeline_context( self, output_log_path=None, marshal_dir=None, run_config=None, executable_dict=None, pipeline_run_dict=None, solid_handle_kwargs=None, instance_ref_dict=None, ): """Reconstitutes a context for dagstermill-managed execution. You'll see this function called to reconstruct a pipeline context within the ``injected parameters`` cell of a dagstermill output notebook. Users should not call this function interactively except when debugging output notebooks. Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a context for interactive exploration and development. This call will be replaced by one to :func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by dagstermill. """ check.opt_str_param(output_log_path, "output_log_path") check.opt_str_param(marshal_dir, "marshal_dir") run_config = check.opt_dict_param(run_config, "run_config", key_type=str) check.dict_param(pipeline_run_dict, "pipeline_run_dict") check.dict_param(executable_dict, "executable_dict") check.dict_param(solid_handle_kwargs, "solid_handle_kwargs") check.dict_param(instance_ref_dict, "instance_ref_dict") pipeline = ReconstructablePipeline.from_dict(executable_dict) pipeline_def = pipeline.get_definition() try: instance_ref = unpack_value(instance_ref_dict) instance = DagsterInstance.from_ref(instance_ref) except Exception as err: # pylint: disable=broad-except six.raise_from( DagstermillError( "Error when attempting to resolve DagsterInstance from serialized InstanceRef" ), err, ) pipeline_run = unpack_value(pipeline_run_dict) solid_handle = SolidHandle.from_dict(solid_handle_kwargs) solid_def = pipeline_def.get_solid(solid_handle).definition self.marshal_dir = marshal_dir self.in_pipeline = True self.solid_def = solid_def self.pipeline = pipeline execution_plan = create_execution_plan( self.pipeline, run_config, mode=pipeline_run.mode, step_keys_to_execute=pipeline_run.step_keys_to_execute, ) with scoped_pipeline_context( execution_plan, run_config, pipeline_run, instance, scoped_resources_builder_cm=self._setup_resources, # Set this flag even though we're not in test for clearer error reporting raise_on_error=True, ) as pipeline_context: self.context = DagstermillRuntimeExecutionContext( pipeline_context=pipeline_context, solid_config=None, resource_keys_to_init=get_required_resource_keys_to_init( execution_plan, pipeline_context.system_storage_def, pipeline_context.intermediate_storage_def, ), ) return self.context
def test_max_concurrency_one(): handle = ExecutionTargetHandle.for_pipeline_python_file( __file__, 'infinite_loop_pipeline') run_id_one = make_new_run_id() run_id_two = make_new_run_id() with safe_tempfile_path() as file_one, safe_tempfile_path() as file_two: instance = DagsterInstance.local_temp() execution_manager = QueueingSubprocessExecutionManager( instance, max_concurrent_runs=1) run_one = instance.create_run( PipelineRun.create_empty_run( pipeline_name=infinite_loop_pipeline.name, run_id=run_id_one, environment_dict={ 'solids': { 'loop': { 'config': { 'file': file_one } } } }, )) run_two = instance.create_run( PipelineRun.create_empty_run( pipeline_name=infinite_loop_pipeline.name, run_id=run_id_two, environment_dict={ 'solids': { 'loop': { 'config': { 'file': file_two } } } }, )) execution_manager.execute_pipeline(handle, infinite_loop_pipeline, run_one, instance) execution_manager.execute_pipeline(handle, infinite_loop_pipeline, run_two, instance) while not os.path.exists(file_one): execution_manager.check() time.sleep(0.1) assert execution_manager.is_active(run_id_one) assert not execution_manager.is_active(run_id_two) assert not os.path.exists(file_two) assert execution_manager.terminate(run_id_one) while not os.path.exists(file_two): execution_manager.check() time.sleep(0.1) assert not execution_manager.is_active(run_id_one) assert execution_manager.is_active(run_id_two) assert execution_manager.terminate(run_id_two)
def sqlite_instance_with_manager_disabled(): with seven.TemporaryDirectory() as temp_dir: yield DagsterInstance.local_temp( tempdir=temp_dir, overrides={'dagit': {'execution_manager': {'disabled': True}}} )
def test_two_runs_running(): run_id_one = make_new_run_id() run_id_two = make_new_run_id() handle = ExecutionTargetHandle.for_pipeline_python_file( __file__, 'infinite_loop_pipeline') with safe_tempfile_path() as file_one, safe_tempfile_path() as file_two: instance = DagsterInstance.local_temp() execution_manager = SubprocessExecutionManager(instance) pipeline_run_one = instance.create_run( PipelineRun.create_empty_run( pipeline_name=infinite_loop_pipeline.name, run_id=run_id_one, environment_dict={ 'solids': { 'loop': { 'config': { 'file': file_one } } } }, )) execution_manager.execute_pipeline(handle, infinite_loop_pipeline, pipeline_run_one, instance) pipeline_run_two = instance.create_run( PipelineRun.create_empty_run( pipeline_name=infinite_loop_pipeline.name, run_id=run_id_two, environment_dict={ 'solids': { 'loop': { 'config': { 'file': file_two } } } }, )) execution_manager.execute_pipeline(handle, infinite_loop_pipeline, pipeline_run_two, instance) # ensure both runs have begun execution while not os.path.exists(file_one) and not os.path.exists(file_two): time.sleep(0.1) assert execution_manager.is_process_running(run_id_one) assert execution_manager.is_process_running(run_id_two) assert execution_manager.terminate(run_id_one) assert not execution_manager.is_process_running(run_id_one) assert execution_manager.is_process_running(run_id_two) assert execution_manager.terminate(run_id_two) assert not execution_manager.is_process_running(run_id_one) assert not execution_manager.is_process_running(run_id_two)
def _check_execute_pipeline_args(pipeline, run_config, mode, preset, tags, instance, solid_selection=None): pipeline = _check_pipeline(pipeline) pipeline_def = pipeline.get_definition() check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition) run_config = check.opt_dict_param(run_config, 'run_config') check.opt_str_param(mode, 'mode') check.opt_str_param(preset, 'preset') check.invariant( not (mode is not None and preset is not None), 'You may set only one of `mode` (got {mode}) or `preset` (got {preset}).' .format(mode=mode, preset=preset), ) tags = check.opt_dict_param(tags, 'tags', key_type=str) check.opt_list_param(solid_selection, 'solid_selection', of_type=str) if preset is not None: pipeline_preset = pipeline_def.get_preset(preset) if pipeline_preset.run_config is not None: check.invariant( (not run_config) or (pipeline_preset.run_config == run_config), 'The environment set in preset \'{preset}\' does not agree with the environment ' 'passed in the `run_config` argument.'.format(preset=preset), ) run_config = pipeline_preset.run_config # load solid_selection from preset if pipeline_preset.solid_selection is not None: check.invariant( solid_selection is None or solid_selection == pipeline_preset.solid_selection, 'The solid_selection set in preset \'{preset}\', {preset_subset}, does not agree with ' 'the `solid_selection` argument: {solid_selection}'.format( preset=preset, preset_subset=pipeline_preset.solid_selection, solid_selection=solid_selection, ), ) solid_selection = pipeline_preset.solid_selection check.invariant( mode is None or mode == pipeline_preset.mode, 'Mode {mode} does not agree with the mode set in preset \'{preset}\': ' '(\'{preset_mode}\')'.format(preset=preset, preset_mode=pipeline_preset.mode, mode=mode), ) mode = pipeline_preset.mode tags = merge_dicts(pipeline_preset.tags, tags) if mode is not None: if not pipeline_def.has_mode_definition(mode): raise DagsterInvariantViolationError(( 'You have attempted to execute pipeline {name} with mode {mode}. ' 'Available modes: {modes}').format( name=pipeline_def.name, mode=mode, modes=pipeline_def.available_modes, )) else: if pipeline_def.is_multi_mode: raise DagsterInvariantViolationError(( 'Pipeline {name} has multiple modes (Available modes: {modes}) and you have ' 'attempted to execute it without specifying a mode. Set ' 'mode property on the PipelineRun object.').format( name=pipeline_def.name, modes=pipeline_def.available_modes)) mode = pipeline_def.get_default_mode_name() tags = merge_dicts(pipeline_def.tags, tags) check.opt_inst_param(instance, 'instance', DagsterInstance) instance = instance or DagsterInstance.ephemeral() # generate pipeline subset from the given solid_selection if solid_selection: pipeline = pipeline.subset_for_execution(solid_selection) return ( pipeline, run_config, instance, mode, tags, pipeline.solids_to_execute, solid_selection, )
def test_in_memory_persist_one_run(): with DagsterInstance.ephemeral() as instance: do_test_single_write_read(instance)
def pipeline_print_command(verbose, **cli_args): with DagsterInstance.get() as instance: return execute_print_command(verbose, cli_args, click.echo, instance)
def test_using_s3_for_subplan(s3_bucket): pipeline_def = define_inty_pipeline() run_config = {'storage': {'s3': {'config': {'s3_bucket': s3_bucket}}}} run_id = make_new_run_id() execution_plan = create_execution_plan(pipeline_def, run_config=run_config) assert execution_plan.get_step_by_key('return_one.compute') step_keys = ['return_one.compute'] instance = DagsterInstance.ephemeral() pipeline_run = PipelineRun(pipeline_name=pipeline_def.name, run_id=run_id, run_config=run_config) return_one_step_events = list( execute_plan( execution_plan.build_subset_plan(step_keys), run_config=run_config, pipeline_run=pipeline_run, instance=instance, )) assert get_step_output(return_one_step_events, 'return_one.compute') with scoped_pipeline_context( execution_plan.build_subset_plan(['return_one.compute']), run_config, pipeline_run, instance, ) as context: intermediates_manager = S3IntermediateStorage( s3_bucket, run_id, s3_session=context.scoped_resources_builder.build( required_resource_keys={'s3'}, ).s3, ) step_output_handle = StepOutputHandle('return_one.compute') assert intermediates_manager.has_intermediate(context, step_output_handle) assert intermediates_manager.get_intermediate( context, Int, step_output_handle).obj == 1 add_one_step_events = list( execute_plan( execution_plan.build_subset_plan(['add_one.compute']), run_config=run_config, pipeline_run=pipeline_run, instance=instance, )) assert get_step_output(add_one_step_events, 'add_one.compute') with scoped_pipeline_context( execution_plan.build_subset_plan(['add_one.compute']), run_config, pipeline_run, instance, ) as context: step_output_handle = StepOutputHandle('add_one.compute') assert intermediates_manager.has_intermediate(context, step_output_handle) assert intermediates_manager.get_intermediate( context, Int, step_output_handle).obj == 2
def pipeline_list_versions_command(**kwargs): with DagsterInstance.get() as instance: execute_list_versions_command(instance, kwargs)
def test_all_step_events(): # pylint: disable=too-many-locals workspace = workspace_from_load_target( PythonFileTarget(__file__, define_test_events_pipeline.__name__)) pipeline_def = define_test_events_pipeline() mode = pipeline_def.get_default_mode_name() instance = DagsterInstance.ephemeral() execution_plan = create_execution_plan(pipeline_def, mode=mode) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline_def, execution_plan=execution_plan, mode=mode) step_levels = execution_plan.topological_step_levels() unhandled_events = STEP_EVENTS.copy() # Exclude types that are not step events ignored_events = { 'LogMessageEvent', 'PipelineStartEvent', 'PipelineSuccessEvent', 'PipelineInitFailureEvent', 'PipelineFailureEvent', } event_counts = defaultdict(int) for step_level in step_levels: for step in step_level: variables = { 'executionParams': { 'selector': { 'repositoryLocationName': 'test_events', 'repositoryName': '<<unnamed>>', 'pipelineName': pipeline_def.name, }, 'runConfigData': { 'storage': { 'filesystem': {} } }, 'mode': mode, 'executionMetadata': { 'runId': pipeline_run.run_id }, 'stepKeys': [step.key], } } res = execute_query( workspace, EXECUTE_PLAN_MUTATION, variables, instance=instance, ) # go through the same dict, decrement all the event records we've seen from the GraphQL # response if not res.get('errors'): assert 'data' in res, res assert 'executePlan' in res['data'], res assert 'stepEvents' in res['data']['executePlan'], res step_events = res['data']['executePlan']['stepEvents'] events = [ dagster_event_from_dict(e, pipeline_def.name) for e in step_events if e['__typename'] not in ignored_events ] for event in events: if event.step_key: key = event.step_key + '.' + event.event_type_value else: key = event.event_type_value event_counts[key] -= 1 unhandled_events -= { DagsterEventType(e.event_type_value) for e in events } else: raise Exception(res['errors']) # build up a dict, incrementing all the event records we've produced in the run storage logs = instance.all_logs(pipeline_run.run_id) for log in logs: if not log.dagster_event or (DagsterEventType( log.dagster_event.event_type_value) not in STEP_EVENTS.union( set([DagsterEventType.ENGINE_EVENT]))): continue if log.dagster_event.step_key: key = log.dagster_event.step_key + '.' + log.dagster_event.event_type_value else: key = log.dagster_event.event_type_value event_counts[key] += 1 # Ensure we've processed all the events that were generated in the run storage assert sum(event_counts.values()) == 0 # Ensure we've handled the universe of event types # Why are these retry events not handled? Because right now there is no way to configure retries # on executePlan -- this needs to change, and we should separate the ExecutionParams that get # sent to executePlan fromm those that get sent to startPipelineExecution and friends assert unhandled_events == { DagsterEventType.STEP_UP_FOR_RETRY, DagsterEventType.STEP_RESTARTED }
def pipeline_launch_command(**kwargs): with DagsterInstance.get() as instance: return execute_launch_command(instance, kwargs)
def launch_scheduled_execution(output_file, schedule_name, override_system_timezone, **kwargs): with (mock_system_timezone(override_system_timezone) if override_system_timezone else nullcontext()): with ipc_write_stream(output_file) as stream: with DagsterInstance.get() as instance: repository_origin = get_repository_origin_from_kwargs(kwargs) job_origin = repository_origin.get_job_origin(schedule_name) # open the tick scope before we load any external artifacts so that # load errors are stored in DB with _schedule_tick_context( instance, stream, JobTickData( job_origin_id=job_origin.get_id(), job_name=schedule_name, job_type=JobType.SCHEDULE, status=JobTickStatus.STARTED, timestamp=time.time(), ), ) as tick_context: with get_repository_location_from_kwargs( kwargs) as repo_location: repo_dict = repo_location.get_repositories() check.invariant( repo_dict and len(repo_dict) == 1, "Passed in arguments should reference exactly one repository, instead there are {num_repos}" .format(num_repos=len(repo_dict)), ) external_repo = next(iter(repo_dict.values())) if not schedule_name in [ schedule.name for schedule in external_repo.get_external_schedules() ]: raise DagsterInvariantViolationError( "Could not find schedule named {schedule_name}" .format(schedule_name=schedule_name), ) external_schedule = external_repo.get_external_schedule( schedule_name) # Validate that either the schedule has no timezone or it matches # the system timezone schedule_timezone = external_schedule.execution_timezone if schedule_timezone: system_timezone = pendulum.now().timezone.name if system_timezone != external_schedule.execution_timezone: raise DagsterInvariantViolationError( "Schedule {schedule_name} is set to execute in {schedule_timezone}, " "but this scheduler can only run in the system timezone, " "{system_timezone}. Use DagsterDaemonScheduler if you want to be able " "to execute schedules in arbitrary timezones." .format( schedule_name=external_schedule.name, schedule_timezone=schedule_timezone, system_timezone=system_timezone, ), ) _launch_scheduled_executions(instance, repo_location, external_repo, external_schedule, tick_context)
def pipeline_list_command(**kwargs): with DagsterInstance.get() as instance: return execute_list_command(kwargs, click.echo, instance)
def test_execute_hammer_through_dagit(): handle = ExecutionTargetHandle.for_pipeline_python_file( script_relative_path( '../../../examples/dagster_examples/toys/hammer.py'), 'hammer_pipeline') instance = DagsterInstance.local_temp() execution_manager = SubprocessExecutionManager(instance) context = DagsterGraphQLContext(handle=handle, execution_manager=execution_manager, instance=instance) executor = SyncExecutor() variables = { 'executionParams': { 'environmentConfigData': { 'storage': { 'filesystem': {} }, 'execution': { 'dask': {} } }, 'selector': { 'name': handle.build_pipeline_definition().name }, 'mode': 'default', } } start_pipeline_result = graphql( request_string=START_PIPELINE_EXECUTION_MUTATION, schema=create_schema(), context=context, variables=variables, executor=executor, ) run_id = start_pipeline_result.data['startPipelineExecution']['run'][ 'runId'] context.execution_manager.join() subscription = execute_dagster_graphql(context, SUBSCRIPTION_QUERY, variables={'runId': run_id}) subscribe_results = [] subscription.subscribe(subscribe_results.append) messages = [ x['__typename'] for x in subscribe_results[0].data['pipelineRunLogs']['messages'] ] assert 'PipelineProcessStartEvent' in messages assert 'PipelineProcessStartedEvent' in messages assert 'PipelineStartEvent' in messages assert 'PipelineSuccessEvent' in messages assert 'PipelineProcessExitedEvent' in messages
def test_run_groups_over_time(): with seven.TemporaryDirectory() as tempdir: instance = DagsterInstance.local_temp(tempdir=tempdir) repo_1 = get_repo_at_time_1() full_evolve_run_id = execute_pipeline( repo_1.get_pipeline('evolving_pipeline'), instance=instance).run_id foo_run_id = execute_pipeline(repo_1.get_pipeline('foo_pipeline'), instance=instance).run_id evolve_a_run_id = execute_pipeline( repo_1.get_pipeline('evolving_pipeline').get_pipeline_subset_def( {'solid_A'}), instance=instance, ).run_id evolve_b_run_id = execute_pipeline( repo_1.get_pipeline('evolving_pipeline').get_pipeline_subset_def( {'solid_B'}), instance=instance, ).run_id context_at_time_1 = define_context_for_file(__file__, 'get_repo_at_time_1', instance) result = execute_dagster_graphql(context_at_time_1, ALL_RUN_GROUPS_QUERY) assert result.data assert 'runGroupsOrError' in result.data assert 'results' in result.data['runGroupsOrError'] assert len(result.data['runGroupsOrError']['results']) == 4 t1_runs = { run['runId']: run for group in result.data['runGroupsOrError']['results'] for run in group['runs'] } # test full_evolve_run_id assert t1_runs[full_evolve_run_id]['pipeline'] == { '__typename': 'PipelineSnapshot', 'name': 'evolving_pipeline', 'solidSelection': None, } # test foo_run_id assert t1_runs[foo_run_id]['pipeline'] == { '__typename': 'PipelineSnapshot', 'name': 'foo_pipeline', 'solidSelection': None, } # test evolve_a_run_id assert t1_runs[evolve_a_run_id]['pipeline'] == { '__typename': 'PipelineSnapshot', 'name': 'evolving_pipeline', 'solidSelection': ['solid_A'], } assert t1_runs[evolve_a_run_id]['pipelineSnapshotId'] # test evolve_b_run_id assert t1_runs[evolve_b_run_id]['pipeline'] == { '__typename': 'PipelineSnapshot', 'name': 'evolving_pipeline', 'solidSelection': ['solid_B'], } context_at_time_2 = define_context_for_file(__file__, 'get_repo_at_time_2', instance) result = execute_dagster_graphql(context_at_time_2, ALL_RUN_GROUPS_QUERY) assert 'runGroupsOrError' in result.data assert 'results' in result.data['runGroupsOrError'] assert len(result.data['runGroupsOrError']['results']) == 4 t2_runs = { run['runId']: run for group in result.data['runGroupsOrError']['results'] for run in group['runs'] } # test full_evolve_run_id assert t2_runs[full_evolve_run_id]['pipeline'] == { '__typename': 'PipelineSnapshot', 'name': 'evolving_pipeline', 'solidSelection': None, } # test evolve_a_run_id assert t2_runs[evolve_a_run_id]['pipeline'] == { '__typename': 'PipelineSnapshot', 'name': 'evolving_pipeline', 'solidSelection': ['solid_A'], } assert t2_runs[evolve_a_run_id]['pipelineSnapshotId'] # names same assert (t1_runs[full_evolve_run_id]['pipeline']['name'] == t2_runs[evolve_a_run_id]['pipeline']['name']) # snapshots differ assert (t1_runs[full_evolve_run_id]['pipelineSnapshotId'] != t2_runs[evolve_a_run_id]['pipelineSnapshotId']) # pipeline name changed assert t2_runs[foo_run_id]['pipeline'] == { '__typename': 'PipelineSnapshot', 'name': 'foo_pipeline', 'solidSelection': None, } # subset no longer valid - b renamed assert t2_runs[evolve_b_run_id]['pipeline'] == { '__typename': 'PipelineSnapshot', 'name': 'evolving_pipeline', 'solidSelection': ['solid_B'], }
def execute_inner(step_key, pipeline_run, instance_ref): instance = DagsterInstance.from_ref(instance_ref) inner_step(instance, pipeline_run, step_key)
def test_using_gcs_for_subplan(gcs_bucket): pipeline_def = define_inty_pipeline() environment_dict = {'storage': {'gcs': {'config': {'gcs_bucket': gcs_bucket}}}} run_id = str(uuid.uuid4()) execution_plan = create_execution_plan( pipeline_def, environment_dict=environment_dict, run_config=RunConfig(run_id=run_id) ) assert execution_plan.get_step_by_key('return_one.compute') step_keys = ['return_one.compute'] instance = DagsterInstance.ephemeral() pipeline_run = PipelineRun.create_empty_run( pipeline_def.name, run_id=run_id, environment_dict=environment_dict ) return_one_step_events = list( execute_plan( execution_plan.build_subset_plan(step_keys), environment_dict=environment_dict, pipeline_run=pipeline_run, instance=instance, ) ) assert get_step_output(return_one_step_events, 'return_one.compute') with scoped_pipeline_context( pipeline_def, environment_dict, pipeline_run, instance, execution_plan.build_subset_plan(['return_one.compute']), ) as context: store = GCSIntermediateStore( gcs_bucket, run_id, client=context.scoped_resources_builder.build( mapper_fn=SolidInvocation.default_resource_mapper_fn, required_resource_keys={'gcs'}, ).gcs.client, ) assert store.has_intermediate(context, 'return_one.compute') assert store.get_intermediate(context, 'return_one.compute', Int).obj == 1 add_one_step_events = list( execute_plan( execution_plan.build_subset_plan(['add_one.compute']), environment_dict=environment_dict, pipeline_run=pipeline_run, instance=instance, ) ) assert get_step_output(add_one_step_events, 'add_one.compute') with scoped_pipeline_context( pipeline_def, environment_dict, pipeline_run, instance, execution_plan.build_subset_plan(['return_one.compute']), ) as context: assert store.has_intermediate(context, 'add_one.compute') assert store.get_intermediate(context, 'add_one.compute', Int).obj == 2
def test_create_app_with_multiple_workspace_files(): workspace = load_workspace_from_yaml_paths([ file_relative_path(__file__, './workspace.yaml'), file_relative_path(__file__, './override.yaml'), ]) assert create_app_from_workspace(workspace, DagsterInstance.ephemeral())
def ExecuteRun(self, request, _context): try: execute_run_args = deserialize_json_to_dagster_namedtuple( request.serialized_execute_run_args) check.inst_param(execute_run_args, "execute_run_args", ExecuteRunArgs) run_id = execute_run_args.pipeline_run_id recon_pipeline = self._recon_pipeline_from_origin( execute_run_args.pipeline_origin) except: # pylint: disable=bare-except yield api_pb2.ExecuteRunEvent( serialized_dagster_event_or_ipc_error_message= serialize_dagster_namedtuple( IPCErrorMessage( serializable_error_info= serializable_error_info_from_exc_info(sys.exc_info()), message="Error during RPC setup for ExecuteRun", ))) return event_queue = multiprocessing.Queue() termination_event = multiprocessing.Event() execution_process = multiprocessing.Process( target=execute_run_in_subprocess, args=[ request.serialized_execute_run_args, recon_pipeline, event_queue, termination_event, ], ) with self._execution_lock: execution_process.start() self._executions[run_id] = ( execution_process, DagsterInstance.from_ref(execute_run_args.instance_ref), ) self._termination_events[run_id] = termination_event done = False while not done: try: # We use `get_nowait()` instead of `get()` so that we can handle the case where the # execution process has died unexpectedly -- `get()` would hang forever in that case dagster_event_or_ipc_error_message_or_done = event_queue.get_nowait( ) except queue.Empty: if not execution_process.is_alive(): # subprocess died unexpectedly yield api_pb2.ExecuteRunEvent( serialized_dagster_event_or_ipc_error_message= serialize_dagster_namedtuple( IPCErrorMessage( serializable_error_info= serializable_error_info_from_exc_info( sys.exc_info()), message= ("GRPC server: Subprocess for {run_id} terminated unexpectedly" ).format(run_id=run_id), ))) done = True time.sleep(EVENT_QUEUE_POLL_INTERVAL) else: if isinstance(dagster_event_or_ipc_error_message_or_done, RunInSubprocessComplete): done = True elif isinstance(dagster_event_or_ipc_error_message_or_done, StartRunInSubprocessSuccessful): continue else: yield api_pb2.ExecuteRunEvent( serialized_dagster_event_or_ipc_error_message= serialize_dagster_namedtuple( dagster_event_or_ipc_error_message_or_done)) with self._execution_lock: if run_id in self._executions: del self._executions[run_id] if run_id in self._termination_events: del self._termination_events[run_id]
def test_multiprocessing_execution_for_composite_solid_with_config_mapping(): environment_dict = { 'solids': { 'composite_with_nested_config_solid_and_config_mapping': { 'config': {'foo': 'baz', 'bar': 3} } } } run_id = make_new_run_id() handle = ExecutionTargetHandle.for_pipeline_python_file( __file__, 'composite_pipeline_with_config_mapping' ) instance = DagsterInstance.local_temp() pipeline_run = instance.create_run( PipelineRun( pipeline_name=composite_pipeline_with_config_mapping.name, run_id=run_id, selector=ExecutionSelector('nonce'), environment_dict=environment_dict, mode='default', reexecution_config=None, step_keys_to_execute=None, tags=None, status=PipelineRunStatus.NOT_STARTED, ) ) execution_manager = SubprocessExecutionManager(instance) execution_manager.execute_pipeline( handle, composite_pipeline_with_config_mapping, pipeline_run, instance ) execution_manager.join() assert instance.get_run_by_id(run_id).status == PipelineRunStatus.SUCCESS environment_dict = { 'solids': { 'composite_with_nested_config_solid_and_config_mapping': { 'config': {'foo': 'baz', 'bar': 3} } }, 'execution': {'multiprocess': {}}, 'storage': {'filesystem': {}}, } run_id = make_new_run_id() pipeline_run = instance.create_run( PipelineRun( pipeline_name=composite_pipeline.name, run_id=run_id, selector=ExecutionSelector('nonce'), environment_dict=environment_dict, mode='default', reexecution_config=None, step_keys_to_execute=None, tags=None, status=PipelineRunStatus.NOT_STARTED, ) ) execution_manager = SubprocessExecutionManager(instance) execution_manager.execute_pipeline(handle, composite_pipeline, pipeline_run, instance) execution_manager.join() assert instance.get_run_by_id(run_id).status == PipelineRunStatus.SUCCESS
def test_compute_log_manager_with_envvar(gcs_bucket): @job def simple(): @op def easy(context): context.log.info("easy") print(HELLO_WORLD) # pylint: disable=print-call return "easy" easy() with open(os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")) as f: with tempfile.TemporaryDirectory() as temp_dir: with environ({"ENV_VAR": f.read(), "DAGSTER_HOME": temp_dir}): run_store = SqliteRunStorage.from_local(temp_dir) event_store = SqliteEventLogStorage(temp_dir) manager = GCSComputeLogManager( bucket=gcs_bucket, prefix="my_prefix", local_dir=temp_dir, json_credentials_envvar="ENV_VAR", ) instance = DagsterInstance( instance_type=InstanceType.PERSISTENT, local_artifact_storage=LocalArtifactStorage(temp_dir), run_storage=run_store, event_storage=event_store, compute_log_manager=manager, run_coordinator=DefaultRunCoordinator(), run_launcher=DefaultRunLauncher(), ref=InstanceRef.from_dir(temp_dir), ) result = simple.execute_in_process(instance=instance) compute_steps = [ event.step_key for event in result.all_node_events if event.event_type == DagsterEventType.STEP_START ] assert len(compute_steps) == 1 step_key = compute_steps[0] stdout = manager.read_logs_file(result.run_id, step_key, ComputeIOType.STDOUT) assert stdout.data == HELLO_WORLD + SEPARATOR stderr = manager.read_logs_file(result.run_id, step_key, ComputeIOType.STDERR) for expected in EXPECTED_LOGS: assert expected in stderr.data # Check GCS directly stderr_gcs = (storage.Client().bucket(gcs_bucket).blob( f"my_prefix/storage/{result.run_id}/compute_logs/easy.err" ).download_as_bytes().decode("utf-8")) for expected in EXPECTED_LOGS: assert expected in stderr_gcs # Check download behavior by deleting locally cached logs compute_logs_dir = os.path.join(temp_dir, result.run_id, "compute_logs") for filename in os.listdir(compute_logs_dir): os.unlink(os.path.join(compute_logs_dir, filename)) stdout = manager.read_logs_file(result.run_id, step_key, ComputeIOType.STDOUT) assert stdout.data == HELLO_WORLD + SEPARATOR stderr = manager.read_logs_file(result.run_id, step_key, ComputeIOType.STDERR) for expected in EXPECTED_LOGS: assert expected in stderr.data
def test_runs_over_time(): with seven.TemporaryDirectory() as temp_dir: instance = DagsterInstance.local_temp(temp_dir) repo_1 = get_repo_at_time_1() full_evolve_run_id = execute_pipeline( repo_1.get_pipeline('evolving_pipeline'), instance=instance).run_id foo_run_id = execute_pipeline(repo_1.get_pipeline('foo_pipeline'), instance=instance).run_id evolve_a_run_id = execute_pipeline( repo_1.get_pipeline('evolving_pipeline').subset_for_execution( ['solid_A']), instance=instance, ).run_id evolve_b_run_id = execute_pipeline( repo_1.get_pipeline('evolving_pipeline').subset_for_execution( ['solid_B']), instance=instance, ).run_id context_at_time_1 = define_context_for_file(__file__, 'get_repo_at_time_1', instance) result = execute_dagster_graphql(context_at_time_1, ALL_RUNS_QUERY) assert result.data t1_runs = { run['runId']: run for run in result.data['pipelineRunsOrError']['results'] } assert t1_runs[full_evolve_run_id]['pipeline'] == { '__typename': 'PipelineSnapshot', 'name': 'evolving_pipeline', 'solidSubset': None, } assert t1_runs[foo_run_id]['pipeline'] == { '__typename': 'PipelineSnapshot', 'name': 'foo_pipeline', 'solidSubset': None, } assert t1_runs[evolve_a_run_id]['pipeline'] == { '__typename': 'PipelineSnapshot', 'name': 'evolving_pipeline', 'solidSubset': ['solid_A'], } assert t1_runs[evolve_b_run_id]['pipeline'] == { '__typename': 'PipelineSnapshot', 'name': 'evolving_pipeline', 'solidSubset': ['solid_B'], } context_at_time_2 = define_context_for_file(__file__, 'get_repo_at_time_2', instance) result = execute_dagster_graphql(context_at_time_2, ALL_RUNS_QUERY) assert result.data t2_runs = { run['runId']: run for run in result.data['pipelineRunsOrError']['results'] } assert t2_runs[full_evolve_run_id]['pipeline'] == { '__typename': 'PipelineSnapshot', 'name': 'evolving_pipeline', 'solidSubset': None, } assert t2_runs[evolve_a_run_id]['pipeline'] == { '__typename': 'PipelineSnapshot', 'name': 'evolving_pipeline', 'solidSubset': ['solid_A'], } # pipeline name changed assert t2_runs[foo_run_id]['pipeline'] == { '__typename': 'PipelineSnapshot', 'name': 'foo_pipeline', 'solidSubset': None, } # subset no longer valid - b renamed assert t2_runs[evolve_b_run_id]['pipeline'] == { '__typename': 'PipelineSnapshot', 'name': 'evolving_pipeline', 'solidSubset': ['solid_B'], }