def test_dict_roundtrip(): handle = SolidHandle("baz", SolidHandle("bar", SolidHandle("foo", None))) assert SolidHandle.from_dict(json.loads(json.dumps( handle._asdict()))) == handle handle = SolidHandle("foo", None) assert SolidHandle.from_dict(json.loads(json.dumps( handle._asdict()))) == handle
def test_dict_roundtrip(): handle = SolidHandle('baz', SolidHandle('bar', SolidHandle('foo', None))) assert SolidHandle.from_dict(json.loads(json.dumps( handle._asdict()))) == handle handle = SolidHandle('foo', None) assert SolidHandle.from_dict(json.loads(json.dumps( handle._asdict()))) == handle
def test_rehydrate_solid_handle(): h = SolidHandle.from_dict({ 'name': 'foo', 'definition_name': 'foo', 'parent': None }) assert h.name == 'foo' assert h.definition_name == 'foo' assert h.parent is None h = SolidHandle.from_dict(json.loads(json.dumps(h._asdict()))) assert h.name == 'foo' assert h.definition_name == 'foo' assert h.parent is None h = SolidHandle.from_dict({ 'name': 'foo', 'definition_name': 'foo', 'parent': ['bar', 'bar', None] }) assert h.name == 'foo' assert h.definition_name == 'foo' assert isinstance(h.parent, SolidHandle) assert h.parent.name == 'bar' assert h.parent.definition_name == 'bar' assert h.parent.parent is None h = SolidHandle.from_dict(json.loads(json.dumps(h._asdict()))) assert h.name == 'foo' assert h.definition_name == 'foo' assert isinstance(h.parent, SolidHandle) assert h.parent.name == 'bar' assert h.parent.definition_name == 'bar' assert h.parent.parent is None h = SolidHandle.from_dict({ 'name': 'foo', 'definition_name': 'foo', 'parent': ['bar', 'bar', ['baz', 'baz', None]] }) assert h.name == 'foo' assert h.definition_name == 'foo' assert isinstance(h.parent, SolidHandle) assert h.parent.name == 'bar' assert h.parent.definition_name == 'bar' assert isinstance(h.parent.parent, SolidHandle) assert h.parent.parent.name == 'baz' assert h.parent.parent.definition_name == 'baz' assert h.parent.parent.parent is None h = SolidHandle.from_dict(json.loads(json.dumps(h._asdict()))) assert h.name == 'foo' assert h.definition_name == 'foo' assert isinstance(h.parent, SolidHandle) assert h.parent.name == 'bar' assert h.parent.definition_name == 'bar' assert isinstance(h.parent.parent, SolidHandle) assert h.parent.parent.name == 'baz' assert h.parent.parent.definition_name == 'baz' assert h.parent.parent.parent is None
def test_rehydrate_solid_handle(): h = SolidHandle.from_dict({"name": "foo", "parent": None}) assert h.name == "foo" assert h.parent is None h = SolidHandle.from_dict(json.loads(json.dumps(h._asdict()))) assert h.name == "foo" assert h.parent is None h = SolidHandle.from_dict({"name": "foo", "parent": ["bar", None]}) assert h.name == "foo" assert isinstance(h.parent, SolidHandle) assert h.parent.name == "bar" assert h.parent.parent is None h = SolidHandle.from_dict(json.loads(json.dumps(h._asdict()))) assert h.name == "foo" assert isinstance(h.parent, SolidHandle) assert h.parent.name == "bar" assert h.parent.parent is None h = SolidHandle.from_dict({ "name": "foo", "parent": ["bar", ["baz", None]] }) assert h.name == "foo" assert isinstance(h.parent, SolidHandle) assert h.parent.name == "bar" assert isinstance(h.parent.parent, SolidHandle) assert h.parent.parent.name == "baz" assert h.parent.parent.parent is None h = SolidHandle.from_dict(json.loads(json.dumps(h._asdict()))) assert h.name == "foo" assert isinstance(h.parent, SolidHandle) assert h.parent.name == "bar" assert isinstance(h.parent.parent, SolidHandle) assert h.parent.parent.name == "baz" assert h.parent.parent.parent is None
def reconstitute_pipeline_context( self, output_log_path=None, marshal_dir=None, run_config=None, executable_dict=None, pipeline_run_dict=None, solid_handle_kwargs=None, instance_ref_dict=None, ): """Reconstitutes a context for dagstermill-managed execution. You'll see this function called to reconstruct a pipeline context within the ``injected parameters`` cell of a dagstermill output notebook. Users should not call this function interactively except when debugging output notebooks. Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a context for interactive exploration and development. This call will be replaced by one to :func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by dagstermill. """ check.opt_str_param(output_log_path, "output_log_path") check.opt_str_param(marshal_dir, "marshal_dir") run_config = check.opt_dict_param(run_config, "run_config", key_type=str) check.dict_param(pipeline_run_dict, "pipeline_run_dict") check.dict_param(executable_dict, "executable_dict") check.dict_param(solid_handle_kwargs, "solid_handle_kwargs") check.dict_param(instance_ref_dict, "instance_ref_dict") pipeline = ReconstructablePipeline.from_dict(executable_dict) pipeline_def = pipeline.get_definition() try: instance_ref = unpack_value(instance_ref_dict) instance = DagsterInstance.from_ref(instance_ref) except Exception as err: # pylint: disable=broad-except raise DagstermillError( "Error when attempting to resolve DagsterInstance from serialized InstanceRef" ) from err pipeline_run = unpack_value(pipeline_run_dict) solid_handle = SolidHandle.from_dict(solid_handle_kwargs) solid_def = pipeline_def.get_solid(solid_handle).definition self.marshal_dir = marshal_dir self.in_pipeline = True self.solid_def = solid_def self.pipeline = pipeline environment_config = EnvironmentConfig.build(pipeline_def, run_config, mode=pipeline_run.mode) execution_plan = ExecutionPlan.build( self.pipeline, environment_config, step_keys_to_execute=pipeline_run.step_keys_to_execute, ) with scoped_pipeline_context( execution_plan, pipeline, run_config, pipeline_run, instance, scoped_resources_builder_cm=self._setup_resources, # Set this flag even though we're not in test for clearer error reporting raise_on_error=True, ) as pipeline_context: self.context = DagstermillRuntimeExecutionContext( pipeline_context=pipeline_context, pipeline_def=pipeline_def, solid_config=run_config.get("solids", {}).get(solid_def.name, {}).get("config"), resource_keys_to_init=get_required_resource_keys_to_init( execution_plan, pipeline_def, environment_config, pipeline_context.intermediate_storage_def, ), solid_name=solid_def.name, ) return self.context
def reconstitute_pipeline_context( self, output_log_path=None, marshal_dir=None, environment_dict=None, handle_kwargs=None, pipeline_run_dict=None, solid_subset=None, solid_handle_kwargs=None, instance_ref_dict=None, ): '''Reconstitutes a context for dagstermill-managed execution. You'll see this function called to reconstruct a pipeline context within the ``injected parameters`` cell of a dagstermill output notebook. Users should not call this function interactively except when debugging output notebooks. Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a context for interactive exploration and development. This call will be replaced by one to :func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by dagstermill. ''' check.opt_str_param(output_log_path, 'output_log_path') check.opt_str_param(marshal_dir, 'marshal_dir') environment_dict = check.opt_dict_param(environment_dict, 'environment_dict', key_type=str) check.dict_param(pipeline_run_dict, 'pipeline_run_dict') check.dict_param(handle_kwargs, 'handle_kwargs') check.opt_list_param(solid_subset, 'solid_subset', of_type=str) check.dict_param(solid_handle_kwargs, 'solid_handle_kwargs') check.dict_param(instance_ref_dict, 'instance_ref_dict') try: handle = load_handle.handle_for_pipeline_cli_args( handle_kwargs, use_default_repository_yaml=False) except (check.CheckError, load_handle.UsageError) as err: six.raise_from( DagstermillError( 'Cannot invoke a dagstermill solid from an in-memory pipeline that was not loaded ' 'from an ExecutionTargetHandle. Run this pipeline using dagit, the dagster CLI, ' 'through dagster-graphql, or in-memory after loading it through an ' 'ExecutionTargetHandle.'), err, ) try: instance_ref = unpack_value(instance_ref_dict) instance = DagsterInstance.from_ref(instance_ref) except Exception as err: # pylint: disable=broad-except six.raise_from( DagstermillError( 'Error when attempting to resolve DagsterInstance from serialized InstanceRef' ), err, ) pipeline_def = check.inst_param( handle.build_pipeline_definition(), 'pipeline_def (from handle {handle_dict})'.format( handle_dict=handle.data._asdict()), PipelineDefinition, ).build_sub_pipeline(solid_subset) solid_handle = SolidHandle.from_dict(solid_handle_kwargs) solid_def = pipeline_def.get_solid(solid_handle) pipeline_run = unpack_value(pipeline_run_dict) self.marshal_dir = marshal_dir self.in_pipeline = True self.solid_def = solid_def self.pipeline_def = pipeline_def with scoped_pipeline_context( self.pipeline_def, environment_dict, pipeline_run, instance=instance, scoped_resources_builder_cm=self._setup_resources, ) as pipeline_context: self.context = DagstermillExecutionContext(pipeline_context) return self.context
def reconstitute_pipeline_context( self, output_log_path=None, marshal_dir=None, environment_dict=None, handle_kwargs=None, run_config_kwargs=None, solid_subset=None, solid_handle_kwargs=None, ): '''Reconstitutes a context for dagstermill-managed execution. You'll see this function called to reconstruct a pipeline context within the ``injected parameters`` cell of a dagstermill output notebook. Users should not call this function interactively except when debugging output notebooks. Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a context for interactive exploration and development. This call will be replaced by one to :func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by dagstermill. ''' check.opt_str_param(output_log_path, 'output_log_path') check.opt_str_param(marshal_dir, 'marshal_dir') environment_dict = check.opt_dict_param(environment_dict, 'environment_dict', key_type=str) check.dict_param(run_config_kwargs, 'run_config_kwargs') check.dict_param(handle_kwargs, 'handle_kwargs') check.opt_list_param(solid_subset, 'solid_subset', of_type=str) check.dict_param(solid_handle_kwargs, 'solid_handle_kwargs') try: handle = load_handle.handle_for_pipeline_cli_args( handle_kwargs, use_default_repository_yaml=False) except (check.CheckError, load_handle.CliUsageError) as err: six.raise_from( DagstermillError( 'Cannot invoke a dagstermill solid from an in-memory pipeline that was not loaded ' 'from an ExecutionTargetHandle. Run this pipeline using dagit, the dagster CLI, ' 'through dagster-graphql, or in-memory after loading it through an ' 'ExecutionTargetHandle.'), err, ) pipeline_def = check.inst_param( handle.build_pipeline_definition(), 'pipeline_def (from handle {handle_dict})'.format( handle_dict=handle.data._asdict()), PipelineDefinition, ).build_sub_pipeline(solid_subset) solid_handle = SolidHandle.from_dict(solid_handle_kwargs) solid_def = pipeline_def.get_solid(solid_handle) run_config = RunConfig(**run_config_kwargs) # since we are rehydrating the SqliteEventSink we will skip the db init run_config = run_config.with_event_sink( SqliteEventSink(output_log_path, skip_db_init=True)) self.marshal_dir = marshal_dir self.in_pipeline = True self.solid_def = solid_def self.pipeline_def = pipeline_def with scoped_pipeline_context( self.pipeline_def, environment_dict, run_config, scoped_resources_builder_cm=self._setup_resources, ) as pipeline_context: self.context = DagstermillExecutionContext(pipeline_context) return self.context
def reconstitute_pipeline_context( self, output_log_path=None, marshal_dir=None, environment_dict=None, executable_dict=None, pipeline_run_dict=None, solid_handle_kwargs=None, instance_ref_dict=None, ): '''Reconstitutes a context for dagstermill-managed execution. You'll see this function called to reconstruct a pipeline context within the ``injected parameters`` cell of a dagstermill output notebook. Users should not call this function interactively except when debugging output notebooks. Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a context for interactive exploration and development. This call will be replaced by one to :func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by dagstermill. ''' check.opt_str_param(output_log_path, 'output_log_path') check.opt_str_param(marshal_dir, 'marshal_dir') environment_dict = check.opt_dict_param(environment_dict, 'environment_dict', key_type=str) check.dict_param(pipeline_run_dict, 'pipeline_run_dict') check.dict_param(executable_dict, 'executable_dict') check.dict_param(solid_handle_kwargs, 'solid_handle_kwargs') check.dict_param(instance_ref_dict, 'instance_ref_dict') pipeline = InterProcessExecutablePipeline.from_dict(executable_dict) pipeline_def = pipeline.get_definition() try: instance_ref = unpack_value(instance_ref_dict) instance = DagsterInstance.from_ref(instance_ref) except Exception as err: # pylint: disable=broad-except six.raise_from( DagstermillError( 'Error when attempting to resolve DagsterInstance from serialized InstanceRef' ), err, ) pipeline_run = unpack_value(pipeline_run_dict) solid_handle = SolidHandle.from_dict(solid_handle_kwargs) solid_def = pipeline_def.get_solid(solid_handle).definition self.marshal_dir = marshal_dir self.in_pipeline = True self.solid_def = solid_def self.pipeline = pipeline execution_plan = create_execution_plan( self.pipeline, environment_dict, mode=pipeline_run.mode, step_keys_to_execute=pipeline_run.step_keys_to_execute, ) with scoped_pipeline_context( execution_plan, environment_dict, pipeline_run, instance, scoped_resources_builder_cm=self._setup_resources, # Set this flag even though we're not in test for clearer error reporting raise_on_error=True, ) as pipeline_context: self.context = DagstermillRuntimeExecutionContext( pipeline_context=pipeline_context, solid_config=None, resource_keys_to_init=get_required_resource_keys_to_init( execution_plan, pipeline_context.system_storage_def), ) return self.context