def test_dict_roundtrip(): handle = NodeHandle("baz", NodeHandle("bar", NodeHandle("foo", None))) assert NodeHandle.from_dict(json.loads(json.dumps( handle._asdict()))) == handle handle = NodeHandle("foo", None) assert NodeHandle.from_dict(json.loads(json.dumps( handle._asdict()))) == handle
def test_rehydrate_solid_handle(): h = NodeHandle.from_dict({"name": "foo", "parent": None}) assert h.name == "foo" assert h.parent is None h = NodeHandle.from_dict(json.loads(json.dumps(h._asdict()))) assert h.name == "foo" assert h.parent is None h = NodeHandle.from_dict({"name": "foo", "parent": ["bar", None]}) assert h.name == "foo" assert isinstance(h.parent, NodeHandle) assert h.parent.name == "bar" assert h.parent.parent is None h = NodeHandle.from_dict(json.loads(json.dumps(h._asdict()))) assert h.name == "foo" assert isinstance(h.parent, NodeHandle) assert h.parent.name == "bar" assert h.parent.parent is None h = NodeHandle.from_dict({"name": "foo", "parent": ["bar", ["baz", None]]}) assert h.name == "foo" assert isinstance(h.parent, NodeHandle) assert h.parent.name == "bar" assert isinstance(h.parent.parent, NodeHandle) assert h.parent.parent.name == "baz" assert h.parent.parent.parent is None h = NodeHandle.from_dict(json.loads(json.dumps(h._asdict()))) assert h.name == "foo" assert isinstance(h.parent, NodeHandle) assert h.parent.name == "bar" assert isinstance(h.parent.parent, NodeHandle) assert h.parent.parent.name == "baz" assert h.parent.parent.parent is None
def reconstitute_pipeline_context( self, output_log_path=None, marshal_dir=None, run_config=None, executable_dict=None, pipeline_run_dict=None, solid_handle_kwargs=None, instance_ref_dict=None, step_key=None, ): """Reconstitutes a context for dagstermill-managed execution. You'll see this function called to reconstruct a pipeline context within the ``injected parameters`` cell of a dagstermill output notebook. Users should not call this function interactively except when debugging output notebooks. Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a context for interactive exploration and development. This call will be replaced by one to :func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by dagstermill. """ check.opt_str_param(output_log_path, "output_log_path") check.opt_str_param(marshal_dir, "marshal_dir") run_config = check.opt_dict_param(run_config, "run_config", key_type=str) check.dict_param(pipeline_run_dict, "pipeline_run_dict") check.dict_param(executable_dict, "executable_dict") check.dict_param(solid_handle_kwargs, "solid_handle_kwargs") check.dict_param(instance_ref_dict, "instance_ref_dict") check.str_param(step_key, "step_key") pipeline = ReconstructablePipeline.from_dict(executable_dict) pipeline_def = pipeline.get_definition() try: instance_ref = unpack_value(instance_ref_dict) instance = DagsterInstance.from_ref(instance_ref) except Exception as err: raise DagstermillError( "Error when attempting to resolve DagsterInstance from serialized InstanceRef" ) from err pipeline_run = unpack_value(pipeline_run_dict) solid_handle = NodeHandle.from_dict(solid_handle_kwargs) solid = pipeline_def.get_solid(solid_handle) solid_def = solid.definition self.marshal_dir = marshal_dir self.in_pipeline = True self.solid_def = solid_def self.pipeline = pipeline resolved_run_config = ResolvedRunConfig.build(pipeline_def, run_config, mode=pipeline_run.mode) execution_plan = ExecutionPlan.build( self.pipeline, resolved_run_config, step_keys_to_execute=pipeline_run.step_keys_to_execute, ) with scoped_pipeline_context( execution_plan, pipeline, run_config, pipeline_run, instance, scoped_resources_builder_cm=self._setup_resources, # Set this flag even though we're not in test for clearer error reporting raise_on_error=True, ) as pipeline_context: self.context = DagstermillRuntimeExecutionContext( pipeline_context=pipeline_context, pipeline_def=pipeline_def, solid_config=run_config.get("solids", {}).get(solid.name, {}).get("config"), resource_keys_to_init=get_required_resource_keys_to_init( execution_plan, pipeline_def, resolved_run_config, ), solid_name=solid.name, solid_handle=solid_handle, step_context=pipeline_context.for_step( execution_plan.get_step_by_key(step_key)), ) return self.context