def test_dict_roundtrip():
handle = SolidHandle("baz", SolidHandle("bar", SolidHandle("foo", None)))
assert SolidHandle.from_dict(json.loads(json.dumps(
handle._asdict()))) == handle
handle = SolidHandle("foo", None)
assert SolidHandle.from_dict(json.loads(json.dumps(
handle._asdict()))) == handle
def test_dict_roundtrip():
handle = SolidHandle('baz', SolidHandle('bar', SolidHandle('foo', None)))
assert SolidHandle.from_dict(json.loads(json.dumps(
handle._asdict()))) == handle
handle = SolidHandle('foo', None)
assert SolidHandle.from_dict(json.loads(json.dumps(
handle._asdict()))) == handle
def test_rehydrate_solid_handle():
h = SolidHandle.from_dict({
'name': 'foo',
'definition_name': 'foo',
'parent': None
})
assert h.name == 'foo'
assert h.definition_name == 'foo'
assert h.parent is None
h = SolidHandle.from_dict(json.loads(json.dumps(h._asdict())))
assert h.name == 'foo'
assert h.definition_name == 'foo'
assert h.parent is None
h = SolidHandle.from_dict({
'name': 'foo',
'definition_name': 'foo',
'parent': ['bar', 'bar', None]
})
assert h.name == 'foo'
assert h.definition_name == 'foo'
assert isinstance(h.parent, SolidHandle)
assert h.parent.name == 'bar'
assert h.parent.definition_name == 'bar'
assert h.parent.parent is None
h = SolidHandle.from_dict(json.loads(json.dumps(h._asdict())))
assert h.name == 'foo'
assert h.definition_name == 'foo'
assert isinstance(h.parent, SolidHandle)
assert h.parent.name == 'bar'
assert h.parent.definition_name == 'bar'
assert h.parent.parent is None
h = SolidHandle.from_dict({
'name': 'foo',
'definition_name': 'foo',
'parent': ['bar', 'bar', ['baz', 'baz', None]]
})
assert h.name == 'foo'
assert h.definition_name == 'foo'
assert isinstance(h.parent, SolidHandle)
assert h.parent.name == 'bar'
assert h.parent.definition_name == 'bar'
assert isinstance(h.parent.parent, SolidHandle)
assert h.parent.parent.name == 'baz'
assert h.parent.parent.definition_name == 'baz'
assert h.parent.parent.parent is None
h = SolidHandle.from_dict(json.loads(json.dumps(h._asdict())))
assert h.name == 'foo'
assert h.definition_name == 'foo'
assert isinstance(h.parent, SolidHandle)
assert h.parent.name == 'bar'
assert h.parent.definition_name == 'bar'
assert isinstance(h.parent.parent, SolidHandle)
assert h.parent.parent.name == 'baz'
assert h.parent.parent.definition_name == 'baz'
assert h.parent.parent.parent is None
def test_rehydrate_solid_handle():
h = SolidHandle.from_dict({"name": "foo", "parent": None})
assert h.name == "foo"
assert h.parent is None
h = SolidHandle.from_dict(json.loads(json.dumps(h._asdict())))
assert h.name == "foo"
assert h.parent is None
h = SolidHandle.from_dict({"name": "foo", "parent": ["bar", None]})
assert h.name == "foo"
assert isinstance(h.parent, SolidHandle)
assert h.parent.name == "bar"
assert h.parent.parent is None
h = SolidHandle.from_dict(json.loads(json.dumps(h._asdict())))
assert h.name == "foo"
assert isinstance(h.parent, SolidHandle)
assert h.parent.name == "bar"
assert h.parent.parent is None
h = SolidHandle.from_dict({
"name": "foo",
"parent": ["bar", ["baz", None]]
})
assert h.name == "foo"
assert isinstance(h.parent, SolidHandle)
assert h.parent.name == "bar"
assert isinstance(h.parent.parent, SolidHandle)
assert h.parent.parent.name == "baz"
assert h.parent.parent.parent is None
h = SolidHandle.from_dict(json.loads(json.dumps(h._asdict())))
assert h.name == "foo"
assert isinstance(h.parent, SolidHandle)
assert h.parent.name == "bar"
assert isinstance(h.parent.parent, SolidHandle)
assert h.parent.parent.name == "baz"
assert h.parent.parent.parent is None
def reconstitute_pipeline_context(
self,
output_log_path=None,
marshal_dir=None,
run_config=None,
executable_dict=None,
pipeline_run_dict=None,
solid_handle_kwargs=None,
instance_ref_dict=None,
):
"""Reconstitutes a context for dagstermill-managed execution.
You'll see this function called to reconstruct a pipeline context within the ``injected
parameters`` cell of a dagstermill output notebook. Users should not call this function
interactively except when debugging output notebooks.
Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a
context for interactive exploration and development. This call will be replaced by one to
:func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by
dagstermill.
"""
check.opt_str_param(output_log_path, "output_log_path")
check.opt_str_param(marshal_dir, "marshal_dir")
run_config = check.opt_dict_param(run_config,
"run_config",
key_type=str)
check.dict_param(pipeline_run_dict, "pipeline_run_dict")
check.dict_param(executable_dict, "executable_dict")
check.dict_param(solid_handle_kwargs, "solid_handle_kwargs")
check.dict_param(instance_ref_dict, "instance_ref_dict")
pipeline = ReconstructablePipeline.from_dict(executable_dict)
pipeline_def = pipeline.get_definition()
try:
instance_ref = unpack_value(instance_ref_dict)
instance = DagsterInstance.from_ref(instance_ref)
except Exception as err: # pylint: disable=broad-except
raise DagstermillError(
"Error when attempting to resolve DagsterInstance from serialized InstanceRef"
) from err
pipeline_run = unpack_value(pipeline_run_dict)
solid_handle = SolidHandle.from_dict(solid_handle_kwargs)
solid_def = pipeline_def.get_solid(solid_handle).definition
self.marshal_dir = marshal_dir
self.in_pipeline = True
self.solid_def = solid_def
self.pipeline = pipeline
environment_config = EnvironmentConfig.build(pipeline_def,
run_config,
mode=pipeline_run.mode)
execution_plan = ExecutionPlan.build(
self.pipeline,
environment_config,
step_keys_to_execute=pipeline_run.step_keys_to_execute,
)
with scoped_pipeline_context(
execution_plan,
pipeline,
run_config,
pipeline_run,
instance,
scoped_resources_builder_cm=self._setup_resources,
# Set this flag even though we're not in test for clearer error reporting
raise_on_error=True,
) as pipeline_context:
self.context = DagstermillRuntimeExecutionContext(
pipeline_context=pipeline_context,
pipeline_def=pipeline_def,
solid_config=run_config.get("solids",
{}).get(solid_def.name,
{}).get("config"),
resource_keys_to_init=get_required_resource_keys_to_init(
execution_plan,
pipeline_def,
environment_config,
pipeline_context.intermediate_storage_def,
),
solid_name=solid_def.name,
)
return self.context
def reconstitute_pipeline_context(
self,
output_log_path=None,
marshal_dir=None,
environment_dict=None,
handle_kwargs=None,
pipeline_run_dict=None,
solid_subset=None,
solid_handle_kwargs=None,
instance_ref_dict=None,
):
'''Reconstitutes a context for dagstermill-managed execution.
You'll see this function called to reconstruct a pipeline context within the ``injected
parameters`` cell of a dagstermill output notebook. Users should not call this function
interactively except when debugging output notebooks.
Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a
context for interactive exploration and development. This call will be replaced by one to
:func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by
dagstermill.
'''
check.opt_str_param(output_log_path, 'output_log_path')
check.opt_str_param(marshal_dir, 'marshal_dir')
environment_dict = check.opt_dict_param(environment_dict,
'environment_dict',
key_type=str)
check.dict_param(pipeline_run_dict, 'pipeline_run_dict')
check.dict_param(handle_kwargs, 'handle_kwargs')
check.opt_list_param(solid_subset, 'solid_subset', of_type=str)
check.dict_param(solid_handle_kwargs, 'solid_handle_kwargs')
check.dict_param(instance_ref_dict, 'instance_ref_dict')
try:
handle = load_handle.handle_for_pipeline_cli_args(
handle_kwargs, use_default_repository_yaml=False)
except (check.CheckError, load_handle.UsageError) as err:
six.raise_from(
DagstermillError(
'Cannot invoke a dagstermill solid from an in-memory pipeline that was not loaded '
'from an ExecutionTargetHandle. Run this pipeline using dagit, the dagster CLI, '
'through dagster-graphql, or in-memory after loading it through an '
'ExecutionTargetHandle.'),
err,
)
try:
instance_ref = unpack_value(instance_ref_dict)
instance = DagsterInstance.from_ref(instance_ref)
except Exception as err: # pylint: disable=broad-except
six.raise_from(
DagstermillError(
'Error when attempting to resolve DagsterInstance from serialized InstanceRef'
),
err,
)
pipeline_def = check.inst_param(
handle.build_pipeline_definition(),
'pipeline_def (from handle {handle_dict})'.format(
handle_dict=handle.data._asdict()),
PipelineDefinition,
).build_sub_pipeline(solid_subset)
solid_handle = SolidHandle.from_dict(solid_handle_kwargs)
solid_def = pipeline_def.get_solid(solid_handle)
pipeline_run = unpack_value(pipeline_run_dict)
self.marshal_dir = marshal_dir
self.in_pipeline = True
self.solid_def = solid_def
self.pipeline_def = pipeline_def
with scoped_pipeline_context(
self.pipeline_def,
environment_dict,
pipeline_run,
instance=instance,
scoped_resources_builder_cm=self._setup_resources,
) as pipeline_context:
self.context = DagstermillExecutionContext(pipeline_context)
return self.context
def reconstitute_pipeline_context(
self,
output_log_path=None,
marshal_dir=None,
environment_dict=None,
handle_kwargs=None,
run_config_kwargs=None,
solid_subset=None,
solid_handle_kwargs=None,
):
'''Reconstitutes a context for dagstermill-managed execution.
You'll see this function called to reconstruct a pipeline context within the ``injected
parameters`` cell of a dagstermill output notebook. Users should not call this function
interactively except when debugging output notebooks.
Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a
context for interactive exploration and development. This call will be replaced by one to
:func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by
dagstermill.
'''
check.opt_str_param(output_log_path, 'output_log_path')
check.opt_str_param(marshal_dir, 'marshal_dir')
environment_dict = check.opt_dict_param(environment_dict,
'environment_dict',
key_type=str)
check.dict_param(run_config_kwargs, 'run_config_kwargs')
check.dict_param(handle_kwargs, 'handle_kwargs')
check.opt_list_param(solid_subset, 'solid_subset', of_type=str)
check.dict_param(solid_handle_kwargs, 'solid_handle_kwargs')
try:
handle = load_handle.handle_for_pipeline_cli_args(
handle_kwargs, use_default_repository_yaml=False)
except (check.CheckError, load_handle.CliUsageError) as err:
six.raise_from(
DagstermillError(
'Cannot invoke a dagstermill solid from an in-memory pipeline that was not loaded '
'from an ExecutionTargetHandle. Run this pipeline using dagit, the dagster CLI, '
'through dagster-graphql, or in-memory after loading it through an '
'ExecutionTargetHandle.'),
err,
)
pipeline_def = check.inst_param(
handle.build_pipeline_definition(),
'pipeline_def (from handle {handle_dict})'.format(
handle_dict=handle.data._asdict()),
PipelineDefinition,
).build_sub_pipeline(solid_subset)
solid_handle = SolidHandle.from_dict(solid_handle_kwargs)
solid_def = pipeline_def.get_solid(solid_handle)
run_config = RunConfig(**run_config_kwargs)
# since we are rehydrating the SqliteEventSink we will skip the db init
run_config = run_config.with_event_sink(
SqliteEventSink(output_log_path, skip_db_init=True))
self.marshal_dir = marshal_dir
self.in_pipeline = True
self.solid_def = solid_def
self.pipeline_def = pipeline_def
with scoped_pipeline_context(
self.pipeline_def,
environment_dict,
run_config,
scoped_resources_builder_cm=self._setup_resources,
) as pipeline_context:
self.context = DagstermillExecutionContext(pipeline_context)
return self.context
def reconstitute_pipeline_context(
self,
output_log_path=None,
marshal_dir=None,
environment_dict=None,
executable_dict=None,
pipeline_run_dict=None,
solid_handle_kwargs=None,
instance_ref_dict=None,
):
'''Reconstitutes a context for dagstermill-managed execution.
You'll see this function called to reconstruct a pipeline context within the ``injected
parameters`` cell of a dagstermill output notebook. Users should not call this function
interactively except when debugging output notebooks.
Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a
context for interactive exploration and development. This call will be replaced by one to
:func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by
dagstermill.
'''
check.opt_str_param(output_log_path, 'output_log_path')
check.opt_str_param(marshal_dir, 'marshal_dir')
environment_dict = check.opt_dict_param(environment_dict,
'environment_dict',
key_type=str)
check.dict_param(pipeline_run_dict, 'pipeline_run_dict')
check.dict_param(executable_dict, 'executable_dict')
check.dict_param(solid_handle_kwargs, 'solid_handle_kwargs')
check.dict_param(instance_ref_dict, 'instance_ref_dict')
pipeline = InterProcessExecutablePipeline.from_dict(executable_dict)
pipeline_def = pipeline.get_definition()
try:
instance_ref = unpack_value(instance_ref_dict)
instance = DagsterInstance.from_ref(instance_ref)
except Exception as err: # pylint: disable=broad-except
six.raise_from(
DagstermillError(
'Error when attempting to resolve DagsterInstance from serialized InstanceRef'
),
err,
)
pipeline_run = unpack_value(pipeline_run_dict)
solid_handle = SolidHandle.from_dict(solid_handle_kwargs)
solid_def = pipeline_def.get_solid(solid_handle).definition
self.marshal_dir = marshal_dir
self.in_pipeline = True
self.solid_def = solid_def
self.pipeline = pipeline
execution_plan = create_execution_plan(
self.pipeline,
environment_dict,
mode=pipeline_run.mode,
step_keys_to_execute=pipeline_run.step_keys_to_execute,
)
with scoped_pipeline_context(
execution_plan,
environment_dict,
pipeline_run,
instance,
scoped_resources_builder_cm=self._setup_resources,
# Set this flag even though we're not in test for clearer error reporting
raise_on_error=True,
) as pipeline_context:
self.context = DagstermillRuntimeExecutionContext(
pipeline_context=pipeline_context,
solid_config=None,
resource_keys_to_init=get_required_resource_keys_to_init(
execution_plan, pipeline_context.system_storage_def),
)
return self.context
