def test_in_pipeline_manager_solid_config():
with in_pipeline_manager() as manager:
assert manager.context.solid_config is None
with in_pipeline_manager(
solid_handle=SolidHandle('hello_world_config', None),
handle_kwargs={
'module_name': 'dagstermill.examples.repository',
'fn_name': 'define_hello_world_config_pipeline',
},
) as manager:
assert manager.context.solid_config == {'greeting': 'hello'}
with in_pipeline_manager(
solid_handle=SolidHandle('hello_world_config', None),
environment_dict={
'solids': {
'hello_world_config': {
'config': {
'greeting': 'bonjour'
}
}
}
},
handle_kwargs={
'module_name': 'dagstermill.examples.repository',
'fn_name': 'define_hello_world_config_pipeline',
},
) as manager:
assert manager.context.solid_config == {'greeting': 'bonjour'}
def test_in_pipeline_manager_solid_config():
with in_pipeline_manager() as manager:
assert manager.context.solid_config is None
with in_pipeline_manager(
pipeline_name='hello_world_config_pipeline',
solid_handle=SolidHandle('hello_world_config', None),
executable_dict=ReconstructablePipeline.for_module(
'dagstermill.examples.repository',
'define_hello_world_config_pipeline',
).to_dict(),
) as manager:
assert manager.context.solid_config == {'greeting': 'hello'}
with in_pipeline_manager(
pipeline_name='hello_world_config_pipeline',
solid_handle=SolidHandle('hello_world_config', None),
run_config={
'solids': {
'hello_world_config': {
'config': {
'greeting': 'bonjour'
}
}
}
},
executable_dict=ReconstructablePipeline.for_module(
'dagstermill.examples.repository',
'define_hello_world_config_pipeline',
).to_dict(),
) as manager:
assert manager.context.solid_config == {'greeting': 'bonjour'}
def test_dict_roundtrip():
handle = SolidHandle("baz", SolidHandle("bar", SolidHandle("foo", None)))
assert SolidHandle.from_dict(json.loads(json.dumps(
handle._asdict()))) == handle
handle = SolidHandle("foo", None)
assert SolidHandle.from_dict(json.loads(json.dumps(
handle._asdict()))) == handle
def test_dict_roundtrip():
handle = SolidHandle('baz', SolidHandle('bar', SolidHandle('foo', None)))
assert SolidHandle.from_dict(json.loads(json.dumps(
handle._asdict()))) == handle
handle = SolidHandle('foo', None)
assert SolidHandle.from_dict(json.loads(json.dumps(
handle._asdict()))) == handle
def parse_from_key(string):
unresolved_match = re.match(r"(.*)\[\?\]", string)
if unresolved_match:
return UnresolvedStepHandle(
SolidHandle.from_string(unresolved_match.group(1)))
resolved_match = re.match(r"(.*)\[(.*)\]", string)
if resolved_match:
return ResolvedFromDynamicStepHandle(
SolidHandle.from_string(resolved_match.group(1)),
resolved_match.group(2))
return StepHandle(SolidHandle.from_string(string))
def test_in_pipeline_manager_bad_solid():
with pytest.raises(
DagsterInvariantViolationError,
match=('Pipeline hello_world_pipeline has no solid named foobar'),
):
with in_pipeline_manager(solid_handle=SolidHandle('foobar', 'foobar', None)) as _manager:
pass
def make_log_string(error):
step_failure_event = DagsterEvent(
event_type_value="STEP_FAILURE",
pipeline_name="my_pipeline",
step_key="solid2",
solid_handle=SolidHandle("solid2", None),
step_kind_value="COMPUTE",
logging_tags={},
event_specific_data=StepFailureData(error=error,
user_failure_data=None),
message='Execution of step "solid2" failed.',
pid=54348,
)
message_props = {
"dagster_event": step_failure_event,
"pipeline_name": "my_pipeline"
}
synth_props = {
"orig_message": step_failure_event.message,
"run_id": "f79a8a93-27f1-41b5-b465-b35d0809b26d",
}
return construct_log_string(message_props=message_props,
logging_tags={},
synth_props=synth_props)
def in_pipeline_manager(pipeline_name='hello_world_pipeline',
solid_handle=SolidHandle('hello_world', 'hello_world',
None),
handle_kwargs=None,
**kwargs):
manager = Manager()
run_id = str(uuid.uuid4())
marshal_dir = tempfile.mkdtemp()
if not handle_kwargs:
handle_kwargs = {
'pipeline_name': pipeline_name,
'module_name': 'dagstermill.examples.repository',
'fn_name': 'define_hello_world_pipeline',
}
try:
with safe_tempfile_path() as output_log_file_path:
context_dict = {
'run_config_kwargs': dict(run_id=run_id, mode='default'),
'solid_handle_kwargs': solid_handle._asdict(),
'handle_kwargs': handle_kwargs,
'marshal_dir': marshal_dir,
'environment_dict': {},
'output_log_path': output_log_file_path,
}
manager.reconstitute_pipeline_context(
**dict(context_dict, **kwargs))
yield manager
finally:
shutil.rmtree(marshal_dir)
def test_construct_log_string_for_event():
step_output_event = DagsterEvent(
event_type_value="STEP_OUTPUT",
pipeline_name="my_pipeline",
step_key="solid2",
solid_handle=SolidHandle("solid2", None),
step_kind_value="COMPUTE",
logging_tags={},
event_specific_data=StepOutputData(
step_output_handle=StepOutputHandle("solid2", "result")),
message=
'Yielded output "result" of type "Any" for step "solid2". (Type check passed).',
pid=54348,
)
message_props = {
"dagster_event": step_output_event,
"pipeline_name": "my_pipeline"
}
synth_props = {
"orig_message": step_output_event.message,
"run_id": "f79a8a93-27f1-41b5-b465-b35d0809b26d",
}
assert (
construct_log_string(message_props=message_props,
logging_tags={},
synth_props=synth_props) ==
'my_pipeline - f79a8a93-27f1-41b5-b465-b35d0809b26d - 54348 - STEP_OUTPUT - Yielded output "result" of type "Any" for step "solid2". (Type check passed).'
)
def _event_record(run_id,
solid_name,
timestamp,
event_type,
event_specific_data=None):
pipeline_name = "pipeline_name"
solid_handle = SolidHandle(solid_name, None)
step_handle = StepHandle(solid_handle)
return EventRecord(
None,
"",
"debug",
"",
run_id,
timestamp,
step_key=step_handle.to_key(),
pipeline_name=pipeline_name,
dagster_event=DagsterEvent(
event_type.value,
pipeline_name,
solid_handle=solid_handle,
step_handle=step_handle,
event_specific_data=event_specific_data,
),
)
def in_pipeline_manager(pipeline_name='hello_world_pipeline',
solid_handle=SolidHandle('hello_world', 'hello_world',
None),
**kwargs):
manager = Manager()
run_id = str(uuid.uuid4())
marshal_dir = tempfile.mkdtemp()
handle = handle_for_pipeline_cli_args({
'pipeline_name':
pipeline_name,
'module_name':
'dagstermill.examples.repository',
'fn_name':
'define_hello_world_pipeline',
})
try:
with tempfile.NamedTemporaryFile() as output_log_file:
context_dict = {
'run_config': RunConfig(run_id=run_id, mode='default'),
'solid_handle': solid_handle,
'handle': handle,
'marshal_dir': marshal_dir,
'environment_dict': {},
'output_log_path': output_log_file.name,
}
manager.reconstitute_pipeline_context(
**dict(context_dict, **kwargs))
yield manager
finally:
shutil.rmtree(marshal_dir)
def test_in_pipeline_manager_bad_solid():
with pytest.raises(
check.CheckError,
match=("hello_world_pipeline has no solid named foobar"),
):
with in_pipeline_manager(
solid_handle=SolidHandle("foobar", None)) as _manager:
pass
def test_in_pipeline_manager_solid_config():
with in_pipeline_manager() as manager:
assert manager.context.solid_config is None
with in_pipeline_manager(
pipeline_name="hello_world_config_pipeline",
solid_handle=SolidHandle("hello_world_config", None),
executable_dict=ReconstructablePipeline.for_module(
"dagstermill.examples.repository", "define_hello_world_config_pipeline",
).to_dict(),
) as manager:
assert manager.context.solid_config == {"greeting": "hello"}
with in_pipeline_manager(
pipeline_name="hello_world_config_pipeline",
solid_handle=SolidHandle("hello_world_config", None),
run_config={
"solids": {
"hello_world_config": {"config": {"greeting": "bonjour"}},
"goodbye_config": {"config": {"farewell": "goodbye"}},
}
},
executable_dict=ReconstructablePipeline.for_module(
"dagstermill.examples.repository", "define_hello_world_config_pipeline",
).to_dict(),
) as manager:
assert manager.context.solid_config == {"greeting": "bonjour"}
with in_pipeline_manager(
pipeline_name="hello_world_config_pipeline",
solid_handle=SolidHandle("goodbye_config", None),
run_config={
"solids": {
"hello_world_config": {"config": {"greeting": "bonjour"},},
"goodbye_config": {"config": {"farewell": "goodbye"}},
}
},
executable_dict=ReconstructablePipeline.for_module(
"dagstermill.examples.repository", "define_hello_world_config_pipeline",
).to_dict(),
) as manager:
assert manager.context.solid_config == {"farewell": "goodbye"}
def construct_solid_dictionary(solid_dict_value, parent_handle=None, config_map=None):
config_map = config_map or {}
for name, value in solid_dict_value.items():
key = SolidHandle(name, None, parent_handle)
config_map[str(key)] = SolidConfig.from_dict(value)
# solids implies a composite solid config
if value.get('solids'):
construct_solid_dictionary(value['solids'], key, config_map)
return config_map
def test_rehydrate_solid_handle():
h = SolidHandle.from_dict({
'name': 'foo',
'definition_name': 'foo',
'parent': None
})
assert h.name == 'foo'
assert h.definition_name == 'foo'
assert h.parent is None
h = SolidHandle.from_dict(json.loads(json.dumps(h._asdict())))
assert h.name == 'foo'
assert h.definition_name == 'foo'
assert h.parent is None
h = SolidHandle.from_dict({
'name': 'foo',
'definition_name': 'foo',
'parent': ['bar', 'bar', None]
})
assert h.name == 'foo'
assert h.definition_name == 'foo'
assert isinstance(h.parent, SolidHandle)
assert h.parent.name == 'bar'
assert h.parent.definition_name == 'bar'
assert h.parent.parent is None
h = SolidHandle.from_dict(json.loads(json.dumps(h._asdict())))
assert h.name == 'foo'
assert h.definition_name == 'foo'
assert isinstance(h.parent, SolidHandle)
assert h.parent.name == 'bar'
assert h.parent.definition_name == 'bar'
assert h.parent.parent is None
h = SolidHandle.from_dict({
'name': 'foo',
'definition_name': 'foo',
'parent': ['bar', 'bar', ['baz', 'baz', None]]
})
assert h.name == 'foo'
assert h.definition_name == 'foo'
assert isinstance(h.parent, SolidHandle)
assert h.parent.name == 'bar'
assert h.parent.definition_name == 'bar'
assert isinstance(h.parent.parent, SolidHandle)
assert h.parent.parent.name == 'baz'
assert h.parent.parent.definition_name == 'baz'
assert h.parent.parent.parent is None
h = SolidHandle.from_dict(json.loads(json.dumps(h._asdict())))
assert h.name == 'foo'
assert h.definition_name == 'foo'
assert isinstance(h.parent, SolidHandle)
assert h.parent.name == 'bar'
assert h.parent.definition_name == 'bar'
assert isinstance(h.parent.parent, SolidHandle)
assert h.parent.parent.name == 'baz'
assert h.parent.parent.definition_name == 'baz'
assert h.parent.parent.parent is None
def test_yield_unserializable_result():
manager = Manager()
assert manager.yield_result(threading.Lock())
with in_pipeline_manager(
solid_handle=SolidHandle('hello_world_output', None),
handle_kwargs={
'module_name': 'dagstermill.examples.repository',
'fn_name': 'define_hello_world_with_output_pipeline',
},
) as manager:
with pytest.raises(TypeError):
manager.yield_result(threading.Lock())
def test_yield_unserializable_result():
manager = Manager()
assert manager.yield_result(threading.Lock())
with in_pipeline_manager(
pipeline_name="hello_world_with_output_pipeline",
solid_handle=SolidHandle("hello_world_output", None),
executable_dict=ReconstructablePipeline.for_module(
"dagstermill.examples.repository", "define_hello_world_with_output_pipeline",
).to_dict(),
) as manager:
with pytest.raises(TypeError):
manager.yield_result(threading.Lock())
def test_handle_path():
handle = SolidHandle("baz", SolidHandle("bar", SolidHandle("foo", None)))
assert handle.path == ["foo", "bar", "baz"]
assert SolidHandle.from_path(handle.path) == handle
handle = SolidHandle("foo", None)
assert handle.path == ["foo"]
assert SolidHandle.from_path(handle.path) == handle
def test_handle_path():
handle = SolidHandle('baz', SolidHandle('bar', SolidHandle('foo', None)))
assert handle.path == ['foo', 'bar', 'baz']
assert SolidHandle.from_path(handle.path) == handle
handle = SolidHandle('foo', None)
assert handle.path == ['foo']
assert SolidHandle.from_path(handle.path) == handle
def in_pipeline_manager(
pipeline_name='hello_world_pipeline',
solid_handle=SolidHandle('hello_world', 'hello_world', None),
handle_kwargs=None,
mode=None,
**kwargs
):
manager = Manager()
run_id = str(uuid.uuid4())
instance = DagsterInstance.local_temp()
marshal_dir = tempfile.mkdtemp()
if not handle_kwargs:
handle_kwargs = {
'pipeline_name': pipeline_name,
'module_name': 'dagstermill.examples.repository',
'fn_name': 'define_hello_world_pipeline',
}
pipeline_run_dict = pack_value(
PipelineRun(
pipeline_name=pipeline_name,
run_id=run_id,
mode=mode or 'default',
environment_dict=None,
selector=None,
reexecution_config=None,
step_keys_to_execute=None,
status=PipelineRunStatus.NOT_STARTED,
)
)
try:
with safe_tempfile_path() as output_log_file_path:
context_dict = {
'pipeline_run_dict': pipeline_run_dict,
'solid_handle_kwargs': solid_handle._asdict(),
'handle_kwargs': handle_kwargs,
'marshal_dir': marshal_dir,
'environment_dict': {},
'output_log_path': output_log_file_path,
'instance_ref_dict': pack_value(instance.get_ref()),
}
manager.reconstitute_pipeline_context(**dict(context_dict, **kwargs))
yield manager
finally:
shutil.rmtree(marshal_dir)
def test_in_pipeline_manager_with_resources():
with tempfile.NamedTemporaryFile() as fd:
path = fd.name
try:
with in_pipeline_manager(
handle=handle_for_pipeline_cli_args({
'pipeline_name':
'resource_pipeline',
'fn_name':
'define_resource_pipeline',
'module_name':
'dagstermill.examples.repository',
}),
solid_handle=SolidHandle('hello_world_resource',
'hello_world_resource', None),
environment_dict={'resources': {
'list': {
'config': path
}
}},
run_config=RunConfig(mode='prod'),
) as manager:
assert len(manager.context.resources._asdict()) == 1
with open(path, 'rb') as fd:
messages = pickle.load(fd)
messages = [message.split(': ') for message in messages]
assert len(messages) == 1
assert messages[0][1] == 'Opened'
manager.teardown_resources()
with open(path, 'rb') as fd:
messages = pickle.load(fd)
messages = [message.split(': ') for message in messages]
assert len(messages) == 2
assert messages[1][1] == 'Closed'
finally:
if os.path.exists(path):
os.unlink(path)
def in_pipeline_manager(
pipeline_name="hello_world_pipeline",
solid_handle=SolidHandle("hello_world", None),
executable_dict=None,
mode=None,
**kwargs,
):
manager = Manager()
run_id = make_new_run_id()
with instance_for_test() as instance:
marshal_dir = tempfile.mkdtemp()
if not executable_dict:
executable_dict = ReconstructablePipeline.for_module(
"dagstermill.examples.repository",
"define_hello_world_pipeline").to_dict()
pipeline_run_dict = pack_value(
PipelineRun(
pipeline_name=pipeline_name,
run_id=run_id,
mode=mode or "default",
run_config=None,
step_keys_to_execute=None,
status=PipelineRunStatus.NOT_STARTED,
))
try:
with safe_tempfile_path() as output_log_file_path:
context_dict = {
"pipeline_run_dict": pipeline_run_dict,
"solid_handle_kwargs": solid_handle._asdict(),
"executable_dict": executable_dict,
"marshal_dir": marshal_dir,
"run_config": {},
"output_log_path": output_log_file_path,
"instance_ref_dict": pack_value(instance.get_ref()),
}
manager.reconstitute_pipeline_context(
**dict(context_dict, **kwargs))
yield manager
finally:
shutil.rmtree(marshal_dir)
def in_pipeline_manager(pipeline_name='hello_world_pipeline',
solid_handle=SolidHandle('hello_world', None),
executable_dict=None,
mode=None,
**kwargs):
manager = Manager()
run_id = make_new_run_id()
instance = DagsterInstance.local_temp()
marshal_dir = tempfile.mkdtemp()
if not executable_dict:
executable_dict = ReconstructablePipeline.for_module(
'dagstermill.examples.repository',
'define_hello_world_pipeline').to_dict()
pipeline_run_dict = pack_value(
PipelineRun(
pipeline_name=pipeline_name,
run_id=run_id,
mode=mode or 'default',
run_config=None,
step_keys_to_execute=None,
status=PipelineRunStatus.NOT_STARTED,
))
try:
with safe_tempfile_path() as output_log_file_path:
context_dict = {
'pipeline_run_dict': pipeline_run_dict,
'solid_handle_kwargs': solid_handle._asdict(),
'executable_dict': executable_dict,
'marshal_dir': marshal_dir,
'run_config': {},
'output_log_path': output_log_file_path,
'instance_ref_dict': pack_value(instance.get_ref()),
}
manager.reconstitute_pipeline_context(
**dict(context_dict, **kwargs))
yield manager
finally:
shutil.rmtree(marshal_dir)
def test_in_pipeline_manager_with_resources():
with tempfile.NamedTemporaryFile() as fd:
path = fd.name
try:
with in_pipeline_manager(
pipeline_name="resource_pipeline",
executable_dict=ReconstructablePipeline.for_module(
"dagstermill.examples.repository",
"define_resource_pipeline",
).to_dict(),
solid_handle=SolidHandle("hello_world_resource", None),
run_config={"resources": {
"list": {
"config": path
}
}},
mode="prod",
) as manager:
assert "list" in manager.context.resources._asdict()
with open(path, "rb") as fd:
messages = pickle.load(fd)
messages = [message.split(": ") for message in messages]
assert len(messages) == 1
assert messages[0][1] == "Opened"
manager.teardown_resources()
with open(path, "rb") as fd:
messages = pickle.load(fd)
messages = [message.split(": ") for message in messages]
assert len(messages) == 2
assert messages[1][1] == "Closed"
finally:
if os.path.exists(path):
os.unlink(path)
def test_in_pipeline_manager_with_resources():
with tempfile.NamedTemporaryFile() as fd:
path = fd.name
try:
with in_pipeline_manager(
pipeline_name='resource_pipeline',
executable_dict=ReconstructablePipeline.for_module(
'dagstermill.examples.repository',
'define_resource_pipeline',
).to_dict(),
solid_handle=SolidHandle('hello_world_resource', None),
run_config={'resources': {
'list': {
'config': path
}
}},
mode='prod',
) as manager:
assert len(manager.context.resources._asdict()) == 1
with open(path, 'rb') as fd:
messages = pickle.load(fd)
messages = [message.split(': ') for message in messages]
assert len(messages) == 1
assert messages[0][1] == 'Opened'
manager.teardown_resources()
with open(path, 'rb') as fd:
messages = pickle.load(fd)
messages = [message.split(': ') for message in messages]
assert len(messages) == 2
assert messages[1][1] == 'Closed'
finally:
if os.path.exists(path):
os.unlink(path)
def test_construct_log_string_with_error():
try:
raise ValueError("some error")
except ValueError:
error = serializable_error_info_from_exc_info(sys.exc_info())
step_failure_event = DagsterEvent(
event_type_value="STEP_FAILURE",
pipeline_name="my_pipeline",
step_key="solid2.compute",
solid_handle=SolidHandle("solid2", None),
step_kind_value="COMPUTE",
logging_tags={},
event_specific_data=StepFailureData(error=error,
user_failure_data=None),
message='Execution of step "solid2.compute" failed.',
pid=54348,
)
message_props = {
"dagster_event": step_failure_event,
"pipeline_name": "my_pipeline"
}
synth_props = {
"orig_message": step_failure_event.message,
"run_id": "f79a8a93-27f1-41b5-b465-b35d0809b26d",
}
log_string = construct_log_string(message_props=message_props,
logging_tags={},
synth_props=synth_props)
expected_start = textwrap.dedent("""
my_pipeline - f79a8a93-27f1-41b5-b465-b35d0809b26d - 54348 - STEP_FAILURE - Execution of step "solid2.compute" failed.
ValueError: some error
File "
""").strip()
assert log_string.startswith(expected_start)
def test_rehydrate_solid_handle():
h = SolidHandle.from_dict({"name": "foo", "parent": None})
assert h.name == "foo"
assert h.parent is None
h = SolidHandle.from_dict(json.loads(json.dumps(h._asdict())))
assert h.name == "foo"
assert h.parent is None
h = SolidHandle.from_dict({"name": "foo", "parent": ["bar", None]})
assert h.name == "foo"
assert isinstance(h.parent, SolidHandle)
assert h.parent.name == "bar"
assert h.parent.parent is None
h = SolidHandle.from_dict(json.loads(json.dumps(h._asdict())))
assert h.name == "foo"
assert isinstance(h.parent, SolidHandle)
assert h.parent.name == "bar"
assert h.parent.parent is None
h = SolidHandle.from_dict({
"name": "foo",
"parent": ["bar", ["baz", None]]
})
assert h.name == "foo"
assert isinstance(h.parent, SolidHandle)
assert h.parent.name == "bar"
assert isinstance(h.parent.parent, SolidHandle)
assert h.parent.parent.name == "baz"
assert h.parent.parent.parent is None
h = SolidHandle.from_dict(json.loads(json.dumps(h._asdict())))
assert h.name == "foo"
assert isinstance(h.parent, SolidHandle)
assert h.parent.name == "bar"
assert isinstance(h.parent.parent, SolidHandle)
assert h.parent.parent.name == "baz"
assert h.parent.parent.parent is None
def reconstitute_pipeline_context(
self,
output_log_path=None,
marshal_dir=None,
run_config=None,
executable_dict=None,
pipeline_run_dict=None,
solid_handle_kwargs=None,
instance_ref_dict=None,
):
"""Reconstitutes a context for dagstermill-managed execution.
You'll see this function called to reconstruct a pipeline context within the ``injected
parameters`` cell of a dagstermill output notebook. Users should not call this function
interactively except when debugging output notebooks.
Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a
context for interactive exploration and development. This call will be replaced by one to
:func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by
dagstermill.
"""
check.opt_str_param(output_log_path, "output_log_path")
check.opt_str_param(marshal_dir, "marshal_dir")
run_config = check.opt_dict_param(run_config,
"run_config",
key_type=str)
check.dict_param(pipeline_run_dict, "pipeline_run_dict")
check.dict_param(executable_dict, "executable_dict")
check.dict_param(solid_handle_kwargs, "solid_handle_kwargs")
check.dict_param(instance_ref_dict, "instance_ref_dict")
pipeline = ReconstructablePipeline.from_dict(executable_dict)
pipeline_def = pipeline.get_definition()
try:
instance_ref = unpack_value(instance_ref_dict)
instance = DagsterInstance.from_ref(instance_ref)
except Exception as err: # pylint: disable=broad-except
raise DagstermillError(
"Error when attempting to resolve DagsterInstance from serialized InstanceRef"
) from err
pipeline_run = unpack_value(pipeline_run_dict)
solid_handle = SolidHandle.from_dict(solid_handle_kwargs)
solid_def = pipeline_def.get_solid(solid_handle).definition
self.marshal_dir = marshal_dir
self.in_pipeline = True
self.solid_def = solid_def
self.pipeline = pipeline
environment_config = EnvironmentConfig.build(pipeline_def,
run_config,
mode=pipeline_run.mode)
execution_plan = ExecutionPlan.build(
self.pipeline,
environment_config,
step_keys_to_execute=pipeline_run.step_keys_to_execute,
)
with scoped_pipeline_context(
execution_plan,
pipeline,
run_config,
pipeline_run,
instance,
scoped_resources_builder_cm=self._setup_resources,
# Set this flag even though we're not in test for clearer error reporting
raise_on_error=True,
) as pipeline_context:
self.context = DagstermillRuntimeExecutionContext(
pipeline_context=pipeline_context,
pipeline_def=pipeline_def,
solid_config=run_config.get("solids",
{}).get(solid_def.name,
{}).get("config"),
resource_keys_to_init=get_required_resource_keys_to_init(
execution_plan,
pipeline_def,
environment_config,
pipeline_context.intermediate_storage_def,
),
solid_name=solid_def.name,
)
return self.context
def reconstitute_pipeline_context(
self,
output_log_path=None,
marshal_dir=None,
environment_dict=None,
handle_kwargs=None,
pipeline_run_dict=None,
solid_subset=None,
solid_handle_kwargs=None,
instance_ref_dict=None,
):
'''Reconstitutes a context for dagstermill-managed execution.
You'll see this function called to reconstruct a pipeline context within the ``injected
parameters`` cell of a dagstermill output notebook. Users should not call this function
interactively except when debugging output notebooks.
Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a
context for interactive exploration and development. This call will be replaced by one to
:func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by
dagstermill.
'''
check.opt_str_param(output_log_path, 'output_log_path')
check.opt_str_param(marshal_dir, 'marshal_dir')
environment_dict = check.opt_dict_param(environment_dict,
'environment_dict',
key_type=str)
check.dict_param(pipeline_run_dict, 'pipeline_run_dict')
check.dict_param(handle_kwargs, 'handle_kwargs')
check.opt_list_param(solid_subset, 'solid_subset', of_type=str)
check.dict_param(solid_handle_kwargs, 'solid_handle_kwargs')
check.dict_param(instance_ref_dict, 'instance_ref_dict')
try:
handle = load_handle.handle_for_pipeline_cli_args(
handle_kwargs, use_default_repository_yaml=False)
except (check.CheckError, load_handle.UsageError) as err:
six.raise_from(
DagstermillError(
'Cannot invoke a dagstermill solid from an in-memory pipeline that was not loaded '
'from an ExecutionTargetHandle. Run this pipeline using dagit, the dagster CLI, '
'through dagster-graphql, or in-memory after loading it through an '
'ExecutionTargetHandle.'),
err,
)
try:
instance_ref = unpack_value(instance_ref_dict)
instance = DagsterInstance.from_ref(instance_ref)
except Exception as err: # pylint: disable=broad-except
six.raise_from(
DagstermillError(
'Error when attempting to resolve DagsterInstance from serialized InstanceRef'
),
err,
)
pipeline_def = check.inst_param(
handle.build_pipeline_definition(),
'pipeline_def (from handle {handle_dict})'.format(
handle_dict=handle.data._asdict()),
PipelineDefinition,
).build_sub_pipeline(solid_subset)
solid_handle = SolidHandle.from_dict(solid_handle_kwargs)
solid_def = pipeline_def.get_solid(solid_handle)
pipeline_run = unpack_value(pipeline_run_dict)
self.marshal_dir = marshal_dir
self.in_pipeline = True
self.solid_def = solid_def
self.pipeline_def = pipeline_def
with scoped_pipeline_context(
self.pipeline_def,
environment_dict,
pipeline_run,
instance=instance,
scoped_resources_builder_cm=self._setup_resources,
) as pipeline_context:
self.context = DagstermillExecutionContext(pipeline_context)
return self.context
def descend(self, solid):
return self._replace(
handle=SolidHandle(solid.name, parent=self.handle))
