class Manager:
def __init__(self):
self.repository_def = None
self.populated_by_papermill = False
self.pipeline_def = None
self.solid_def = None
self.marshal_dir = None
self.context = None
self.input_name_type_dict = None
self.output_name_type_dict = None
self.solid_def_name = None
self.resources_stack = None
def register_repository(self, repository_def):
self.repository_def = repository_def
def deregister_repository(self):
# This function is intended to support test cases, and should not be invoked
# from user notebooks.
self.repository_def = None
@contextmanager
def setup_resources(self, pipeline_def, environment_config, run_config, log_manager):
'''This context manager is a drop-in replacement for
dagster.core.execution.context_creation_pipeline.create_resources. It uses the Manager's
instance of ResourceStack to create resources, but does not tear them down when the
context manager returns -- teardown must be managed manually using Manager.teardown().
'''
# pylint: disable=protected-access
self.resources_stack = ResourcesStack(
pipeline_def, environment_config, run_config, log_manager
)
yield self.resources_stack.create()
def define_out_of_pipeline_context(self, config=None):
'''Defines a context to be used in a notebook (i.e., not in pipeline execution).
'''
config = check.opt_dict_param(config, 'config')
pipeline_def = PipelineDefinition([], name='Ephemeral Notebook Pipeline')
if config.keys():
warnings.warn(
'Config keys will not be respected for in-notebook '
'execution: [{keys}]'.format(
keys=', '.join(['\'{key}\''.format(key=key) for key in config.keys()])
)
)
config = {}
run_config = RunConfig()
with scoped_pipeline_context(
pipeline_def, config, run_config, scoped_resources_builder_cm=self.setup_resources
) as pipeline_context:
self.context = DagstermillInNotebookExecutionContext(
pipeline_context, out_of_pipeline=True
)
if self.context.resources: # pylint: disable=protected-access
warnings.warn(
'Call dagstermill.teardown() to finalize resources attached to the context.'
)
return self.context
def yield_result(self, value, output_name='result'):
if not self.populated_by_papermill:
return value
if self.solid_def is None:
if output_name not in self.output_name_type_dict:
raise DagstermillError(
'Solid {solid_name} does not have output named {output_name}'.format(
solid_name=self.solid_def_name, output_name=output_name
)
)
runtime_type_enum = self.output_name_type_dict[output_name]
if runtime_type_enum == SerializableRuntimeType.SCALAR:
scrapbook.glue(output_name, value)
elif runtime_type_enum == SerializableRuntimeType.ANY and is_json_serializable(value):
scrapbook.glue(output_name, value)
elif runtime_type_enum == SerializableRuntimeType.PICKLE_SERIALIZABLE:
out_file = os.path.join(self.marshal_dir, 'output-{}'.format(output_name))
PickleSerializationStrategy().serialize_to_file(value, out_file)
scrapbook.glue(output_name, out_file)
else:
raise DagstermillError(
# Discuss this in the docs and improve error message
# https://github.com/dagster-io/dagster/issues/1275
# https://github.com/dagster-io/dagster/issues/1276
'Output Definition for output {output_name} requires repo registration '
'since it has a complex serialization format'.format(output_name=output_name)
)
else:
if not self.solid_def.has_output(output_name):
raise DagstermillError(
'Solid {solid_name} does not have output named {output_name}'.format(
solid_name=self.solid_def.name, output_name=output_name
)
)
runtime_type = self.solid_def.output_def_named(output_name).runtime_type
out_file = os.path.join(self.marshal_dir, 'output-{}'.format(output_name))
scrapbook.glue(output_name, write_value(runtime_type, value, out_file))
def yield_event(self, dagster_event):
if not self.populated_by_papermill:
return dagster_event
event_id = 'event-{event_uuid}'.format(event_uuid=str(uuid.uuid4()))
out_file_path = os.path.join(self.marshal_dir, event_id)
with open(out_file_path, 'wb') as fd:
fd.write(pickle.dumps(dagster_event, PICKLE_PROTOCOL))
scrapbook.glue(event_id, out_file_path)
def populate_context(
self,
run_id=None,
mode=None,
solid_def_name=None,
pipeline_name=None,
marshal_dir=None,
environment_config=None,
input_name_type_dict=None,
output_name_type_dict=None,
output_log_path=None,
**_kwargs
):
check.str_param(run_id, 'run_id')
check.str_param(mode, 'mode')
check.str_param(solid_def_name, 'solid_def_name')
check.str_param(pipeline_name, 'pipeline_name')
check.str_param(marshal_dir, 'marshal_dir')
check.dict_param(environment_config, 'environment_config')
check.dict_param(input_name_type_dict, 'input_name_type_dict')
check.dict_param(output_name_type_dict, 'output_name_type_dict')
check.str_param(output_log_path, 'output_log_path')
self.populated_by_papermill = True
self.solid_def_name = solid_def_name
self.marshal_dir = marshal_dir
logger_def = construct_logger(output_log_path)
loggers = {'dagstermill': logger_def}
if self.repository_def is None:
self.solid_def = None
self.pipeline_def = PipelineDefinition(
[],
mode_definitions=[ModeDefinition(loggers=loggers)],
name='Dummy Pipeline (No Repo Registration)',
)
self.input_name_type_dict = dict_to_enum(input_name_type_dict)
self.output_name_type_dict = dict_to_enum(output_name_type_dict)
for _, runtime_type_enum in self.input_name_type_dict.items():
if runtime_type_enum == SerializableRuntimeType.NONE:
raise DagstermillError(
'If Dagstermill solids have inputs that require serialization strategies '
'that are not pickling, then you must register a repository within '
'notebook by calling dagstermill.register_repository(repository_def)'
)
for _, runtime_type_enum in self.output_name_type_dict.items():
if runtime_type_enum == SerializableRuntimeType.NONE:
raise DagstermillError(
'If Dagstermill solids have outputs that require serialization strategies '
'that are not pickling, then you must register a repository within '
'notebook by calling dagstermill.register_repository(repository_def).'
)
environment_config = {'loggers': {'dagstermill': {}}}
run_config = RunConfig(run_id=run_id, mode=mode)
else:
self.pipeline_def = self.repository_def.get_pipeline(pipeline_name)
check.invariant(
self.pipeline_def.has_solid_def(solid_def_name),
'solid {} not found'.format(solid_def_name),
)
self.solid_def = self.pipeline_def.solid_def_named(solid_def_name)
logger = logger_def.logger_fn(
InitLoggerContext({}, self.pipeline_def, logger_def, run_id)
)
run_config = RunConfig(run_id, loggers=[logger], mode=mode)
with scoped_pipeline_context(
self.pipeline_def,
environment_config,
run_config,
scoped_resources_builder_cm=self.setup_resources,
) as pipeline_context:
self.context = DagstermillInNotebookExecutionContext(pipeline_context)
return self.context
def teardown_resources(self):
if self.resources_stack is not None:
self.resources_stack.teardown()
class Manager:
def __init__(self):
self.handle = None
self.pipeline_def = None
self.solid_def = None
self.in_pipeline = False
self.marshal_dir = None
self.context = None
self.resources_stack = None
@contextmanager
def _setup_resources(self, pipeline_def, environment_config, run_config,
log_manager):
'''This context manager is a drop-in replacement for
dagster.core.execution.context_creation_pipeline.create_resources. It uses the Manager's
instance of ResourceStack to create resources, but does not tear them down when the
context manager returns -- teardown must be managed manually using Manager.teardown().
'''
# pylint: disable=protected-access
self.resources_stack = ResourcesStack(pipeline_def, environment_config,
run_config, log_manager)
yield self.resources_stack.create()
def reconstitute_pipeline_context(
self,
output_log_path=None,
marshal_dir=None,
environment_dict=None,
handle=None,
run_config=None,
solid_subset=None,
solid_handle=None,
):
'''Reconstitutes a context for dagstermill-managed execution.
You'll see this function called to reconstruct a pipeline context within the ``injected
parameters`` cell of a dagstermill output notebook. Users should not call this function
interactively except when debugging output notebooks.
Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a
context for interactive exploration and development. This call will be replaced by one to
:func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by
dagstermill.
'''
check.opt_str_param(output_log_path, 'output_log_path')
check.opt_str_param(marshal_dir, 'marshal_dir')
environment_dict = check.opt_dict_param(environment_dict,
'environment_dict',
key_type=str)
check.inst_param(run_config, 'run_config', RunConfig)
check.inst_param(handle, 'handle', ExecutionTargetHandle)
check.opt_list_param(solid_subset, 'solid_subset', of_type=str)
check.inst_param(solid_handle, 'solid_handle', SolidHandle)
pipeline_def = check.inst_param(
handle.build_pipeline_definition(),
'pipeline_def (from handle {handle_dict})'.format(
handle_dict=handle.data._asdict()),
PipelineDefinition,
).build_sub_pipeline(solid_subset)
solid_def = pipeline_def.get_solid(solid_handle)
mode_def = pipeline_def.get_mode_definition(run_config.mode)
shim_mode_def = ModeDefinition(
name=mode_def.name,
logger_defs=dict(
mode_def.loggers,
dagstermill=construct_sqlite_logger(output_log_path)),
resource_defs=mode_def.resource_defs,
)
pipeline_def = pipeline_def.new_with(mode_defs=[shim_mode_def])
if 'loggers' not in environment_dict:
environment_dict['loggers'] = {'dagstermill': {}}
if 'dagstermill' not in environment_dict['loggers']:
environment_dict['loggers']['dagstermill'] = {}
self.marshal_dir = marshal_dir
self.in_pipeline = True
self.solid_def = solid_def
self.pipeline_def = pipeline_def
with scoped_pipeline_context(
self.pipeline_def,
environment_dict,
run_config,
scoped_resources_builder_cm=self._setup_resources,
) as pipeline_context:
self.context = DagstermillExecutionContext(pipeline_context)
return self.context
def get_context(self,
solid_config=None,
mode_def=None,
environment_dict=None):
'''Get a dagstermill execution context for interactive exploration and development.
Args:
solid_config (Optional[Any]): If specified, this value will be made available on the
context as its ``solid_config`` property.
mode_def (Optional[:class:`dagster.ModeDefinition`]): If specified, defines the mode to
use to construct the context. Specify this if you would like a context constructed
with specific ``resource_defs`` or ``logger_defs``. By default, an ephemeral mode
with a console logger will be constructed.
environment_dict(Optional[dict]): The environment config dict with which to construct
the context.
Returns:
:class:`dagstermill.DagstermillExecutionContext`
'''
check.opt_inst_param(mode_def, 'mode_def', ModeDefinition)
environment_dict = check.opt_dict_param(environment_dict,
'environment_dict',
key_type=str)
solid_def = SolidDefinition(
name='this_solid',
input_defs=[],
compute_fn=lambda *args, **kwargs: None,
output_defs=[],
description=
'Ephemeral solid constructed by dagstermill.get_context()',
)
if not mode_def:
mode_def = ModeDefinition(
logger_defs={'dagstermill': colored_console_logger})
environment_dict['loggers'] = {'dagstermill': {}}
pipeline_def = PipelineDefinition(
[solid_def],
mode_defs=[mode_def],
name='ephemeral_dagstermill_pipeline')
run_config = RunConfig(mode=mode_def.name)
self.in_pipeline = False
self.solid_def = solid_def
self.pipeline_def = pipeline_def
with scoped_pipeline_context(
self.pipeline_def,
environment_dict,
run_config,
scoped_resources_builder_cm=self._setup_resources,
) as pipeline_context:
self.context = DagstermillExecutionContext(pipeline_context,
solid_config)
return self.context
def yield_result(self, value, output_name='result'):
'''Yield a result directly from notebook code.
When called interactively or in development, returns its input.
Args:
value (Any): The value to yield.
output_name (Optional[str]): The name of the result to yield (default: ``'result'``).
'''
if not self.in_pipeline:
return value
# deferred import for perf
import scrapbook
if not self.solid_def.has_output(output_name):
raise DagstermillError(
'Solid {solid_name} does not have output named {output_name}'.
format(solid_name=self.solid_def.name,
output_name=output_name))
runtime_type = self.solid_def.output_def_named(
output_name).runtime_type
out_file = os.path.join(self.marshal_dir,
'output-{}'.format(output_name))
scrapbook.glue(output_name, write_value(runtime_type, value, out_file))
def yield_event(self, dagster_event):
'''Yield a dagster event directly from notebook code.
When called interactively or in development, returns its input.
Args:
dagster_event (Union[:class:`dagster.Materialization`, :class:`dagster.ExpectationResult`, :class:`dagster.TypeCheck`, :class:`dagster.Failure`]):
An event to yield back to Dagster.
'''
check.inst_param(
dagster_event, 'dagster_event',
(Materialization, ExpectationResult, TypeCheck, Failure))
if not self.in_pipeline:
return dagster_event
# deferred import for perf
import scrapbook
event_id = 'event-{event_uuid}'.format(event_uuid=str(uuid.uuid4()))
out_file_path = os.path.join(self.marshal_dir, event_id)
with open(out_file_path, 'wb') as fd:
fd.write(pickle.dumps(dagster_event, PICKLE_PROTOCOL))
scrapbook.glue(event_id, out_file_path)
def teardown_resources(self):
if self.resources_stack is not None:
self.resources_stack.teardown()
def load_parameter(self, input_name, input_value):
input_def = self.solid_def.input_def_named(input_name)
return read_value(input_def.runtime_type, input_value)
class Manager:
def __init__(self):
self.handle = None
self.pipeline_def = None
self.solid_def = None
self.in_pipeline = False
self.marshal_dir = None
self.context = None
self.resources_stack = None
@contextmanager
def _setup_resources(self, pipeline_def, environment_config, pipeline_run,
log_manager):
'''This context manager is a drop-in replacement for
dagster.core.execution.context_creation_pipeline.create_resources. It uses the Manager's
instance of ResourceStack to create resources, but does not tear them down when the
context manager returns -- teardown must be managed manually using Manager.teardown().
'''
# pylint: disable=protected-access
self.resources_stack = ResourcesStack(pipeline_def, environment_config,
pipeline_run, log_manager)
yield self.resources_stack.create()
def reconstitute_pipeline_context(
self,
output_log_path=None,
marshal_dir=None,
environment_dict=None,
handle_kwargs=None,
pipeline_run_dict=None,
solid_subset=None,
solid_handle_kwargs=None,
instance_ref_dict=None,
):
'''Reconstitutes a context for dagstermill-managed execution.
You'll see this function called to reconstruct a pipeline context within the ``injected
parameters`` cell of a dagstermill output notebook. Users should not call this function
interactively except when debugging output notebooks.
Use :func:`dagstermill.get_context` in the ``parameters`` cell of your notebook to define a
context for interactive exploration and development. This call will be replaced by one to
:func:`dagstermill.reconstitute_pipeline_context` when the notebook is executed by
dagstermill.
'''
check.opt_str_param(output_log_path, 'output_log_path')
check.opt_str_param(marshal_dir, 'marshal_dir')
environment_dict = check.opt_dict_param(environment_dict,
'environment_dict',
key_type=str)
check.dict_param(pipeline_run_dict, 'pipeline_run_dict')
check.dict_param(handle_kwargs, 'handle_kwargs')
check.opt_list_param(solid_subset, 'solid_subset', of_type=str)
check.dict_param(solid_handle_kwargs, 'solid_handle_kwargs')
check.dict_param(instance_ref_dict, 'instance_ref_dict')
try:
handle = load_handle.handle_for_pipeline_cli_args(
handle_kwargs, use_default_repository_yaml=False)
except (check.CheckError, load_handle.UsageError) as err:
six.raise_from(
DagstermillError(
'Cannot invoke a dagstermill solid from an in-memory pipeline that was not loaded '
'from an ExecutionTargetHandle. Run this pipeline using dagit, the dagster CLI, '
'through dagster-graphql, or in-memory after loading it through an '
'ExecutionTargetHandle.'),
err,
)
try:
instance_ref = unpack_value(instance_ref_dict)
instance = DagsterInstance.from_ref(instance_ref)
except Exception as err: # pylint: disable=broad-except
six.raise_from(
DagstermillError(
'Error when attempting to resolve DagsterInstance from serialized InstanceRef'
),
err,
)
pipeline_def = check.inst_param(
handle.build_pipeline_definition(),
'pipeline_def (from handle {handle_dict})'.format(
handle_dict=handle.data._asdict()),
PipelineDefinition,
).build_sub_pipeline(solid_subset)
solid_handle = SolidHandle.from_dict(solid_handle_kwargs)
solid_def = pipeline_def.get_solid(solid_handle)
pipeline_run = unpack_value(pipeline_run_dict)
self.marshal_dir = marshal_dir
self.in_pipeline = True
self.solid_def = solid_def
self.pipeline_def = pipeline_def
with scoped_pipeline_context(
self.pipeline_def,
environment_dict,
pipeline_run,
instance=instance,
scoped_resources_builder_cm=self._setup_resources,
) as pipeline_context:
self.context = DagstermillExecutionContext(pipeline_context)
return self.context
def get_context(self,
solid_config=None,
mode_def=None,
environment_dict=None):
'''Get a dagstermill execution context for interactive exploration and development.
Args:
solid_config (Optional[Any]): If specified, this value will be made available on the
context as its ``solid_config`` property.
mode_def (Optional[:class:`dagster.ModeDefinition`]): If specified, defines the mode to
use to construct the context. Specify this if you would like a context constructed
with specific ``resource_defs`` or ``logger_defs``. By default, an ephemeral mode
with a console logger will be constructed.
environment_dict(Optional[dict]): The environment config dict with which to construct
the context.
Returns:
:class:`dagstermill.DagstermillExecutionContext`
'''
check.opt_inst_param(mode_def, 'mode_def', ModeDefinition)
environment_dict = check.opt_dict_param(environment_dict,
'environment_dict',
key_type=str)
solid_def = SolidDefinition(
name='this_solid',
input_defs=[],
compute_fn=lambda *args, **kwargs: None,
output_defs=[],
description=
'Ephemeral solid constructed by dagstermill.get_context()',
)
if not mode_def:
mode_def = ModeDefinition(
logger_defs={'dagstermill': colored_console_logger})
environment_dict['loggers'] = {'dagstermill': {}}
pipeline_def = PipelineDefinition(
[solid_def],
mode_defs=[mode_def],
name='ephemeral_dagstermill_pipeline')
run_id = str(uuid.uuid4())
# construct stubbed PipelineRun for notebook exploration...
# The actual pipeline run during pipeline execution will be serialized and reconstituted
# in the `reconstitute_pipeline_context` call
pipeline_run = PipelineRun(
pipeline_name=pipeline_def.name,
run_id=run_id,
environment_dict=environment_dict,
mode=mode_def.name,
reexecution_config=None,
selector=None,
step_keys_to_execute=None,
status=PipelineRunStatus.NOT_STARTED,
tags=None,
)
self.in_pipeline = False
self.solid_def = solid_def
self.pipeline_def = pipeline_def
with scoped_pipeline_context(
self.pipeline_def,
environment_dict,
pipeline_run,
instance=DagsterInstance.ephemeral(),
scoped_resources_builder_cm=self._setup_resources,
) as pipeline_context:
self.context = DagstermillExecutionContext(pipeline_context,
solid_config)
return self.context
def yield_result(self, value, output_name='result'):
'''Yield a result directly from notebook code.
When called interactively or in development, returns its input.
Args:
value (Any): The value to yield.
output_name (Optional[str]): The name of the result to yield (default: ``'result'``).
'''
if not self.in_pipeline:
return value
# deferred import for perf
import scrapbook
if not self.solid_def.has_output(output_name):
raise DagstermillError(
'Solid {solid_name} does not have output named {output_name}'.
format(solid_name=self.solid_def.name,
output_name=output_name))
runtime_type = self.solid_def.output_def_named(
output_name).runtime_type
out_file = os.path.join(self.marshal_dir,
'output-{}'.format(output_name))
scrapbook.glue(output_name, write_value(runtime_type, value, out_file))
def yield_event(self, dagster_event):
'''Yield a dagster event directly from notebook code.
When called interactively or in development, returns its input.
Args:
dagster_event (Union[:class:`dagster.Materialization`, :class:`dagster.ExpectationResult`, :class:`dagster.TypeCheck`, :class:`dagster.Failure`]):
An event to yield back to Dagster.
'''
check.inst_param(
dagster_event, 'dagster_event',
(Materialization, ExpectationResult, TypeCheck, Failure))
if not self.in_pipeline:
return dagster_event
# deferred import for perf
import scrapbook
event_id = 'event-{event_uuid}'.format(event_uuid=str(uuid.uuid4()))
out_file_path = os.path.join(self.marshal_dir, event_id)
with open(out_file_path, 'wb') as fd:
fd.write(pickle.dumps(dagster_event, PICKLE_PROTOCOL))
scrapbook.glue(event_id, out_file_path)
def teardown_resources(self):
if self.resources_stack is not None:
self.resources_stack.teardown()
def load_parameter(self, input_name, input_value):
input_def = self.solid_def.input_def_named(input_name)
return read_value(input_def.runtime_type, input_value)
class Manager:
def __init__(self):
self.handle = None
self.pipeline_def = None
self.solid_def = None
self.in_pipeline = False
self.marshal_dir = None
self.context = None
self.resources_stack = None
@contextmanager
def _setup_resources(self, pipeline_def, environment_config, run_config,
log_manager):
'''This context manager is a drop-in replacement for
dagster.core.execution.context_creation_pipeline.create_resources. It uses the Manager's
instance of ResourceStack to create resources, but does not tear them down when the
context manager returns -- teardown must be managed manually using Manager.teardown().
'''
# pylint: disable=protected-access
self.resources_stack = ResourcesStack(pipeline_def, environment_config,
run_config, log_manager)
yield self.resources_stack.create()
def reconstitute_pipeline_context(
self,
output_log_path=None,
marshal_dir=None,
environment_dict=None,
handle=None,
run_config=None,
solid_handle=None,
):
'''Reconstitutes a context for dagstermill-managed execution.
This function is called to reconstruct a pipeline context within the injected parameters cell
of a dagstermill notebook. Users should not call this function interactively except when
debugging output notebooks.
Use dagstermill.get_context in the ``parameters`` cell of your notebook when defining a
context for interactive exploration and development.
'''
check.opt_str_param(output_log_path, 'output_log_path')
check.opt_str_param(marshal_dir, 'marshal_dir')
environment_dict = check.opt_dict_param(environment_dict,
'environment_dict',
key_type=str)
check.inst_param(run_config, 'run_config', RunConfig)
check.inst_param(handle, 'handle', ExecutionTargetHandle)
check.inst_param(solid_handle, 'solid_handle', SolidHandle)
pipeline_def = check.inst_param(
handle.build_pipeline_definition(),
'pipeline_def (from handle {handle_dict})'.format(
handle_dict=handle.data._asdict()),
PipelineDefinition,
)
solid_def = pipeline_def.get_solid(solid_handle)
mode_def = pipeline_def.get_mode_definition(run_config.mode)
shim_mode_def = ModeDefinition(
name=mode_def.name,
logger_defs=dict(
mode_def.loggers,
dagstermill=construct_sqlite_logger(output_log_path)),
resource_defs=mode_def.resource_defs,
)
pipeline_def.mode_definitions = [shim_mode_def]
if 'loggers' not in environment_dict:
environment_dict['loggers'] = {'dagstermill': {}}
if 'dagstermill' not in environment_dict['loggers']:
environment_dict['loggers']['dagstermill'] = {}
self.marshal_dir = marshal_dir
self.in_pipeline = True
self.solid_def = solid_def
self.pipeline_def = pipeline_def
with scoped_pipeline_context(
self.pipeline_def,
environment_dict,
run_config,
scoped_resources_builder_cm=self._setup_resources,
) as pipeline_context:
self.context = DagstermillInPipelineExecutionContext(
pipeline_context)
return self.context
def get_context(self,
solid_def=None,
mode_def=None,
environment_dict=None):
check.opt_inst_param(solid_def, 'solid_def', SolidDefinition)
check.opt_inst_param(mode_def, 'mode_def', ModeDefinition)
environment_dict = check.opt_dict_param(environment_dict,
'environment_dict',
key_type=str)
if solid_def is None:
solid_def = SolidDefinition(
name='this_solid',
input_defs=[],
compute_fn=lambda *args, **kwargs: None,
output_defs=[],
description=
'Ephemeral solid constructed by dagstermill.get_context()',
)
if not mode_def:
mode_def = ModeDefinition(
logger_defs={'dagstermill': colored_console_logger})
environment_dict['loggers'] = {'dagstermill': {}}
pipeline_def = PipelineDefinition(
[solid_def],
mode_defs=[mode_def],
name='ephemeral_dagstermill_pipeline')
run_config = RunConfig(mode=mode_def.name)
self.in_pipeline = False
self.solid_def = solid_def
self.pipeline_def = pipeline_def
with scoped_pipeline_context(
self.pipeline_def,
environment_dict,
run_config,
scoped_resources_builder_cm=self._setup_resources,
) as pipeline_context:
self.context = DagstermillInPipelineExecutionContext(
pipeline_context)
return self.context
def yield_result(self, value, output_name='result'):
if not self.in_pipeline:
return value
# deferred import for perf
import scrapbook
if not self.solid_def.has_output(output_name):
raise DagstermillError(
'Solid {solid_name} does not have output named {output_name}'.
format(solid_name=self.solid_def.name,
output_name=output_name))
runtime_type = self.solid_def.output_def_named(
output_name).runtime_type
out_file = os.path.join(self.marshal_dir,
'output-{}'.format(output_name))
scrapbook.glue(output_name, write_value(runtime_type, value, out_file))
def yield_event(self, dagster_event):
if not self.in_pipeline:
return dagster_event
# deferred import for perf
import scrapbook
event_id = 'event-{event_uuid}'.format(event_uuid=str(uuid.uuid4()))
out_file_path = os.path.join(self.marshal_dir, event_id)
with open(out_file_path, 'wb') as fd:
fd.write(pickle.dumps(dagster_event, PICKLE_PROTOCOL))
scrapbook.glue(event_id, out_file_path)
def teardown_resources(self):
if self.resources_stack is not None:
self.resources_stack.teardown()