def in_pipeline_manager(pipeline_name='hello_world_pipeline',
solid_handle=SolidHandle('hello_world', 'hello_world',
None),
handle_kwargs=None,
**kwargs):
manager = Manager()
run_id = str(uuid.uuid4())
marshal_dir = tempfile.mkdtemp()
if not handle_kwargs:
handle_kwargs = {
'pipeline_name': pipeline_name,
'module_name': 'dagstermill.examples.repository',
'fn_name': 'define_hello_world_pipeline',
}
try:
with safe_tempfile_path() as output_log_file_path:
context_dict = {
'run_config_kwargs': dict(run_id=run_id, mode='default'),
'solid_handle_kwargs': solid_handle._asdict(),
'handle_kwargs': handle_kwargs,
'marshal_dir': marshal_dir,
'environment_dict': {},
'output_log_path': output_log_file_path,
}
manager.reconstitute_pipeline_context(
**dict(context_dict, **kwargs))
yield manager
finally:
shutil.rmtree(marshal_dir)
def in_pipeline_manager(pipeline_name='hello_world_pipeline',
solid_handle=SolidHandle('hello_world', 'hello_world',
None),
**kwargs):
manager = Manager()
run_id = str(uuid.uuid4())
marshal_dir = tempfile.mkdtemp()
handle = handle_for_pipeline_cli_args({
'pipeline_name':
pipeline_name,
'module_name':
'dagstermill.examples.repository',
'fn_name':
'define_hello_world_pipeline',
})
try:
with tempfile.NamedTemporaryFile() as output_log_file:
context_dict = {
'run_config': RunConfig(run_id=run_id, mode='default'),
'solid_handle': solid_handle,
'handle': handle,
'marshal_dir': marshal_dir,
'environment_dict': {},
'output_log_path': output_log_file.name,
}
manager.reconstitute_pipeline_context(
**dict(context_dict, **kwargs))
yield manager
finally:
shutil.rmtree(marshal_dir)
def test_yield_unserializable_result():
manager = Manager()
assert manager.yield_result(threading.Lock())
with in_pipeline_manager(
solid_handle=SolidHandle('hello_world_output', None),
handle_kwargs={
'module_name': 'dagstermill.examples.repository',
'fn_name': 'define_hello_world_with_output_pipeline',
},
) as manager:
with pytest.raises(TypeError):
manager.yield_result(threading.Lock())
def test_yield_unserializable_result():
manager = Manager()
assert manager.yield_result(threading.Lock())
with in_pipeline_manager(
pipeline_name="hello_world_with_output_pipeline",
solid_handle=SolidHandle("hello_world_output", None),
executable_dict=ReconstructablePipeline.for_module(
"dagstermill.examples.repository", "define_hello_world_with_output_pipeline",
).to_dict(),
) as manager:
with pytest.raises(TypeError):
manager.yield_result(threading.Lock())
def in_pipeline_manager(
pipeline_name='hello_world_pipeline',
solid_handle=SolidHandle('hello_world', 'hello_world', None),
handle_kwargs=None,
mode=None,
**kwargs
):
manager = Manager()
run_id = str(uuid.uuid4())
instance = DagsterInstance.local_temp()
marshal_dir = tempfile.mkdtemp()
if not handle_kwargs:
handle_kwargs = {
'pipeline_name': pipeline_name,
'module_name': 'dagstermill.examples.repository',
'fn_name': 'define_hello_world_pipeline',
}
pipeline_run_dict = pack_value(
PipelineRun(
pipeline_name=pipeline_name,
run_id=run_id,
mode=mode or 'default',
environment_dict=None,
selector=None,
reexecution_config=None,
step_keys_to_execute=None,
status=PipelineRunStatus.NOT_STARTED,
)
)
try:
with safe_tempfile_path() as output_log_file_path:
context_dict = {
'pipeline_run_dict': pipeline_run_dict,
'solid_handle_kwargs': solid_handle._asdict(),
'handle_kwargs': handle_kwargs,
'marshal_dir': marshal_dir,
'environment_dict': {},
'output_log_path': output_log_file_path,
'instance_ref_dict': pack_value(instance.get_ref()),
}
manager.reconstitute_pipeline_context(**dict(context_dict, **kwargs))
yield manager
finally:
shutil.rmtree(marshal_dir)
def in_pipeline_manager(
pipeline_name="hello_world_pipeline",
solid_handle=NodeHandle("hello_world", None),
step_key="hello_world",
executable_dict=None,
mode=None,
**kwargs,
):
manager = Manager()
run_id = make_new_run_id()
with instance_for_test() as instance:
marshal_dir = tempfile.mkdtemp()
if not executable_dict:
executable_dict = ReconstructablePipeline.for_module(
"dagstermill.examples.repository",
"hello_world_pipeline").to_dict()
pipeline_run_dict = pack_value(
PipelineRun(
pipeline_name=pipeline_name,
run_id=run_id,
mode=mode or "default",
run_config=None,
step_keys_to_execute=None,
status=PipelineRunStatus.NOT_STARTED,
))
try:
with safe_tempfile_path() as output_log_file_path:
context_dict = {
"pipeline_run_dict": pipeline_run_dict,
"solid_handle_kwargs": solid_handle._asdict(),
"executable_dict": executable_dict,
"marshal_dir": marshal_dir,
"run_config": {},
"output_log_path": output_log_file_path,
"instance_ref_dict": pack_value(instance.get_ref()),
"step_key": step_key,
}
manager.reconstitute_pipeline_context(
**dict(context_dict, **kwargs))
yield manager
finally:
shutil.rmtree(marshal_dir)
def in_pipeline_manager(pipeline_name='hello_world_pipeline',
solid_handle=SolidHandle('hello_world', None),
executable_dict=None,
mode=None,
**kwargs):
manager = Manager()
run_id = make_new_run_id()
instance = DagsterInstance.local_temp()
marshal_dir = tempfile.mkdtemp()
if not executable_dict:
executable_dict = ReconstructablePipeline.for_module(
'dagstermill.examples.repository',
'define_hello_world_pipeline').to_dict()
pipeline_run_dict = pack_value(
PipelineRun(
pipeline_name=pipeline_name,
run_id=run_id,
mode=mode or 'default',
run_config=None,
step_keys_to_execute=None,
status=PipelineRunStatus.NOT_STARTED,
))
try:
with safe_tempfile_path() as output_log_file_path:
context_dict = {
'pipeline_run_dict': pipeline_run_dict,
'solid_handle_kwargs': solid_handle._asdict(),
'executable_dict': executable_dict,
'marshal_dir': marshal_dir,
'run_config': {},
'output_log_path': output_log_file_path,
'instance_ref_dict': pack_value(instance.get_ref()),
}
manager.reconstitute_pipeline_context(
**dict(context_dict, **kwargs))
yield manager
finally:
shutil.rmtree(marshal_dir)
def test_out_of_pipeline_manager_yield_complex_result():
class Foo:
pass
manager = Manager()
assert isinstance(manager.yield_result(Foo()), Foo)
def test_out_of_pipeline_manager_yield_result():
manager = Manager()
assert manager.yield_result("foo") == "foo"
def test_out_of_pipeline_yield_event():
manager = Manager()
assert manager.yield_event(
AssetMaterialization("foo")) == AssetMaterialization("foo")
def test_out_of_pipeline_yield_event():
manager = Manager()
assert manager.yield_event(
Materialization('foo')) == Materialization('foo')