def in_pipeline_manager(pipeline_name='hello_world_pipeline', solid_handle=SolidHandle('hello_world', 'hello_world', None), handle_kwargs=None, **kwargs): manager = Manager() run_id = str(uuid.uuid4()) marshal_dir = tempfile.mkdtemp() if not handle_kwargs: handle_kwargs = { 'pipeline_name': pipeline_name, 'module_name': 'dagstermill.examples.repository', 'fn_name': 'define_hello_world_pipeline', } try: with safe_tempfile_path() as output_log_file_path: context_dict = { 'run_config_kwargs': dict(run_id=run_id, mode='default'), 'solid_handle_kwargs': solid_handle._asdict(), 'handle_kwargs': handle_kwargs, 'marshal_dir': marshal_dir, 'environment_dict': {}, 'output_log_path': output_log_file_path, } manager.reconstitute_pipeline_context( **dict(context_dict, **kwargs)) yield manager finally: shutil.rmtree(marshal_dir)
def in_pipeline_manager(pipeline_name='hello_world_pipeline', solid_handle=SolidHandle('hello_world', 'hello_world', None), **kwargs): manager = Manager() run_id = str(uuid.uuid4()) marshal_dir = tempfile.mkdtemp() handle = handle_for_pipeline_cli_args({ 'pipeline_name': pipeline_name, 'module_name': 'dagstermill.examples.repository', 'fn_name': 'define_hello_world_pipeline', }) try: with tempfile.NamedTemporaryFile() as output_log_file: context_dict = { 'run_config': RunConfig(run_id=run_id, mode='default'), 'solid_handle': solid_handle, 'handle': handle, 'marshal_dir': marshal_dir, 'environment_dict': {}, 'output_log_path': output_log_file.name, } manager.reconstitute_pipeline_context( **dict(context_dict, **kwargs)) yield manager finally: shutil.rmtree(marshal_dir)
def in_pipeline_manager( pipeline_name='hello_world_pipeline', solid_handle=SolidHandle('hello_world', 'hello_world', None), handle_kwargs=None, mode=None, **kwargs ): manager = Manager() run_id = str(uuid.uuid4()) instance = DagsterInstance.local_temp() marshal_dir = tempfile.mkdtemp() if not handle_kwargs: handle_kwargs = { 'pipeline_name': pipeline_name, 'module_name': 'dagstermill.examples.repository', 'fn_name': 'define_hello_world_pipeline', } pipeline_run_dict = pack_value( PipelineRun( pipeline_name=pipeline_name, run_id=run_id, mode=mode or 'default', environment_dict=None, selector=None, reexecution_config=None, step_keys_to_execute=None, status=PipelineRunStatus.NOT_STARTED, ) ) try: with safe_tempfile_path() as output_log_file_path: context_dict = { 'pipeline_run_dict': pipeline_run_dict, 'solid_handle_kwargs': solid_handle._asdict(), 'handle_kwargs': handle_kwargs, 'marshal_dir': marshal_dir, 'environment_dict': {}, 'output_log_path': output_log_file_path, 'instance_ref_dict': pack_value(instance.get_ref()), } manager.reconstitute_pipeline_context(**dict(context_dict, **kwargs)) yield manager finally: shutil.rmtree(marshal_dir)
def in_pipeline_manager( pipeline_name="hello_world_pipeline", solid_handle=NodeHandle("hello_world", None), step_key="hello_world", executable_dict=None, mode=None, **kwargs, ): manager = Manager() run_id = make_new_run_id() with instance_for_test() as instance: marshal_dir = tempfile.mkdtemp() if not executable_dict: executable_dict = ReconstructablePipeline.for_module( "dagstermill.examples.repository", "hello_world_pipeline").to_dict() pipeline_run_dict = pack_value( PipelineRun( pipeline_name=pipeline_name, run_id=run_id, mode=mode or "default", run_config=None, step_keys_to_execute=None, status=PipelineRunStatus.NOT_STARTED, )) try: with safe_tempfile_path() as output_log_file_path: context_dict = { "pipeline_run_dict": pipeline_run_dict, "solid_handle_kwargs": solid_handle._asdict(), "executable_dict": executable_dict, "marshal_dir": marshal_dir, "run_config": {}, "output_log_path": output_log_file_path, "instance_ref_dict": pack_value(instance.get_ref()), "step_key": step_key, } manager.reconstitute_pipeline_context( **dict(context_dict, **kwargs)) yield manager finally: shutil.rmtree(marshal_dir)
def in_pipeline_manager(pipeline_name='hello_world_pipeline', solid_handle=SolidHandle('hello_world', None), executable_dict=None, mode=None, **kwargs): manager = Manager() run_id = make_new_run_id() instance = DagsterInstance.local_temp() marshal_dir = tempfile.mkdtemp() if not executable_dict: executable_dict = ReconstructablePipeline.for_module( 'dagstermill.examples.repository', 'define_hello_world_pipeline').to_dict() pipeline_run_dict = pack_value( PipelineRun( pipeline_name=pipeline_name, run_id=run_id, mode=mode or 'default', run_config=None, step_keys_to_execute=None, status=PipelineRunStatus.NOT_STARTED, )) try: with safe_tempfile_path() as output_log_file_path: context_dict = { 'pipeline_run_dict': pipeline_run_dict, 'solid_handle_kwargs': solid_handle._asdict(), 'executable_dict': executable_dict, 'marshal_dir': marshal_dir, 'run_config': {}, 'output_log_path': output_log_file_path, 'instance_ref_dict': pack_value(instance.get_ref()), } manager.reconstitute_pipeline_context( **dict(context_dict, **kwargs)) yield manager finally: shutil.rmtree(marshal_dir)