def test_handle_to_from_string(): handle = SolidHandle("baz", SolidHandle("bar", SolidHandle("foo", None))) assert handle.to_string() == "foo.bar.baz" assert SolidHandle.from_string(handle.to_string()) == handle handle = SolidHandle("foo", None) assert handle.to_string() == "foo" assert SolidHandle.from_string(handle.to_string()) == handle
def test_handle_to_from_string(): handle = SolidHandle('baz', SolidHandle('bar', SolidHandle('foo', None))) assert handle.to_string() == 'foo.bar.baz' assert SolidHandle.from_string(handle.to_string()) == handle handle = SolidHandle('foo', None) assert handle.to_string() == 'foo' assert SolidHandle.from_string(handle.to_string()) == handle
def parse_from_key(string): unresolved_match = re.match(r"(.*)\[\?\]", string) if unresolved_match: return UnresolvedStepHandle( SolidHandle.from_string(unresolved_match.group(1))) resolved_match = re.match(r"(.*)\[(.*)\]", string) if resolved_match: return ResolvedFromDynamicStepHandle( SolidHandle.from_string(resolved_match.group(1)), resolved_match.group(2)) return StepHandle(SolidHandle.from_string(string))
def from_key(string): return StepHandle(SolidHandle.from_string(string))
def execute_in_process( node: NodeDefinition, run_config: Optional[dict] = None, resources: Optional[Dict[str, ResourceDefinition]] = None, loggers: Optional[Dict[str, LoggerDefinition]] = None, input_values: Optional[Dict[str, Any]] = None, instance: DagsterInstance = None, output_capturing_enabled: Optional[bool] = True, ) -> NodeExecutionResult: node = check.inst_param(node, "node", NodeDefinition) resources = check.opt_dict_param(resources, "resources", key_type=str, value_type=ResourceDefinition) loggers = check.opt_dict_param(loggers, "logger", key_type=str, value_type=LoggerDefinition) run_config = check.opt_dict_param(run_config, "run_config", key_type=str) input_values = check.opt_dict_param(input_values, "input_values", key_type=str) node_defs = [node] dependencies: Dict[str, Dict[str, DependencyDefinition]] = defaultdict(dict) for input_name, input_value in input_values.items(): dependencies[node.name][input_name] = DependencyDefinition(input_name) node_defs.append(_create_value_solid(input_name, input_value)) mode_def = ModeDefinition( "created", resource_defs=merge_dicts(resources, {EPHEMERAL_IO_MANAGER_KEY: mem_io_manager}), logger_defs=loggers, ) pipeline_def = PipelineDefinition( node_defs, name=f"ephemeral_{node.name}_node_pipeline", mode_defs=[mode_def], dependencies=dependencies, ) pipeline = InMemoryPipeline(pipeline_def) execution_plan = create_execution_plan(pipeline, run_config=run_config, mode=mode_def.name) recorder: Dict[StepOutputHandle, Any] = {} with ephemeral_instance_if_missing(instance) as execute_instance: pipeline_run = execute_instance.create_run_for_pipeline( pipeline_def=pipeline_def, run_config=run_config, mode=mode_def.name, ) _execute_run_iterable = ExecuteRunWithPlanIterable( execution_plan=execution_plan, iterator=pipeline_execution_iterator, execution_context_manager=PipelineExecutionContextManager( pipeline=pipeline, execution_plan=execution_plan, pipeline_run=pipeline_run, instance=execute_instance, run_config=run_config, output_capture=recorder if output_capturing_enabled else None, ), ) event_list = list(_execute_run_iterable) top_level_node_handle = SolidHandle.from_string(node.name) event_list_for_top_lvl_node = [ event for event in event_list if event.solid_handle and event.solid_handle.is_or_descends_from(top_level_node_handle) ] if isinstance(node, SolidDefinition): return InProcessSolidResult(node, SolidHandle(node.name, None), event_list_for_top_lvl_node, recorder) else: return InProcessGraphResult(node, SolidHandle(node.name, None), event_list_for_top_lvl_node, recorder)
def execute_in_process( node: NodeDefinition, run_config: Optional[dict] = None, resources: Optional[Dict[str, ResourceDefinition]] = None, loggers: Optional[Dict[str, LoggerDefinition]] = None, instance: DagsterInstance = None, ) -> ExecutionResult: node = check.inst_param(node, "node", NodeDefinition) resources = check.opt_dict_param(resources, "resources", key_type=str, value_type=ResourceDefinition) loggers = check.opt_dict_param(loggers, "logger", key_type=str, value_type=LoggerDefinition) run_config = check.opt_dict_param(run_config, "run_config", key_type=str) node_defs = [node] mode_def = ModeDefinition( "created", resource_defs=resources, logger_defs=loggers, ) pipeline_def = PipelineDefinition( node_defs, name=f"ephemeral_{node.name}_node_pipeline", mode_defs=[mode_def], ) pipeline = InMemoryPipeline(pipeline_def) execution_plan = create_execution_plan(pipeline, run_config=run_config, mode=mode_def.name) with ephemeral_instance_if_missing(instance) as execute_instance: pipeline_run = execute_instance.create_run_for_pipeline( pipeline_def=pipeline_def, run_config=run_config, mode=mode_def.name, ) _execute_run_iterable = ExecuteRunWithPlanIterable( execution_plan=execution_plan, iterator=pipeline_execution_iterator, execution_context_manager=PipelineExecutionContextManager( execution_plan=execution_plan, pipeline_run=pipeline_run, instance=execute_instance, run_config=run_config, ), ) event_list = list(_execute_run_iterable) top_level_node_handle = SolidHandle.from_string(node.name) event_list_for_top_lvl_node = [ event for event in event_list if event.solid_handle and event.solid_handle.is_or_descends_from(top_level_node_handle) ] return ExecutionResult(node, event_list_for_top_lvl_node)