def _to_transformer( # noqa: C901 obj: Any, schema: Any = None) -> Union[Transformer, CoTransformer]: exp: Optional[Exception] = None try: return copy.copy(to_instance(obj, Transformer)) except Exception as e: exp = e try: return copy.copy(to_instance(obj, CoTransformer)) except Exception as e: exp = e try: f = to_function(obj) # this is for string expression of function with decorator if isinstance(f, Transformer): return copy.copy(f) # this is for functions without decorator return _FuncAsTransformer.from_func(f, schema) except Exception as e: exp = e try: f = to_function(obj) # this is for string expression of function with decorator if isinstance(f, CoTransformer): return copy.copy(f) # this is for functions without decorator return _FuncAsCoTransformer.from_func(f, schema) except Exception as e: exp = e raise FugueInterfacelessError(f"{obj} is not a valid transformer", exp)
def _to_transformer( # noqa: C901 obj: Any, schema: Any = None, global_vars: Optional[Dict[str, Any]] = None, local_vars: Optional[Dict[str, Any]] = None, validation_rules: Optional[Dict[str, Any]] = None, func_transformer_type: Type = _FuncAsTransformer, func_cotransformer_type: Type = _FuncAsCoTransformer, ) -> Union[Transformer, CoTransformer]: global_vars, local_vars = get_caller_global_local_vars( global_vars, local_vars) exp: Optional[Exception] = None if validation_rules is None: validation_rules = {} try: return copy.copy( to_instance(obj, Transformer, global_vars=global_vars, local_vars=local_vars)) except Exception as e: exp = e try: return copy.copy( to_instance(obj, CoTransformer, global_vars=global_vars, local_vars=local_vars)) except Exception as e: exp = e try: f = to_function(obj, global_vars=global_vars, local_vars=local_vars) # this is for string expression of function with decorator if isinstance(f, Transformer): return copy.copy(f) # this is for functions without decorator return func_transformer_type.from_func( f, schema, validation_rules=validation_rules) except Exception as e: exp = e try: f = to_function(obj, global_vars=global_vars, local_vars=local_vars) # this is for string expression of function with decorator if isinstance(f, CoTransformer): return copy.copy(f) # this is for functions without decorator return func_cotransformer_type.from_func( f, schema, validation_rules=validation_rules) except Exception as e: exp = e raise FugueInterfacelessError(f"{obj} is not a valid transformer", exp)
def _to_processor( obj: Any, schema: Any = None, global_vars: Optional[Dict[str, Any]] = None, local_vars: Optional[Dict[str, Any]] = None, validation_rules: Optional[Dict[str, Any]] = None, ) -> Processor: global_vars, local_vars = get_caller_global_local_vars( global_vars, local_vars) obj = _PROCESSOR_REGISTRY.get(obj) exp: Optional[Exception] = None if validation_rules is None: validation_rules = {} try: return copy.copy( to_instance(obj, Processor, global_vars=global_vars, local_vars=local_vars)) except Exception as e: exp = e try: f = to_function(obj, global_vars=global_vars, local_vars=local_vars) # this is for string expression of function with decorator if isinstance(f, Processor): return copy.copy(f) # this is for functions without decorator return _FuncAsProcessor.from_func(f, schema, validation_rules=validation_rules) except Exception as e: exp = e raise FugueInterfacelessError(f"{obj} is not a valid processor", exp)
def _to_creator( obj: Any, schema: Any = None, global_vars: Optional[Dict[str, Any]] = None, local_vars: Optional[Dict[str, Any]] = None, ) -> Creator: global_vars, local_vars = get_caller_global_local_vars( global_vars, local_vars) exp: Optional[Exception] = None try: return copy.copy( to_instance(obj, Creator, global_vars=global_vars, local_vars=local_vars)) except Exception as e: exp = e try: f = to_function(obj, global_vars=global_vars, local_vars=local_vars) # this is for string expression of function with decorator if isinstance(f, Creator): return copy.copy(f) # this is for functions without decorator return _FuncAsCreator.from_func(f, schema) except Exception as e: exp = e raise FugueInterfacelessError(f"{obj} is not a valid creator", exp)
def _to_outputter( obj: Any, global_vars: Optional[Dict[str, Any]] = None, local_vars: Optional[Dict[str, Any]] = None, validation_rules: Optional[Dict[str, Any]] = None, ) -> Outputter: global_vars, local_vars = get_caller_global_local_vars(global_vars, local_vars) exp: Optional[Exception] = None if validation_rules is None: validation_rules = {} try: return copy.copy( to_instance(obj, Outputter, global_vars=global_vars, local_vars=local_vars) ) except Exception as e: exp = e try: f = to_function(obj, global_vars=global_vars, local_vars=local_vars) # this is for string expression of function with decorator if isinstance(f, Outputter): return copy.copy(f) # this is for functions without decorator return _FuncAsOutputter.from_func(f, validation_rules=validation_rules) except Exception as e: exp = e raise FugueInterfacelessError(f"{obj} is not a valid outputter", exp)
def _interfaceless_wrapper(ctx: TaskContext) -> None: ctx.ensure_all_ready() func = to_function(ctx.metadata.get_or_throw("__interfaceless_func", object)) o = func(**ctx.inputs, **ctx.configs) res = list(o) if isinstance(o, tuple) else [o] n = 0 for k in ctx.outputs.keys(): ctx.outputs[k] = res[n] n += 1
def get_tunable() -> Tunable: if isinstance(obj, Tunable): return copy.copy(obj) try: f = to_function(obj, global_vars=global_vars, local_vars=local_vars) # this is for string expression of function with decorator if isinstance(f, Tunable): return copy.copy(f) # this is for functions without decorator return _FuncAsTunable.from_func(f, distributable) except Exception as e: exp = e raise FugueTuneCompileError(f"{obj} is not a valid tunable function", exp)
def _to_outputter(obj: Any) -> Outputter: exp: Optional[Exception] = None try: return copy.copy(to_instance(obj, Outputter)) except Exception as e: exp = e try: f = to_function(obj) # this is for string expression of function with decorator if isinstance(f, Outputter): return copy.copy(f) # this is for functions without decorator return _FuncAsOutputter.from_func(f) except Exception as e: exp = e raise FugueInterfacelessError(f"{obj} is not a valid outputter", exp)
def _to_processor(obj: Any, schema: Any = None) -> Processor: exp: Optional[Exception] = None try: return copy.copy(to_instance(obj, Processor)) except Exception as e: exp = e try: f = to_function(obj) # this is for string expression of function with decorator if isinstance(f, Processor): return copy.copy(f) # this is for functions without decorator return _FuncAsProcessor.from_func(f, schema) except Exception as e: exp = e raise FugueInterfacelessError(f"{obj} is not a valid processor", exp)
def __init__( self, configs: Any, inputs: Any, outputs: Any, func: Any, metadata: Any = None, deterministic: bool = True, lazy: bool = False, ): self.configs = self._parse_spec_collection(configs, ConfigSpec) self.inputs = self._parse_spec_collection(inputs, InputSpec) self.outputs = self._parse_spec_collection(outputs, OutputSpec) self.metadata = ParamDict(metadata, deep=True) self.func = to_function(func) self.deterministic = deterministic self.lazy = lazy self._node_spec: Optional["_NodeSpec"] = None
def _to_module( obj: Any, global_vars: Optional[Dict[str, Any]] = None, local_vars: Optional[Dict[str, Any]] = None, ) -> "_ModuleFunctionWrapper": if isinstance(obj, _ModuleFunctionWrapper): return obj global_vars, local_vars = get_caller_global_local_vars(global_vars, local_vars) try: f = to_function(obj, global_vars=global_vars, local_vars=local_vars) # this is for string expression of function with decorator if isinstance(f, _ModuleFunctionWrapper): return copy.copy(f) # this is for functions without decorator return _ModuleFunctionWrapper(f) except Exception as e: exp = e raise FugueInterfacelessError(f"{obj} is not a valid module", exp)
def to_noniterative_objective( obj: Any, min_better: bool = True, global_vars: Optional[Dict[str, Any]] = None, local_vars: Optional[Dict[str, Any]] = None, ) -> NonIterativeObjectiveFunc: if isinstance(obj, NonIterativeObjectiveFunc): return copy.copy(obj) global_vars, local_vars = get_caller_global_local_vars(global_vars, local_vars) try: f = to_function(obj, global_vars=global_vars, local_vars=local_vars) # this is for string expression of function with decorator if isinstance(f, NonIterativeObjectiveFunc): return copy.copy(f) # this is for functions without decorator return _NonIterativeObjectiveFuncWrapper.from_func(f, min_better) except Exception as e: exp = e raise TuneCompileError(f"{obj} is not a valid tunable function", exp)
def visitFugueOutputTransformTask( self, ctx: fp.FugueOutputTransformTaskContext) -> None: data = self.get_dict(ctx, "partition", "dfs", "using", "params", "callback") if "dfs" not in data: data["dfs"] = WorkflowDataFrames(self.last) using = _to_output_transformer( data["using"], global_vars=self.global_vars, local_vars=self.local_vars, ) # ignore errors is not implemented self.workflow.out_transform( data["dfs"], using=using, params=data.get("params"), pre_partition=data.get("partition"), callback=to_function(data["callback"], self.global_vars, self.local_vars) if "callback" in data else None, )
def visitFugueTransformTask( self, ctx: fp.FugueTransformTaskContext) -> WorkflowDataFrame: data = self.get_dict(ctx, "partition", "dfs", "params", "callback") if "dfs" not in data: data["dfs"] = WorkflowDataFrames(self.last) p = data["params"] using = _to_transformer( p["using"], schema=p.get("schema"), global_vars=self.global_vars, local_vars=self.local_vars, ) __modified_exception__ = self.to_runtime_error(ctx) # noqa # TODO: ignore errors is not implemented return self.workflow.transform( data["dfs"], using=using, params=p.get("params"), pre_partition=data.get("partition"), callback=to_function(data["callback"], self.global_vars, self.local_vars) if "callback" in data else None, )
def test_obj_to_function(): def _mock(): pass assert _mock == to_function("_mock") f = to_function("dummy_for_test") assert f == dummy_for_test f = to_function(dummy_for_test) assert f == dummy_for_test f = to_function("open") assert f == open f = to_function("tests.utils.test_convert.dummy_for_test") assert f == dummy_for_test f = to_function("triad.utils.convert.to_instance") assert f == to_instance raises(AttributeError, lambda: to_function(None)) raises(AttributeError, lambda: to_function("asdfasdf")) raises(AttributeError, lambda: to_function("BaseClass")) assert to_function("min") == builtins.min class _Mock(object): def x(self, p=10): return p * 10 @property def xx(self): return 0 m = _Mock() assert to_function("m.x") == m.x assert 30 == to_function("m.x")(3) raises(AttributeError, lambda: to_function("m.xx"))