def _to_transformer( # noqa: C901
obj: Any, schema: Any = None) -> Union[Transformer, CoTransformer]:
exp: Optional[Exception] = None
try:
return copy.copy(to_instance(obj, Transformer))
except Exception as e:
exp = e
try:
return copy.copy(to_instance(obj, CoTransformer))
except Exception as e:
exp = e
try:
f = to_function(obj)
# this is for string expression of function with decorator
if isinstance(f, Transformer):
return copy.copy(f)
# this is for functions without decorator
return _FuncAsTransformer.from_func(f, schema)
except Exception as e:
exp = e
try:
f = to_function(obj)
# this is for string expression of function with decorator
if isinstance(f, CoTransformer):
return copy.copy(f)
# this is for functions without decorator
return _FuncAsCoTransformer.from_func(f, schema)
except Exception as e:
exp = e
raise FugueInterfacelessError(f"{obj} is not a valid transformer", exp)
def _to_transformer( # noqa: C901
obj: Any,
schema: Any = None,
global_vars: Optional[Dict[str, Any]] = None,
local_vars: Optional[Dict[str, Any]] = None,
validation_rules: Optional[Dict[str, Any]] = None,
func_transformer_type: Type = _FuncAsTransformer,
func_cotransformer_type: Type = _FuncAsCoTransformer,
) -> Union[Transformer, CoTransformer]:
global_vars, local_vars = get_caller_global_local_vars(
global_vars, local_vars)
exp: Optional[Exception] = None
if validation_rules is None:
validation_rules = {}
try:
return copy.copy(
to_instance(obj,
Transformer,
global_vars=global_vars,
local_vars=local_vars))
except Exception as e:
exp = e
try:
return copy.copy(
to_instance(obj,
CoTransformer,
global_vars=global_vars,
local_vars=local_vars))
except Exception as e:
exp = e
try:
f = to_function(obj, global_vars=global_vars, local_vars=local_vars)
# this is for string expression of function with decorator
if isinstance(f, Transformer):
return copy.copy(f)
# this is for functions without decorator
return func_transformer_type.from_func(
f, schema, validation_rules=validation_rules)
except Exception as e:
exp = e
try:
f = to_function(obj, global_vars=global_vars, local_vars=local_vars)
# this is for string expression of function with decorator
if isinstance(f, CoTransformer):
return copy.copy(f)
# this is for functions without decorator
return func_cotransformer_type.from_func(
f, schema, validation_rules=validation_rules)
except Exception as e:
exp = e
raise FugueInterfacelessError(f"{obj} is not a valid transformer", exp)
def _to_processor(
obj: Any,
schema: Any = None,
global_vars: Optional[Dict[str, Any]] = None,
local_vars: Optional[Dict[str, Any]] = None,
validation_rules: Optional[Dict[str, Any]] = None,
) -> Processor:
global_vars, local_vars = get_caller_global_local_vars(
global_vars, local_vars)
obj = _PROCESSOR_REGISTRY.get(obj)
exp: Optional[Exception] = None
if validation_rules is None:
validation_rules = {}
try:
return copy.copy(
to_instance(obj,
Processor,
global_vars=global_vars,
local_vars=local_vars))
except Exception as e:
exp = e
try:
f = to_function(obj, global_vars=global_vars, local_vars=local_vars)
# this is for string expression of function with decorator
if isinstance(f, Processor):
return copy.copy(f)
# this is for functions without decorator
return _FuncAsProcessor.from_func(f,
schema,
validation_rules=validation_rules)
except Exception as e:
exp = e
raise FugueInterfacelessError(f"{obj} is not a valid processor", exp)
def _to_creator(
obj: Any,
schema: Any = None,
global_vars: Optional[Dict[str, Any]] = None,
local_vars: Optional[Dict[str, Any]] = None,
) -> Creator:
global_vars, local_vars = get_caller_global_local_vars(
global_vars, local_vars)
exp: Optional[Exception] = None
try:
return copy.copy(
to_instance(obj,
Creator,
global_vars=global_vars,
local_vars=local_vars))
except Exception as e:
exp = e
try:
f = to_function(obj, global_vars=global_vars, local_vars=local_vars)
# this is for string expression of function with decorator
if isinstance(f, Creator):
return copy.copy(f)
# this is for functions without decorator
return _FuncAsCreator.from_func(f, schema)
except Exception as e:
exp = e
raise FugueInterfacelessError(f"{obj} is not a valid creator", exp)
def _to_outputter(
obj: Any,
global_vars: Optional[Dict[str, Any]] = None,
local_vars: Optional[Dict[str, Any]] = None,
validation_rules: Optional[Dict[str, Any]] = None,
) -> Outputter:
global_vars, local_vars = get_caller_global_local_vars(global_vars, local_vars)
exp: Optional[Exception] = None
if validation_rules is None:
validation_rules = {}
try:
return copy.copy(
to_instance(obj, Outputter, global_vars=global_vars, local_vars=local_vars)
)
except Exception as e:
exp = e
try:
f = to_function(obj, global_vars=global_vars, local_vars=local_vars)
# this is for string expression of function with decorator
if isinstance(f, Outputter):
return copy.copy(f)
# this is for functions without decorator
return _FuncAsOutputter.from_func(f, validation_rules=validation_rules)
except Exception as e:
exp = e
raise FugueInterfacelessError(f"{obj} is not a valid outputter", exp)
def _interfaceless_wrapper(ctx: TaskContext) -> None:
ctx.ensure_all_ready()
func = to_function(ctx.metadata.get_or_throw("__interfaceless_func", object))
o = func(**ctx.inputs, **ctx.configs)
res = list(o) if isinstance(o, tuple) else [o]
n = 0
for k in ctx.outputs.keys():
ctx.outputs[k] = res[n]
n += 1
def get_tunable() -> Tunable:
if isinstance(obj, Tunable):
return copy.copy(obj)
try:
f = to_function(obj,
global_vars=global_vars,
local_vars=local_vars)
# this is for string expression of function with decorator
if isinstance(f, Tunable):
return copy.copy(f)
# this is for functions without decorator
return _FuncAsTunable.from_func(f, distributable)
except Exception as e:
exp = e
raise FugueTuneCompileError(f"{obj} is not a valid tunable function",
exp)
def _to_outputter(obj: Any) -> Outputter:
exp: Optional[Exception] = None
try:
return copy.copy(to_instance(obj, Outputter))
except Exception as e:
exp = e
try:
f = to_function(obj)
# this is for string expression of function with decorator
if isinstance(f, Outputter):
return copy.copy(f)
# this is for functions without decorator
return _FuncAsOutputter.from_func(f)
except Exception as e:
exp = e
raise FugueInterfacelessError(f"{obj} is not a valid outputter", exp)
def _to_processor(obj: Any, schema: Any = None) -> Processor:
exp: Optional[Exception] = None
try:
return copy.copy(to_instance(obj, Processor))
except Exception as e:
exp = e
try:
f = to_function(obj)
# this is for string expression of function with decorator
if isinstance(f, Processor):
return copy.copy(f)
# this is for functions without decorator
return _FuncAsProcessor.from_func(f, schema)
except Exception as e:
exp = e
raise FugueInterfacelessError(f"{obj} is not a valid processor", exp)
def __init__(
self,
configs: Any,
inputs: Any,
outputs: Any,
func: Any,
metadata: Any = None,
deterministic: bool = True,
lazy: bool = False,
):
self.configs = self._parse_spec_collection(configs, ConfigSpec)
self.inputs = self._parse_spec_collection(inputs, InputSpec)
self.outputs = self._parse_spec_collection(outputs, OutputSpec)
self.metadata = ParamDict(metadata, deep=True)
self.func = to_function(func)
self.deterministic = deterministic
self.lazy = lazy
self._node_spec: Optional["_NodeSpec"] = None
def _to_module(
obj: Any,
global_vars: Optional[Dict[str, Any]] = None,
local_vars: Optional[Dict[str, Any]] = None,
) -> "_ModuleFunctionWrapper":
if isinstance(obj, _ModuleFunctionWrapper):
return obj
global_vars, local_vars = get_caller_global_local_vars(global_vars, local_vars)
try:
f = to_function(obj, global_vars=global_vars, local_vars=local_vars)
# this is for string expression of function with decorator
if isinstance(f, _ModuleFunctionWrapper):
return copy.copy(f)
# this is for functions without decorator
return _ModuleFunctionWrapper(f)
except Exception as e:
exp = e
raise FugueInterfacelessError(f"{obj} is not a valid module", exp)
def to_noniterative_objective(
obj: Any,
min_better: bool = True,
global_vars: Optional[Dict[str, Any]] = None,
local_vars: Optional[Dict[str, Any]] = None,
) -> NonIterativeObjectiveFunc:
if isinstance(obj, NonIterativeObjectiveFunc):
return copy.copy(obj)
global_vars, local_vars = get_caller_global_local_vars(global_vars, local_vars)
try:
f = to_function(obj, global_vars=global_vars, local_vars=local_vars)
# this is for string expression of function with decorator
if isinstance(f, NonIterativeObjectiveFunc):
return copy.copy(f)
# this is for functions without decorator
return _NonIterativeObjectiveFuncWrapper.from_func(f, min_better)
except Exception as e:
exp = e
raise TuneCompileError(f"{obj} is not a valid tunable function", exp)
def visitFugueOutputTransformTask(
self, ctx: fp.FugueOutputTransformTaskContext) -> None:
data = self.get_dict(ctx, "partition", "dfs", "using", "params",
"callback")
if "dfs" not in data:
data["dfs"] = WorkflowDataFrames(self.last)
using = _to_output_transformer(
data["using"],
global_vars=self.global_vars,
local_vars=self.local_vars,
)
# ignore errors is not implemented
self.workflow.out_transform(
data["dfs"],
using=using,
params=data.get("params"),
pre_partition=data.get("partition"),
callback=to_function(data["callback"], self.global_vars,
self.local_vars)
if "callback" in data else None,
)
def visitFugueTransformTask(
self, ctx: fp.FugueTransformTaskContext) -> WorkflowDataFrame:
data = self.get_dict(ctx, "partition", "dfs", "params", "callback")
if "dfs" not in data:
data["dfs"] = WorkflowDataFrames(self.last)
p = data["params"]
using = _to_transformer(
p["using"],
schema=p.get("schema"),
global_vars=self.global_vars,
local_vars=self.local_vars,
)
__modified_exception__ = self.to_runtime_error(ctx) # noqa
# TODO: ignore errors is not implemented
return self.workflow.transform(
data["dfs"],
using=using,
params=p.get("params"),
pre_partition=data.get("partition"),
callback=to_function(data["callback"], self.global_vars,
self.local_vars)
if "callback" in data else None,
)
def test_obj_to_function():
def _mock():
pass
assert _mock == to_function("_mock")
f = to_function("dummy_for_test")
assert f == dummy_for_test
f = to_function(dummy_for_test)
assert f == dummy_for_test
f = to_function("open")
assert f == open
f = to_function("tests.utils.test_convert.dummy_for_test")
assert f == dummy_for_test
f = to_function("triad.utils.convert.to_instance")
assert f == to_instance
raises(AttributeError, lambda: to_function(None))
raises(AttributeError, lambda: to_function("asdfasdf"))
raises(AttributeError, lambda: to_function("BaseClass"))
assert to_function("min") == builtins.min
class _Mock(object):
def x(self, p=10):
return p * 10
@property
def xx(self):
return 0
m = _Mock()
assert to_function("m.x") == m.x
assert 30 == to_function("m.x")(3)
raises(AttributeError, lambda: to_function("m.xx"))
