def _parse_spec(self, obj: Any, to_type: Type[T]) -> T:
if isinstance(obj, to_type):
return obj
if isinstance(obj, str):
obj = json.loads(obj)
aot(isinstance(obj, dict), lambda: f"{obj} is not dict")
return to_type(**obj)
def to_sk_model(obj: Any) -> Type:
if isinstance(obj, str):
obj = to_type(obj)
assert_or_throw(
is_classifier(obj) or is_regressor(obj),
TypeError(f"{obj} is neither a sklearn classifier or regressor"),
)
return obj
def test_obj_to_type():
assert to_type(None) is type(None)
raises(TypeError, lambda: to_type(None, str))
assert issubclass(to_type(123), int)
raises(TypeError, lambda: to_type(123, str))
i = to_type("tests.utils.Class2")
assert issubclass(i, Class2)
i = to_type(Class2)
assert issubclass(i, Class2)
raises(TypeError, lambda: to_type(Class2, BaseClass))
raises(TypeError, lambda: to_type("tests.utils.Class2", BaseClass))
assert ex.__Dummy__ is not __Dummy__
assert to_type("tests.utils.convert_examples.__Dummy__") is ex.__Dummy__
def __init__(self,
name: str,
data_type: Any,
nullable: bool,
metadata: Any = None):
self.name = assert_triad_var_name(name)
self.data_type = to_type(data_type)
self.nullable = nullable
self.metadata = ParamDict(metadata, deep=True)
self.metadata.set_readonly()
def _parse_param( # noqa: C901
self,
annotation: Any,
param: Optional[inspect.Parameter],
none_as_other: bool = True,
) -> "_FuncParam":
if annotation == type(None): # noqa: E721
return _NoneParam(param)
if annotation == inspect.Parameter.empty:
if param is not None and param.kind == param.VAR_POSITIONAL:
return _PositionalParam(param)
if param is not None and param.kind == param.VAR_KEYWORD:
return _KeywordParam(param)
return _OtherParam(param) if none_as_other else _NoneParam(param)
if (annotation == Callable or annotation == callable # pylint: disable=comparison-with-callable
or str(annotation).startswith("typing.Callable")):
return _CallableParam(param)
if (annotation == Optional[Callable]
or annotation == Optional[callable]
or str(annotation).startswith("typing.Union[typing.Callable")):
return _OptionalCallableParam(param)
for _, c in _ANNOTATION_CONVERTERS:
if c.check(annotation):
return c.convert(param)
if annotation == to_type("fugue.execution.ExecutionEngine"):
# to prevent cyclic import
return ExecutionEngineParam(param, "ExecutionEngine", annotation)
if annotation == DataFrames:
return _DataFramesParam(param)
if annotation == LocalDataFrame:
return _LocalDataFrameParam(param)
if annotation == DataFrame:
return DataFrameParam(param)
if annotation == pd.DataFrame:
return _PandasParam(param)
if annotation == List[List[Any]]:
return _ListListParam(param)
if annotation == Iterable[List[Any]]:
return _IterableListParam(param)
if annotation == EmptyAwareIterable[List[Any]]:
return _EmptyAwareIterableListParam(param)
if annotation == List[Dict[str, Any]]:
return _ListDictParam(param)
if annotation == Iterable[Dict[str, Any]]:
return _IterableDictParam(param)
if annotation == EmptyAwareIterable[Dict[str, Any]]:
return _EmptyAwareIterableDictParam(param)
if annotation == Iterable[pd.DataFrame]:
return _IterablePandasParam(param)
if param is not None and param.kind == param.VAR_POSITIONAL:
return _PositionalParam(param)
if param is not None and param.kind == param.VAR_KEYWORD:
return _KeywordParam(param)
return _OtherParam(param)
def _to_model(obj: Any) -> Any:
if isinstance(obj, str):
parts = obj.split(".")
if len(parts) > 1:
import_module(".".join(parts[:-1]))
obj = to_type(obj)
assert_or_throw(
is_classifier(obj) or is_regressor(obj),
TypeError(f"{obj} is neither a sklearn classifier or regressor"),
)
return obj
def make_rpc_server(conf: Any) -> RPCServer:
"""Make :class:`~.RPCServer` based on configuration.
If '`fugue.rpc.server`` is set, then the value will be used as
the server type for the initialization. Otherwise, a
:class:`~.NativeRPCServer` instance will be returned
:param conf: |FugueConfig|
:return: the RPC server
"""
conf = ParamDict(conf)
tp = conf.get_or_none("fugue.rpc.server", str)
t_server = NativeRPCServer if tp is None else to_type(tp, RPCServer)
return t_server(conf) # type: ignore
def visitFugueSqlEngine(
self, ctx: fp.FugueSqlEngineContext) -> Tuple[Any, Dict[str, Any]]:
data = self.get_dict(ctx, "using", "params")
try:
engine: Any = to_type(
data["using"],
SQLEngine,
global_vars=self.global_vars,
local_vars=self.local_vars,
)
except TypeError:
engine = str(data["using"])
return engine, data.get("params", {})
def _parse_param( # noqa: C901
self,
annotation: Any,
param: Optional[inspect.Parameter],
none_as_other: bool = True,
) -> "_FuncParam":
if annotation is type(None): # noqa: E721
return _NoneParam(param)
if annotation == inspect.Parameter.empty:
if param is not None and param.kind == param.VAR_POSITIONAL:
return _PositionalParam(param)
if param is not None and param.kind == param.VAR_KEYWORD:
return _KeywordParam(param)
return _OtherParam(param) if none_as_other else _NoneParam(param)
if (annotation is Callable or annotation is callable
or str(annotation).startswith("typing.Callable")):
return _CallableParam(param)
if (annotation is Optional[Callable]
or annotation is Optional[callable]
or str(annotation).startswith("typing.Union[typing.Callable")):
return _OptionalCallableParam(param)
if annotation is to_type("fugue.execution.ExecutionEngine"):
# to prevent cyclic import
return _ExecutionEngineParam(param)
if annotation is DataFrames:
return _DataFramesParam(param)
if annotation is LocalDataFrame:
return _LocalDataFrameParam(param)
if annotation is DataFrame:
return _DataFrameParam(param)
if annotation is pd.DataFrame:
return _PandasParam(param)
if annotation is List[List[Any]]:
return _ListListParam(param)
if annotation is Iterable[List[Any]]:
return _IterableListParam(param)
if annotation is EmptyAwareIterable[List[Any]]:
return _EmptyAwareIterableListParam(param)
if annotation is List[Dict[str, Any]]:
return _ListDictParam(param)
if annotation is Iterable[Dict[str, Any]]:
return _IterableDictParam(param)
if annotation is EmptyAwareIterable[Dict[str, Any]]:
return _EmptyAwareIterableDictParam(param)
if annotation is Iterable[pd.DataFrame]:
return _IterablePandasParam(param)
if param is not None and param.kind == param.VAR_POSITIONAL:
return _PositionalParam(param)
if param is not None and param.kind == param.VAR_KEYWORD:
return _KeywordParam(param)
return _OtherParam(param)
def process(self, dfs: DataFrames) -> DataFrame:
df = dfs[0]
tf = _to_transformer(
self.params.get_or_none("transformer", object),
self.params.get_or_none("schema", object),
)
tf._workflow_conf = self.execution_engine.conf
tf._params = self.params.get("params", ParamDict()) # type: ignore
tf._partition_spec = self.partition_spec # type: ignore
ie = self.params.get("ignore_errors", [])
self._ignore_errors = [to_type(x, Exception) for x in ie]
if isinstance(tf, Transformer):
return self.transform(df, tf)
else:
return self.cotransform(df, tf)
def process(self, dfs: DataFrames) -> DataFrame:
df = dfs[0]
tf = _to_transformer(
self.params.get_or_none("transformer", object),
self.params.get_or_none("schema", object),
)
tf._workflow_conf = self.execution_engine.conf
tf._params = self.params.get("params", ParamDict()) # type: ignore
tf._partition_spec = self.partition_spec
rpc_handler = to_rpc_handler(self.params.get_or_throw("rpc_handler", object))
if not isinstance(rpc_handler, EmptyRPCHandler):
tf._rpc_client = self.execution_engine.rpc_server.make_client(rpc_handler)
ie = self.params.get("ignore_errors", [])
self._ignore_errors = [to_type(x, Exception) for x in ie]
tf.validate_on_runtime(df)
if isinstance(tf, Transformer):
return self.transform(df, tf)
else:
return self.cotransform(df, tf)
def to_keras_spec(obj: Any) -> Type[KerasTrainingSpec]:
if isinstance(obj, str) and obj in _TYPE_DICT:
return _TYPE_DICT[obj]
return to_type(obj, KerasTrainingSpec)
def invoke_to_type(exp):
return to_type(exp)
