def _parse_spec(self, obj: Any, to_type: Type[T]) -> T: if isinstance(obj, to_type): return obj if isinstance(obj, str): obj = json.loads(obj) aot(isinstance(obj, dict), lambda: f"{obj} is not dict") return to_type(**obj)
def to_sk_model(obj: Any) -> Type: if isinstance(obj, str): obj = to_type(obj) assert_or_throw( is_classifier(obj) or is_regressor(obj), TypeError(f"{obj} is neither a sklearn classifier or regressor"), ) return obj
def test_obj_to_type(): assert to_type(None) is type(None) raises(TypeError, lambda: to_type(None, str)) assert issubclass(to_type(123), int) raises(TypeError, lambda: to_type(123, str)) i = to_type("tests.utils.Class2") assert issubclass(i, Class2) i = to_type(Class2) assert issubclass(i, Class2) raises(TypeError, lambda: to_type(Class2, BaseClass)) raises(TypeError, lambda: to_type("tests.utils.Class2", BaseClass)) assert ex.__Dummy__ is not __Dummy__ assert to_type("tests.utils.convert_examples.__Dummy__") is ex.__Dummy__
def __init__(self, name: str, data_type: Any, nullable: bool, metadata: Any = None): self.name = assert_triad_var_name(name) self.data_type = to_type(data_type) self.nullable = nullable self.metadata = ParamDict(metadata, deep=True) self.metadata.set_readonly()
def _parse_param( # noqa: C901 self, annotation: Any, param: Optional[inspect.Parameter], none_as_other: bool = True, ) -> "_FuncParam": if annotation == type(None): # noqa: E721 return _NoneParam(param) if annotation == inspect.Parameter.empty: if param is not None and param.kind == param.VAR_POSITIONAL: return _PositionalParam(param) if param is not None and param.kind == param.VAR_KEYWORD: return _KeywordParam(param) return _OtherParam(param) if none_as_other else _NoneParam(param) if (annotation == Callable or annotation == callable # pylint: disable=comparison-with-callable or str(annotation).startswith("typing.Callable")): return _CallableParam(param) if (annotation == Optional[Callable] or annotation == Optional[callable] or str(annotation).startswith("typing.Union[typing.Callable")): return _OptionalCallableParam(param) for _, c in _ANNOTATION_CONVERTERS: if c.check(annotation): return c.convert(param) if annotation == to_type("fugue.execution.ExecutionEngine"): # to prevent cyclic import return ExecutionEngineParam(param, "ExecutionEngine", annotation) if annotation == DataFrames: return _DataFramesParam(param) if annotation == LocalDataFrame: return _LocalDataFrameParam(param) if annotation == DataFrame: return DataFrameParam(param) if annotation == pd.DataFrame: return _PandasParam(param) if annotation == List[List[Any]]: return _ListListParam(param) if annotation == Iterable[List[Any]]: return _IterableListParam(param) if annotation == EmptyAwareIterable[List[Any]]: return _EmptyAwareIterableListParam(param) if annotation == List[Dict[str, Any]]: return _ListDictParam(param) if annotation == Iterable[Dict[str, Any]]: return _IterableDictParam(param) if annotation == EmptyAwareIterable[Dict[str, Any]]: return _EmptyAwareIterableDictParam(param) if annotation == Iterable[pd.DataFrame]: return _IterablePandasParam(param) if param is not None and param.kind == param.VAR_POSITIONAL: return _PositionalParam(param) if param is not None and param.kind == param.VAR_KEYWORD: return _KeywordParam(param) return _OtherParam(param)
def _to_model(obj: Any) -> Any: if isinstance(obj, str): parts = obj.split(".") if len(parts) > 1: import_module(".".join(parts[:-1])) obj = to_type(obj) assert_or_throw( is_classifier(obj) or is_regressor(obj), TypeError(f"{obj} is neither a sklearn classifier or regressor"), ) return obj
def make_rpc_server(conf: Any) -> RPCServer: """Make :class:`~.RPCServer` based on configuration. If '`fugue.rpc.server`` is set, then the value will be used as the server type for the initialization. Otherwise, a :class:`~.NativeRPCServer` instance will be returned :param conf: |FugueConfig| :return: the RPC server """ conf = ParamDict(conf) tp = conf.get_or_none("fugue.rpc.server", str) t_server = NativeRPCServer if tp is None else to_type(tp, RPCServer) return t_server(conf) # type: ignore
def visitFugueSqlEngine( self, ctx: fp.FugueSqlEngineContext) -> Tuple[Any, Dict[str, Any]]: data = self.get_dict(ctx, "using", "params") try: engine: Any = to_type( data["using"], SQLEngine, global_vars=self.global_vars, local_vars=self.local_vars, ) except TypeError: engine = str(data["using"]) return engine, data.get("params", {})
def _parse_param( # noqa: C901 self, annotation: Any, param: Optional[inspect.Parameter], none_as_other: bool = True, ) -> "_FuncParam": if annotation is type(None): # noqa: E721 return _NoneParam(param) if annotation == inspect.Parameter.empty: if param is not None and param.kind == param.VAR_POSITIONAL: return _PositionalParam(param) if param is not None and param.kind == param.VAR_KEYWORD: return _KeywordParam(param) return _OtherParam(param) if none_as_other else _NoneParam(param) if (annotation is Callable or annotation is callable or str(annotation).startswith("typing.Callable")): return _CallableParam(param) if (annotation is Optional[Callable] or annotation is Optional[callable] or str(annotation).startswith("typing.Union[typing.Callable")): return _OptionalCallableParam(param) if annotation is to_type("fugue.execution.ExecutionEngine"): # to prevent cyclic import return _ExecutionEngineParam(param) if annotation is DataFrames: return _DataFramesParam(param) if annotation is LocalDataFrame: return _LocalDataFrameParam(param) if annotation is DataFrame: return _DataFrameParam(param) if annotation is pd.DataFrame: return _PandasParam(param) if annotation is List[List[Any]]: return _ListListParam(param) if annotation is Iterable[List[Any]]: return _IterableListParam(param) if annotation is EmptyAwareIterable[List[Any]]: return _EmptyAwareIterableListParam(param) if annotation is List[Dict[str, Any]]: return _ListDictParam(param) if annotation is Iterable[Dict[str, Any]]: return _IterableDictParam(param) if annotation is EmptyAwareIterable[Dict[str, Any]]: return _EmptyAwareIterableDictParam(param) if annotation is Iterable[pd.DataFrame]: return _IterablePandasParam(param) if param is not None and param.kind == param.VAR_POSITIONAL: return _PositionalParam(param) if param is not None and param.kind == param.VAR_KEYWORD: return _KeywordParam(param) return _OtherParam(param)
def process(self, dfs: DataFrames) -> DataFrame: df = dfs[0] tf = _to_transformer( self.params.get_or_none("transformer", object), self.params.get_or_none("schema", object), ) tf._workflow_conf = self.execution_engine.conf tf._params = self.params.get("params", ParamDict()) # type: ignore tf._partition_spec = self.partition_spec # type: ignore ie = self.params.get("ignore_errors", []) self._ignore_errors = [to_type(x, Exception) for x in ie] if isinstance(tf, Transformer): return self.transform(df, tf) else: return self.cotransform(df, tf)
def process(self, dfs: DataFrames) -> DataFrame: df = dfs[0] tf = _to_transformer( self.params.get_or_none("transformer", object), self.params.get_or_none("schema", object), ) tf._workflow_conf = self.execution_engine.conf tf._params = self.params.get("params", ParamDict()) # type: ignore tf._partition_spec = self.partition_spec rpc_handler = to_rpc_handler(self.params.get_or_throw("rpc_handler", object)) if not isinstance(rpc_handler, EmptyRPCHandler): tf._rpc_client = self.execution_engine.rpc_server.make_client(rpc_handler) ie = self.params.get("ignore_errors", []) self._ignore_errors = [to_type(x, Exception) for x in ie] tf.validate_on_runtime(df) if isinstance(tf, Transformer): return self.transform(df, tf) else: return self.cotransform(df, tf)
def to_keras_spec(obj: Any) -> Type[KerasTrainingSpec]: if isinstance(obj, str) and obj in _TYPE_DICT: return _TYPE_DICT[obj] return to_type(obj, KerasTrainingSpec)
def invoke_to_type(exp): return to_type(exp)