def _to_transformer( # noqa: C901 obj: Any, schema: Any = None) -> Union[Transformer, CoTransformer]: exp: Optional[Exception] = None try: return copy.copy(to_instance(obj, Transformer)) except Exception as e: exp = e try: return copy.copy(to_instance(obj, CoTransformer)) except Exception as e: exp = e try: f = to_function(obj) # this is for string expression of function with decorator if isinstance(f, Transformer): return copy.copy(f) # this is for functions without decorator return _FuncAsTransformer.from_func(f, schema) except Exception as e: exp = e try: f = to_function(obj) # this is for string expression of function with decorator if isinstance(f, CoTransformer): return copy.copy(f) # this is for functions without decorator return _FuncAsCoTransformer.from_func(f, schema) except Exception as e: exp = e raise FugueInterfacelessError(f"{obj} is not a valid transformer", exp)
def _to_transformer( # noqa: C901 obj: Any, schema: Any = None, global_vars: Optional[Dict[str, Any]] = None, local_vars: Optional[Dict[str, Any]] = None, validation_rules: Optional[Dict[str, Any]] = None, func_transformer_type: Type = _FuncAsTransformer, func_cotransformer_type: Type = _FuncAsCoTransformer, ) -> Union[Transformer, CoTransformer]: global_vars, local_vars = get_caller_global_local_vars( global_vars, local_vars) exp: Optional[Exception] = None if validation_rules is None: validation_rules = {} try: return copy.copy( to_instance(obj, Transformer, global_vars=global_vars, local_vars=local_vars)) except Exception as e: exp = e try: return copy.copy( to_instance(obj, CoTransformer, global_vars=global_vars, local_vars=local_vars)) except Exception as e: exp = e try: f = to_function(obj, global_vars=global_vars, local_vars=local_vars) # this is for string expression of function with decorator if isinstance(f, Transformer): return copy.copy(f) # this is for functions without decorator return func_transformer_type.from_func( f, schema, validation_rules=validation_rules) except Exception as e: exp = e try: f = to_function(obj, global_vars=global_vars, local_vars=local_vars) # this is for string expression of function with decorator if isinstance(f, CoTransformer): return copy.copy(f) # this is for functions without decorator return func_cotransformer_type.from_func( f, schema, validation_rules=validation_rules) except Exception as e: exp = e raise FugueInterfacelessError(f"{obj} is not a valid transformer", exp)
def _to_processor( obj: Any, schema: Any = None, global_vars: Optional[Dict[str, Any]] = None, local_vars: Optional[Dict[str, Any]] = None, validation_rules: Optional[Dict[str, Any]] = None, ) -> Processor: global_vars, local_vars = get_caller_global_local_vars( global_vars, local_vars) obj = _PROCESSOR_REGISTRY.get(obj) exp: Optional[Exception] = None if validation_rules is None: validation_rules = {} try: return copy.copy( to_instance(obj, Processor, global_vars=global_vars, local_vars=local_vars)) except Exception as e: exp = e try: f = to_function(obj, global_vars=global_vars, local_vars=local_vars) # this is for string expression of function with decorator if isinstance(f, Processor): return copy.copy(f) # this is for functions without decorator return _FuncAsProcessor.from_func(f, schema, validation_rules=validation_rules) except Exception as e: exp = e raise FugueInterfacelessError(f"{obj} is not a valid processor", exp)
def _to_creator( obj: Any, schema: Any = None, global_vars: Optional[Dict[str, Any]] = None, local_vars: Optional[Dict[str, Any]] = None, ) -> Creator: global_vars, local_vars = get_caller_global_local_vars( global_vars, local_vars) exp: Optional[Exception] = None try: return copy.copy( to_instance(obj, Creator, global_vars=global_vars, local_vars=local_vars)) except Exception as e: exp = e try: f = to_function(obj, global_vars=global_vars, local_vars=local_vars) # this is for string expression of function with decorator if isinstance(f, Creator): return copy.copy(f) # this is for functions without decorator return _FuncAsCreator.from_func(f, schema) except Exception as e: exp = e raise FugueInterfacelessError(f"{obj} is not a valid creator", exp)
def __init__( self, execution_engine: Any = None, cache: Any = NoOpCache, workflow_engine: Any = None, hooks: Any = WorkflowHooks, ): if execution_engine is None: ee: ExecutionEngine = NativeExecutionEngine() else: ee = to_instance(execution_engine, ExecutionEngine) self._fugue_engine = ee self._lock = RLock() self._results: Dict[Any, DataFrame] = {} if workflow_engine is None: workflow_engine = ParallelExecutionEngine( self.execution_engine.conf.get("fugue.workflow.concurrency", 1), self) super().__init__( cache=cache, engine=workflow_engine, hooks=hooks, logger=ee.log, config=ee.conf, )
def __init__( self, execution_engine: Any = None, cache: Any = NoOpCache, workflow_engine: Any = None, hooks: Any = WorkflowHooks, ): if execution_engine is None: ee: ExecutionEngine = NativeExecutionEngine() else: ee = to_instance(execution_engine, ExecutionEngine) self._fugue_engine = ee self._lock = RLock() self._results: Dict[Any, DataFrame] = {} if workflow_engine is None: workflow_engine = ParallelExecutionEngine( self.execution_engine.conf.get( FUGUE_CONF_WORKFLOW_CONCURRENCY, FUGUE_DEFAULT_CONF[FUGUE_CONF_WORKFLOW_CONCURRENCY], ), self, ) super().__init__( cache=cache, engine=workflow_engine, hooks=hooks, logger=ee.log, config=ee.conf, )
def _to_outputter( obj: Any, global_vars: Optional[Dict[str, Any]] = None, local_vars: Optional[Dict[str, Any]] = None, validation_rules: Optional[Dict[str, Any]] = None, ) -> Outputter: global_vars, local_vars = get_caller_global_local_vars(global_vars, local_vars) exp: Optional[Exception] = None if validation_rules is None: validation_rules = {} try: return copy.copy( to_instance(obj, Outputter, global_vars=global_vars, local_vars=local_vars) ) except Exception as e: exp = e try: f = to_function(obj, global_vars=global_vars, local_vars=local_vars) # this is for string expression of function with decorator if isinstance(f, Outputter): return copy.copy(f) # this is for functions without decorator return _FuncAsOutputter.from_func(f, validation_rules=validation_rules) except Exception as e: exp = e raise FugueInterfacelessError(f"{obj} is not a valid outputter", exp)
def process(self, dfs: DataFrames) -> DataFrame: statement = self.params.get_or_throw("statement", str) engine = self.params.get_or_none("sql_engine", object) if engine is None: engine = self.execution_engine.default_sql_engine elif not isinstance(engine, SQLEngine): engine = to_instance(engine, SQLEngine, args=[self.execution_engine]) return engine.select(dfs, statement)
def _setup_fugue_notebook(ipython: Any, setup_obj: Any) -> None: s = NotebookSetup() if setup_obj is None else to_instance( setup_obj, NotebookSetup) magics = _FugueSQLMagics(ipython, dict(s.get_pre_conf()), dict(s.get_post_conf())) ipython.register_magics(magics) s.register_execution_engines() Show.set_hook(s.get_pretty_print())
def make_engine(engine: Any) -> ExecutionEngine: if isinstance(engine, str) and engine in self._funcs: return self._funcs[engine](conf, **kwargs) for k, f in self._type_funcs.items(): if isinstance(engine, k): return f(engine, conf, **kwargs) if isinstance(engine, ExecutionEngine): if conf is not None: engine.compile_conf.update(conf) engine.compile_conf.update(kwargs) return engine return to_instance( engine, ExecutionEngine, kwargs=dict(conf=conf, **kwargs) )
def _to_outputter(obj: Any) -> Outputter: exp: Optional[Exception] = None try: return copy.copy(to_instance(obj, Outputter)) except Exception as e: exp = e try: f = to_function(obj) # this is for string expression of function with decorator if isinstance(f, Outputter): return copy.copy(f) # this is for functions without decorator return _FuncAsOutputter.from_func(f) except Exception as e: exp = e raise FugueInterfacelessError(f"{obj} is not a valid outputter", exp)
def _to_processor(obj: Any, schema: Any = None) -> Processor: exp: Optional[Exception] = None try: return copy.copy(to_instance(obj, Processor)) except Exception as e: exp = e try: f = to_function(obj) # this is for string expression of function with decorator if isinstance(f, Processor): return copy.copy(f) # this is for functions without decorator return _FuncAsProcessor.from_func(f, schema) except Exception as e: exp = e raise FugueInterfacelessError(f"{obj} is not a valid processor", exp)
def make_execution_engine( self, engine: Any = None, conf: Any = None, **kwargs: Any ) -> ExecutionEngine: if engine is None: engine = "" if isinstance(engine, str) and engine in self._funcs: return self._funcs[engine](conf, **kwargs) if isinstance(engine, ExecutionEngine): assert_or_throw( conf is None and len(kwargs) == 0, lambda: ValueError( f"{engine} is an instance, " f"can't take arguments conf={conf}, kwargs={kwargs}" ), ) return engine return to_instance(engine, ExecutionEngine, kwargs=dict(conf=conf, **kwargs))
def make_sql_engine( self, engine: Any = None, execution_engine: Optional[ExecutionEngine] = None, **kwargs: Any, ) -> SQLEngine: if engine is None: engine = "" if isinstance(engine, str) and engine in self._sql_funcs: return self._sql_funcs[engine](execution_engine, **kwargs) if isinstance(engine, SQLEngine): assert_or_throw( execution_engine is None and len(kwargs) == 0, lambda: ValueError( f"{engine} is an instance, can't take arguments " f"execution_engine={execution_engine}, kwargs={kwargs}" ), ) return engine return to_instance( engine, SQLEngine, kwargs=dict(execution_engine=execution_engine, **kwargs) )
def test_obj_to_instance(): raises(ValueError, lambda: to_instance(None)) i = to_instance("tests.utils.Class2", args=[1, 2, 3]) assert isinstance(i, Class2) assert i.s == "123" i = to_instance(Class2, kwargs={"a": 1, "b": 2, "c": 3}) assert isinstance(i, Class2) assert i.s == "123" i = to_instance(Class2) assert isinstance(i, Class2) i = to_instance(Class2, args=[1, 2, 3]) assert isinstance(i, Class2) raises(TypeError, lambda: to_instance(Class2, BaseClass, args=[1, 2, 3])) i = to_instance(Class2("a:int")) assert isinstance(i, Class2) raises(TypeError, lambda: to_instance(Class2(), BaseClass)) raises(ValueError, lambda: to_instance(Class2(), args=[1])) raises(ValueError, lambda: to_instance(Class2(), kwargs={"a": 1})) assert ex.__Dummy__ is not __Dummy__ assert type(to_instance("__Dummy__")) is __Dummy__ assert type( to_instance("tests.utils.convert_examples.__Dummy__")) is ex.__Dummy__ assert type( to_instance("tests.utils.convert_examples.__Dummy__")) is ex.__Dummy__
def _parse_config(self, data: Any, tp: Type[WFMT], args: List[Any]) -> WFMT: if isinstance(data, tp): return data return cast(WFMT, to_instance(data, expected_base_type=tp, args=args))