def _to_transformer( # noqa: C901
obj: Any, schema: Any = None) -> Union[Transformer, CoTransformer]:
exp: Optional[Exception] = None
try:
return copy.copy(to_instance(obj, Transformer))
except Exception as e:
exp = e
try:
return copy.copy(to_instance(obj, CoTransformer))
except Exception as e:
exp = e
try:
f = to_function(obj)
# this is for string expression of function with decorator
if isinstance(f, Transformer):
return copy.copy(f)
# this is for functions without decorator
return _FuncAsTransformer.from_func(f, schema)
except Exception as e:
exp = e
try:
f = to_function(obj)
# this is for string expression of function with decorator
if isinstance(f, CoTransformer):
return copy.copy(f)
# this is for functions without decorator
return _FuncAsCoTransformer.from_func(f, schema)
except Exception as e:
exp = e
raise FugueInterfacelessError(f"{obj} is not a valid transformer", exp)
def _to_transformer( # noqa: C901
obj: Any,
schema: Any = None,
global_vars: Optional[Dict[str, Any]] = None,
local_vars: Optional[Dict[str, Any]] = None,
validation_rules: Optional[Dict[str, Any]] = None,
func_transformer_type: Type = _FuncAsTransformer,
func_cotransformer_type: Type = _FuncAsCoTransformer,
) -> Union[Transformer, CoTransformer]:
global_vars, local_vars = get_caller_global_local_vars(
global_vars, local_vars)
exp: Optional[Exception] = None
if validation_rules is None:
validation_rules = {}
try:
return copy.copy(
to_instance(obj,
Transformer,
global_vars=global_vars,
local_vars=local_vars))
except Exception as e:
exp = e
try:
return copy.copy(
to_instance(obj,
CoTransformer,
global_vars=global_vars,
local_vars=local_vars))
except Exception as e:
exp = e
try:
f = to_function(obj, global_vars=global_vars, local_vars=local_vars)
# this is for string expression of function with decorator
if isinstance(f, Transformer):
return copy.copy(f)
# this is for functions without decorator
return func_transformer_type.from_func(
f, schema, validation_rules=validation_rules)
except Exception as e:
exp = e
try:
f = to_function(obj, global_vars=global_vars, local_vars=local_vars)
# this is for string expression of function with decorator
if isinstance(f, CoTransformer):
return copy.copy(f)
# this is for functions without decorator
return func_cotransformer_type.from_func(
f, schema, validation_rules=validation_rules)
except Exception as e:
exp = e
raise FugueInterfacelessError(f"{obj} is not a valid transformer", exp)
def _to_processor(
obj: Any,
schema: Any = None,
global_vars: Optional[Dict[str, Any]] = None,
local_vars: Optional[Dict[str, Any]] = None,
validation_rules: Optional[Dict[str, Any]] = None,
) -> Processor:
global_vars, local_vars = get_caller_global_local_vars(
global_vars, local_vars)
obj = _PROCESSOR_REGISTRY.get(obj)
exp: Optional[Exception] = None
if validation_rules is None:
validation_rules = {}
try:
return copy.copy(
to_instance(obj,
Processor,
global_vars=global_vars,
local_vars=local_vars))
except Exception as e:
exp = e
try:
f = to_function(obj, global_vars=global_vars, local_vars=local_vars)
# this is for string expression of function with decorator
if isinstance(f, Processor):
return copy.copy(f)
# this is for functions without decorator
return _FuncAsProcessor.from_func(f,
schema,
validation_rules=validation_rules)
except Exception as e:
exp = e
raise FugueInterfacelessError(f"{obj} is not a valid processor", exp)
def _to_creator(
obj: Any,
schema: Any = None,
global_vars: Optional[Dict[str, Any]] = None,
local_vars: Optional[Dict[str, Any]] = None,
) -> Creator:
global_vars, local_vars = get_caller_global_local_vars(
global_vars, local_vars)
exp: Optional[Exception] = None
try:
return copy.copy(
to_instance(obj,
Creator,
global_vars=global_vars,
local_vars=local_vars))
except Exception as e:
exp = e
try:
f = to_function(obj, global_vars=global_vars, local_vars=local_vars)
# this is for string expression of function with decorator
if isinstance(f, Creator):
return copy.copy(f)
# this is for functions without decorator
return _FuncAsCreator.from_func(f, schema)
except Exception as e:
exp = e
raise FugueInterfacelessError(f"{obj} is not a valid creator", exp)
def __init__(
self,
execution_engine: Any = None,
cache: Any = NoOpCache,
workflow_engine: Any = None,
hooks: Any = WorkflowHooks,
):
if execution_engine is None:
ee: ExecutionEngine = NativeExecutionEngine()
else:
ee = to_instance(execution_engine, ExecutionEngine)
self._fugue_engine = ee
self._lock = RLock()
self._results: Dict[Any, DataFrame] = {}
if workflow_engine is None:
workflow_engine = ParallelExecutionEngine(
self.execution_engine.conf.get("fugue.workflow.concurrency",
1), self)
super().__init__(
cache=cache,
engine=workflow_engine,
hooks=hooks,
logger=ee.log,
config=ee.conf,
)
def __init__(
self,
execution_engine: Any = None,
cache: Any = NoOpCache,
workflow_engine: Any = None,
hooks: Any = WorkflowHooks,
):
if execution_engine is None:
ee: ExecutionEngine = NativeExecutionEngine()
else:
ee = to_instance(execution_engine, ExecutionEngine)
self._fugue_engine = ee
self._lock = RLock()
self._results: Dict[Any, DataFrame] = {}
if workflow_engine is None:
workflow_engine = ParallelExecutionEngine(
self.execution_engine.conf.get(
FUGUE_CONF_WORKFLOW_CONCURRENCY,
FUGUE_DEFAULT_CONF[FUGUE_CONF_WORKFLOW_CONCURRENCY],
),
self,
)
super().__init__(
cache=cache,
engine=workflow_engine,
hooks=hooks,
logger=ee.log,
config=ee.conf,
)
def _to_outputter(
obj: Any,
global_vars: Optional[Dict[str, Any]] = None,
local_vars: Optional[Dict[str, Any]] = None,
validation_rules: Optional[Dict[str, Any]] = None,
) -> Outputter:
global_vars, local_vars = get_caller_global_local_vars(global_vars, local_vars)
exp: Optional[Exception] = None
if validation_rules is None:
validation_rules = {}
try:
return copy.copy(
to_instance(obj, Outputter, global_vars=global_vars, local_vars=local_vars)
)
except Exception as e:
exp = e
try:
f = to_function(obj, global_vars=global_vars, local_vars=local_vars)
# this is for string expression of function with decorator
if isinstance(f, Outputter):
return copy.copy(f)
# this is for functions without decorator
return _FuncAsOutputter.from_func(f, validation_rules=validation_rules)
except Exception as e:
exp = e
raise FugueInterfacelessError(f"{obj} is not a valid outputter", exp)
def process(self, dfs: DataFrames) -> DataFrame:
statement = self.params.get_or_throw("statement", str)
engine = self.params.get_or_none("sql_engine", object)
if engine is None:
engine = self.execution_engine.default_sql_engine
elif not isinstance(engine, SQLEngine):
engine = to_instance(engine, SQLEngine, args=[self.execution_engine])
return engine.select(dfs, statement)
def _setup_fugue_notebook(ipython: Any, setup_obj: Any) -> None:
s = NotebookSetup() if setup_obj is None else to_instance(
setup_obj, NotebookSetup)
magics = _FugueSQLMagics(ipython, dict(s.get_pre_conf()),
dict(s.get_post_conf()))
ipython.register_magics(magics)
s.register_execution_engines()
Show.set_hook(s.get_pretty_print())
def make_engine(engine: Any) -> ExecutionEngine:
if isinstance(engine, str) and engine in self._funcs:
return self._funcs[engine](conf, **kwargs)
for k, f in self._type_funcs.items():
if isinstance(engine, k):
return f(engine, conf, **kwargs)
if isinstance(engine, ExecutionEngine):
if conf is not None:
engine.compile_conf.update(conf)
engine.compile_conf.update(kwargs)
return engine
return to_instance(
engine, ExecutionEngine, kwargs=dict(conf=conf, **kwargs)
)
def _to_outputter(obj: Any) -> Outputter:
exp: Optional[Exception] = None
try:
return copy.copy(to_instance(obj, Outputter))
except Exception as e:
exp = e
try:
f = to_function(obj)
# this is for string expression of function with decorator
if isinstance(f, Outputter):
return copy.copy(f)
# this is for functions without decorator
return _FuncAsOutputter.from_func(f)
except Exception as e:
exp = e
raise FugueInterfacelessError(f"{obj} is not a valid outputter", exp)
def _to_processor(obj: Any, schema: Any = None) -> Processor:
exp: Optional[Exception] = None
try:
return copy.copy(to_instance(obj, Processor))
except Exception as e:
exp = e
try:
f = to_function(obj)
# this is for string expression of function with decorator
if isinstance(f, Processor):
return copy.copy(f)
# this is for functions without decorator
return _FuncAsProcessor.from_func(f, schema)
except Exception as e:
exp = e
raise FugueInterfacelessError(f"{obj} is not a valid processor", exp)
def make_execution_engine(
self, engine: Any = None, conf: Any = None, **kwargs: Any
) -> ExecutionEngine:
if engine is None:
engine = ""
if isinstance(engine, str) and engine in self._funcs:
return self._funcs[engine](conf, **kwargs)
if isinstance(engine, ExecutionEngine):
assert_or_throw(
conf is None and len(kwargs) == 0,
lambda: ValueError(
f"{engine} is an instance, "
f"can't take arguments conf={conf}, kwargs={kwargs}"
),
)
return engine
return to_instance(engine, ExecutionEngine, kwargs=dict(conf=conf, **kwargs))
def make_sql_engine(
self,
engine: Any = None,
execution_engine: Optional[ExecutionEngine] = None,
**kwargs: Any,
) -> SQLEngine:
if engine is None:
engine = ""
if isinstance(engine, str) and engine in self._sql_funcs:
return self._sql_funcs[engine](execution_engine, **kwargs)
if isinstance(engine, SQLEngine):
assert_or_throw(
execution_engine is None and len(kwargs) == 0,
lambda: ValueError(
f"{engine} is an instance, can't take arguments "
f"execution_engine={execution_engine}, kwargs={kwargs}"
),
)
return engine
return to_instance(
engine, SQLEngine, kwargs=dict(execution_engine=execution_engine, **kwargs)
)
def test_obj_to_instance():
raises(ValueError, lambda: to_instance(None))
i = to_instance("tests.utils.Class2", args=[1, 2, 3])
assert isinstance(i, Class2)
assert i.s == "123"
i = to_instance(Class2, kwargs={"a": 1, "b": 2, "c": 3})
assert isinstance(i, Class2)
assert i.s == "123"
i = to_instance(Class2)
assert isinstance(i, Class2)
i = to_instance(Class2, args=[1, 2, 3])
assert isinstance(i, Class2)
raises(TypeError, lambda: to_instance(Class2, BaseClass, args=[1, 2, 3]))
i = to_instance(Class2("a:int"))
assert isinstance(i, Class2)
raises(TypeError, lambda: to_instance(Class2(), BaseClass))
raises(ValueError, lambda: to_instance(Class2(), args=[1]))
raises(ValueError, lambda: to_instance(Class2(), kwargs={"a": 1}))
assert ex.__Dummy__ is not __Dummy__
assert type(to_instance("__Dummy__")) is __Dummy__
assert type(
to_instance("tests.utils.convert_examples.__Dummy__")) is ex.__Dummy__
assert type(
to_instance("tests.utils.convert_examples.__Dummy__")) is ex.__Dummy__
def _parse_config(self, data: Any, tp: Type[WFMT],
args: List[Any]) -> WFMT:
if isinstance(data, tp):
return data
return cast(WFMT, to_instance(data, expected_base_type=tp, args=args))
