def from_func(func: Callable, schema: Any,
validation_rules: Dict[str, Any]) -> "_FuncAsProcessor":
if schema is None:
schema = parse_output_schema_from_comment(func)
validation_rules.update(parse_validation_rules_from_comment(func))
tr = _FuncAsProcessor()
tr._wrapper = FunctionWrapper(func, "^e?(c|[dlspq]+)x*z?$",
"^[dlspq]$") # type: ignore
tr._engine_param = (tr._wrapper._params.get_value_by_index(0) if
tr._wrapper.input_code.startswith("e") else None)
tr._use_dfs = "c" in tr._wrapper.input_code
tr._need_output_schema = tr._wrapper.need_output_schema
tr._validation_rules = validation_rules
tr._output_schema = Schema(schema)
if len(tr._output_schema) == 0:
assert_or_throw(
tr._need_output_schema is None or not tr._need_output_schema,
FugueInterfacelessError(
f"schema must be provided for return type {tr._wrapper._rt}"
),
)
else:
assert_or_throw(
tr._need_output_schema is None or tr._need_output_schema,
FugueInterfacelessError(
f"schema must not be provided for return type {tr._wrapper._rt}"
),
)
return tr
def test_function_wrapper_copy():
class Test(object):
def __init__(self):
self.n = 0
def t(self) -> None:
self.n += 1
test = Test()
w1 = FunctionWrapper(test.t, "", "n")
w2 = copy.copy(w1)
w3 = copy.deepcopy(w1)
w1.run([], {}, output=False)
w2.run([], {}, output=False)
w3.run([], {}, output=False)
assert 3 == test.n
def from_func(func: Callable, schema: Any,
validation_rules: Dict[str, Any]) -> "_FuncAsCoTransformer":
assert_or_throw(
len(validation_rules) == 0,
NotImplementedError(
"CoTransformer does not support validation rules"),
)
if schema is None:
schema = parse_output_schema_from_comment(func)
if isinstance(schema, Schema): # to be less strict on determinism
schema = str(schema)
if isinstance(schema, str):
assert_or_throw(
"*" not in schema,
FugueInterfacelessError(
"* can't be used on cotransformer output schema"),
)
assert_arg_not_none(schema, "schema")
tr = _FuncAsCoTransformer()
tr._wrapper = FunctionWrapper( # type: ignore
func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspq]$")
tr._dfs_input = tr._wrapper.input_code[0] == "c" # type: ignore
tr._output_schema_arg = schema # type: ignore
tr._validation_rules = {} # type: ignore
tr._uses_callback = "f" in tr._wrapper.input_code.lower(
) # type: ignore
tr._requires_callback = "F" in tr._wrapper.input_code # type: ignore
return tr
def from_func(func: Callable, schema: Any) -> "_FuncAsCreator":
# pylint: disable=W0201
if schema is None:
schema = parse_output_schema_from_comment(func)
tr = _FuncAsCreator()
tr._wrapper = FunctionWrapper(func, "^e?x*z?$",
"^[dlspq]$") # type: ignore
tr._need_engine = tr._wrapper.input_code.startswith("e")
tr._need_output_schema = "s" == tr._wrapper.output_code
tr._output_schema = Schema(schema)
if len(tr._output_schema) == 0:
assert_or_throw(
not tr._need_output_schema,
FugueInterfacelessError(
f"schema must be provided for return type {tr._wrapper._rt}"
),
)
else:
assert_or_throw(
tr._need_output_schema,
FugueInterfacelessError(
f"schema must not be provided for return type {tr._wrapper._rt}"
),
)
return tr
def from_func(func: Callable) -> "_FuncAsOutputter":
tr = _FuncAsOutputter()
tr._wrapper = FunctionWrapper(func, "^e?(c|[dlsp]+)x*$",
"^n$") # type: ignore
tr._need_engine = tr._wrapper.input_code.startswith("e")
tr._use_dfs = "c" in tr._wrapper.input_code
return tr
def from_func(func: Callable, schema: Any) -> "_FuncAsTransformer":
if schema is None:
schema = parse_output_schema_from_comment(func)
if isinstance(schema, Schema): # to be less strict on determinism
schema = str(schema)
assert_arg_not_none(schema, "schema")
tr = _FuncAsTransformer()
tr._wrapper = FunctionWrapper(func, "^[lsp]x*$",
"^[lsp]$") # type: ignore
tr._output_schema_arg = schema # type: ignore
return tr
def from_func(
func: Callable, validation_rules: Dict[str, Any]
) -> "_FuncAsOutputter":
validation_rules.update(parse_validation_rules_from_comment(func))
tr = _FuncAsOutputter()
tr._wrapper = FunctionWrapper( # type: ignore
func, "^e?(c|[dlspq]+)x*z?$", "^n$"
)
tr._need_engine = tr._wrapper.input_code.startswith("e")
tr._use_dfs = "c" in tr._wrapper.input_code
tr._validation_rules = validation_rules
return tr
def from_func(
func: Callable, schema: Any,
validation_rules: Dict[str, Any]) -> "_FuncAsOutputTransformer":
assert_or_throw(schema is None,
"schema must be None for output transformers")
validation_rules.update(parse_validation_rules_from_comment(func))
tr = _FuncAsOutputTransformer()
tr._wrapper = FunctionWrapper( # type: ignore
func, "^[lspq][fF]?x*z?$", "^[lspnq]$")
tr._output_schema_arg = None # type: ignore
tr._validation_rules = validation_rules # type: ignore
tr._uses_callback = "f" in tr._wrapper.input_code.lower(
) # type: ignore
tr._requires_callback = "F" in tr._wrapper.input_code # type: ignore
return tr
def from_func(func: Callable, schema: Any,
validation_rules: Dict[str, Any]) -> "_FuncAsTransformer":
if schema is None:
schema = parse_output_schema_from_comment(func)
if isinstance(schema, Schema): # to be less strict on determinism
schema = str(schema)
validation_rules.update(parse_validation_rules_from_comment(func))
assert_arg_not_none(schema, "schema")
tr = _FuncAsTransformer()
tr._wrapper = FunctionWrapper( # type: ignore
func, "^[lspq][fF]?x*z?$", "^[lspq]$")
tr._output_schema_arg = schema # type: ignore
tr._validation_rules = validation_rules # type: ignore
tr._uses_callback = "f" in tr._wrapper.input_code.lower(
) # type: ignore
tr._requires_callback = "F" in tr._wrapper.input_code # type: ignore
return tr
def from_func(
func: Callable, schema: Any,
validation_rules: Dict[str, Any]) -> "_FuncAsOutputCoTransformer":
assert_or_throw(schema is None,
"schema must be None for output cotransformers")
assert_or_throw(
len(validation_rules) == 0,
NotImplementedError(
"CoTransformer does not support validation rules"),
)
tr = _FuncAsOutputCoTransformer()
tr._wrapper = FunctionWrapper( # type: ignore
func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspnq]$")
tr._dfs_input = tr._wrapper.input_code[0] == "c" # type: ignore
tr._output_schema_arg = None # type: ignore
tr._validation_rules = {} # type: ignore
tr._uses_callback = "f" in tr._wrapper.input_code.lower(
) # type: ignore
tr._requires_callback = "F" in tr._wrapper.input_code # type: ignore
return tr
def _parse_function(f, params_re, return_re):
FunctionWrapper(f, params_re, return_re)
def test_function_wrapper_determinism():
w1 = FunctionWrapper(f20, "^[ldsp][ldsp]$", "[ldsp]")
w2 = FunctionWrapper(f20, "^[ldsp][ldsp]$", "[ldsp]")
assert w1 is not w2
assert to_uuid(w1) == to_uuid(w2)
def test_function_wrapper():
for f in [f20, f21, f212, f22, f23, f24, f25, f26, f30, f31, f32, f35]:
df = ArrayDataFrame([[0]], "a:int")
w = FunctionWrapper(f, "^[ldsp][ldsp]$", "[ldspq]")
res = w.run([df], dict(a=df), ignore_unknown=False, output_schema="a:int")
df_eq(res, [[0], [0]], "a:int", throw=True)
w.run([df], dict(a=df), ignore_unknown=False, output=False)
# test other data types, simple operations
w = FunctionWrapper(f27)
assert 3 == w(1, 2)
assert 3 == w.run([1, 2], dict(), ignore_unknown=False)
assert 3 == w.run([5], dict(a=1, b=2), ignore_unknown=True) # dict will overwrite
assert 3 == w.run([], dict(a=1, b=2, c=4), ignore_unknown=True)
raises(ValueError, lambda: w.run([], dict(a=1, b=2, c=4), ignore_unknown=False))
# test default and required
w = FunctionWrapper(f28)
assert 3 == w.run([], dict(a=1, b=2), ignore_unknown=False)
assert 2 == w.run([], dict(a=1), ignore_unknown=False)
assert 3 == w.run([], dict(a=1, b=2), ignore_unknown=True)
assert 3 == w.run([], dict(a=1, b=2, c=4), ignore_unknown=True)
raises(ValueError, lambda: w.run([], dict(a=1, b=2, c=4), ignore_unknown=False))
raises(ValueError, lambda: w.run([], dict(b=2), ignore_unknown=True))
# test kwargs
w = FunctionWrapper(f29)
assert 3 == w.run([], dict(a=1, b=2), ignore_unknown=False)
assert 1 == w.run([], dict(a=1), ignore_unknown=False)
assert 3 == w.run([], dict(a=1, b=2), ignore_unknown=True)
assert 7 == w.run([], dict(a=1, b=2, c=4), ignore_unknown=True)
assert 7 == w.run([], dict(a=1, b=2, c=4), ignore_unknown=False)
# test method inside class
class Test(object):
def t(self, a=1, b=2) -> int:
return a + b
test = Test()
# instance method test
w = FunctionWrapper(test.t, "^0?.*", ".*")
assert 4 == w.run([], kwargs={"b": 3}, ignore_unknown=True)
assert 5 == w.run([2], kwargs={"b": 3}, ignore_unknown=True)
def test_function_wrapper():
for f in [f20, f21, f22, f23, f24, f25, f26, f30, f31, f32]:
df = ArrayDataFrame([[0]], "a:int")
w = FunctionWrapper(f, "^[ldsp][ldsp]$", "[ldsp]")
res = w.run([df],
dict(a=df),
ignore_unknown=False,
output_schema="a:int")
df_eq(res, [[0], [0]], "a:int", throw=True)
# test other data types, simple operations
w = FunctionWrapper(f27)
assert 3 == w(1, 2)
assert 3 == w.run([1, 2], dict(), ignore_unknown=False)
assert 3 == w.run([5], dict(a=1, b=2),
ignore_unknown=True) # dict will overwrite
assert 3 == w.run([], dict(a=1, b=2, c=4), ignore_unknown=True)
raises(ValueError,
lambda: w.run([], dict(a=1, b=2, c=4), ignore_unknown=False))
# test default and required
w = FunctionWrapper(f28)
assert 3 == w.run([], dict(a=1, b=2), ignore_unknown=False)
assert 2 == w.run([], dict(a=1), ignore_unknown=False)
assert 3 == w.run([], dict(a=1, b=2), ignore_unknown=True)
assert 3 == w.run([], dict(a=1, b=2, c=4), ignore_unknown=True)
raises(ValueError,
lambda: w.run([], dict(a=1, b=2, c=4), ignore_unknown=False))
raises(ValueError, lambda: w.run([], dict(b=2), ignore_unknown=True))
# test kwargs
w = FunctionWrapper(f29)
assert 3 == w.run([], dict(a=1, b=2), ignore_unknown=False)
assert 1 == w.run([], dict(a=1), ignore_unknown=False)
assert 3 == w.run([], dict(a=1, b=2), ignore_unknown=True)
assert 7 == w.run([], dict(a=1, b=2, c=4), ignore_unknown=True)
assert 7 == w.run([], dict(a=1, b=2, c=4), ignore_unknown=False)
