def test_call_dynamic_args_kwargs(self):
"""expect"""
self.assertTrue(call_callable_dynamic_args(lambda a, b: True, 1, b=2))
self.assertTrue(call_callable_dynamic_args(lambda a, b: True, a=1,
b=2))
self.assertTrue(call_callable_dynamic_args(lambda a, b: True, 1, 2))
self.assertRaises(
Exception,
lambda: call_callable_dynamic_args(lambda a, b: True, 1))
self.assertRaises(
Exception,
lambda: call_callable_dynamic_args(lambda a, b: True, 1, c=1))
def plot_to_html_img(plotter: Callable, **kwargs):
import matplotlib.pyplot as plt
with io.BytesIO() as f:
try:
from pandas_ml_common.utils.callable_utils import call_callable_dynamic_args
call_callable_dynamic_args(plotter, **kwargs)
fig = plt.gcf()
fig.savefig(f, format="png", bbox_inches='tight')
image = base64.encodebytes(f.getvalue()).decode("utf-8")
plt.close(fig)
return f'data:image/png;base64, {image}'
except TypeError:
return traceback.print_exc()
def test_final_kwargs(self):
kwargs = call_callable_dynamic_args(
lambda x, y, **kwargs: list(kwargs.keys()),
x=1,
y=2,
z=3,
foo="bar")
self.assertListEqual(['z', 'foo'], kwargs)
def test_call_dynamic_args(self):
"""when"""
arguments = list(range(4))
l1 = lambda a: a
l2 = lambda a, b: a + b
l3 = lambda a, *b: a + sum(b)
l4 = lambda *a: sum(a)
def f1(a, b, c, d):
return a + b + c + d
def f2(a, *b, **kwargs):
return a + sum(b)
def f3(a, b):
return a + b
"""then"""
self.assertEqual(call_callable_dynamic_args(l1, *arguments), 0)
self.assertEqual(call_callable_dynamic_args(l2, *arguments), 1)
self.assertEqual(call_callable_dynamic_args(l3, *arguments), 6)
self.assertEqual(call_callable_dynamic_args(l4, *arguments), 6)
self.assertEqual(call_callable_dynamic_args(f1, *arguments), 6)
self.assertEqual(call_callable_dynamic_args(f2, *arguments), 6)
self.assertEqual(call_callable_dynamic_args(f3, *arguments), 1)
def __call__(self,
df: pd.DataFrame,
extractor: callable = extract_feature_labels_weights,
*args,
**kwargs) -> Union[pd.DataFrame, Tuple[pd.DataFrame, pd.DataFrame, pd.Series]]:
# Basic concept:
# we call an extractor(df, **{**self.kwargs, **kwargs})
# this extractor uses 'get_pandas_object' which itself can handle lambdas with dependecies
# injected from available kwargs
return call_callable_dynamic_args(extractor, df, self, **{**self.kwargs, **kwargs})
def plot_to_html_img(plotter, **kwargs):
import matplotlib.pyplot as plt
from pandas_ml_common.utils.callable_utils import call_callable_dynamic_args
if callable(plotter):
ret_fig = call_callable_dynamic_args(plotter, **kwargs)
fig = ret_fig if isinstance(ret_fig, plt.Figure) else plt.gcf()
else:
fig = plotter
image = serialize_figure(fig, format="png", bbox_inches='tight')
image = base64.encodebytes(image).decode("utf-8")
return f'data:image/png;base64, {image}'
def __call__(
self,
df: pd.DataFrame,
extractor: callable = None,
*args,
**kwargs
) -> Union[pd.DataFrame, Tuple[pd.DataFrame, pd.DataFrame, pd.Series]]:
# Basic concept:
# we call an extractor(df, **{**self.kwargs, **kwargs})
# this extractor uses 'get_pandas_object' which itself can handle lambdas with dependecies
# injected from available kwargs
if extractor is None or extractor in ['training', 'train']:
from pandas_ml_utils.ml.data.extraction.features_and_labels_extractor import extract_feature_labels_weights
extractor = extract_feature_labels_weights
elif extractor in ['eval', 'evaluate', 'predict']:
from pandas_ml_utils.ml.data.extraction.features_and_labels_extractor import extract_features
extractor = extract_features
return call_callable_dynamic_args(extractor, df, self, **{
**self.kwargs,
**kwargs
})
def get_pandas_object(po: PandasObject, item, **kwargs):
if item is None:
_log.info("passed item was None")
return None
elif isinstance(item, Constant):
return pd.Series(np.full(len(po), item.value), name=f"{item.value}", index=po.index)
elif isinstance(item, PandasObject):
return item
else:
if isinstance(item, Callable):
# also allow callables where we pass kwargs and such ...
res = call_callable_dynamic_args(item, po, **kwargs)
if isinstance(res, PandasObject):
return res
else:
return pd.Series(res, index=po.index)
if isinstance(item, List):
res = None
for sub_item in item:
sub_po = get_pandas_object(po, sub_item, **kwargs)
if sub_po is None:
pass # do nothing
if isinstance(sub_po, pd.Series):
if res is None:
res = sub_po.to_frame()
else:
if sub_po.name in res.columns:
raise ValueError(f"{sub_po.name} already in {res.columns}")
else:
res = res.join(sub_po)
elif isinstance(sub_po, pd.DataFrame):
if res is None:
res = sub_po
else:
duplicates = [col for col in sub_po.columns if col in res.columns]
if len(duplicates) > 0:
raise ValueError(f"{duplicates} already in {res.columns}")
else:
res = res.join(sub_po)
else:
raise ValueError(f"Unknown pandas object {type(sub_po)}")
return res
else:
try:
if has_indexed_columns(po):
if item in po.columns:
return po[item]
else:
if isinstance(po.columns, pd.MultiIndex):
# try partial match
cols = {col: col.index(item) for col in po.columns.tolist() if item in col}
if len(cols) <= 0:
# try regex
cols = {col: i for col in po.columns.tolist() for i, part in enumerate(col) if re.compile(item).match(part)}
levels = set(cols.values())
if len(levels) == 1:
return po[cols.keys()].swaplevel(0, levels.pop(), axis=1)
else:
return po[cols.keys()]
else:
# try regex
return po[list(filter(re.compile(item).match, po.columns))]
else:
# return po[item]
raise KeyError("should never get here ?")
except KeyError:
raise KeyError(f"{item} not found in {po.columns if hasattr(po, 'columns') else po}")
def extract(self, func: callable, *args, **kwargs):
return call_callable_dynamic_args(func, self.df, *args, **kwargs)