def test_call_dynamic_args_kwargs(self): """expect""" self.assertTrue(call_callable_dynamic_args(lambda a, b: True, 1, b=2)) self.assertTrue(call_callable_dynamic_args(lambda a, b: True, a=1, b=2)) self.assertTrue(call_callable_dynamic_args(lambda a, b: True, 1, 2)) self.assertRaises( Exception, lambda: call_callable_dynamic_args(lambda a, b: True, 1)) self.assertRaises( Exception, lambda: call_callable_dynamic_args(lambda a, b: True, 1, c=1))
def plot_to_html_img(plotter: Callable, **kwargs): import matplotlib.pyplot as plt with io.BytesIO() as f: try: from pandas_ml_common.utils.callable_utils import call_callable_dynamic_args call_callable_dynamic_args(plotter, **kwargs) fig = plt.gcf() fig.savefig(f, format="png", bbox_inches='tight') image = base64.encodebytes(f.getvalue()).decode("utf-8") plt.close(fig) return f'data:image/png;base64, {image}' except TypeError: return traceback.print_exc()
def test_final_kwargs(self): kwargs = call_callable_dynamic_args( lambda x, y, **kwargs: list(kwargs.keys()), x=1, y=2, z=3, foo="bar") self.assertListEqual(['z', 'foo'], kwargs)
def test_call_dynamic_args(self): """when""" arguments = list(range(4)) l1 = lambda a: a l2 = lambda a, b: a + b l3 = lambda a, *b: a + sum(b) l4 = lambda *a: sum(a) def f1(a, b, c, d): return a + b + c + d def f2(a, *b, **kwargs): return a + sum(b) def f3(a, b): return a + b """then""" self.assertEqual(call_callable_dynamic_args(l1, *arguments), 0) self.assertEqual(call_callable_dynamic_args(l2, *arguments), 1) self.assertEqual(call_callable_dynamic_args(l3, *arguments), 6) self.assertEqual(call_callable_dynamic_args(l4, *arguments), 6) self.assertEqual(call_callable_dynamic_args(f1, *arguments), 6) self.assertEqual(call_callable_dynamic_args(f2, *arguments), 6) self.assertEqual(call_callable_dynamic_args(f3, *arguments), 1)
def __call__(self, df: pd.DataFrame, extractor: callable = extract_feature_labels_weights, *args, **kwargs) -> Union[pd.DataFrame, Tuple[pd.DataFrame, pd.DataFrame, pd.Series]]: # Basic concept: # we call an extractor(df, **{**self.kwargs, **kwargs}) # this extractor uses 'get_pandas_object' which itself can handle lambdas with dependecies # injected from available kwargs return call_callable_dynamic_args(extractor, df, self, **{**self.kwargs, **kwargs})
def plot_to_html_img(plotter, **kwargs): import matplotlib.pyplot as plt from pandas_ml_common.utils.callable_utils import call_callable_dynamic_args if callable(plotter): ret_fig = call_callable_dynamic_args(plotter, **kwargs) fig = ret_fig if isinstance(ret_fig, plt.Figure) else plt.gcf() else: fig = plotter image = serialize_figure(fig, format="png", bbox_inches='tight') image = base64.encodebytes(image).decode("utf-8") return f'data:image/png;base64, {image}'
def __call__( self, df: pd.DataFrame, extractor: callable = None, *args, **kwargs ) -> Union[pd.DataFrame, Tuple[pd.DataFrame, pd.DataFrame, pd.Series]]: # Basic concept: # we call an extractor(df, **{**self.kwargs, **kwargs}) # this extractor uses 'get_pandas_object' which itself can handle lambdas with dependecies # injected from available kwargs if extractor is None or extractor in ['training', 'train']: from pandas_ml_utils.ml.data.extraction.features_and_labels_extractor import extract_feature_labels_weights extractor = extract_feature_labels_weights elif extractor in ['eval', 'evaluate', 'predict']: from pandas_ml_utils.ml.data.extraction.features_and_labels_extractor import extract_features extractor = extract_features return call_callable_dynamic_args(extractor, df, self, **{ **self.kwargs, **kwargs })
def get_pandas_object(po: PandasObject, item, **kwargs): if item is None: _log.info("passed item was None") return None elif isinstance(item, Constant): return pd.Series(np.full(len(po), item.value), name=f"{item.value}", index=po.index) elif isinstance(item, PandasObject): return item else: if isinstance(item, Callable): # also allow callables where we pass kwargs and such ... res = call_callable_dynamic_args(item, po, **kwargs) if isinstance(res, PandasObject): return res else: return pd.Series(res, index=po.index) if isinstance(item, List): res = None for sub_item in item: sub_po = get_pandas_object(po, sub_item, **kwargs) if sub_po is None: pass # do nothing if isinstance(sub_po, pd.Series): if res is None: res = sub_po.to_frame() else: if sub_po.name in res.columns: raise ValueError(f"{sub_po.name} already in {res.columns}") else: res = res.join(sub_po) elif isinstance(sub_po, pd.DataFrame): if res is None: res = sub_po else: duplicates = [col for col in sub_po.columns if col in res.columns] if len(duplicates) > 0: raise ValueError(f"{duplicates} already in {res.columns}") else: res = res.join(sub_po) else: raise ValueError(f"Unknown pandas object {type(sub_po)}") return res else: try: if has_indexed_columns(po): if item in po.columns: return po[item] else: if isinstance(po.columns, pd.MultiIndex): # try partial match cols = {col: col.index(item) for col in po.columns.tolist() if item in col} if len(cols) <= 0: # try regex cols = {col: i for col in po.columns.tolist() for i, part in enumerate(col) if re.compile(item).match(part)} levels = set(cols.values()) if len(levels) == 1: return po[cols.keys()].swaplevel(0, levels.pop(), axis=1) else: return po[cols.keys()] else: # try regex return po[list(filter(re.compile(item).match, po.columns))] else: # return po[item] raise KeyError("should never get here ?") except KeyError: raise KeyError(f"{item} not found in {po.columns if hasattr(po, 'columns') else po}")
def extract(self, func: callable, *args, **kwargs): return call_callable_dynamic_args(func, self.df, *args, **kwargs)