def calculate(self, feature, **kwargs):
"""Use the metric function passed at initialization.
Note:
If default_return was set, the wrapper will suppress any errors
raised by the wrapped function.
Args:
feature: feature/column of pandas dataset requires it.
**kwargs: any keyword arguments to metric function
Returns:
The metric computation defined by the metric.
Raises:
re_raise: If default return is not set the metric then re-raise
"""
try:
self._last_call = self.fn(feature, **kwargs)
except Exception as re_raise:
logging.debug("There was an exception when calling {}".format(
self.fn))
if self.default_return is not None:
return self.default_return
else:
raise re_raise
return self._last_call if not self.invert else (1.0 - self._last_call)
def inverse_transform(self, X, *args, **kwargs):
"""Give original inputs using fitted transformer. Pandas enabled.
See transformer
Args:
X: transformed inputs
*args: arguments to transformer
**kwargs: keyword arguments to transformer
Returns:
original inputs
Raises:
ValueError: If not a valid output type from transformer.
"""
df = check_df(X)
init_cols = [str(col) for col in df]
func = super(DFTransformer, self).inverse_transform
out = func(df, *args, **kwargs)
# determine name of new columns
name = getattr(self, "name", type(self).__name__)
out_is_transformer = hasattr(out, "__class__") and is_transformer(
out.__class__, method="issubclass") # noqa: E127
# check if the output
# returned by the scikit-learn public function is a transformer or
# not. It will be a transformer in fit calls.
if not (out_is_transformer):
# if the output is a transformer, we do nothing.
if isinstance(out,
pd.DataFrame): # custom handling based on the
# type returned by the sklearn transformer function call
out, graph = _df_post_process(out, init_cols, name)
elif isinstance(out, np.ndarray):
out, graph = _ndarray_post_process(out, df, init_cols,
name)
elif scipy.sparse.issparse(out):
out = out.toarray()
out, graph = _ndarray_post_process(out, df, init_cols,
name)
elif isinstance(out, pd.Series):
graph = [] # just return the series
else:
raise ValueError("undefined input {0}".format(type(out)))
if getattr(self, "keep_columns", False):
out = _keep_columns_process(out, df, name, graph)
if getattr(self, "cache_manager", None) is not None: # only
# used when part of the Foreshadow flow.
for column in X:
self.cache_manager["graph", column] = graph
else:
logging.debug("cache_manager is not set for: "
"{}".format(self))
return out # TODO output is a DataFrame, make it detect based
def fit_transform(self, X, *args, **kwargs):
df = check_df(X)
kwargs.pop("full_df", None)
init_cols = [str(col) for col in df]
func = super(DFTransformer, self).fit_transform
out = func(df, *args, **kwargs)
# determine name of new columns
name = getattr(self, "name", type(self).__name__)
out_is_transformer = hasattr(out, "__class__") and is_transformer(
out.__class__, method="issubclass") # noqa: E127
# check if the output returned by the scikit-learn public function
# is a transformer or not. It will be a transformer in fit calls.
if not (out_is_transformer) and not isinstance(out, pd.DataFrame):
# out_is_transformer: if the output is a transformer,
# we do nothing.
# pd.DataFrame: fit_transform will likely be
# passed to the TransformerMixin fit_transform, which just
# calls .fit and .transform. Processing will be handled
# there
if isinstance(out, np.ndarray): # output was not yet
# transformed to DataFrame
out, graph = _ndarray_post_process(out, df, init_cols,
name)
elif scipy.sparse.issparse(out):
out = out.toarray()
out, graph = _ndarray_post_process(out, df, init_cols,
name)
elif isinstance(out, pd.Series):
graph = [] # just return the series
else:
raise ValueError("undefined input {0}".format(type(out)))
if getattr(self, "keep_columns", False):
out = _keep_columns_process(out, df, name, graph)
if getattr(self, "cache_manager", None) is not None: # only
# used when part of the Foreshadow flow.
for column in X:
self.cache_manager["graph", column] = graph
else:
logging.debug("cache_manager is not set for: "
"{}".format(self))
return out
def pick_transformer(self, X, y=None, **fit_params):
"""Get best transformer for a given column.
Args:
X: input DataFrame
y: input labels
**fit_params: fit_params
Returns:
Best data cleaning transformer.
"""
# TODO do we want to parallize this step?
cleaners = config.get_cleaners(cleaners=True)
user_provided_cleaners = self.cache_manager[
AcceptedKey.CUSTOMIZED_TRANSFORMERS][ConfigKey.CUSTOMIZED_CLEANERS]
if len(user_provided_cleaners) > 0:
cleaners.extend(user_provided_cleaners)
best_score = 0
best_cleaner = None
logging.debug("Picking cleaners...")
# The sampling is to speed up the metric score calculation as it may
# not be necessary to scan every row in the data frame to generate a
# score.
sampled_df = self.sample_data_frame(df=X)
# TODO if this improvement is not sufficient, we can try using
# multiprocessing to get the scores instead of doing it sequentially.
for cleaner in cleaners:
cleaner = cleaner()
score = cleaner.metric_score(sampled_df)
if score > best_score:
best_score = score
best_cleaner = cleaner
if best_cleaner is None:
return NoTransform()
logging.debug("Picked...")
return best_cleaner
def resolve(self, X, *args, **kwargs):
"""Resolve the underlying concrete transformer.
Sets self.cache_manager with the domain tag.
Args:
X: input DataFrame
*args: args to super
**kwargs: kwargs to super
Returns:
Return from super.
"""
ret = super().resolve(X, *args, **kwargs)
if self.cache_manager is not None:
self.cache_manager[
"domain", X.columns[0]] = self.transformer.__class__.__name__
else:
logging.debug("cache_manager was None")
return ret
def metric_score(self, X):
"""Compute the score for this cleaner using confidence_computation.
confidence_computation is passed through init for each subclass.
The confidence determines which cleaner/flattener is picked in an
OVR fashion.
Args:
X: input DataFrame.
Returns:
float: confidence value.
"""
# TODO can we also do a sampling here?
logging.debug("Calculating scores....")
scores = []
for metric_wrapper, weight in self.confidence_computation.items():
scores.append(
metric_wrapper.calculate(X, cleaner=self.transform_row)
* weight
)
logging.debug("End calculating scores...")
return sum(scores)