def extract_feature_labels_weights(
df: pd.DataFrame, features_and_labels, **kwargs
) -> Tuple[Tuple[pd.DataFrame, int], pd.DataFrame, pd.DataFrame, pd.Series,
pd.Series]:
features = get_pandas_object(df, features_and_labels.features,
**kwargs).dropna()
labels = get_pandas_object(df, features_and_labels.labels,
**kwargs).dropna()
targets = call_if_not_none(
get_pandas_object(df, features_and_labels.targets, **kwargs), 'dropna')
sample_weights = call_if_not_none(
get_pandas_object(df, features_and_labels.sample_weights, **kwargs),
'dropna')
gross_loss = call_if_not_none(
get_pandas_object(df, features_and_labels.gross_loss, **kwargs),
'dropna')
common_index = intersection_of_index(features, labels, targets,
sample_weights, gross_loss)
if features_and_labels.label_type is not None:
labels = labels.astype(features_and_labels.label_type)
return ((features.loc[common_index], len(df) - len(features) + 1),
labels.loc[common_index], loc_if_not_none(targets, common_index),
loc_if_not_none(sample_weights, common_index),
loc_if_not_none(gross_loss, common_index))
def test_call_if_not_none(self):
df1 = pd.DataFrame({"a": [np.nan]})
df2 = None
self.assertIsNone(call_if_not_none(df2, 'dropna'))
self.assertEqual(0, len(call_if_not_none(df1, 'dropna')))
self.assertEqual(1, len(df1))
def extract_feature_labels_weights(df: Typing.PatchedDataFrame,
features_and_labels,
**kwargs) -> FeaturesWithLabels:
features, targets, latent = extract_features(df, features_and_labels,
**kwargs)
labels = extract_labels(df, features_and_labels, **kwargs)
sample_weights = call_if_not_none(
get_pandas_object(df, features_and_labels.sample_weights, **kwargs),
'dropna')
gross_loss = call_if_not_none(
get_pandas_object(df, features_and_labels.gross_loss, **kwargs),
'dropna')
# do some sanity check for any non numeric values in any of the data frames
for frame in [features, labels, targets, sample_weights, gross_loss]:
if frame is not None:
# we could have nested arrays so we need to use the un-nested values
values = flatten_nested_list(frame._.values, np.max)
max_value = max([v.max() for v in values])
if np.isscalar(max_value) and np.isinf(max_value):
_log.warning(
f"features containing infinit number\n"
f"{frame[frame.apply(lambda r: np.isinf(r.values).any(), axis=1)]}"
)
frame.replace([np.inf, -np.inf], np.nan, inplace=True)
frame.dropna(inplace=True)
# now get the common index and return the filtered data frames
common_index = intersection_of_index(features, labels, targets,
sample_weights, gross_loss)
return FeaturesWithLabels(
FeaturesWithRequiredSamples(
tuple([f.loc[common_index] for f in features]) if isinstance(
features, tuple) else features.loc[common_index],
len(df) - len(features) + 1, len(features.columns)),
labels.loc[common_index], loc_if_not_none(latent, common_index),
loc_if_not_none(targets, common_index),
loc_if_not_none(sample_weights, common_index),
loc_if_not_none(gross_loss, common_index))
def extract_feature_labels_weights(
df: Typing.PatchedDataFrame, features_and_labels, **kwargs
) -> Tuple[Tuple[pd.DataFrame, int], pd.DataFrame, pd.DataFrame, pd.Series,
pd.Series]:
_, features, targets = extract_features(df, features_and_labels, **kwargs)
labels = get_pandas_object(df, features_and_labels.labels,
**kwargs).dropna()
sample_weights = call_if_not_none(
get_pandas_object(df, features_and_labels.sample_weights, **kwargs),
'dropna')
gross_loss = call_if_not_none(
get_pandas_object(df, features_and_labels.gross_loss, **kwargs),
'dropna')
if features_and_labels.label_type is not None:
labels = labels.astype(features_and_labels.label_type)
# do some sanity check for any non numeric values in any of the data frames
for frame in [features, labels, targets, sample_weights, gross_loss]:
if frame is not None:
max = frame._.max()
if np.isscalar(max) and np.isinf(max):
_log.warning(
"features containing infinit number\n",
frame[frame.apply(lambda r: np.isinf(r.values).any(),
axis=1)])
frame.replace([np.inf, -np.inf], np.nan, inplace=True)
frame.dropna(inplace=True)
# now get the common index and return the filtered data frames
common_index = intersection_of_index(features, labels, targets,
sample_weights, gross_loss)
return ((tuple([f.loc[common_index] for f in features])
if isinstance(features, tuple) else features.loc[common_index],
len(df) - len(features) + 1), labels.loc[common_index],
loc_if_not_none(targets, common_index),
loc_if_not_none(sample_weights, common_index),
loc_if_not_none(gross_loss, common_index))
def extract_features(df: pd.DataFrame, features_and_labels,
**kwargs) -> Tuple[List, pd.DataFrame, pd.DataFrame]:
features = get_pandas_object(df, features_and_labels.features,
**kwargs).dropna()
targets = call_if_not_none(
get_pandas_object(df, features_and_labels.targets, **kwargs), 'dropna')
common_index = intersection_of_index(features, targets)
if len(features) <= 0:
raise ValueError("not enough data!")
return (features_and_labels.label_columns, features.loc[common_index],
loc_if_not_none(targets, common_index))
def extract_feature_labels_weights(
df: Typing.PatchedDataFrame, features_and_labels, **kwargs
) -> Tuple[Tuple[pd.DataFrame, int], pd.DataFrame, pd.DataFrame, pd.Series,
pd.Series]:
features = get_pandas_object(df, features_and_labels.features,
**kwargs).dropna()
labels = get_pandas_object(df, features_and_labels.labels,
**kwargs).dropna()
targets = call_if_not_none(
get_pandas_object(df, features_and_labels.targets, **kwargs), 'dropna')
sample_weights = call_if_not_none(
get_pandas_object(df, features_and_labels.sample_weights, **kwargs),
'dropna')
gross_loss = call_if_not_none(
get_pandas_object(df, features_and_labels.gross_loss, **kwargs),
'dropna')
if features_and_labels.label_type is not None:
labels = labels.astype(features_and_labels.label_type)
for frame in [features, labels, targets, sample_weights, gross_loss]:
if frame is not None:
max = frame._.values.max()
if np.isscalar(max) and np.isinf(max):
_log.warning(
"features containing infinit number\n",
frame[frame.apply(lambda r: np.isinf(r.values).any(),
axis=1)])
frame.replace([np.inf, -np.inf], np.nan, inplace=True)
frame.dropna(inplace=True)
common_index = intersection_of_index(features, labels, targets,
sample_weights, gross_loss)
return ((features.loc[common_index], len(df) - len(features) + 1),
labels.loc[common_index], loc_if_not_none(targets, common_index),
loc_if_not_none(sample_weights, common_index),
loc_if_not_none(gross_loss, common_index))
def extract_features(df: pd.DataFrame, features_and_labels,
**kwargs) -> FeaturesWithTargets:
if isinstance(features_and_labels.features, tuple):
# allow multiple feature sets i.e. for multi input layered networks
features = MultiFrameDecorator([
get_pandas_object(df, f, **kwargs).dropna()
for f in features_and_labels.features
], True)
else:
features = get_pandas_object(df, features_and_labels.features,
**kwargs).dropna()
targets = call_if_not_none(
get_pandas_object(df, features_and_labels.targets, **kwargs), 'dropna')
latent = call_if_not_none(
get_pandas_object(df, features_and_labels.latent, **kwargs), 'dropna')
common_index = intersection_of_index(features, targets)
if len(features) <= 0:
raise ValueError("not enough data!")
return FeaturesWithTargets(features.loc[common_index],
loc_if_not_none(targets, common_index),
loc_if_not_none(latent, common_index))