def _eliminate_cleaner(self, train_features, train_targets, test_features):
"""Function to remove data missing or useless.
Parameters
----------
train_features : array
The array of training features.
train_targets : array
A list of training target values.
test_features : array
The array of test features.
"""
train_features = np.array(train_features)
if test_features is not None:
test_features = np.array(test_features)
# Identify clean and informative features.
finite = clean_infinite(train=train_features,
test=test_features,
targets=train_targets)
informative = clean_variance(train=train_features, test=test_features)
# Join lists of features to keep.
self.clean_index = np.intersect1d(finite['index'],
informative['index'])
if test_features is None:
return train_features[:, self.clean_index], train_targets, \
test_features
return train_features[:, self.clean_index], train_targets, \
test_features[:, self.clean_index]
def _eliminate_cleaner(self, train_features, train_targets, test_features):
"""Function to remove data missing or useless.
Parameters
----------
train_features : array
The array of training features.
train_targets : array
A list of training target values.
test_features : array
The array of test features.
"""
reshape_targets = False
if len(np.shape(train_targets)) == 1:
train_targets = np.reshape(train_targets, (len(train_targets), 1))
reshape_targets = True
cleaned = clean_infinite(train=train_features,
test=test_features,
targets=train_targets)
# Assign cleaned training target data.
train_targets = cleaned['targets']
if reshape_targets:
train_targets = np.reshape(train_targets, (len(train_targets), ))
# Keep the indexes to delete.
self.eliminate_index = cleaned['index']
cleaned = clean_variance(train=cleaned['train'], test=cleaned['test'])
# Join lists of features to eliminate.
self.eliminate_index += cleaned['index']
return cleaned['train'], train_targets, cleaned['test']
def clean_infinite(self, df_train, df_test, labels):
"""
"""
# | - clean_infinite
verbose = self.verbose
# #####################################################################
if verbose:
print("Cleaning infinite:")
print("train_data.shape:", df_train.shape)
cleaned_data = clean_infinite(df_train,
test=df_test,
targets=None,
labels=labels,
mask=None,
max_impute_fraction=0,
strategy='mean')
df_train = cleaned_data["train"]
df_test = cleaned_data["test"]
labels = cleaned_data["labels"]
if verbose:
print("train_data.shape:", df_train.shape)
print("")
out_dict = {
"df_train": df_train,
"df_test": df_test,
"labels": labels,
}
return (out_dict)
def test_inf(self):
"""Test cleaning inf variable features."""
features = np.random.random_sample((50, 5))
features[0, 0] = np.nan
features[40:, 1] = np.nan
test = np.random.random_sample((100, 5))
finite = clean.clean_infinite(features,
test=test,
max_impute_fraction=0.1)
self.assertTrue(np.shape(finite['train']) == (50, 4))
self.assertTrue(np.shape(finite['test']) == (100, 4))
def clean_features(features, scale=False):
remove_indices = {
'train': np.array([], dtype=int),
'test': np.array([], dtype=int)
}
for key, feature_set in features.items():
if feature_set is None or len(feature_set) == 0:
continue
bad_structure_indices = \
np.where(np.isfinite(feature_set).all(axis=1) == False)
for b in bad_structure_indices:
if len(np.where(np.isfinite(feature_set[b]) == False)) > 1:
remove_indices[key] = np.append(remove_indices[key], b)
features[key] = np.delete(feature_set, remove_indices[key], axis=0)
if not 'test' in features:
features['test'] = None
# Finite features
features = clean_infinite(features['train'], features['test'])
# Clean variance
features = clean_variance(features['train'], features['test'])
# Clean skewness & standardize
if features['test'] is None or len(features['test']) == 0:
features = clean_skewness(features['train'], skewness=3)
if scale:
features = standardize(features['train'])
else:
features = clean_skewness(features['train'],
features['test'],
skewness=3)
if scale:
features = standardize(features['train'], features['test'])
return features, remove_indices
# In[4]: voro = VoronoiFingerprintGenerator(alist) data_frame = voro.generate() # In cases, where the generated featues does not apply to the input data, `NaN`s are returned. # There are various ways of filling in this kind of data and the simplest is simply to remove features containing infinite values. # # The conventional data format in CatLearn is a matrix, so we first convert the Pandas dataframe into a numpy array. # In[5]: matrix = data_frame.to_numpy() finite_numeric_data = clean_infinite(matrix) print(np.shape(finite_numeric_data['train'])) # Furthermore, you might have data sets where certain features have completely the same value. Use `clean_variance` to get rid of those meaningless features. # In[6]: useful_data = clean_variance(finite_numeric_data['train']) print(np.shape(useful_data['train'])) # We only selected the first 10 data points in this example, so there are likely to be some invariant features across those 10.
if site is None:
site = 'AA'
r.site = site
syss = reactionlist
# Generate the fingreprints for all systems.
trainfingers = np.asarray([return_features(sys) for sys in syss])
train_name = np.asarray([sys['surface'] + ' ' + sys['a'] for sys in syss])
# Get the target values for training and test.
for prop in ['reaction_energy']:
trainval = [float(getattr(sys, prop)) for sys in syss]
# Clean up the data
data_dict0 = clean_infinite(trainfingers)
print(data_dict0['index'])
data_dict1 = clean_variance(data_dict0['train'])
print(data_dict1['index'])
allfingers = data_dict1['train']
# Save all the data.
print('Saving', np.shape(allfingers), 'all data matrix.')
np.save(file=data_path + 'catapp_features.npy',
arr=allfingers,
allow_pickle=True,
fix_imports=True)
np.save(file=data_path + 'catapp_targets.npy',
arr=trainval,
allow_pickle=True,
fix_imports=True)