def test_sklearnmodel(self):
X = pd.DataFrame(np.random.uniform(low=0.0, high=100, size=(50, 5)))
y = pd.Series(np.random.uniform(low=0.0, high=100, size=(50, )))
model = SklearnModel(model='LinearRegression')
model.fit(X=X, y=y)
ypred = model.predict(X=X, as_frame=True)
self.assertEqual(ypred.shape, y.shape)
return
def fit(self, X, y, model, cv=None, savepath=None):
rst = dict()
param_dict = self._get_grid_param_dict()
if savepath is None:
savepath = os.getcwd()
estimator_name = model.model.__class__.__name__
param_dict = self._search_space_generator(param_dict)
if cv is None:
cv = ms.RepeatedKFold()
metrics = Metrics(metrics_list=None)._metric_zoo()
if self.scoring is None:
scoring = make_scorer(
metrics['mean_absolute_error'][1],
greater_is_better=metrics['mean_absolute_error'][0]
) # Note using True b/c if False then sklearn multiplies by -1
else:
scoring = make_scorer(
metrics[self.scoring][1],
greater_is_better=metrics[self.scoring][0]
) # Note using True b/c if False then sklearn multiplies by -1
model = GridSearchCV(model.model,
param_dict,
scoring=scoring,
cv=cv,
refit=True,
n_jobs=self.n_jobs,
verbose=0)
try:
rst[estimator_name] = model.fit(X, y)
except:
print(
'Hyperparameter optimization failed, likely due to inappropriate domain of values to optimize'
' one or more parameters over. Please check your input file and the sklearn docs for the mode'
' you are optimizing for the domain of correct values')
exit()
best_estimator = rst[estimator_name].best_estimator_
self._save_output(savepath, rst)
# Need to rebuild the estimator as SklearnModel
best_estimator = SklearnModel(model=best_estimator.__class__.__name__,
**best_estimator.get_params())
return best_estimator
def test_nosplit(self):
X = pd.DataFrame(np.random.uniform(low=0.0, high=100, size=(10, 10)))
y = pd.Series(np.random.uniform(low=0.0, high=100, size=(10, )))
model = SklearnModel(model='LinearRegression')
splitter = NoSplit()
splitter.evaluate(X=X,
y=y,
models=[model],
savepath=os.getcwd(),
plots=list())
for d in splitter.splitdirs:
self.assertTrue(os.path.exists(d))
shutil.rmtree(d)
return
def test_leaveoutcluster(self):
X = pd.DataFrame(np.random.uniform(low=0.0, high=100, size=(5, 10)))
y = pd.Series(np.random.uniform(low=0.0, high=100, size=(5, )))
splitter = LeaveOutClusterCV(cluster='KMeans', n_clusters=5)
model = SklearnModel(model='LinearRegression')
splitter.evaluate(X=X,
y=y,
models=[model],
savepath=os.getcwd(),
plots=list())
for d in splitter.splitdirs:
self.assertTrue(os.path.exists(d))
shutil.rmtree(d)
return
def test_bootstrap(self):
X = pd.DataFrame(np.random.uniform(low=0.0, high=100, size=(25, 10)))
y = pd.Series(np.random.uniform(low=0.0, high=100, size=(25, )))
splitter = Bootstrap(n=25, n_bootstraps=3, train_size=0.5)
model = SklearnModel(model='LinearRegression')
splitter.evaluate(X=X,
y=y,
models=[model],
groups=None,
savepath=os.getcwd(),
plots=list())
for d in splitter.splitdirs:
self.assertTrue(os.path.exists(d))
shutil.rmtree(d)
return
def test_leaveoutpercent(self):
X = pd.DataFrame(np.random.uniform(low=0.0, high=100, size=(25, 10)))
y = pd.Series(np.random.uniform(low=0.0, high=100, size=(25, )))
splitter = LeaveOutPercent(percent_leave_out=0.20, n_repeats=5)
model = SklearnModel(model='LinearRegression')
splitter.evaluate(X=X,
y=y,
models=[model],
groups=None,
savepath=os.getcwd(),
plots=list())
for d in splitter.splitdirs:
self.assertTrue(os.path.exists(d))
shutil.rmtree(d)
return
def test_justeachgroup(self):
X = pd.DataFrame(np.random.uniform(low=0.0, high=100, size=(5, 10)))
y = pd.Series(np.random.uniform(low=0.0, high=100, size=(5, )))
groups = pd.DataFrame.from_dict({'groups': [0, 1, 1, 0, 1]})
X = pd.concat([X, groups], axis=1)
splitter = JustEachGroup()
model = SklearnModel(model='LinearRegression')
splitter.evaluate(X=X,
y=y,
models=[model],
groups=X['groups'],
savepath=os.getcwd(),
plots=list())
for d in splitter.splitdirs:
self.assertTrue(os.path.exists(d))
shutil.rmtree(d)
return
def test_close_comps(self):
# Make entries at a 10% spacing
composition_df = pd.DataFrame(
{'composition': ['Al{}Cu{}'.format(i, 10 - i) for i in range(11)]})
X = pd.DataFrame(np.random.uniform(low=0.0, high=100, size=(11, 10)))
y = pd.Series(np.random.uniform(low=0.0, high=100, size=(11, )))
# Generate test splits with a 5% distance cutoff
splitter = LeaveCloseCompositionsOut(composition_df=composition_df,
dist_threshold=0.5)
model = SklearnModel(model='LinearRegression')
splitter.evaluate(X=X,
y=y,
models=[model],
savepath=os.getcwd(),
plots=list())
for d in splitter.splitdirs:
self.assertTrue(os.path.exists(d))
shutil.rmtree(d)
return
def test_leaveouttwins(self):
# SAME TEST AS OTHER SPLITTERS
X = pd.DataFrame(np.random.uniform(low=0.0, high=100, size=(25, 10)))
y = pd.Series(np.random.uniform(low=0.0, high=100, size=(25, )))
splitter = LeaveOutTwinCV(threshold=0, auto_threshold=True)
model = SklearnModel(model='LinearRegression')
splitter.evaluate(X=X,
y=y,
models=[model],
groups=None,
savepath=os.getcwd(),
plots=list())
for d in splitter.splitdirs:
self.assertTrue(os.path.exists(d))
shutil.rmtree(d)
# Setup to check exact twins
splitter = LeaveOutTwinCV(threshold=0, auto_threshold=True)
model = SklearnModel(model='LinearRegression')
n_datapoints = 25
n_features = 5
# CASE 1: every datapoint is an exact twin, twins in both X and y
X = pd.DataFrame(
np.random.choice(range(-n_features * n_datapoints,
n_features * n_datapoints),
size=(n_datapoints, n_features),
replace=False)
) # This generates random numbers without repetitions
# Pull out last row to be y
y = X[n_features - 1]
X.drop(columns=n_features - 1, inplace=True)
# Create duplicates
X = X.append(X.copy(), ignore_index=True)
y = y.append(y.copy(), ignore_index=True)
ret = splitter.split(X, y)
numTwins = n_datapoints * 2
for r in ret:
self.assertTrue(
len(r[0]) == numTwins or len(r[1]) == numTwins
) # check that everything was counted as a twin in each split
# CASE 2: half datapoint is an exact twin, twins in both X and y
X = pd.DataFrame(
np.random.choice(range(-n_features * n_datapoints,
n_features * n_datapoints),
size=(n_datapoints, n_features),
replace=False))
y = X[n_features - 1]
X.drop(columns=n_features - 1, inplace=True)
X = X.append(X[0:int(n_datapoints / 2)].copy(), ignore_index=True)
y = y.append(y[0:int(n_datapoints / 2)].copy(), ignore_index=True)
ret = splitter.split(X, y)
numTwins = int(n_datapoints / 2) * 2
for r in ret:
self.assertTrue(
len(r[0]) == numTwins
or len(r[1]) == numTwins) # check correct number of twins
# Setup to check twins generated by sci kit learn blobs
splitter = LeaveOutTwinCV(threshold=2,
debug=False,
ceiling=1,
auto_threshold=True)
model = SklearnModel(model='LinearRegression')
n_datapoints = 25
n_features = 5
cen = []
for i in range(3):
for j in range(3):
cen.append((i * 10, j * 10))
v = sk.make_blobs(n_samples=100,
n_features=2,
centers=cen,
cluster_std=.1,
shuffle=True,
random_state=None)
X = pd.DataFrame(v[0])
y = pd.DataFrame(
np.random.choice(range(-500, 500), size=(100, 1), replace=False))
ret = splitter.split(X, y)
numTwins = 100
for r in ret:
self.assertTrue(len(r[0]) == numTwins or len(r[1]) == numTwins)
# plt.scatter(v[0][:, 0], v[0][:, 1], c=v[1])
# plt.savefig("./test.png")
# self.assertTrue(False, msg="In Progress")
return