def test_frequency_selection():
# Test frequency selection over grid-search and cross_validation
model = Pipeline(steps=[
('driver',
ExtractDriver(fs=fs,
low_fq=4.,
max_low_fq=7.,
low_fq_width=low_fq_width,
random_state=0)),
('add', AddDriverDelay()),
('dar', DARSklearn(fs=fs, ordar=20, ordriv=1, max_ordar=20)),
])
param_grid = {
'driver__low_fq': [3., 5., 7.],
'driver__low_fq_width': [0.25, 0.5, 1.],
}
gscv = GridSearchCVProgressBar(model,
param_grid=param_grid,
return_train_score=False,
verbose=1)
X = MultipleArray(raw_signal, None)
gscv.fit(X)
assert gscv.best_params_['driver__low_fq'] == 5
assert gscv.best_params_['driver__low_fq_width'] == 1
def run_grid_search(model, path, param_grid, X, y, cv=3):
start = time.time()
search = GridSearchCVProgressBar(model,
param_grid,
scoring='roc_auc',
cv=cv,
n_jobs=-1,
verbose=2)
search.fit(X, y)
print("Total Runtime for Grid Search: {:.4} seconds".format(
round(time.time() - start, 2)))
best_score = search.best_score_
best_params = search.best_params_
print("Testing Accuracy: {:.4}%".format(best_score * 100))
print("\nOptimal Parameters: {}".format(best_params))
search_results = pd.DataFrame.from_dict(search.cv_results_)
search_results.to_csv('./grid_search_results/' + path + '_' +
str(round(best_score, 4)).replace('.', '') + '_' +
time.asctime().replace(' ', '_'))
return search_results, best_score, best_params
def build_model():
"""
Builds a model, runs through paramaters with GridSearch and loads the best set
"""
pipeline = Pipeline([
('features', FeatureUnion([
('text_pipeline', Pipeline([
('vect', CountVectorizer(tokenizer=tokenize)),
('tfidf', TfidfTransformer())
])),
])),
('clf', MultiOutputClassifier(OneVsRestClassifier(SGDClassifier())))
])
parameters = {
'features__text_pipeline__vect__ngram_range': ((1,1),(1,2),(1,3)),
'features__text_pipeline__tfidf__use_idf': (True, False),
'features__text_pipeline__tfidf__smooth_idf': (True, False),
'features__transformer_weights': (
{'text_pipeline': 1},
{'text_pipeline': 0.5},
{'text_pipeline': 0.2}
),
'clf__estimator__estimator__n_jobs': [50, 100, 200],
'clf__estimator__estimator__alpha': [0.0001] #0.001,0.01]
}
cv = GridSearchCVProgressBar(pipeline, param_grid=parameters, n_jobs=-1)
return cv
def train_learning_model_NB(learning_model, hyperparameters, X, y,
classifier_string): #TFIDF FOR FACT OR OPINION
path_dictionary = {}
learning_model = learning_model
clf = GridSearchCVProgressBar(learning_model,
hyperparameters,
cv=10,
verbose=0)
best_model = clf.fit(X, y)
grid_df = pd.DataFrame(best_model.cv_results_)
best_model_path = '../Classification_models/' + classifier_string + '/' + classifier_string + '_' + '.pkl'
grid_df_path = '../Classification_models/' + classifier_string + '/' + 'grid_search_' + classifier_string + '_' + '.csv'
grid_df.to_csv(grid_df_path)
joblib.dump(best_model.best_estimator_, best_model_path)
print('Saved Model!')
path_dictionary = (best_model_path, grid_df_path)
print('Saved Grid Search!')
return path_dictionary
def train_learning_model(learning_model, hyperparameters, all_d2v_models,
classifier_string):
path_dictionary = {}
for d2v_model in all_d2v_models:
print('Training ' + d2v_model)
X_resampled, y_resampled = resampled_SMOTE(d2v_model)
learning_model = learning_model
clf = GridSearchCVProgressBar(learning_model,
hyperparameters,
cv=10,
verbose=0)
best_model = clf.fit(X_resampled, y_resampled)
grid_df = pd.DataFrame(best_model.cv_results_)
best_model_path = '../Classification_models/' + classifier_string + '/' + classifier_string + '_' + d2v_model + '.pkl'
grid_df_path = '../Classification_models/' + classifier_string + '/' + 'grid_search_' + classifier_string + '_' + d2v_model + '.csv'
grid_df.to_csv(grid_df_path)
joblib.dump(best_model.best_estimator_, best_model_path)
print('Saved Model!')
path_dictionary[d2v_model] = (best_model_path, grid_df_path)
print('Saved Grid Search!')
return path_dictionary
def optimize_hyperparams(self,
params_dict,
n_iter=10,
n_folds=5,
search_type='random'):
tmp_classifier = LGBM_classifier(feature_names=self.model_features)
# Set boosting type according to classifier name.
if self.classifier_name == 'random_forest':
tmp_classifier.set_params(**{
'boosting': 'rf',
'bagging_freq': 1,
'bagging_fraction': 0.7
})
elif self.classifier_name == 'lgbm':
tmp_classifier.set_params(**{'boosting': 'gbdt'})
if search_type == 'random':
self.model_selection = RandomizedSearchCV(
estimator=tmp_classifier,
param_distributions=params_dict,
refit=False,
random_state=2020,
n_iter=n_iter,
cv=n_folds,
verbose=10,
n_jobs=2)
elif search_type == 'grid':
self.model_selection = GridSearchCVProgressBar(
estimator=tmp_classifier,
param_grid=params_dict,
refit=False,
cv=n_folds,
verbose=10,
n_jobs=2)
self.model_selection.fit(self.X_train, self.y_train)
return self.model_selection.cv_results_
def SVM_I(X, Y, grid):
"""SVM classifier."""
print(DIVIDER)
# define model
mSVM = svm.SVC(random_state=RANDOM_STATE)
# create cache
cachedir = mkdtemp()
# set-up pipeline: normalize, reduce components, model
pipe = Pipeline(steps=[('scale', None), ('pca', None), ('model', mSVM)],
memory=cachedir)
# grid search parameters
grid = {
'scale': [None],
'pca': [None],
'model__kernel': ['rbf', 'linear'],
'model__C': numpy.logspace(-4, 4, 5),
'model__gamma': numpy.logspace(-4, 4, 5)
}
# define grid search and fit the values
estimator = GridSearchCVProgressBar(pipe,
grid,
scoring='accuracy',
n_jobs=-1,
verbose=2)
estimator.fit(X.values, Y.values.ravel())
# store the results of grid search in CSV
best_df = pandas.DataFrame.from_dict(estimator.cv_results_)
best_df.to_csv('{0}/svm_I.csv'.format(CSV_ROOT))
# prepare variables for printing
means = 100 * estimator.cv_results_['mean_test_score']
stds = 100 * estimator.cv_results_['std_test_score']
params = estimator.cv_results_['params']
i = estimator.best_index_
print("SVM Best Results: %.2f%% (%.2f%%) with %r" %
(means[i], stds[i], params[i]))
print(DIVIDER)
# remove cache
rmtree(cachedir)
def _model(self,
X_train,
y_train,
model_name: str = 'Logistic Regression',
apply_reduction: bool = False) -> str:
"""
Trains the models, saves the pickled object to the model_path.
:param: X_train: Training feature data
:param: y_train: Output feature data
:param model_name: which model to be used?
:param apply_reduction: whether to apply PCA reduction
:return: Classification report!
"""
param_grid = {}
steps = [('scaler', StandardScaler())]
if apply_reduction:
steps.append(('dimension_reduction', PCA()))
param_grid['dimension_reduction__n_components'] = [5, 10]
if model_name == 'Logistic Regression':
steps.append(('logistic', LogisticRegression()))
param_grid['logistic__C'] = [
1e-4, 1e-3, 1e-2, 1e-1, 1e1, 1e2, 1e3, 1e4
]
param_grid['logistic__penalty'] = ['l1', 'l2', 'elasticnet']
param_grid['logistic__solver'] = [
'newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'
]
param_grid['logistic__multi_class'] = ['auto', 'ovr']
elif model_name == 'Random Forest':
steps.append(
('random_forest', RandomForestClassifier(random_state=1769)))
param_grid['random_forest__max_depth'] = [20, 50, 100]
param_grid['random_forest__n_estimators'] = [10, 50, 100]
param_grid['random_forest__criterion'] = ['gini', 'entropy']
param_grid['random_forest__min_samples_split'] = [2, 0.3]
param_grid['random_forest__max_features'] = ['auto', 'sqrt']
estimator = GridSearchCVProgressBar(Pipeline(steps),
param_grid=param_grid,
cv=3,
refit=True)
print(estimator.get_params().keys())
estimator.fit(X_train, y_train)
if not os.path.exists(os.path.join(self.model_path, model_name)):
os.makedirs(os.path.join(self.model_path, model_name))
pickle.dump(
estimator,
open(
os.path.join(self.model_path, model_name, f"{model_name}.pkl"),
'wb'))
predictions = estimator.predict(X_train)
return classification_report(y_train, predictions, output_dict=True)
class Model():
def __init__(self, **kwargs):
#self.model_params = kwargs
self.classifier_name = kwargs.get('classifier', 'lgbm')
self.categorical_features = kwargs.get('categorical_features', [])
self.numerical_features = kwargs.get('numerical_features', [])
self.text_features = kwargs.get('text_features', [])
self.sequence_features = kwargs.get('sequence_features', [])
self.scale_numerical = kwargs.get('scale_numerical', False)
self.accepts_sparse = kwargs.get('accepts_sparse', True)
self.pipeline = CustomPipeline(
categorical_features=self.categorical_features,
numerical_features=self.numerical_features,
text_features=self.text_features,
sequence_features=self.sequence_features,
accepts_sparse=self.accepts_sparse,
scale_numerical=self.scale_numerical).build_pipeline()
def get_model_params(self, deep=True):
return {
'classifier': self.classifier_name,
'classifier_params': self.get_classifier_params(),
'categorical_features': self.categorical_features,
'numerical_features': self.numerical_features,
'text_features': self.text_features,
'sequence_features': self.sequence_features,
'scale_numerical': self.scale_numerical,
'accepts_sparse': self.accepts_sparse
}
def get_classifier_params(self):
return self.classifier.get_params()
@property
def model_features(self):
feature_names = []
transformers = self.pipeline.named_steps[
'feature_engineering'].transformer_list
for transformer in transformers:
transformer_features = transformer[1].get_feature_names()
feature_names.extend(transformer_features)
#HACK
feature_names = [
"".join(c if c.isascii() and c.isalnum() else "_" for c in str(x))
for x in feature_names
]
return feature_names
@property
def n_features(self):
return (len(self.model_features))
def transform(self, data, transform_test=False):
print('Fitting pipeline...')
self.pipeline.fit(data.train.X)
print('Transforming data...')
self.X_train = self.pipeline.transform(data.train.X)
self.y_train = data.train.y
self.X_val = self.pipeline.transform(data.val.X)
self.y_val = data.val.y
if transform_test:
self.X_test = self.pipeline.transform(data.test.X)
self.y_test = data.test.y
def fit_classifier(self, **kwargs):
if self.classifier_name == 'lgbm':
boosting_name = 'gbdt'
elif self.classifier_name == 'random_forest':
boosting_name = 'rf'
self.classifier = LGBM_classifier(feature_names=self.model_features)
#Set boosting type according to classifier name.
if self.classifier_name == 'random_forest':
self.classifier.set_params(**{
'boosting': 'rf',
'bagging_freq': 1,
'bagging_fraction': 0.7
})
elif self.classifier_name == 'lgbm':
self.classifier.set_params(**{'boosting': 'gbdt'})
#set other paramaters based on kwargs if any.
self.classifier.set_params(**kwargs)
#train classifier
print('Training classifier')
self.classifier.fit(self.X_train, self.y_train, self.X_val, self.y_val)
def fit_best_classifier(self):
assert hasattr(self, 'model_selection')
self.fit_classifier(**self.model_selection.best_params_)
def predict(self, X_transf):
return self.classifier.predict(X_transf)
def score(self, X_transf, y):
return self.classifier.score(X_transf, y)
@property
def get_feature_importance(self):
return {
'feature_importance': self.classifier.feature_importance_,
'feature_name': self.model_features
}
def plot_feature_importance(self, n_features=30):
importance_df = pd.DataFrame(self.get_feature_importance).sort_values(
by='feature_importance', ascending=True)
importance_df[-n_features:].plot.barh(x=1,
y=0,
title='Feature Importance',
legend=False,
figsize=(8, 10))
def get_performance_metrics(self):
auc_train = self.score(self.X_train, self.y_train)
auc_val = self.score(self.X_val, self.y_val)
print('training AUC ROC score: ', auc_train)
print('validation AUC ROC score: ', auc_val)
overfitting = abs(auc_train - auc_val) / auc_train
print('relative over-fitting: ', overfitting)
return {
'auc_training': auc_train,
'auc_validation': auc_val,
'overfitting': overfitting
}
def optimize_hyperparams(self,
params_dict,
n_iter=10,
n_folds=5,
search_type='random'):
tmp_classifier = LGBM_classifier(feature_names=self.model_features)
# Set boosting type according to classifier name.
if self.classifier_name == 'random_forest':
tmp_classifier.set_params(**{
'boosting': 'rf',
'bagging_freq': 1,
'bagging_fraction': 0.7
})
elif self.classifier_name == 'lgbm':
tmp_classifier.set_params(**{'boosting': 'gbdt'})
if search_type == 'random':
self.model_selection = RandomizedSearchCV(
estimator=tmp_classifier,
param_distributions=params_dict,
refit=False,
random_state=2020,
n_iter=n_iter,
cv=n_folds,
verbose=10,
n_jobs=2)
elif search_type == 'grid':
self.model_selection = GridSearchCVProgressBar(
estimator=tmp_classifier,
param_grid=params_dict,
refit=False,
cv=n_folds,
verbose=10,
n_jobs=2)
self.model_selection.fit(self.X_train, self.y_train)
return self.model_selection.cv_results_
def get_model_selection_results(self):
results = self.model_selection.cv_results_
results_df = pd.DataFrame(
{key: results[key]
for key in results if key != 'params'})
return results_df
'driver__low_fq_width': [0.25, 0.5, 1.],
}
###############################################################################
# Then we plug the model into GridSearchCV and we fit it.
#
# This performs a grid-search with cross-validation: First, multiple train and
# test sets are defined by the splitting strategy, as defined by the parameter
# `cv` in GridSearchCV. Then, GridSearchCV will loop over each parameter
# configuration, fitting the model on one train set and evaluating it on the
# corresponding test set.
# Plug the model and the parameter grid into a GridSearchCV estimator
# (GridSearchCVProgressBar is identical to GridSearchCV, but it adds a nice
# progress bar to monitor progress.)
gscv = GridSearchCVProgressBar(model, param_grid=param_grid, cv=3,
return_train_score=False, verbose=1)
# Fit the grid-search. We use `MultipleArray` to put together low_sig and
# high_sig. If high_sig is None, we use low_sig for both the driver and the
# modeled signal.
X = MultipleArray(low_sig, None)
gscv.fit(X)
###############################################################################
# Print the results of the grid search.
print("\nBest parameters set found over cross-validation:\n")
print(gscv.best_params_)
###############################################################################
# Plot the results of the grid search.