def get_classifier(self, X, y, word_index): print('===== MLP Keras =====') # generate embedding matrix self._embedding_matrix = self.get_embeddings(word_index) def create_model(neurons=1): input_dim = X.shape[1] model = Sequential() model.add( layers.Embedding(input_dim=self._vocab_size, output_dim=self._embedding_dim, weights=[self._embedding_matrix], input_length=self._maxlen, trainable=True)) model.add(layers.LSTM(units=neurons)) #model.add(layers.Flatten()) #model.add(layers.Dense(neurons, activation='relu')) model.add(layers.Dense(1, activation='sigmoid')) model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"]) #model.summary() return model print('===== Keras hyperparameter optimization =====') model = KerasClassifier(build_fn=create_model, epochs=150, verbose=0) params = {'neurons': [1, 10, 20, 30, 50]} cfl = GridSearchCV(model, params, cv=2, scoring='accuracy') cfl.fit(X, y) for param, value in cfl.best_params_.items(): print("%s : %s" % (param, value)) model = KerasClassifier(build_fn=create_model, epochs=150, verbose=0) model.set_params(**cfl.best_params_) return model
def get_classifier(self, X, y): print('===== MLP Keras =====') def create_model(neurons=1): input_dim = X.shape[1] model = Sequential() model.add( layers.Dense(neurons, input_dim=input_dim, activation='relu')) model.add(layers.Dense(1, activation='sigmoid')) model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"]) #model.summary() return model print('===== Keras hyperparameter optimization =====') model = KerasClassifier(build_fn=create_model, epochs=150, verbose=0) params = {'neurons': [1, 10, 20, 30, 50]} cfl = GridSearchCV(model, params, cv=2, scoring='accuracy') cfl.fit(X, y) for param, value in cfl.best_params_.items(): print("%s : %s" % (param, value)) model = KerasClassifier(build_fn=create_model, epochs=150, verbose=0) model.set_params(**cfl.best_params_) return model
class model(): def __init__(self): self.model = KerasClassifier(build_fn=self.create_nn, verbose=0) self.params= {'optimizer':['rmsprop', 'adam'], 'init':['glorot_uniform', 'normal', 'uniform'], 'epochs':[50, 100, 150], 'batch_size':[5, 10, 20]} def create_nn(self,optimizer='rmsprop', init='glorot_uniform'): model = Sequential() model.add(Dense(12, input_dim=8, kernel_initializer=init, activation='relu')) model.add(Dense(8, kernel_initializer=init, activation='relu')) model.add(Dense(1, kernel_initializer=init, activation='sigmoid')) #compile model model.compile(loss='binary_crossentropy',optimizer=optimizer,metrics=['accuracy']) return model def grid_search(self, train_dataset): gs_clf = GridSearchCV(estimator=self.model, param_grid=self.params) gs_clf = gs_clf.fit(train_dataset.x, train_dataset.y) # summarize results print("Best: %f using %s" % (gs_clf.best_score_, gs_clf.best_params_)) means = gs_clf.cv_results_['mean_test_score'] stds = gs_clf.cv_results_['std_test_score'] params = gs_clf.cv_results_['params'] for mean, stdev, param in zip(means, stds, params): print("%f (%f) with: %r" % (mean, stdev, param)) self.save_model(gs_clf, 'keras_model.pkl') self.model.set_params(**gs_clf.best_params_) def cross_validation(self, train_dataset): predicted = cross_val_predict(self.model, train_dataset.x, train_dataset.y, cv=10) print(metrics.classification_report(train_dataset.y, predicted, target_names=train_dataset.target_names)) fpr,tpr,thresholds = metrics.roc_curve(train_dataset.y, predicted) print("AUC is: " + str(metrics.auc(fpr,tpr)) + "\n") def save_model(self, gs_clf, file_name): joblib.dump(gs_clf.best_estimator_, file_name) def load_model_params(self, file_name): self.model.set_params(**joblib.load(file_name).best_params_)
def best_keras_clf_estimator( y_type, best_nn_build_fn, nb_epoch, input_dim, labels, batch_size=None): best_model_estim = None best_model_estim = KerasClassifier( build_fn=best_nn_build_fn, nb_epoch=nb_epoch, input_dim=input_dim, verbose=0) if y_type == 'multiclass': if labels is None: raise ValueError("%r is not a valid type for var 'labels'" % labels) elif not isinstance(labels, list): raise TypeError("Multiclass keras models need a list of string labels.") else: output_dim = len(labels) best_model_estim.set_params(output_dim=output_dim) if batch_size is not None and isinstance(batch_size, int): best_model_estim.set_params(batch_size=batch_size) return best_model_estim
class FinalModelATC(BaseEstimator, TransformerMixin): def __init__(self, model, model_name=None, ml_for_analytics=False, type_of_estimator='classifier', output_column=None, name=None, _scorer=None, training_features=None, column_descriptions=None, feature_learning=False, uncertainty_model=None, uc_results=None, training_prediction_intervals=False, min_step_improvement=0.0001, interval_predictors=None, keep_cat_features=False, is_hp_search=None, X_test=None, y_test=None): self.model = model self.model_name = model_name self.ml_for_analytics = ml_for_analytics self.type_of_estimator = type_of_estimator self.name = name self.training_features = training_features self.column_descriptions = column_descriptions self.feature_learning = feature_learning self.uncertainty_model = uncertainty_model self.uc_results = uc_results self.training_prediction_intervals = training_prediction_intervals self.min_step_improvement = min_step_improvement self.interval_predictors = interval_predictors self.is_hp_search = is_hp_search self.keep_cat_features = keep_cat_features self.X_test = X_test self.y_test = y_test if self.type_of_estimator == 'classifier': self._scorer = _scorer else: self._scorer = _scorer def get(self, prop_name, default=None): try: return getattr(self, prop_name) except AttributeError: return default def fit(self, X, y): global keras_imported, KerasRegressor, KerasClassifier, EarlyStopping, ModelCheckpoint, TerminateOnNaN, keras_load_model self.model_name = get_name_from_model(self.model) X_fit = X if self.model_name[:12] == 'DeepLearning' or self.model_name in [ 'BayesianRidge', 'LassoLars', 'OrthogonalMatchingPursuit', 'ARDRegression', 'Perceptron', 'PassiveAggressiveClassifier', 'SGDClassifier', 'RidgeClassifier', 'LogisticRegression' ]: if scipy.sparse.issparse(X_fit): X_fit = X_fit.todense() if self.model_name[:12] == 'DeepLearning': if keras_imported == False: # Suppress some level of logs os.environ['TF_CPP_MIN_VLOG_LEVEL'] = '3' os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' from keras.callbacks import EarlyStopping, ModelCheckpoint, TerminateOnNaN from keras.models import load_model as keras_load_model from keras.wrappers.scikit_learn import KerasRegressor, KerasClassifier keras_imported = True # For Keras, we need to tell it how many input nodes to expect, which is our num_cols num_cols = X_fit.shape[1] model_params = self.model.get_params() del model_params['build_fn'] try: del model_params['feature_learning'] except: pass try: del model_params['num_cols'] except: pass if self.type_of_estimator == 'regressor': self.model = KerasRegressor( build_fn=utils_models.make_deep_learning_model, num_cols=num_cols, feature_learning=self.feature_learning, **model_params) elif self.type_of_estimator == 'classifier': self.model = KerasClassifier( build_fn=utils_models.make_deep_learning_classifier, num_cols=num_cols, feature_learning=self.feature_learning, **model_params) if self.model_name[:12] == 'DeepLearning': try: if self.is_hp_search == True: patience = 5 verbose = 0 else: patience = 25 verbose = 2 X_fit, y, X_test, y_test = self.get_X_test(X_fit, y) try: X_test = X_test.toarray() except AttributeError as e: pass if not self.is_hp_search: print( '\nWe will stop training early if we have not seen an improvement in validation accuracy in {} epochs' .format(patience)) print( 'To measure validation accuracy, we will split off a random 10 percent of your training data set' ) early_stopping = EarlyStopping(monitor='val_loss', patience=patience, verbose=verbose) terminate_on_nan = TerminateOnNaN() now_time = datetime.datetime.now() time_string = str(now_time.year) + '_' + str( now_time.month) + '_' + str(now_time.day) + '_' + str( now_time.hour) + '_' + str(now_time.minute) temp_file_name = 'tmp_dl_model_checkpoint_' + time_string + str( random.random()) + '.h5' model_checkpoint = ModelCheckpoint(temp_file_name, monitor='val_loss', save_best_only=True, mode='min', period=1) callbacks = [early_stopping, terminate_on_nan] if not self.is_hp_search: callbacks.append(model_checkpoint) self.model.fit(X_fit, y, callbacks=callbacks, validation_data=(X_test, y_test), verbose=verbose) # TODO: give some kind of logging on how the model did here! best epoch, best accuracy, etc. if self.is_hp_search is False: self.model = keras_load_model(temp_file_name) try: os.remove(temp_file_name) except OSError as e: pass except KeyboardInterrupt as e: print( 'Stopping training at this point because we heard a KeyboardInterrupt' ) print( 'If the deep learning model is functional at this point, we will output the model in its latest form' ) print( 'Note that this feature is an unofficial beta-release feature that is known to fail on occasion' ) if self.is_hp_search is False: self.model = keras_load_model(temp_file_name) try: os.remove(temp_file_name) except OSError as e: pass elif self.model_name[:4] == 'LGBM': X_fit = X.toarray() X_fit, y, X_test, y_test = self.get_X_test(X_fit, y) try: X_test = X_test.toarray() except AttributeError as e: pass if self.type_of_estimator == 'regressor': eval_metric = 'rmse' elif self.type_of_estimator == 'classifier': if len(set(y_test)) > 2: eval_metric = 'multi_logloss' else: eval_metric = 'binary_logloss' verbose = True if self.is_hp_search == True: verbose = False if self.X_test is not None: eval_name = 'X_test_the_user_passed_in' else: eval_name = 'random_holdout_set_from_training_data' cat_feature_indices = self.get_categorical_feature_indices() if cat_feature_indices is None: self.model.fit(X_fit, y, eval_set=[(X_test, y_test)], early_stopping_rounds=100, eval_metric=eval_metric, eval_names=[eval_name], verbose=verbose) else: self.model.fit(X_fit, y, eval_set=[(X_test, y_test)], early_stopping_rounds=100, eval_metric=eval_metric, eval_names=[eval_name], categorical_feature=cat_feature_indices, verbose=verbose) elif self.model_name[:8] == 'CatBoost': X_fit = X_fit.toarray() if self.type_of_estimator == 'classifier' and len( pd.Series(y).unique()) > 2: # TODO: we might have to modify the format of the y values, converting them all to ints, then back again (sklearn has a useful inverse_transform on some preprocessing classes) self.model.set_params(loss_function='MultiClass') cat_feature_indices = self.get_categorical_feature_indices() self.model.fit(X_fit, y, cat_features=cat_feature_indices) elif self.model_name[:16] == 'GradientBoosting': if not sklearn_version > '0.18.1': X_fit = X_fit.toarray() patience = 20 best_val_loss = -10000000000 num_worse_rounds = 0 best_model = deepcopy(self.model) X_fit, y, X_test, y_test = self.get_X_test(X_fit, y) # Add a variable number of trees each time, depending how far into the process we are if os.environ.get('is_test_suite', False) == 'True': num_iters = list(range(1, 50, 1)) + list(range( 50, 100, 2)) + list(range(100, 250, 3)) else: num_iters = list(range( 1, 50, 1)) + list(range(50, 100, 2)) + list( range(100, 250, 3)) + list(range(250, 500, 5)) + list( range(500, 1000, 10)) + list(range( 1000, 2000, 20)) + list(range( 2000, 10000, 100)) # TODO: get n_estimators from the model itself, and reduce this list to only those values that come under the value from the model try: for num_iter in num_iters: warm_start = True if num_iter == 1: warm_start = False self.model.set_params(n_estimators=num_iter, warm_start=warm_start) self.model.fit(X_fit, y) if self.training_prediction_intervals == True: val_loss = self.model.score(X_test, y_test) else: try: val_loss = self._scorer.score(self, X_test, y_test) except Exception as e: val_loss = self.model.score(X_test, y_test) if val_loss - self.min_step_improvement > best_val_loss: best_val_loss = val_loss num_worse_rounds = 0 best_model = deepcopy(self.model) else: num_worse_rounds += 1 print( '[' + str(num_iter) + '] random_holdout_set_from_training_data\'s score is: ' + str(round(val_loss, 3))) if num_worse_rounds >= patience: break except KeyboardInterrupt: print( 'Heard KeyboardInterrupt. Stopping training, and using the best checkpointed GradientBoosting model' ) pass self.model = best_model print( 'The number of estimators that were the best for this training dataset: ' + str(self.model.get_params()['n_estimators'])) print('The best score on the holdout set: ' + str(best_val_loss)) else: self.model.fit(X_fit, y) if self.X_test is not None: del self.X_test del self.y_test return self def remove_categorical_values(self, features): clean_features = set([]) for feature in features: if '=' not in feature: clean_features.add(feature) else: clean_features.add(feature[:feature.index('=')]) return clean_features def verify_features(self, X, raw_features_only=False): if self.column_descriptions is None: print( 'This feature is not enabled by default. Depending on the shape of the training data, it can add hundreds of KB to the saved file size.' ) print( 'Please pass in `ml_predictor.train(data, verify_features=True)` when training a model, and we will enable this function, at the cost of a potentially larger file size.' ) warnings.warn( 'Please pass verify_features=True when invoking .train() on the ml_predictor instance.' ) return None print( '\n\nNow verifying consistency between training features and prediction features' ) if isinstance(X, dict): prediction_features = set(X.keys()) elif isinstance(X, pd.DataFrame): prediction_features = set(X.columns) # If the user passed in categorical features, we will effectively one-hot-encode them ourselves here # Note that this assumes we're using the "=" as the separater in DictVectorizer/DataFrameVectorizer date_col_names = [] categorical_col_names = [] for key, value in self.column_descriptions.items(): if value == 'categorical' and 'day_part' not in key: try: # This covers the case that the user passes in a value in column_descriptions that is not present in their prediction data column_vals = X[key].unique() for val in column_vals: prediction_features.add(key + '=' + str(val)) categorical_col_names.append(key) except: print( '\nFound a column in your column_descriptions that is not present in your prediction data:' ) print(key) elif 'day_part' in key: # We have found a date column. Make sure this date column is in our prediction data # It is outside the scope of this function to make sure that the same date parts are available in both our training and testing data raw_date_col_name = key[:key.index('day_part') - 1] date_col_names.append(raw_date_col_name) elif value == 'output': try: prediction_features.remove(key) except KeyError: pass # Now that we've added in all the one-hot-encoded categorical columns (name=val1, name=val2), remove the base name from our prediction data prediction_features = prediction_features - set(categorical_col_names) # Get only the unique raw_date_col_names date_col_names = set(date_col_names) training_features = set(self.training_features) # Remove all of the transformed date column feature names from our training data features_to_remove = [] for feature in training_features: for raw_date_col_name in date_col_names: if raw_date_col_name in feature: features_to_remove.append(feature) training_features = training_features - set(features_to_remove) # Make sure the raw_date_col_name is in our training data after we have removed all the transformed feature names training_features = training_features | date_col_names # MVP means ignoring text features print_nlp_warning = False nlp_example = None for feature in training_features: if 'nlp_' in feature: print_nlp_warning = True nlp_example = feature training_features.remove(feature) if print_nlp_warning == True: print('\n\nWe found an NLP column in the training data') print( 'verify_features() currently does not support checking all of the values within an NLP column, so if the text of your NLP column has dramatically changed, you will have to check that yourself.' ) print( 'Here is one example of an NLP feature in the training data:') print(nlp_example) training_not_prediction = training_features - prediction_features if raw_features_only == True: training_not_prediction = self.remove_categorical_values( training_not_prediction) if len(training_not_prediction) > 0: print( '\n\nHere are the features this model was trained on that were not present in this prediction data:' ) print(sorted(list(training_not_prediction))) else: print( 'All of the features this model was trained on are included in the prediction data' ) prediction_not_training = prediction_features - training_features if raw_features_only == True: prediction_not_training = self.remove_categorical_values( prediction_not_training) if len(prediction_not_training) > 0: # Separate out those values we were told to ignore by column_descriptions ignored_features = [] for feature in prediction_not_training: if self.column_descriptions.get(feature, 'False') == 'ignore': ignored_features.append(feature) prediction_not_training = prediction_not_training - set( ignored_features) print( '\n\nHere are the features available in the prediction data that were not part of the training data:' ) print(sorted(list(prediction_not_training))) if len(ignored_features) > 0: print( '\n\nAdditionally, we found features in the prediction data that we were told to ignore in the training data' ) print(sorted(list(ignored_features))) else: print( 'All of the features in the prediction data were in this model\'s training data' ) print('\n\n') return { 'training_not_prediction': training_not_prediction, 'prediction_not_training': prediction_not_training } def score(self, X, y, verbose=False): # At the time of writing this, GradientBoosting does not support sparse matrices for predictions if (self.model_name[:16] == 'GradientBoosting' or self.model_name in [ 'BayesianRidge', 'LassoLars', 'OrthogonalMatchingPursuit', 'ARDRegression' ]) and scipy.sparse.issparse(X): X = X.todense() if self._scorer is not None: if self.type_of_estimator == 'regressor': return self._scorer.score(self, X, y) elif self.type_of_estimator == 'classifier': return self._scorer.score(self, X, y) else: return self.model.score(X, y) def predict_proba(self, X, verbose=False): if (self.model_name[:16] == 'GradientBoosting' or self.model_name[:12] == 'DeepLearning' or self.model_name in [ 'BayesianRidge', 'LassoLars', 'OrthogonalMatchingPursuit', 'ARDRegression' ]) and scipy.sparse.issparse(X): X = X.todense() elif (self.model_name[:8] == 'CatBoost' or self.model_name[:4] == 'LGBM') and scipy.sparse.issparse(X): X = X.toarray() try: if self.model_name[:4] == 'LGBM': try: best_iteration = self.model.best_iteration except AttributeError: best_iteration = self.model.best_iteration_ predictions = self.model.predict_proba( X, num_iteration=best_iteration) else: predictions = self.model.predict_proba(X) except AttributeError as e: try: predictions = self.model.predict(X) except TypeError as e: if scipy.sparse.issparse(X): X = X.todense() predictions = self.model.predict(X) except TypeError as e: if scipy.sparse.issparse(X): X = X.todense() predictions = self.model.predict_proba(X) # If this model does not have predict_proba, and we have fallen back on predict, we want to make sure we give results back in the same format the user would expect for predict_proba, namely each prediction is a list of predicted probabilities for each class. # Note that this DOES NOT WORK for multi-label problems, or problems that are not reduced to 0,1 # If this is not an iterable (ignoring strings, which might be iterable), then we will want to turn our predictions into tupled predictions if not (hasattr(predictions[0], '__iter__') and not isinstance(predictions[0], str)): tupled_predictions = [] for prediction in predictions: if prediction == 1: tupled_predictions.append([0, 1]) else: tupled_predictions.append([1, 0]) predictions = tupled_predictions # This handles an annoying edge case with libraries like Keras that, for a binary classification problem, with return a single predicted probability in a list, rather than the probability of both classes in a list if len(predictions[0]) == 1: tupled_predictions = [] for prediction in predictions: tupled_predictions.append([1 - prediction[0], prediction[0]]) predictions = tupled_predictions if X.shape[0] == 1: return predictions[0] else: return predictions def predict(self, X, verbose=False): if (self.model_name[:16] == 'GradientBoosting' or self.model_name[:12] == 'DeepLearning' or self.model_name in [ 'BayesianRidge', 'LassoLars', 'OrthogonalMatchingPursuit', 'ARDRegression' ]) and scipy.sparse.issparse(X): X_predict = X.todense() elif self.model_name[:8] == 'CatBoost' and scipy.sparse.issparse(X): X_predict = X.toarray() else: X_predict = X if self.model_name[:4] == 'LGBM': try: best_iteration = self.model.best_iteration except AttributeError: best_iteration = self.model.best_iteration_ predictions = self.model.predict(X, num_iteration=best_iteration) else: predictions = self.model.predict(X_predict) # Handle cases of getting a prediction for a single item. # It makes a cleaner interface just to get just the single prediction back, rather than a list with the prediction hidden inside. if isinstance(predictions, np.ndarray): predictions = predictions.tolist() if isinstance(predictions, float) or isinstance( predictions, int) or isinstance(predictions, str): return predictions if isinstance(predictions[0], list) and len(predictions[0]) == 1: predictions = [row[0] for row in predictions] if len(predictions) == 1: return predictions[0] else: return predictions def predict_intervals(self, X, return_type=None): if self.interval_predictors is None: print( '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!' ) print('This model was not trained to predict intervals') print( 'Please follow the documentation to tell this model at training time to learn how to predict intervals' ) print( '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!' ) raise ValueError('This model was not trained to predict intervals') base_prediction = self.predict(X) result = {'prediction': base_prediction} for tup in self.interval_predictors: predictor_name = tup[0] predictor = tup[1] result[predictor_name] = predictor.predict(X) if scipy.sparse.issparse(X): len_input = X.shape[0] else: len_input = len(X) if (len_input == 1 and return_type is None) or return_type == 'dict': return result elif (len_input > 1 and return_type is None ) or return_type == 'df' or return_type == 'dataframe': return pd.DataFrame(result) elif return_type == 'list': if len_input == 1: list_result = [base_prediction] for tup in self.interval_predictors: list_result.append(result[tup[0]]) else: list_result = [] for idx in range(len_input): row_result = [base_prediction[idx]] for tup in self.interval_predictors: row_result.append(result[tup[0]][idx]) list_result.append(row_result) return list_result else: print( 'Please pass in a return_type value of one of the following: ["dict", "dataframe", "df", "list"]' ) raise (ValueError( 'Please pass in a return_type value of one of the following: ["dict", "dataframe", "df", "list"]' )) # transform is initially designed to be used with feature_learning def transform(self, X): predicted_features = self.predict(X) predicted_features = list(predicted_features) X = scipy.sparse.hstack([X, predicted_features], format='csr') return X # Allows the user to get the fully transformed data def transform_only(self, X): return X def predict_uncertainty(self, X): if self.uncertainty_model is None: print( '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!' ) print('This model was not trained to predict uncertainties') print( 'Please follow the documentation to tell this model at training time to learn how to predict uncertainties' ) print( '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!' ) raise ValueError( 'This model was not trained to predict uncertainties') base_predictions = self.predict(X) if isinstance(base_predictions, Iterable): base_predictions_col = [[val] for val in base_predictions] base_predictions_col = np.array(base_predictions_col) else: base_predictions_col = [base_predictions] X_combined = scipy.sparse.hstack([X, base_predictions_col], format='csr') uncertainty_predictions = self.uncertainty_model.predict_proba( X_combined) results = { 'base_prediction': base_predictions, 'uncertainty_prediction': uncertainty_predictions } if isinstance(base_predictions, Iterable): results['uncertainty_prediction'] = [ row[1] for row in results['uncertainty_prediction'] ] results = pd.DataFrame.from_dict(results, orient='columns') if self.uc_results is not None: calibration_results = {} # grab the relevant properties from our uc_results, and make them each their own list in calibration_results for key, value in self.uc_results[1].items(): calibration_results[key] = [] for proba in results['uncertainty_prediction']: max_bucket_proba = 0 bucket_num = 1 while proba > max_bucket_proba: calibration_result = self.uc_results[bucket_num] max_bucket_proba = self.uc_results[bucket_num][ 'max_proba'] bucket_num += 1 for key, value in calibration_result.items(): calibration_results[key].append(value) # TODO: grab the uncertainty_calibration data for DataFrames df_calibration_results = pd.DataFrame.from_dict( calibration_results, orient='columns') del df_calibration_results['max_proba'] results = pd.concat([results, df_calibration_results], axis=1) else: if self.uc_results is not None: # TODO: grab the uncertainty_calibration data for dictionaries for bucket_name, bucket_result in self.uc_results.items(): if proba > bucket_result['max_proba']: break results.update(bucket_result) del results['max_proba'] return results def score_uncertainty(self, X, y, verbose=False): return self.uncertainty_model.score(X, y, verbose=False) def get_categorical_feature_indices(self): cat_feature_indices = None if self.keep_cat_features == True: cat_feature_names = [ k for k, v in self.column_descriptions.items() if v == 'categorical' ] cat_feature_indices = [ self.training_features.index(cat_name) for cat_name in cat_feature_names ] return cat_feature_indices def get_X_test(self, X_fit, y): if self.X_test is not None: return X_fit, y, self.X_test, self.y_test else: X_fit, X_test, y, y_test = train_test_split(X_fit, y, test_size=0.15) return X_fit, y, X_test, y_test
'\u03BB', 'parameters', col_names_nn_Keras_classifier, row_names_nn_Keras_classifier, True, savefig=True, figname='Images/NN_clas_accuracy_2' + wine_type + '.png') #refit best NN classifier print(clf.best_params_) nnKerasBest = KerasClassifier(build_fn=build_network, n_outputs=y_onehot.shape[1], output_activation='softmax', loss="categorical_crossentropy", verbose=0) nnKerasBest.set_params(**clf.best_params_) hist = nnKerasBest.fit(Xtrain, ytrain_onehot, validation_data=(Xtest, ytest_onehot)) pred_nnKerasBest_train = nnKerasBest.predict(Xtrain) pred_nnKerasBest_test = nnKerasBest.predict(Xtest) print('Neural network classifier accuracy train: %g' % accuracy_score(ytrain, pred_nnKerasBest_train)) print('Neural network classifier accuracy test: %g' % accuracy_score(ytest, pred_nnKerasBest_test)) #learning chart for best model (accuracy and loss) and confusion matrix plot_several(np.tile(np.arange(clf.best_params_['epochs'])[:, None], [1, 2]), np.concatenate((np.reshape(hist.history['accuracy'], (clf.best_params_['epochs'], 1)), np.reshape(hist.history['val_accuracy'],
class FinalModelATC(BaseEstimator, TransformerMixin): def __init__(self, model, model_name=None, ml_for_analytics=False, type_of_estimator='classifier', output_column=None, name=None, _scorer=None, training_features=None, column_descriptions=None, feature_learning=False, uncertainty_model=None, uc_results = None): self.model = model self.model_name = model_name self.ml_for_analytics = ml_for_analytics self.type_of_estimator = type_of_estimator self.name = name self.training_features = training_features self.column_descriptions = column_descriptions self.feature_learning = feature_learning self.uncertainty_model = uncertainty_model self.uc_results = uc_results if self.type_of_estimator == 'classifier': self._scorer = _scorer else: self._scorer = _scorer def get(self, prop_name, default=None): try: return getattr(self, prop_name) except AttributeError: return default def fit(self, X, y): self.model_name = get_name_from_model(self.model) X_fit = X if self.model_name[:12] == 'DeepLearning' or self.model_name in ['BayesianRidge', 'LassoLars', 'OrthogonalMatchingPursuit', 'ARDRegression', 'Perceptron', 'PassiveAggressiveClassifier', 'SGDClassifier', 'RidgeClassifier', 'LogisticRegression']: if scipy.sparse.issparse(X_fit): X_fit = X_fit.todense() if self.model_name[:12] == 'DeepLearning': # For Keras, we need to tell it how many input nodes to expect, which is our num_cols num_cols = X_fit.shape[1] model_params = self.model.get_params() del model_params['build_fn'] if self.type_of_estimator == 'regressor': self.model = KerasRegressor(build_fn=utils_models.make_deep_learning_model, num_cols=num_cols, feature_learning=self.feature_learning, **model_params) elif self.type_of_estimator == 'classifier': self.model = KerasClassifier(build_fn=utils_models.make_deep_learning_classifier, num_cols=num_cols, feature_learning=self.feature_learning, **model_params) try: if self.model_name[:12] == 'DeepLearning': print('\nWe will stop training early if we have not seen an improvement in training accuracy in 25 epochs') from keras.callbacks import EarlyStopping early_stopping = EarlyStopping(monitor='loss', patience=25, verbose=1) self.model.fit(X_fit, y, callbacks=[early_stopping]) elif self.model_name[:16] == 'GradientBoosting': if scipy.sparse.issparse(X_fit): X_fit = X_fit.todense() patience = 20 best_val_loss = -10000000000 num_worse_rounds = 0 best_model = deepcopy(self.model) X_fit, X_test, y, y_test = train_test_split(X_fit, y, test_size=0.15) # Add a variable number of trees each time, depending how far into the process we are num_iters = list(range(1, 50, 1)) + list(range(50, 100, 2)) + list(range(100, 250, 3)) + list(range(250, 500, 5)) + list(range(500, 1000, 10)) + list(range(1000, 2000, 20)) + list(range(2000, 10000, 100)) try: for num_iter in num_iters: warm_start = True if num_iter == 1: warm_start = False self.model.set_params(n_estimators=num_iter, warm_start=warm_start) self.model.fit(X_fit, y) try: val_loss = self._scorer.score(self, X_test, y_test) except Exception as e: val_loss = self.model.score(X_test, y_test) if val_loss > best_val_loss: best_val_loss = val_loss num_worse_rounds = 0 best_model = deepcopy(self.model) else: num_worse_rounds += 1 if num_worse_rounds >= patience: break except KeyboardInterrupt: print('Heard KeyboardInterrupt. Stopping training, and using the best checkpointed GradientBoosting model') pass self.model = best_model print('The number of estimators that were the best for this training dataset: ' + str(self.model.get_params()['n_estimators'])) print('The best score on a random 15 percent holdout set of the training data: ' + str(best_val_loss)) else: self.model.fit(X_fit, y) except TypeError as e: if scipy.sparse.issparse(X_fit): X_fit = X_fit.todense() self.model.fit(X_fit, y) except KeyboardInterrupt as e: print('Stopping training at this point because we heard a KeyboardInterrupt') print('If the model is functional at this point, we will output the model in its latest form') print('Note that not all models can be interrupted and still used, and that this feature generally is an unofficial beta-release feature that is known to fail on occasion') pass return self def remove_categorical_values(self, features): clean_features = set([]) for feature in features: if '=' not in feature: clean_features.add(feature) else: clean_features.add(feature[:feature.index('=')]) return clean_features def verify_features(self, X, raw_features_only=False): if self.column_descriptions is None: print('This feature is not enabled by default. Depending on the shape of the training data, it can add hundreds of KB to the saved file size.') print('Please pass in `ml_predictor.train(data, verify_features=True)` when training a model, and we will enable this function, at the cost of a potentially larger file size.') warnings.warn('Please pass verify_features=True when invoking .train() on the ml_predictor instance.') return None print('\n\nNow verifying consistency between training features and prediction features') if isinstance(X, dict): prediction_features = set(X.keys()) elif isinstance(X, pd.DataFrame): prediction_features = set(X.columns) # If the user passed in categorical features, we will effectively one-hot-encode them ourselves here # Note that this assumes we're using the "=" as the separater in DictVectorizer/DataFrameVectorizer date_col_names = [] categorical_col_names = [] for key, value in self.column_descriptions.items(): if value == 'categorical' and 'day_part' not in key: try: # This covers the case that the user passes in a value in column_descriptions that is not present in their prediction data column_vals = X[key].unique() for val in column_vals: prediction_features.add(key + '=' + str(val)) categorical_col_names.append(key) except: print('\nFound a column in your column_descriptions that is not present in your prediction data:') print(key) elif 'day_part' in key: # We have found a date column. Make sure this date column is in our prediction data # It is outside the scope of this function to make sure that the same date parts are available in both our training and testing data raw_date_col_name = key[:key.index('day_part') - 1] date_col_names.append(raw_date_col_name) elif value == 'output': try: prediction_features.remove(key) except KeyError: pass # Now that we've added in all the one-hot-encoded categorical columns (name=val1, name=val2), remove the base name from our prediction data prediction_features = prediction_features - set(categorical_col_names) # Get only the unique raw_date_col_names date_col_names = set(date_col_names) training_features = set(self.training_features) # Remove all of the transformed date column feature names from our training data features_to_remove = [] for feature in training_features: for raw_date_col_name in date_col_names: if raw_date_col_name in feature: features_to_remove.append(feature) training_features = training_features - set(features_to_remove) # Make sure the raw_date_col_name is in our training data after we have removed all the transformed feature names training_features = training_features | date_col_names # MVP means ignoring text features print_nlp_warning = False nlp_example = None for feature in training_features: if 'nlp_' in feature: print_nlp_warning = True nlp_example = feature training_features.remove(feature) if print_nlp_warning == True: print('\n\nWe found an NLP column in the training data') print('verify_features() currently does not support checking all of the values within an NLP column, so if the text of your NLP column has dramatically changed, you will have to check that yourself.') print('Here is one example of an NLP feature in the training data:') print(nlp_example) training_not_prediction = training_features - prediction_features if raw_features_only == True: training_not_prediction = self.remove_categorical_values(training_not_prediction) if len(training_not_prediction) > 0: print('\n\nHere are the features this model was trained on that were not present in this prediction data:') print(sorted(list(training_not_prediction))) else: print('All of the features this model was trained on are included in the prediction data') prediction_not_training = prediction_features - training_features if raw_features_only == True: prediction_not_training = self.remove_categorical_values(prediction_not_training) if len(prediction_not_training) > 0: # Separate out those values we were told to ignore by column_descriptions ignored_features = [] for feature in prediction_not_training: if self.column_descriptions.get(feature, 'False') == 'ignore': ignored_features.append(feature) prediction_not_training = prediction_not_training - set(ignored_features) print('\n\nHere are the features available in the prediction data that were not part of the training data:') print(sorted(list(prediction_not_training))) if len(ignored_features) > 0: print('\n\nAdditionally, we found features in the prediction data that we were told to ignore in the training data') print(sorted(list(ignored_features))) else: print('All of the features in the prediction data were in this model\'s training data') print('\n\n') return { 'training_not_prediction': training_not_prediction , 'prediction_not_training': prediction_not_training } def score(self, X, y, verbose=False): # At the time of writing this, GradientBoosting does not support sparse matrices for predictions if (self.model_name[:16] == 'GradientBoosting' or self.model_name in ['BayesianRidge', 'LassoLars', 'OrthogonalMatchingPursuit', 'ARDRegression']) and scipy.sparse.issparse(X): X = X.todense() if self._scorer is not None: if self.type_of_estimator == 'regressor': return self._scorer.score(self, X, y) elif self.type_of_estimator == 'classifier': return self._scorer.score(self, X, y) else: return self.model.score(X, y) def predict_proba(self, X, verbose=False): if (self.model_name[:16] == 'GradientBoosting' or self.model_name[:12] == 'DeepLearning' or self.model_name in ['BayesianRidge', 'LassoLars', 'OrthogonalMatchingPursuit', 'ARDRegression']) and scipy.sparse.issparse(X): X = X.todense() try: predictions = self.model.predict_proba(X) except AttributeError as e: try: predictions = self.model.predict(X) except TypeError as e: if scipy.sparse.issparse(X): X = X.todense() predictions = self.model.predict(X) except TypeError as e: if scipy.sparse.issparse(X): X = X.todense() predictions = self.model.predict_proba(X) # If this model does not have predict_proba, and we have fallen back on predict, we want to make sure we give results back in the same format the user would expect for predict_proba, namely each prediction is a list of predicted probabilities for each class. # Note that this DOES NOT WORK for multi-label problems, or problems that are not reduced to 0,1 # If this is not an iterable (ignoring strings, which might be iterable), then we will want to turn our predictions into tupled predictions if not (hasattr(predictions[0], '__iter__') and not isinstance(predictions[0], str)): tupled_predictions = [] for prediction in predictions: if prediction == 1: tupled_predictions.append([0,1]) else: tupled_predictions.append([1,0]) predictions = tupled_predictions # This handles an annoying edge case with libraries like Keras that, for a binary classification problem, with return a single predicted probability in a list, rather than the probability of both classes in a list if len(predictions[0]) == 1: tupled_predictions = [] for prediction in predictions: tupled_predictions.append([1 - prediction[0], prediction[0]]) predictions = tupled_predictions if X.shape[0] == 1: return predictions[0] else: return predictions def predict(self, X, verbose=False): if (self.model_name[:16] == 'GradientBoosting' or self.model_name[:12] == 'DeepLearning' or self.model_name in ['BayesianRidge', 'LassoLars', 'OrthogonalMatchingPursuit', 'ARDRegression']) and scipy.sparse.issparse(X): X_predict = X.todense() else: X_predict = X prediction = self.model.predict(X_predict) # Handle cases of getting a prediction for a single item. # It makes a cleaner interface just to get just the single prediction back, rather than a list with the prediction hidden inside. if isinstance(prediction, np.ndarray): prediction = prediction.tolist() if isinstance(prediction, float) or isinstance(prediction, int) or isinstance(prediction, str): return prediction if len(prediction) == 1: return prediction[0] else: return prediction # transform is initially designed to be used with feature_learning def transform(self, X): predicted_features = self.predict(X) predicted_features = list(predicted_features) X = scipy.sparse.hstack([X, predicted_features], format='csr') return X def predict_uncertainty(self, X): if self.uncertainty_model is None: print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') print('This model was not trained to predict uncertainties') print('Please follow the documentation to tell this model at training time to learn how to predict uncertainties') print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') raise ValueError('This model was not trained to predict uncertainties') base_predictions = self.predict(X) if isinstance(base_predictions, Iterable): base_predictions_col = [[val] for val in base_predictions] base_predictions_col = np.array(base_predictions_col) else: base_predictions_col = [base_predictions] X_combined = scipy.sparse.hstack([X, base_predictions_col], format='csr') uncertainty_predictions = self.uncertainty_model.predict_proba(X_combined) results = { 'base_prediction': base_predictions , 'uncertainty_prediction': uncertainty_predictions } if isinstance(base_predictions, Iterable): results['uncertainty_prediction'] = [row[1] for row in results['uncertainty_prediction']] results = pd.DataFrame.from_dict(results, orient='columns') if self.uc_results is not None: calibration_results = {} # grab the relevant properties from our uc_results, and make them each their own list in calibration_results for key, value in self.uc_results[1].items(): calibration_results[key] = [] for proba in results['uncertainty_prediction']: max_bucket_proba = 0 bucket_num = 1 while proba > max_bucket_proba: calibration_result = self.uc_results[bucket_num] max_bucket_proba = self.uc_results[bucket_num]['max_proba'] bucket_num += 1 for key, value in calibration_result.items(): calibration_results[key].append(value) # TODO: grab the uncertainty_calibration data for DataFrames df_calibration_results = pd.DataFrame.from_dict(calibration_results, orient='columns') del df_calibration_results['max_proba'] results = pd.concat([results, df_calibration_results], axis=1) else: if self.uc_results is not None: # TODO: grab the uncertainty_calibration data for dictionaries for bucket_name, bucket_result in self.uc_results.items(): if proba > bucket_result['max_proba']: break results.update(bucket_result) del results['max_proba'] return results def score_uncertainty(self, X, y, verbose=False): return self.uncertainty_model.score(X, y, verbose=False)
class BaseKerasSklearnModel(base_model.BaseModel): ''' base keras model based on keras's model(without sklearn) ''' ## def __init__(self, data_file, delimiter, lst_x_keys, lst_y_keys, log_filename=DEFAULT_LOG_FILENAME, model_path=DEFAULT_MODEL_PATH, create_model_func=create_model_demo): ## ''' ## init ## ''' ## import framework.tools.log as log ## loger = log.init_log(log_filename) ## self.load_data(data_file, delimiter, lst_x_keys, lst_y_keys) ## self.model_path = model_path ## self.create_model_func=create_model_func def __init__(self, **kargs): ''' init ''' import framework.tools.log as log self.kargs = kargs log_filename = self.kargs["basic_params"]["log_filename"] model_path = self.kargs["basic_params"]["model_path"] self.load_data_func = self.kargs["load_data"]["method"] self.create_model_func = self.kargs["create_model"]["method"] loger = log.init_log(log_filename) (self.dataset, self.X, self.Y, self.X_evaluation, self.Y_evaluation) = self.load_data_func( **self.kargs["load_data"]["params"]) self.model_path = model_path self.dic_params = {} def load_data(self, data_file, delimiter, lst_x_keys, lst_y_keys): ''' load data ''' # Load the dataset self.dataset = numpy.loadtxt(data_file, delimiter=",") self.X = self.dataset[:, lst_x_keys] self.Y = self.dataset[:, lst_y_keys] def init_callbacks(self): ''' init all callbacks ''' os.system("mkdir -p %s" % (self.model_path)) checkpoint_callback = ModelCheckpoint(self.model_path + '/weights.{epoch:02d}-{acc:.2f}.hdf5', \ monitor='acc', save_best_only=False) history_callback = LossHistory() callbacks_list = [checkpoint_callback, history_callback] self.dic_params["callbacks"] = callbacks_list def init_model(self): ''' init model ''' train_params = {"nb_epoch": 10, "batch_size": 10} self.dic_params.update(train_params) self.model = KerasClassifier(build_fn=self.create_model_func, **self.kargs["create_model"]["params"]) # self.model = KerasClassifier(build_fn=self.create_model_func) self.model.set_params(**self.dic_params) def train_model(self): ''' train model ''' X = self.X Y = self.Y X_evaluation = self.X_evaluation Y_evaluation = self.Y_evaluation seed = 7 numpy.random.seed(seed) # Load the dataset history = self.model.fit(X, Y) scores = self.model.score(X, Y) #history_callback = self.dic_params["callbacks"][1] # print dir(history_callback) # logging.info(str(history_callback.losses)) logging.info("final : %.2f%%" % (scores * 100)) logging.info(str(history.history)) def process(self): ''' process ''' self.init_callbacks() self.init_model() self.train_model()
classifier.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch) print('Testing score function') score = classifier.score(X_train, Y_train) print('Score: ', score) print('Testing predict function') preds = classifier.predict(X_test) print('Preds.shape: ', preds.shape) print('Testing predict proba function') proba = classifier.predict_proba(X_test) print('Proba.shape: ', proba.shape) print('Testing get params') print(classifier.get_params()) print('Testing set params') classifier.set_params(optimizer='sgd', loss='mse') print(classifier.get_params()) print('Testing attributes') print('Classes') print(classifier.classes_) print('Config') print(classifier.config_) print('Weights') print(classifier.weights_) print('Test script complete.')
def _fit_and_score_keras2(method, X, y, scorer, train, test, verbose, parameters, fit_params, type="Classification", return_train_score=False, return_parameters=False, return_n_test_samples=False, return_times=False, error_score='raise'): """Fit estimator and compute scores for a given dataset split for KerasClassifier and KerasRegressor. Parameters ---------- estimator : estimator object implementing 'fit' The object to use to fit the data. X : array-like of shape at least 2D The data to fit. y : array-like, optional, default: None The target variable to try to predict in the case of supervised learning. scorer : A single callable or dict mapping scorer name to the callable If it is a single callable, the return value for ``train_scores`` and ``test_scores`` is a single float. For a dict, it should be one mapping the scorer name to the scorer callable object / function. The callable object / fn should have signature ``scorer(estimator, X, y)``. train : array-like, shape (n_train_samples,) Indices of training samples. test : array-like, shape (n_test_samples,) Indices of test samples. verbose : integer The verbosity level. error_score : 'raise' (default) or numeric Value to assign to the score if an error occurs in estimator fitting. If set to 'raise', the error is raised. If a numeric value is given, FitFailedWarning is raised. This parameter does not affect the refit step, which will always raise the error. parameters : dict or None Parameters to be set on the estimator. fit_params : dict or None Parameters that will be passed to ``estimator.fit``. return_train_score : boolean, optional, default: False Compute and return score on training set. return_parameters : boolean, optional, default: False Return parameters that has been used for the estimator. return_n_test_samples : boolean, optional, default: False Whether to return the ``n_test_samples`` return_times : boolean, optional, default: False Whether to return the fit/score times. session : Keras backend with a tensorflow session attached The keras backend session for applying K.clear_session() after the classifier or regressor has been train and scored given the split. This is mainly required to avoid posible Out Of Memory errors with tensorflow not deallocating the GPU memory after each iteration of the Cross Validation. Returns ------- train_scores : dict of scorer name -> float, optional Score on training set (for all the scorers), returned only if `return_train_score` is `True`. test_scores : dict of scorer name -> float, optional Score on testing set (for all the scorers). n_test_samples : int Number of test samples. fit_time : float Time spent for fitting in seconds. score_time : float Time spent for scoring in seconds. parameters : dict or None, optional The parameters that have been evaluated. """ from keras import backend as K import tensorflow as tf tf.logging.set_verbosity( tf.logging.ERROR) # This is useful to avoid the info log of tensorflow # The next 4 lines are for avoiding tensorflow to allocate all the GPU memory config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.set_session(sess) if verbose > 1: if parameters is None: msg = '' else: msg = '%s' % (', '.join('%s=%s' % (k, v) for k, v in parameters.items())) print("[CV] %s %s" % (msg, (64 - len(msg)) * '.')) # Adjust length of sample weights fit_params = fit_params if fit_params is not None else {} fit_params = dict([(k, _index_param_value(X, v, train)) for k, v in fit_params.items()]) test_scores = {} train_scores = {} estimator = None if type == "Classification": from keras.wrappers.scikit_learn import KerasClassifier estimator = KerasClassifier(build_fn=method, verbose=0) else: from keras.wrappers.scikit_learn import KerasRegressor estimator = KerasRegressor(build_fn=method, verbose=0) if parameters is not None: estimator.set_params(**parameters) start_time = time.time() X_train, y_train = _safe_split(estimator, X, y, train) X_test, y_test = _safe_split(estimator, X, y, test, train) is_multimetric = not callable(scorer) n_scorers = len(scorer.keys()) if is_multimetric else 1 try: if y_train is None: estimator.fit(X_train, **fit_params) else: estimator.fit(X_train, y_train, **fit_params) except Exception as e: # Note fit time as time until error fit_time = time.time() - start_time score_time = 0.0 if error_score == 'raise': raise elif isinstance(error_score, numbers.Number): if is_multimetric: test_scores = dict( zip(scorer.keys(), [ error_score, ] * n_scorers)) if return_train_score: train_scores = dict( zip(scorer.keys(), [ error_score, ] * n_scorers)) else: test_scores = error_score if return_train_score: train_scores = error_score warnings.warn( "Classifier fit failed. The score on this train-test" " partition for these parameters will be set to %f. " "Details: \n%r" % (error_score, e), FitFailedWarning) else: raise ValueError("error_score must be the string 'raise' or a" " numeric value. (Hint: if using 'raise', please" " make sure that it has been spelled correctly.)") else: fit_time = time.time() - start_time # _score will return dict if is_multimetric is True test_scores = _score(estimator, X_test, y_test, scorer, is_multimetric) score_time = time.time() - start_time - fit_time if return_train_score: train_scores = _score(estimator, X_train, y_train, scorer, is_multimetric) if verbose > 2: if is_multimetric: for scorer_name, score in test_scores.items(): msg += ", %s=%s" % (scorer_name, score) else: msg += ", score=%s" % test_scores if verbose > 1: total_time = score_time + fit_time end_msg = "%s, total=%s" % (msg, logger.short_format_time(total_time)) print("[CV] %s %s" % ((64 - len(end_msg)) * '.', end_msg)) ret = [train_scores, test_scores] if return_train_score else [test_scores] if return_n_test_samples: ret.append(_num_samples(X_test)) if return_times: ret.extend([fit_time, score_time]) if return_parameters: ret.append(parameters) # The estimator is erased del estimator # We assign the keras backend # Clean the session K.clear_session() # The garbage collector is called in order to ensure that the estimator is erased from memory for i in range(15): gc.collect() return ret
def neural_network_learning(x_train, y_train, x_test, y_test, listoftraintestsplits, config_learning: ConfigurationLearning, num_classes: int): # A. Preprocessing for i in range(len(listoftraintestsplits)): sc = preprocessing.StandardScaler().fit( listoftraintestsplits[i][0].todense()) listoftraintestsplits[i][0] = sc.transform( listoftraintestsplits[i][0].todense()) listoftraintestsplits[i][1] = sc.transform( listoftraintestsplits[i][1].todense()) listoftraintestsplits[i][2] = to_categorical( listoftraintestsplits[i][2], num_classes=num_classes) listoftraintestsplits[i][3] = to_categorical( listoftraintestsplits[i][3], num_classes=num_classes) listoftraintestsplits[i][0], listoftraintestsplits[i][2] = shuffle( listoftraintestsplits[i][0], listoftraintestsplits[i][2], random_state=31 * 9) listoftraintestsplits[i][1], listoftraintestsplits[i][3] = shuffle( listoftraintestsplits[i][1], listoftraintestsplits[i][3], random_state=31 * 9) sc = preprocessing.StandardScaler().fit(x_train.todense()) x_train = sc.transform(x_train.todense()) x_test = sc.transform(x_test.todense()) y_train_c = to_categorical(y_train, num_classes=num_classes) y_test_c = to_categorical(y_test, num_classes=num_classes) x_train, y_train_c = shuffle(x_train, y_train_c, random_state=31 * 5) x_test, y_test_c, y_test = shuffle(x_test, y_test_c, y_test, random_state=31 * 7) # B. Grid search to get best params # activation = ['relu', 'tanh', 'sigmoid'] if config_learning.hyperparameters is None: neurons_eq = [(25, 25), (25, 25, 25), (50, 50), (50, 50, 50), (100, 100), (100, 100, 100), (175, 175), (175, 175, 175), (200, 200), (200, 200, 200), (300, 300)] param_grid = { "optimizer_eq": ['RMSprop'], # ['SGD', 'RMSprop', 'Adagrad', 'Adadelta', 'Adam'], "epochs": [100, 200, 300, 400, 500], "neurons_eq": neurons_eq, #50 "dropout_eq": [0, 0.01, 0.1, 0.25, 0.5], } else: param_grid = config_learning.hyperparameters kerasclf = KerasClassifier(build_fn=_my_keras_model, batch_size=20, verbose=0) best_params, best_params_acc = customized_grid_search_dnn( param_grid=param_grid, clf=kerasclf, listoftraintestsplits=listoftraintestsplits) # C. Learn on best params clf_best = KerasClassifier(build_fn=_my_keras_model, batch_size=30, input_dim_eq=x_train.shape[1], output_dim_eq=y_train_c.shape[1], verbose=0) clf_best.set_params(**best_params) clf_best.fit(x_train, y_train_c) acc = np.mean(np.argmax(y_test_c, axis=1) == clf_best.predict(x_test)) print("DNN-Acc:", acc) return acc, clf_best, sc
class BaseKerasSklearnModel(base_model.BaseModel): ''' base keras model based on keras's model(without sklearn) ''' ## def __init__(self, data_file, delimiter, lst_x_keys, lst_y_keys, log_filename=DEFAULT_LOG_FILENAME, model_path=DEFAULT_MODEL_PATH, create_model_func=create_model_demo): ## ''' ## init ## ''' ## import framework.tools.log as log ## loger = log.init_log(log_filename) ## self.load_data(data_file, delimiter, lst_x_keys, lst_y_keys) ## self.model_path = model_path ## self.create_model_func=create_model_func def __init__(self, **kargs): ''' init ''' import framework.tools.log as log self.kargs = kargs log_filename = self.kargs["basic_params"]["log_filename"] model_path = self.kargs["basic_params"]["model_path"] self.load_data_func = self.kargs["load_data"]["method"] self.create_model_func = self.kargs["create_model"]["method"] loger = log.init_log(log_filename) (self.dataset, self.X, self.Y, self.X_evaluation, self.Y_evaluation) = self.load_data_func(**self.kargs["load_data"]["params"]) self.model_path = model_path self.dic_params = {} def load_data(self, data_file, delimiter, lst_x_keys, lst_y_keys): ''' load data ''' # Load the dataset self.dataset = numpy.loadtxt(data_file, delimiter=",") self.X = self.dataset[:, lst_x_keys] self.Y = self.dataset[:, lst_y_keys] def init_callbacks(self): ''' init all callbacks ''' os.system("mkdir -p %s" % (self.model_path)) checkpoint_callback = ModelCheckpoint(self.model_path + '/weights.{epoch:02d}-{acc:.2f}.hdf5', \ monitor='acc', save_best_only=False) history_callback = LossHistory() callbacks_list = [checkpoint_callback, history_callback] self.dic_params["callbacks"] = callbacks_list def init_model(self): ''' init model ''' train_params = {"nb_epoch": 10, "batch_size": 10} self.dic_params.update(train_params) self.model = KerasClassifier(build_fn=self.create_model_func, **self.kargs["create_model"]["params"]) # self.model = KerasClassifier(build_fn=self.create_model_func) self.model.set_params(**self.dic_params) def train_model(self): ''' train model ''' X = self.X Y = self.Y X_evaluation = self.X_evaluation Y_evaluation = self.Y_evaluation seed = 7 numpy.random.seed(seed) # Load the dataset history = self.model.fit(X, Y) scores = self.model.score(X, Y) #history_callback = self.dic_params["callbacks"][1] # print dir(history_callback) # logging.info(str(history_callback.losses)) logging.info("final : %.2f%%" % (scores * 100)) logging.info(str(history.history)) def process(self): ''' process ''' self.init_callbacks() self.init_model() self.train_model()
batch_size = [20, 25, 30] epochs = [20, 25, 30] learn_rate = [0.005, 0.01, 0.015] momentum = [0.85, 0.9, 0.95] grid = dict(epochs=epochs, batch_size=batch_size, learn_rate=learn_rate, momentum=momentum) t1 = time.time() scores = [] model_tt = KerasClassifier(build_fn=create_model, verbose=0) for g in ParameterGrid(grid): model_tt.set_params(**g) model_tt.fit(X_train, Y_train) scores.append(dict(params=g, score=model_tt.score(X_test, Y_test))) print('model#', len(scores), scores[-1]) t2 = time.time() print("Training time:", t2 - t1, 'sec') df = pandas.DataFrame([{**row['params'], **row} for row in scores]) df = df.drop('params', axis=1) df.sort_values('score') model_tt.model.save('my_model.h5') model = keras.models.load_model('my_model.h5')
def r_neural_network_learning(x_train, y_train, x_test, y_test, listoftraintestsplits, config_learning: ConfigurationLearningRNN, num_classes: int): # A. Preprocessing for i in range(len(listoftraintestsplits)): if config_learning.scale: sc = preprocessing.StandardScaler().fit( listoftraintestsplits[i][0].todense()) listoftraintestsplits[i][0] = sc.transform( listoftraintestsplits[i][0].todense()) listoftraintestsplits[i][1] = sc.transform( listoftraintestsplits[i][1].todense()) else: listoftraintestsplits[i][0] = np.array( listoftraintestsplits[i][0].todense()) listoftraintestsplits[i][1] = np.array( listoftraintestsplits[i][1].todense()) listoftraintestsplits[i].append( to_categorical(listoftraintestsplits[i][2], num_classes=num_classes)) listoftraintestsplits[i].append( to_categorical(listoftraintestsplits[i][3], num_classes=num_classes)) # listoftraintestsplits[i][0], listoftraintestsplits[i][2] = shuffle(listoftraintestsplits[i][0], # listoftraintestsplits[i][2], # random_state=31 * 9) # listoftraintestsplits[i][1], listoftraintestsplits[i][3] = shuffle(listoftraintestsplits[i][1], # listoftraintestsplits[i][3], # random_state=31 * 9) trainsplitshape = (listoftraintestsplits[i][0].shape[0], 1, listoftraintestsplits[i][0].shape[1]) testsplitshape = (listoftraintestsplits[i][1].shape[0], 1, listoftraintestsplits[i][1].shape[1]) listoftraintestsplits[i][0] = listoftraintestsplits[i][0].reshape( trainsplitshape) listoftraintestsplits[i][1] = listoftraintestsplits[i][1].reshape( testsplitshape) assert len(listoftraintestsplits[0]) == 6 if config_learning.scale: sc = preprocessing.StandardScaler().fit(x_train.todense()) x_train = sc.transform(x_train.todense()) x_test = sc.transform(x_test.todense()) else: sc = None x_train = np.array(x_train.todense()) x_test = np.array(x_test.todense()) y_train_c = to_categorical(y_train, num_classes=num_classes) y_test_c = to_categorical(y_test, num_classes=num_classes) feature_dim = x_train.shape[1] x_train = x_train.reshape(x_train.shape[0], 1, x_train.shape[1]) x_test = x_test.reshape(x_test.shape[0], 1, x_test.shape[1]) # x_train, y_train_c = shuffle(x_train, y_train_c, random_state=31 * 5) # x_test, y_test_c, y_test = shuffle(x_test, y_test_c, y_test, random_state=31 * 7) # "optimizer": ['RMSprop'], # ['SGD', 'RMSprop', 'Adagrad', 'Adadelta', 'Adam'], if config_learning.hyperparameters is None: param_grid = { "RNN_epochs": [100, 200, 300, 350, 450, 500], #350], #50], "RNN_nounits": [32, 128, 196, 256, 288], #, feature_dim], "RNN_dropout": [0.6], "RNN_lstmlayersno": [3], "RNN_denselayersno": [3], "RNN_l2reg": [0.00001], "RNN_denseneurons": [round(0.45 * feature_dim)] } else: param_grid = config_learning.hyperparameters param_grid['RNN_denseneurons'] = [ round(x * feature_dim) for x in param_grid['RNN_denseneurons'] ] if config_learning.cv_optimize_rlf_params: param_grid_rf = { "RF_n_estimators": [250], "RF_max_features": [0.3, 0.6, 'auto'], "RF_max_depth": [10, 25, 50, 75, None], "RF_min_samples_leaf": [6, 12, 1], } param_grid.update(param_grid_rf) kerasclf = KerasClassifier(build_fn=my_model, batch_size=128, input_dim_eq=feature_dim, output_dim_eq=num_classes, optimizer="Adam", verbose=0) best_params_, best_params_acc = customized_grid_search_rnn( param_grid=param_grid, clf=kerasclf, listoftraintestsplits=listoftraintestsplits, cv_use_rnn_output=config_learning.cv_use_rnn_output, noofparallelthreads=config_learning.noofparallelthreads) best_params_rnn, best_params_rf = split_params_into_rnn_rf( params=best_params_) early_stop = keras.callbacks.EarlyStopping(monitor="loss", patience=20, verbose=1, min_delta=0.0) # param['callbacks'] = [early_stop] # param['validation_data'] = (x_test.reshape(x_test.shape[0], feature_dim, 1), y_test_c) # C. Learn on best params clf_best = KerasClassifier(build_fn=my_model, batch_size=128, input_dim_eq=feature_dim, output_dim_eq=num_classes, optimizer=keras.optimizers.Adam(lr=10e-4), callbacks=[early_stop], verbose=1) clf_best.set_params(**best_params_rnn) rnnhist = clf_best.fit(x_train, y_train_c) rnnacc: float = np.mean( np.argmax(y_test_c, axis=1) == clf_best.predict(x_test)) # D. Learn RF rlf_deep, rfaccuracy = compute_rlf_on_rnn( clf_best=clf_best, x_train=x_train, x_test=x_test, y_train=y_train, y_test=y_test, params=best_params_rf, noofparallelthreads=config_learning.noofparallelthreads) print("DNN-Acc: {}% // RF-Acc: {}%".format(round(rnnacc * 100, 2), round(rfaccuracy * 100, 2))) return rnnacc, clf_best, sc, rlf_deep, rfaccuracy, rnnhist