def C_Hyp(trainDir2, valDir2, testDir2, modelEpochs=20):
import skopt
from skopt import gp_minimize
from skopt import gp_minimize, forest_minimize
from skopt.space import Real, Categorical, Integer
from skopt.plots import plot_convergence
from skopt.plots import plot_objective, plot_evaluations
from tensorflow.python.keras import backend as K
from keras.models import Model, load_model, Sequential
from keras.layers import Input, Dense, LSTM, RepeatVector, TimeDistributed
from skopt.utils import use_named_args
import numpy as np
path_best_model = '19_best_model.keras'
dim_learning_rate = Real(low=1e-6, high=1e-2, prior='log-uniform',
name='learning_rate')
dim_activation = Categorical(categories=['relu', 'softmax'],
name='activation_function')
optimizer = Categorical(categories=['Adadelta', 'Adagrad', 'Adam', 'Adamax', 'Nadam', 'SGD'], name='optimizer')
loss = Categorical(
categories=['binary_crossentropy', 'categorical_crossentropy', 'categorical_hinge', 'mean_absolute_error',
'mean_absolute_percentage_error', 'mean_squared_error', 'mean_squared_logarithmic_error'],
name='loss')
dim_num_dense_layers = Integer(low=0, high=1, name='num_dense_layers')
batch_size = Integer(low=32, high=128, name='batch_size')
dimensions = [dim_learning_rate,
dim_activation,
optimizer,
loss,
dim_num_dense_layers,
batch_size]
trainDir2 = pd.DataFrame(trainDir2)
valDir2 = pd.DataFrame(valDir2)
@use_named_args(dimensions=dimensions)
def C_hyper(learning_rate, activation_function, optimizer, loss, num_dense_layers, batch_size):
modelEpochs = 20
valDir = valDir2
sequenceLength2 = 100
trainDir = trainDir2
df = trainDir
fault = 'Class'
col = df.columns
for elements in col:
if "FaultNumber" or "Class" or "class" or "faultnumber" in elements:
fault = elements
# df = df.loc[df[fault]==0]
df = df.drop([fault], axis=1)
trainingData = pd.DataFrame(df)
print("loading_data")
#global numOfColumns
# global valDir
testDir = testDir2
# global sequenceLength
def data_3d_reshape(Data):
print(Data.shape, "DATA_3D_Reshaping")
sequenceLength = sequenceLength2
numOfSequences = int(Data.shape[0] / sequenceLength)
numOfColumns = int(Data.shape[1])
print("sequenceLength: ", sequenceLength)
print("numOfSequences: ", numOfSequences)
print("numOfColumns: ", numOfColumns)
boot = Data.shape[0]
bootr = sequenceLength * numOfSequences
print(bootr)
final = Data[0:bootr, :]
print(final.shape, 'shape of final ')
return final.reshape(numOfSequences, sequenceLength, numOfColumns)
def myround(x, base):
a = base * np.round(x / base)
if a > x:
return a - 50
else:
return a
# return base*np.round(x/base)
valx = myround(trainingData.shape[0], sequenceLength2)
valx = int(valx)
print(valx, "heeee")
trainingData = trainingData.iloc[0:valx, :]
trainDataColumns = trainingData.columns
print(trainingData.shape, "HEREE")
# setting up number of sequences and number of columns
numOfSequences = int(len(trainingData.index) / sequenceLength2)
numOfColumns = len(trainingData.columns)
# scaling the data & saving the scaler for later use
scaler = MinMaxScaler()
scaled_trainingData = scaler.fit_transform(trainingData)
# scaler_filename = "scaler.save"
# joblib.dump(scaler, scaler_filename)
scaler = scaler
dfval = valDir
col = dfval.columns
valDir = dfval
valdirfault = valDir[fault]
valDir = valDir.drop([fault], axis=1)
valval = myround(valDir.shape[0], sequenceLength2)
valval = int(valval)
valDir = valDir.iloc[:valval, :]
valDir = scaler.fit_transform(valDir)
alDir = data_3d_reshape(valDir)
# for test
testfault = testDir[fault]
testDir = testDir.drop([fault], axis=1)
valtest = myround(testDir.shape[0], sequenceLength2)
valtest = int(valtest)
testDir = testDir.iloc[:valtest, :]
testfault = testfault.iloc[:valtest]
testDir = scaler.fit_transform(testDir)
testDir = data_3d_reshape(testDir)
# converting the scaled data to dataFrame
trainingData = data_3d_reshape(scaled_trainingData)
scaledTrainingData = trainingData
sequenceLength = sequenceLength2
numOfColumns = numOfColumns
def autoencoder_model(trainingData):
"""
Desctiption:
------------
This function creates the lstm autoencoder model with the given class parameters settings.
"""
lstm_autoencoder = Sequential()
# Encoder
lstm_autoencoder.add(
LSTM(sequenceLength, activation=activation_function, input_shape=(None, numOfColumns), return_sequences=True))
lstm_autoencoder.add(LSTM(120, activation=activation_function, return_sequences=True))
lstm_autoencoder.add(LSTM(120, activation=activation_function, return_sequences=True))
lstm_autoencoder.add(LSTM(60, activation=activation_function))
lstm_autoencoder.add(RepeatVector(sequenceLength))
# Decoder
lstm_autoencoder.add(LSTM(sequenceLength, activation=activation_function, return_sequences=True))
lstm_autoencoder.add(LSTM(120, activation='relu', return_sequences=True))
lstm_autoencoder.add(LSTM(120, activation='relu', return_sequences=True))
lstm_autoencoder.add(TimeDistributed(Dense(numOfColumns)))
# global modelEpochs
lstm_autoencoder.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])
history = lstm_autoencoder.fit(trainingData, trainingData, epochs=modelEpochs, verbose=1,
batch_size=batch_size)
history_dict = history.history
accuracy = history.history['accuracy'][-1]
return accuracy
accuracy = autoencoder_model(trainingData)
global best_accuracyC
global path_best_model
# If the classification accuracy of the saved model is improved ...
if accuracy > best_accuracyC:
# Save the new model to harddisk.
# autoencoder.save(path_best_model)learning_rate, activation_function, optimizer, loss ,num_dense_layers, batch_size
bestresultC.append(learning_rate)
bestresultC.append(activation_function)
bestresultC.append(optimizer)
bestresultC.append(loss)
bestresultC.append(num_dense_layers)
bestresultC.append(batch_size)
# Update the classification accuracy.
best_accuracyC = accuracy
# Delete the Keras model with these hyper-parameters from memory.
#del lstm_autoencoder
# Clear the Keras session, otherwise it will keep adding new
# models to the same TensorFlow graph each time we create
# a model with a different set of hyper-parameters.
K.clear_session()
# NOTE: Scikit-optimize does minimization so it tries to
# find a set of hyper-parameters with the LOWEST fitness-value.
# Because we are interested in the HIGHEST classification
# accuracy, we need to negate this number so it can be minimized.
return -accuracy
default_parameters = [1e-5, 'relu', 'Adam', 'mean_absolute_error', 1, 83]
search_result = gp_minimize(func=C_hyper, dimensions=dimensions, acq_func='EI', n_calls=50, x0=default_parameters)
print(bestresultC)
print(best_accuracyC)
return bestresultC[-6:]
def SVMhyp(X_train,
X_val,
X_test,
y_train,
y_val,
y_test,
threshold_fn_percentage=0.10):
min_max_scaler = preprocessing.MinMaxScaler()
X_train = min_max_scaler.fit_transform(X_train.astype(np.float))
X_val = min_max_scaler.transform(X_val.astype(np.float))
X_test = min_max_scaler.transform(X_test.astype(np.float))
threshold_fn_percentage = threshold_fn_percentage
X_train = X_train
X_val = X_val
X_test = X_test
y_train = y_train
y_val = y_val
y_test = y_test
# model params
nu = Categorical(categories=[
'0.1', '0.2', '0.3', '0.4', '0.5', '0.6', '0.7', '0.8', '0.9'
],
name='nu')
kernel = Categorical(categories=['linear', 'poly', 'rbf'], name='kernel')
# gamma = Integer(low=1, high=5, name='gamma')
gamma = Real(low=1e-3, high=2, prior='log-uniform', name='gamma')
dimensions = [kernel, gamma, nu]
@use_named_args(dimensions=dimensions)
def svmHyper(kernel, gamma, nu):
nu = float(nu)
# global best_resultssvm
clf = svm.OneClassSVM(gamma=gamma, kernel=kernel, nu=nu)
print('\ntraining the classifier start time: ', str(datetime.now()))
print('\n', clf)
clf = clf.fit(X_train)
val_score = clf.score_samples(X_val)
threshold = 0
best_threshold = 0
acceptable_n_FP = threshold_fn_percentage * len(y_train)
print("\nacceptable_n_FP: ", acceptable_n_FP)
TN = 0
FP = 0
print("\ncalculating threshold......")
while (threshold <= 1):
# print ('**************************')
# print (threshold)
threshold += 0.005
y_pred = [1 if e > threshold else 0 for e in val_score]
y_Pred = np.array(y_pred)
# Confusion Matrix
from sklearn.metrics import confusion_matrix, precision_recall_fscore_support
conf_matrix = confusion_matrix(y_val, y_pred, labels=[0, 1])
tn, fp, fn, tp = conf_matrix.ravel()
# print(conf_matrix)
# print("tn: " , tn)
# print("fp: " ,fp)
if fp < acceptable_n_FP:
break
best_threshold = threshold
y_pred = [1 if e >= best_threshold else 0 for e in val_score]
from sklearn.metrics import confusion_matrix, precision_recall_fscore_support
precision_W, recall_W, fscore_W, xyz = precision_recall_fscore_support(
y_val, y_pred, average='weighted')
global best_resultssvm
# global bestresultssvmlist
print(fscore_W)
if fscore_W > best_resultssvm:
# Save the new model to harddisk.
# autoencoder.save(path_best_model)learning_rate, activation_function, optimizer, loss ,num_dense_layers, batch_size
bestresultssvmlist.append(kernel)
bestresultssvmlist.append(gamma)
bestresultssvmlist.append(nu)
# Update the classification accuracy.
best_resultssvm = fscore_W
# accuracy, we need to negate this number so it can be minimized.
return -fscore_W
default_parameters = ['rbf', 1e-3, '0.2']
search_result = gp_minimize(func=svmHyper,
dimensions=dimensions,
acq_func='EI',
n_calls=50,
x0=default_parameters)
return bestresultssvmlist[-3:]
# In[26]: embedding_matrix_glove = get_GloVe_embedding_matrix(glove_model) embedding_matrix_word2vec = get_word_embedding_matrix(word2vec_model,300) embedding_matrix_fast_text = get_word_embedding_matrix(fast_text_model,100) embedding_matrix_godin = get_word_embedding_matrix(godin_model,400) # In[29]: para_learning_rate = Real(low=1e-4, high=1e-2, prior='log-uniform',name='learning_rate') para_dropout = Categorical(categories=[0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9],name = 'dropout') # para_em = Categorical(categories=['embedding_matrix_fast_text','embedding_matrix_godin','embedding_matrix_word2vec','embedding_matrix_glove'],name='em') para_em = Categorical(categories=['embedding_matrix_word2vec'],name='em') para_em_trainable_flag = Categorical(categories=[True,False],name='em_trainable_flag') para_batch_size = Categorical(categories=[8,16,32,64],name='batch_size') para_epoch = Categorical(categories=[5,10,20,50,100],name='epoch') # para_units_out = Categorical(categories=[64,128,256,512], name='units_out') # para_dropout_cnn_lstm = Categorical(categories=[0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9],name = 'dropout') para_n_dense = Categorical(categories=[100,200,300,400], name='n_dense') para_n_filters = Categorical(categories=[32,100,200,300],name='n_filters') para_filter_size = Integer(low=1,high=8,name = 'filter_size')
count = 0
for y_p, y_t in zip(y_predict, y_target):
if y_p == y_t:
count += 1
return float(count) / len(y_predict)
"""
search space for hyper-parameters
"""
search_space = [
Real(low=0, high=1, name='dropout_rate'),
Integer(low=1, high=10, name='n_hidden_layers'),
Integer(low=32, high=150, name='dim_B'),
Integer(low=32, high=150, name='dim_C'),
Categorical(categories=['8', '16', '32', '64'], name='batch_size'),
Real(low=1e-6, high=1e-2, prior='log-uniform', name='learning_rate')
]
@use_named_args(dimensions=search_space)
def fitness(dropout_rate, n_hidden_layers, dim_B, dim_C, batch_size,
learning_rate):
"""
tuning process for a single group of hyper-parameters
"""
weight_decay = 0
batch_size = int(batch_size)
model = skorch.classifier.NeuralNetBinaryClassifier(
module=DiscriminatorMLPPremiumForTuning,
module__hidden_size=HIDDEN_SIZE,
def test_space_consistency():
# Reals (uniform)
s1 = Space([Real(0.0, 1.0)])
s2 = Space([Real(0.0, 1.0)])
s3 = Space([Real(0, 1)])
s4 = Space([(0.0, 1.0)])
s5 = Space([(0.0, 1.0, "uniform")])
s6 = Space([(0, 1.0)])
s7 = Space([(np.float64(0.0), 1.0)])
s8 = Space([(0, np.float64(1.0))])
a1 = s1.rvs(n_samples=10, random_state=0)
a2 = s2.rvs(n_samples=10, random_state=0)
a3 = s3.rvs(n_samples=10, random_state=0)
a4 = s4.rvs(n_samples=10, random_state=0)
a5 = s5.rvs(n_samples=10, random_state=0)
assert_equal(s1, s2)
assert_equal(s1, s3)
assert_equal(s1, s4)
assert_equal(s1, s5)
assert_equal(s1, s6)
assert_equal(s1, s7)
assert_equal(s1, s8)
assert_array_equal(a1, a2)
assert_array_equal(a1, a3)
assert_array_equal(a1, a4)
assert_array_equal(a1, a5)
# Reals (log-uniform)
s1 = Space([Real(10**-3.0, 10**3.0, prior="log-uniform")])
s2 = Space([Real(10**-3.0, 10**3.0, prior="log-uniform")])
s3 = Space([Real(10**-3, 10**3, prior="log-uniform")])
s4 = Space([(10**-3.0, 10**3.0, "log-uniform")])
s5 = Space([(np.float64(10**-3.0), 10**3.0, "log-uniform")])
a1 = s1.rvs(n_samples=10, random_state=0)
a2 = s2.rvs(n_samples=10, random_state=0)
a3 = s3.rvs(n_samples=10, random_state=0)
a4 = s4.rvs(n_samples=10, random_state=0)
assert_equal(s1, s2)
assert_equal(s1, s3)
assert_equal(s1, s4)
assert_equal(s1, s5)
assert_array_equal(a1, a2)
assert_array_equal(a1, a3)
assert_array_equal(a1, a4)
# Integers
s1 = Space([Integer(1, 5)])
s2 = Space([Integer(1.0, 5.0)])
s3 = Space([(1, 5)])
s4 = Space([(np.int64(1.0), 5)])
s5 = Space([(1, np.int64(5.0))])
a1 = s1.rvs(n_samples=10, random_state=0)
a2 = s2.rvs(n_samples=10, random_state=0)
a3 = s3.rvs(n_samples=10, random_state=0)
assert_equal(s1, s2)
assert_equal(s1, s3)
assert_equal(s1, s4)
assert_equal(s1, s5)
assert_array_equal(a1, a2)
assert_array_equal(a1, a3)
# Categoricals
s1 = Space([Categorical(["a", "b", "c"])])
s2 = Space([Categorical(["a", "b", "c"])])
s3 = Space([["a", "b", "c"]])
a1 = s1.rvs(n_samples=10, random_state=0)
a2 = s2.rvs(n_samples=10, random_state=0)
a3 = s3.rvs(n_samples=10, random_state=0)
assert_equal(s1, s2)
assert_array_equal(a1, a2)
assert_equal(s1, s3)
assert_array_equal(a1, a3)
s1 = Space([(True, False)])
s2 = Space([Categorical([True, False])])
s3 = Space([np.array([True, False])])
assert s1 == s2 == s3
# %% global variables
X_labelled = pd.read_csv(PATH_X_LABELLED, index_col=INDEX_COLS)
y_labelled = pd.read_csv(PATH_Y_LABELLED, index_col=INDEX_COLS)
X_pool = pd.read_csv(PATH_X_POOL, index_col=INDEX_COLS)
y_pool = pd.read_csv(PATH_Y_POOL, index_col=INDEX_COLS)
runs_skopt = 0
N_FEATURES_AUD = X_labelled.filter(regex='aud', axis=1).shape[-1]
N_FEATURES_VID = X_labelled.filter(regex='vid', axis=1).shape[-1]
pool_size = 10
SEQUENCE_LENGTH = 100
# %%
X_labelled.head()
# %% search space
space = [Categorical([8, 16, 32], name='batch_size'),
Real(0.1, 0.4, name='dropout'),
Real(0.0, 0.5, name='rec_dropout'),
Real(1e-06, 1e-04, prior='log-uniform', name='rec_l2'),
Real(1e-06, 1e-04, prior='log-uniform', name='kernel_l2'),
Integer(42, 44, name='n_neurons_hid_aud'),
Integer(90, 180, name='n_neurons_hid_vid')
]
# %%
early_stopper = EarlyStopping(
monitor='val_pred_reg_arou_loss', mode='min',
min_delta=0.001, patience=70, verbose=0,
)
# %% Start hyperparameters:
x_0 = [32, 0.12, 0.08, 2e-06, 2e-06, 44, 120]
def runParameterSearch_Collaborative(recommender_class,
URM_train,
URM_train_last_test=None,
metric_to_optimize="PRECISION",
evaluator_validation=None,
evaluator_test=None,
evaluator_validation_earlystopping=None,
output_folder_path="result_experiments/",
parallelizeKNN=True,
n_cases=35,
n_random_starts=5,
resume_from_saved=False,
save_model="best",
allow_weighting=True,
similarity_type_list=None):
# If directory does not exist, create
if not os.path.exists(output_folder_path):
os.makedirs(output_folder_path)
earlystopping_keywargs = {
"validation_every_n": 5,
"stop_on_validation": True,
"evaluator_object": evaluator_validation_earlystopping,
"lower_validations_allowed": 5,
"validation_metric": metric_to_optimize,
}
URM_train = URM_train.copy()
if URM_train_last_test is not None:
URM_train_last_test = URM_train_last_test.copy()
try:
output_file_name_root = recommender_class.RECOMMENDER_NAME
parameterSearch = SearchBayesianSkopt(
recommender_class,
evaluator_validation=evaluator_validation,
evaluator_test=evaluator_test)
if recommender_class in [TopPop, GlobalEffects, Random]:
"""
TopPop, GlobalEffects and Random have no parameters therefore only one evaluation is needed
"""
parameterSearch = SearchSingleCase(
recommender_class,
evaluator_validation=evaluator_validation,
evaluator_test=evaluator_test)
recommender_input_args = SearchInputRecommenderArgs(
CONSTRUCTOR_POSITIONAL_ARGS=[URM_train],
CONSTRUCTOR_KEYWORD_ARGS={},
FIT_POSITIONAL_ARGS=[],
FIT_KEYWORD_ARGS={})
if URM_train_last_test is not None:
recommender_input_args_last_test = recommender_input_args.copy(
)
recommender_input_args_last_test.CONSTRUCTOR_POSITIONAL_ARGS[
0] = URM_train_last_test
else:
recommender_input_args_last_test = None
parameterSearch.search(
recommender_input_args,
recommender_input_args_last_test=
recommender_input_args_last_test,
fit_hyperparameters_values={},
output_folder_path=output_folder_path,
output_file_name_root=output_file_name_root,
resume_from_saved=resume_from_saved,
save_model=save_model,
)
return
##########################################################################################################
if recommender_class in [ItemKNNCFRecommender, UserKNNCFRecommender]:
if similarity_type_list is None:
similarity_type_list = [
'cosine', 'jaccard', "asymmetric", "dice", "tversky"
]
recommender_input_args = SearchInputRecommenderArgs(
CONSTRUCTOR_POSITIONAL_ARGS=[URM_train],
CONSTRUCTOR_KEYWORD_ARGS={},
FIT_POSITIONAL_ARGS=[],
FIT_KEYWORD_ARGS={})
if URM_train_last_test is not None:
recommender_input_args_last_test = recommender_input_args.copy(
)
recommender_input_args_last_test.CONSTRUCTOR_POSITIONAL_ARGS[
0] = URM_train_last_test
else:
recommender_input_args_last_test = None
run_KNNCFRecommender_on_similarity_type_partial = partial(
run_KNNRecommender_on_similarity_type,
recommender_input_args=recommender_input_args,
parameter_search_space={},
parameterSearch=parameterSearch,
n_cases=n_cases,
n_random_starts=n_random_starts,
resume_from_saved=resume_from_saved,
save_model=save_model,
output_folder_path=output_folder_path,
output_file_name_root=output_file_name_root,
metric_to_optimize=metric_to_optimize,
allow_weighting=allow_weighting,
recommender_input_args_last_test=
recommender_input_args_last_test)
if parallelizeKNN:
pool = multiprocessing.Pool(
processes=multiprocessing.cpu_count(), maxtasksperchild=1)
pool.map(run_KNNCFRecommender_on_similarity_type_partial,
similarity_type_list)
pool.close()
pool.join()
else:
for similarity_type in similarity_type_list:
run_KNNCFRecommender_on_similarity_type_partial(
similarity_type)
return
##########################################################################################################
if recommender_class is P3alphaRecommender:
hyperparameters_range_dictionary = {}
hyperparameters_range_dictionary["topK"] = Integer(5, 1000)
hyperparameters_range_dictionary["alpha"] = Real(low=0,
high=2,
prior='uniform')
hyperparameters_range_dictionary[
"normalize_similarity"] = Categorical([True, False])
recommender_input_args = SearchInputRecommenderArgs(
CONSTRUCTOR_POSITIONAL_ARGS=[URM_train],
CONSTRUCTOR_KEYWORD_ARGS={},
FIT_POSITIONAL_ARGS=[],
FIT_KEYWORD_ARGS={})
##########################################################################################################
if recommender_class is RP3betaRecommender:
hyperparameters_range_dictionary = {}
hyperparameters_range_dictionary["topK"] = Integer(3, 50)
hyperparameters_range_dictionary["alpha"] = Real(low=0,
high=2,
prior='uniform')
hyperparameters_range_dictionary["beta"] = Real(low=0,
high=2,
prior='uniform')
hyperparameters_range_dictionary[
"normalize_similarity"] = Categorical([1, 0])
recommender_input_args = SearchInputRecommenderArgs(
CONSTRUCTOR_POSITIONAL_ARGS=[URM_train],
CONSTRUCTOR_KEYWORD_ARGS={},
FIT_POSITIONAL_ARGS=[],
FIT_KEYWORD_ARGS={})
##########################################################################################################
if recommender_class is MatrixFactorization_FunkSVD_Cython:
hyperparameters_range_dictionary = {}
hyperparameters_range_dictionary["sgd_mode"] = Categorical(
["sgd", "adagrad", "adam"])
hyperparameters_range_dictionary["epochs"] = Categorical([500])
hyperparameters_range_dictionary["use_bias"] = Categorical(
[True, False])
hyperparameters_range_dictionary["batch_size"] = Categorical(
[1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024])
hyperparameters_range_dictionary["num_factors"] = Categorical(
[200])
hyperparameters_range_dictionary["item_reg"] = Real(
low=1e-5, high=1e-2, prior='log-uniform')
hyperparameters_range_dictionary["user_reg"] = Real(
low=1e-5, high=1e-2, prior='log-uniform')
hyperparameters_range_dictionary["learning_rate"] = Real(
low=1e-4, high=1e-1, prior='log-uniform')
hyperparameters_range_dictionary[
"negative_interactions_quota"] = Real(low=0.0,
high=0.5,
prior='uniform')
recommender_input_args = SearchInputRecommenderArgs(
CONSTRUCTOR_POSITIONAL_ARGS=[URM_train],
CONSTRUCTOR_KEYWORD_ARGS={},
FIT_POSITIONAL_ARGS=[],
FIT_KEYWORD_ARGS=earlystopping_keywargs)
##########################################################################################################
if recommender_class is MatrixFactorization_AsySVD_Cython:
hyperparameters_range_dictionary = {}
hyperparameters_range_dictionary["sgd_mode"] = Categorical(
["sgd", "adagrad", "adam"])
hyperparameters_range_dictionary["epochs"] = Categorical([500])
hyperparameters_range_dictionary["use_bias"] = Categorical(
[True, False])
hyperparameters_range_dictionary["batch_size"] = Categorical([1])
hyperparameters_range_dictionary["num_factors"] = Integer(1, 200)
hyperparameters_range_dictionary["item_reg"] = Real(
low=1e-5, high=1e-2, prior='log-uniform')
hyperparameters_range_dictionary["user_reg"] = Real(
low=1e-5, high=1e-2, prior='log-uniform')
hyperparameters_range_dictionary["learning_rate"] = Real(
low=1e-4, high=1e-1, prior='log-uniform')
hyperparameters_range_dictionary[
"negative_interactions_quota"] = Real(low=0.0,
high=0.5,
prior='uniform')
recommender_input_args = SearchInputRecommenderArgs(
CONSTRUCTOR_POSITIONAL_ARGS=[URM_train],
CONSTRUCTOR_KEYWORD_ARGS={},
FIT_POSITIONAL_ARGS=[],
FIT_KEYWORD_ARGS=earlystopping_keywargs)
##########################################################################################################
if recommender_class is MatrixFactorization_BPR_Cython:
hyperparameters_range_dictionary = {}
hyperparameters_range_dictionary["sgd_mode"] = Categorical(
["sgd", "adagrad", "adam"])
hyperparameters_range_dictionary["epochs"] = Integer(20, 500)
hyperparameters_range_dictionary["num_factors"] = Integer(1, 50)
hyperparameters_range_dictionary["batch_size"] = Categorical(
[1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024])
hyperparameters_range_dictionary["positive_reg"] = Real(
low=1e-5, high=1e-2, prior='log-uniform')
hyperparameters_range_dictionary["negative_reg"] = Real(
low=1e-5, high=1e-2, prior='log-uniform')
hyperparameters_range_dictionary["learning_rate"] = Real(
low=1e-4, high=1e-1, prior='log-uniform')
recommender_input_args = SearchInputRecommenderArgs(
CONSTRUCTOR_POSITIONAL_ARGS=[URM_train],
CONSTRUCTOR_KEYWORD_ARGS={},
FIT_POSITIONAL_ARGS=[],
FIT_KEYWORD_ARGS={
**earlystopping_keywargs, "positive_threshold_BPR": None
})
##########################################################################################################
if recommender_class is IALSRecommender:
hyperparameters_range_dictionary = {}
hyperparameters_range_dictionary["num_factors"] = Integer(1, 200)
hyperparameters_range_dictionary[
"confidence_scaling"] = Categorical(["linear", "log"])
hyperparameters_range_dictionary["alpha"] = Real(
low=1e-3, high=50.0, prior='log-uniform')
hyperparameters_range_dictionary["epsilon"] = Real(
low=1e-3, high=10.0, prior='log-uniform')
hyperparameters_range_dictionary["reg"] = Real(low=1e-5,
high=1e-2,
prior='log-uniform')
recommender_input_args = SearchInputRecommenderArgs(
CONSTRUCTOR_POSITIONAL_ARGS=[URM_train],
CONSTRUCTOR_KEYWORD_ARGS={},
FIT_POSITIONAL_ARGS=[],
FIT_KEYWORD_ARGS=earlystopping_keywargs)
##########################################################################################################
if recommender_class is PureSVDRecommender:
hyperparameters_range_dictionary = {}
hyperparameters_range_dictionary["num_factors"] = Integer(1, 1000)
recommender_input_args = SearchInputRecommenderArgs(
CONSTRUCTOR_POSITIONAL_ARGS=[URM_train],
CONSTRUCTOR_KEYWORD_ARGS={},
FIT_POSITIONAL_ARGS=[],
FIT_KEYWORD_ARGS={})
##########################################################################################################
if recommender_class is NMFRecommender:
hyperparameters_range_dictionary = {}
hyperparameters_range_dictionary["num_factors"] = Integer(1, 100)
hyperparameters_range_dictionary["solver"] = Categorical(
["coordinate_descent", "multiplicative_update"])
hyperparameters_range_dictionary["init_type"] = Categorical(
["random", "nndsvda"])
hyperparameters_range_dictionary["beta_loss"] = Categorical(
["frobenius", "kullback-leibler"])
recommender_input_args = SearchInputRecommenderArgs(
CONSTRUCTOR_POSITIONAL_ARGS=[URM_train],
CONSTRUCTOR_KEYWORD_ARGS={},
FIT_POSITIONAL_ARGS=[],
FIT_KEYWORD_ARGS={})
#########################################################################################################
if recommender_class is SLIM_BPR_Cython:
hyperparameters_range_dictionary = {}
hyperparameters_range_dictionary["topK"] = Integer(5, 2000)
hyperparameters_range_dictionary["epochs"] = Categorical(
[1200, 1500, 1700])
hyperparameters_range_dictionary["symmetric"] = Categorical(
[True, False])
hyperparameters_range_dictionary["sgd_mode"] = Categorical(
["adagrad", "adam"])
hyperparameters_range_dictionary["lambda_i"] = Real(
low=1e-7, high=1e1, prior='log-uniform')
hyperparameters_range_dictionary["lambda_j"] = Real(
low=1e-7, high=1e1, prior='log-uniform')
hyperparameters_range_dictionary["learning_rate"] = Real(
low=1e-6, high=1e-3, prior='log-uniform')
recommender_input_args = SearchInputRecommenderArgs(
CONSTRUCTOR_POSITIONAL_ARGS=[URM_train],
CONSTRUCTOR_KEYWORD_ARGS={},
FIT_POSITIONAL_ARGS=[],
FIT_KEYWORD_ARGS={
**earlystopping_keywargs, "positive_threshold_BPR": None,
'train_with_sparse_weights': None
})
##########################################################################################################
if recommender_class is SLIMElasticNetRecommender:
hyperparameters_range_dictionary = {}
hyperparameters_range_dictionary["topK"] = Integer(5, 1000)
hyperparameters_range_dictionary["l1_ratio"] = Real(
low=1e-5, high=1.0, prior='log-uniform')
hyperparameters_range_dictionary["alpha"] = Real(low=1e-3,
high=1.0,
prior='uniform')
recommender_input_args = SearchInputRecommenderArgs(
CONSTRUCTOR_POSITIONAL_ARGS=[URM_train],
CONSTRUCTOR_KEYWORD_ARGS={},
FIT_POSITIONAL_ARGS=[],
FIT_KEYWORD_ARGS={})
#########################################################################################################
if URM_train_last_test is not None:
recommender_input_args_last_test = recommender_input_args.copy()
recommender_input_args_last_test.CONSTRUCTOR_POSITIONAL_ARGS[
0] = URM_train_last_test
else:
recommender_input_args_last_test = None
## Final step, after the hyperparameter range has been defined for each type of algorithm
parameterSearch.search(
recommender_input_args,
parameter_search_space=hyperparameters_range_dictionary,
n_cases=n_cases,
n_random_starts=n_random_starts,
resume_from_saved=resume_from_saved,
save_model=save_model,
output_folder_path=output_folder_path,
output_file_name_root=output_file_name_root,
metric_to_optimize=metric_to_optimize,
recommender_input_args_last_test=recommender_input_args_last_test)
except Exception as e:
print("On recommender {} Exception {}".format(recommender_class,
str(e)))
traceback.print_exc()
error_file = open(output_folder_path + "ErrorLog.txt", "a")
error_file.write("On recommender {} Exception {}\n".format(
recommender_class, str(e)))
error_file.close()
def get_sk_dimensions(api_config, transform="normalize"):
"""Help routine to setup skopt search space in constructor.
Take api_config as argument so this can be static.
"""
# The ordering of iteration prob makes no difference, but just to be
# safe and consistnent with space.py, I will make sorted.
param_list = sorted(api_config.keys())
sk_dims = []
round_to_values = {}
for param_name in param_list:
param_config = api_config[param_name]
param_type = param_config["type"]
param_space = param_config.get("space", None)
param_range = param_config.get("range", None)
param_values = param_config.get("values", None)
# Some setup for case that whitelist of values is provided:
values_only_type = param_type in ("cat", "ordinal")
if (param_values is not None) and (not values_only_type):
assert param_range is None
param_values = np.unique(param_values)
param_range = (param_values[0], param_values[-1])
round_to_values[param_name] = interp1d(
param_values,
param_values,
kind="nearest",
fill_value="extrapolate")
if param_type == "int":
# Integer space in sklearn does not support any warping => Need
# to leave the warping as linear in skopt.
sk_dims.append(
Integer(param_range[0],
param_range[-1],
transform=transform,
name=param_name))
elif param_type == "bool":
assert param_range is None
assert param_values is None
sk_dims.append(
Integer(0, 1, transform=transform, name=param_name))
elif param_type in ("cat", "ordinal"):
assert param_range is None
# Leave x-form to one-hot as per skopt default
sk_dims.append(Categorical(param_values, name=param_name))
elif param_type == "real":
# Skopt doesn't support all our warpings, so need to pick
# closest substitute it does support.
prior = "log-uniform" if param_space in (
"log", "logit") else "uniform"
sk_dims.append(
Real(param_range[0],
param_range[-1],
prior=prior,
transform=transform,
name=param_name))
else:
assert False, "type %s not handled in API" % param_type
return sk_dims, round_to_values
__version__ = "1.0.0"
from skopt.space import Categorical, Integer, Real
from sklearn.pipeline import Pipeline
from sklearn.tree import DecisionTreeClassifier
from utility import HyperParameters, Runner
from model import load_sample_data_frame, ordinal_data_mapper
sample = None
iterations = 24
hyper_parameters = HyperParameters({
'dt__criterion':
Categorical(['gini', 'entropy']),
'dt__max_depth':
Integer(4, 24),
'dt__min_samples_leaf':
Real(0.000001, 0.001),
'dt__min_samples_split':
Real(0.000002, 0.002)
})
decision_tree_basic = Pipeline([('mapper', ordinal_data_mapper),
('dt', DecisionTreeClassifier())])
def test_decision_tree_basic():
runner = Runner('model/experiment/output/decision_tree_basic',
load_sample_data_frame(), 'violation', decision_tree_basic,
)
Y_train = np.load(
'/Users/kefei/Documents/Dataset/NTU/single_poses/labels/y_train_fall_aug_trimmed.npy'
)
#X_train = X_train[:1000]
#Y_train = Y_train[:1000]
print(X_train.shape)
Y_train = np.asarray([1 if x == 43 else 0 for x in Y_train])
return X_train, Y_train
X_train, y_train = load_data()
#model parameter
input_dim = 51
#hidden_dims = Integer(low=512, high=1024, name='hidden_dim')
hidden_dims = Categorical([256, 512, 768, 1024, 1280], name='hidden_dim')
#latent_dims = Integer(low=128, high=256, name='latent_dim')
latent_dims = Categorical([64, 128, 192, 256], name='latent_dim')
W_regularizer_vals = Real(low=0.001, high=0.01, name='hidden_W_regularizer')
dropout_rates = Real(low=0.01, high=0.12, name='dropout_rate')
#classifier_dense_dims = Integer(low=32, high=128, name='classifier_dense_dim')
classifier_dense_dims = Categorical([32, 64, 96, 128],
name='classifier_dense_dim')
learning_rates = Real(low=0.00001, high=0.0005, name='learning_rate')
param_space = [
hidden_dims, latent_dims, W_regularizer_vals, dropout_rates,
classifier_dense_dims, learning_rates
]
#for simplicty sake, just do annealing first
def get_model(self, X, y):
search_space = {'solver' : Categorical(['svd', 'lsqr', 'eigen'])}
model = BayesSearchCV(LinearDiscriminantAnalysis(), search_space, random_state = 0, n_iter = 1, cv = 3, n_jobs = -1)
model.fit(X, y)
return model
from skopt.space import Real, Integer, Categorical
from skopt.utils import use_named_args
from skopt import gp_minimize
# This defines the hyperparameter search space, add more parameters here
# as you see fit :)
space = [
Real(1e-6, 0.01, "log-uniform", name='learning_rate'),
Real(0.1, 0.8, name='dropout'),
Real(0.8, 1.0, name='momentum'),
Real(0.9, 1.0, name='beta_1'),
Real(0.99, 1.0, name='beta_2'),
Integer(low=5, high=20, name='epochs'),
Integer(low=50, high=225, name='num_dense_nodes'),
Categorical(categories=['SGD', 'Adam'], name='optimizer_type')
]
# Define a Callback class that stops training once accuracy reaches 90%
#class myCallback(tf.keras.callbacks.Callback):
#def on_epoch_end(self, epoch, logs={}):
# if(logs.get('acc')>0.87):
#print("\nReached 90% accuracy so cancelling training!")
#self.model.stop_training = True
# make the model - isolating this as a function to be called for each HP search
# iteration to pass in the updated parameters
def make_model(learning_rate, dropout, momentum, beta_1, beta_2,
num_dense_nodes, optimizer_type):
#param drid models for BayessearchCV
from skopt import BayesSearchCV
from skopt.space import Real, Categorical, Integer
#skopt
lr_param_grid = {
'lr__C': Real(1e-3, 100, 'log-uniform'),
'lr__fit_intercept': Categorical([True, False]),
'lr__max_iter': Integer(1e+2, 1e+5, 'log-uniform'),
'lr__penalty': Categorical(['l1', 'l2']),
'lr__solver': Categorical(['liblinear', 'saga']),
'lr__tol': Real(1e-5, 1e-3, 'log-uniform'),
'lr__class_weight': Categorical([None, 'balanced']),
}
#skopt
xgb_param_grid = {
'xgb__colsample_bylevel' : Real(1e-1, 1, 'uniform'),
'xgb__colsample_bytree' : Real(6e-1, 1, 'uniform'),
'xgb__gamma' : Real(5e-1, 6, 'log-uniform'),
'xgb__learning_rate' : Real(10**-5, 10**0, "log-uniform"),
'xgb__max_depth' : Integer(1, 25, 'uniform'),
'xgb__min_child_weight' : Integer(1, 10, 'uniform'),
'xgb__n_estimators' : Integer(50, 400, 'log-uniform'),
'xgb__reg_alpha' : Real(1e-2, 1, 'log-uniform'),
'xgb__reg_lambda' : Real(1e-2, 1, 'log-uniform'),
'xgb__subsample' : Real(6e-1, 1, 'uniform'),
def optimize(self, x, y):
self.iterations = []
space = [
Integer(5, 20, name='hidden_layer_sizes'),
Categorical(['constant', 'invscaling', 'adaptive'],
name='learning_rate'),
Real(1e-5, 1e-3, name='learning_rate_init'),
Integer(5, 20, name='max_iter'),
Real(1e-5, 1e-3, name='tol')
]
@use_named_args(space)
def objective(hidden_layer_sizes, learning_rate, learning_rate_init,
max_iter, tol):
try:
scores = []
params = {
'hidden_layer_sizes': int(hidden_layer_sizes),
'learning_rate': learning_rate,
'learning_rate_init': learning_rate_init,
'max_iter': int(max_iter),
'tol': tol,
'random_state': self.random_state
}
if isinstance(self.fixed_parameters, dict):
params.update(self.fixed_parameters)
skf = StratifiedKFold(self.n_folds,
shuffle=self.shuffle,
random_state=self.random_state)
for train_index, valid_index in skf.split(x, y):
x_train, y_train = x[train_index, :], y[train_index]
x_valid, y_valid = x[valid_index, :], y[valid_index]
mlp = MLPClassifier(**params)
mlp.fit(x_train, y_train)
y_valid_hat = mlp.predict(x_valid)
loss_valid = log_loss(y_valid, y_valid_hat)
scores.append(loss_valid)
result = np.mean(scores)
self.iterations.append((params, result))
return result
except ValueError:
return np.inf
return self.execute_optimization(objective, space)
def optimize(self, x, y):
self.iterations = []
space = [
Integer(1, 30, name='n_neighbors'),
Categorical(['uniform', 'distance'], name='weights'),
Integer(1, 30, name='leaf_size'),
Integer(1, 5, name='p')]
@use_named_args(space)
def objective(
n_neighbors,
weights,
leaf_size,
p
):
try:
scores = []
params = {
'n_neighbors': int(n_neighbors),
'weights': weights,
'leaf_size': int(leaf_size),
'p': int(p),
'n_jobs': self.n_jobs}
if isinstance(self.fixed_parameters, dict):
params.update(self.fixed_parameters)
skf = StratifiedKFold(self.n_folds,
shuffle=self.shuffle,
random_state=self.random_state)
for train_index, valid_index in skf.split(x, y):
x_train, y_train = x[train_index, :], y[train_index]
x_valid, y_valid = x[valid_index, :], y[valid_index]
knn = KNeighborsClassifier(**params)
knn.fit(x_train, y_train)
y_valid_hat = knn.predict(x_valid)
loss_valid = log_loss(y_valid, y_valid_hat)
scores.append(loss_valid)
result = np.mean(scores)
self.iterations.append((params, result))
return result
except ValueError:
return np.inf
return self.execute_optimization(objective, space)
'Own-child': 0,
'Other-relative': 0,
'Husband': 1,
'Wife': 1
})
data.head()
X = data[[
'workclass_num', 'education.num', 'marital_num', 'race_num', 'sex_num',
'rel_num', 'capital.gain', 'capital.loss'
]]
y = data.over50K
start = timeit.default_timer()
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.3,
random_state=42)
opt = BayesSearchCV(SVC(), {
'C': Real(0.001, 10, prior='log-uniform'),
'gamma': Real(0.001, 1, prior='log-uniform'),
'kernel': Categorical(['linear', 'rbf']),
},
n_iter=40)
opt.fit(X_train, y_train)
stop = timeit.default_timer()
print(opt.score(X_test, y_test))
print(opt.best_params_)
print('Time: ', stop - start)
def pow10map(x):
return 10.0**x
def pow2intmap(x):
return int(2.0**x)
def nop(x):
return x
nnparams = {
# up to 1024 neurons
'hidden_layer_sizes': (Real(1.0, 10.0), pow2intmap),
'activation': (Categorical(['identity', 'logistic', 'tanh', 'relu']), nop),
'solver': (Categorical(['lbfgs', 'sgd', 'adam']), nop),
'alpha': (Real(-5.0, -1), pow10map),
'batch_size': (Real(5.0, 10.0), pow2intmap),
'learning_rate': (Categorical(['constant', 'invscaling',
'adaptive']), nop),
'max_iter': (Real(5.0, 8.0), pow2intmap),
'learning_rate_init': (Real(-5.0, -1), pow10map),
'power_t': (Real(0.01, 0.99), nop),
'momentum': (Real(0.1, 0.98), nop),
'nesterovs_momentum': (Categorical([True, False]), nop),
'beta_1': (Real(0.1, 0.98), nop),
'beta_2': (Real(0.1, 0.9999999), nop),
}
MODELS = {
def _Losgistic(self, df):
print("Start Logistic...")
param_grid = [
Categorical(["newton-cg", "lbfgs", "liblinear"], name="solver"),
Categorical(["l2"], name="penalty"),
Real(1e-5, 100, name="C"),
]
# set up the logistic regressoion classifier
lgt = LogisticRegression(random_state=0)
kernel = 1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-1, 10.0))
gpr = GaussianProcessRegressor(kernel=kernel,
normalize_y=True,
noise="gaussian",
n_restarts_optimizer=2)
# the decorator allows our objective function to receive the parameters as
@use_named_args(param_grid)
def objective(**params):
# model with new parameters
lgt.set_params(**params)
# optimization function (hyperparam response function)
value = np.mean(
cross_val_score(
lgt,
df.feature_train,
df.target_train,
cv=5,
n_jobs=-4,
scoring="roc_auc",
) # "accuracy"
)
# negate because we need to minimize
return -value
gp_ = gp_minimize(
objective,
dimensions=param_grid,
base_estimator=gpr,
n_initial_points=5,
acq_optimizer="sampling",
random_state=42,
)
params = {
"solver": gp_.x[0],
"penalty": gp_.x[1],
"C": gp_.x[2],
}
final_model = LogisticRegression(
solver=gp_.x[0],
penalty=gp_.x[1],
C=gp_.x[2],
)
final_model.fit(df.feature_train, df.target_train)
y_preds = final_model.predict(df.feature_test)
joblib.dump(final_model, "./log/Logistic.pkl")
return {
"Logistic": {
"Best_score": metrics.roc_auc_score(y_preds, df.target_test),
"Best_accuracy":
metrics.accuracy_score(y_preds, df.target_test),
"params": params,
"model": final_model,
}
}
from experiment.experiment import Experiment
from experiment.hyper_param_opt import GridSearch
from models.tensorflow.model import Model
from models.tensorflow.tf_train_eval import TfTrainEvalModelFactory
if __name__ == '__main__':
exp = Experiment('density/synthetic/sin_normal')
conf.num_workers = 4
conf.visible_device_list = [0, 1]
conf.eval_batch_size = {'0': 10000, '1': 10000}
exp.data_loader = registry.sin_normal_noise()
exp.model_factory = TfTrainEvalModelFactory(Model(name="MONDE_AR_MADE"))
exp.hyper_param_search = GridSearch([
Categorical(['sigm'], name='tr'),
Categorical([32, 64, 128], name='sh'),
Categorical([1, 2, 3], name='nh'),
Categorical([16], name='xs'),
Categorical([128], name='bs'),
Categorical([1], name='rs'),
Categorical(['AdamOptimizer'], name='opt'),
Categorical([1e-4, 1e-3, 1e-2], name='opt_lr'),
])
exp.early_stopping = EarlyStop(monitor_every_epoch=1, patience=[30])
exp.run()
def _RandomForest(self, df):
print("Start RandomForest...")
param_grid = [
Categorical(["sqrt", "log2", None], name="max_features"),
Integer(120, 1200, name="n_estimators"),
Integer(5, 30, name="max_depth"),
Integer(2, 15, name="min_samples_split"),
Integer(1, 10, name="min_samples_leaf"),
]
# set up the logistic regressoion classifier
rf = RandomForestClassifier(random_state=0)
kernel = 1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-1, 10.0))
gpr = GaussianProcessRegressor(kernel=kernel,
normalize_y=True,
noise="gaussian",
n_restarts_optimizer=2)
# the decorator allows our objective function to receive the parameters as
@use_named_args(param_grid)
def objective(**params):
# model with new parameters
rf.set_params(**params)
# optimization function (hyperparam response function)
value = np.mean(
cross_val_score(
rf,
df.feature_train,
df.target_train,
cv=5,
n_jobs=-4,
scoring="roc_auc",
) # "accuracy"
)
# negate because we need to minimize
return -value
gp_ = gp_minimize(
objective,
dimensions=param_grid,
base_estimator=gpr,
n_initial_points=5,
acq_optimizer="sampling",
random_state=42,
)
params = {
"max_features": gp_.x[0],
"n_estimators": gp_.x[1],
"max_depth": gp_.x[2],
"min_samples_split": gp_.x[3],
"min_samples_leaf": gp_.x[4],
}
final_model = RandomForestClassifier(
max_features=gp_.x[0],
n_estimators=gp_.x[1],
max_depth=gp_.x[2],
min_samples_split=gp_.x[3],
min_samples_leaf=gp_.x[4],
)
final_model.fit(df.feature_train, df.target_train)
y_preds = final_model.predict(df.feature_test)
joblib.dump(final_model, "./log/RandomForest.pkl")
return {
"RandomForest": {
"Best_score": metrics.roc_auc_score(y_preds, df.target_test),
"Best_accuracy":
metrics.accuracy_score(y_preds, df.target_test),
"params": params,
"model": final_model,
}
}
def run_KNNRecommender_on_similarity_type(
similarity_type,
parameterSearch,
parameter_search_space,
recommender_input_args,
n_cases,
n_random_starts,
resume_from_saved,
save_model,
output_folder_path,
output_file_name_root,
metric_to_optimize,
allow_weighting=False,
recommender_input_args_last_test=None):
original_parameter_search_space = parameter_search_space
hyperparameters_range_dictionary = {}
hyperparameters_range_dictionary["topK"] = Integer(5, 1000)
hyperparameters_range_dictionary["shrink"] = Integer(0, 1000)
hyperparameters_range_dictionary["similarity"] = Categorical(
[similarity_type])
hyperparameters_range_dictionary["normalize"] = Categorical([1, 0])
is_set_similarity = similarity_type in [
"tversky", "dice", "jaccard", "tanimoto"
]
if similarity_type == "asymmetric":
hyperparameters_range_dictionary["asymmetric_alpha"] = Real(
low=0, high=2, prior='uniform')
hyperparameters_range_dictionary["normalize"] = Categorical([1])
elif similarity_type == "tversky":
hyperparameters_range_dictionary["tversky_alpha"] = Real(
low=0, high=2, prior='uniform')
hyperparameters_range_dictionary["tversky_beta"] = Real(
low=0, high=2, prior='uniform')
hyperparameters_range_dictionary["normalize"] = Categorical([1])
elif similarity_type == "euclidean":
hyperparameters_range_dictionary["normalize"] = Categorical([1, 0])
hyperparameters_range_dictionary["normalize_avg_row"] = Categorical(
[1, 0])
hyperparameters_range_dictionary[
"similarity_from_distance_mode"] = Categorical(
["lin", "log", "exp"])
if not is_set_similarity:
if allow_weighting:
hyperparameters_range_dictionary[
"feature_weighting"] = Categorical(["none", "BM25", "TF-IDF"])
local_parameter_search_space = {
**hyperparameters_range_dictionary,
**original_parameter_search_space
}
parameterSearch.search(
recommender_input_args,
parameter_search_space=local_parameter_search_space,
n_cases=n_cases,
n_random_starts=n_random_starts,
resume_from_saved=resume_from_saved,
save_model=save_model,
output_folder_path=output_folder_path,
output_file_name_root=output_file_name_root + "_" + similarity_type,
metric_to_optimize=metric_to_optimize,
recommender_input_args_last_test=recommender_input_args_last_test)
def _GradientBoosting(self, df):
print("Start GradientBoosting...")
param_grid = [
Integer(10, 120, name="n_estimators"),
Real(0, 0.999, name="min_samples_split"),
Integer(1, 5, name="max_depth"),
Categorical(["deviance", "exponential"], name="loss"),
]
# set up the gradient boosting classifier
gbm = GradientBoostingClassifier(random_state=0)
kernel = 1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-1, 10.0))
gpr = GaussianProcessRegressor(kernel=kernel,
normalize_y=True,
noise="gaussian",
n_restarts_optimizer=2)
# the decorator allows our objective function to receive the parameters as
@use_named_args(param_grid)
def objective(**params):
# model with new parameters
gbm.set_params(**params)
# optimization function (hyperparam response function)
value = np.mean(
cross_val_score(
gbm,
df.feature_train,
df.target_train,
cv=5,
n_jobs=-4,
scoring="roc_auc",
) # "accuracy"
)
# negate because we need to minimize
return -value
gp_ = gp_minimize(
objective,
dimensions=param_grid,
base_estimator=gpr,
n_initial_points=5,
acq_optimizer="sampling",
random_state=42,
)
params = {
"n_estimators": gp_.x[0],
"min_samples_split": gp_.x[1],
"max_depth": gp_.x[2],
"loss": gp_.x[3],
}
final_model = GradientBoostingClassifier(
n_estimators=gp_.x[0],
min_samples_split=gp_.x[1],
max_depth=gp_.x[2],
loss=gp_.x[3],
)
final_model.fit(df.feature_train, df.target_train)
y_preds = final_model.predict(df.feature_test)
joblib.dump(final_model, "./log/GradientBoosting.pkl")
return {
"GradientBoosting": {
"Best_score": metrics.roc_auc_score(y_preds, df.target_test),
"Best_accuracy":
metrics.accuracy_score(y_preds, df.target_test),
"params": params,
"model": final_model,
}
}
assert_equal(reals.distance(4.1234, i), abs(4.1234 - i))
@pytest.mark.parametrize("dimension, bounds",
[(Real, (2, 1)), (Integer, (2, 1)),
(Real, (2, 2)), (Integer, (2, 2))])
def test_dimension_bounds(dimension, bounds):
with pytest.raises(ValueError) as exc:
dim = dimension(*bounds)
assert "has to be less than the upper bound " in exc.value.args[0]
@pytest.mark.parametrize("dimension, name",
[(Real(1, 2, name="learning rate"), "learning rate"),
(Integer(1, 100, name="no of trees"), "no of trees"),
(Categorical(["red, blue"], name="colors"), "colors")])
def test_dimension_name(dimension, name):
assert dimension.name == name
@pytest.mark.parametrize("dimension",
[Real(1, 2), Integer(1, 100), Categorical(["red, blue"])])
def test_dimension_name_none(dimension):
assert dimension.name is None
def test_dimension_name():
notnames = [1, 1., True]
for n in notnames:
with pytest.raises(ValueError) as exc:
real = Real(1, 2, name=n)
K.set_learning_phase(True)
setCPUCores(4)
if args.test_only:
runParamTests(args)
exit()
logfile = "skopt_current.txt"
holder = Holder(args.dirs, log_to=logfile)
max_batch = 128
space = [
Real(10**-10, 10**-3, "log-uniform", name='l2_reg'),
Categorical(["low", "mid", "high", "up", "down"], name='dropouts'),
Real(10**-9, 10**-1, "log-uniform", name='learning_rate'),
Integer(5, max_batch, name='batch_size'),
Categorical(["RMSProp", "Adagrad", "Adadelta", "Adam"],
name='optimizer'),
]
with open(logfile, 'a') as f:
f.write("#{} {} {}\n".format("FC", "DK", datetime.datetime.now()))
x0 = None
y0 = None
if args.prev is not None:
x0, y0 = previousRuns(args.prev)
res_gp = gp_minimize(holder,
def check_categorical(vals, random_val):
x = Categorical(vals)
assert_equal(x, Categorical(vals))
assert_not_equal(x, Categorical(vals[:-1] + ("zzz",)))
assert_equal(x.rvs(random_state=1), random_val)
from experiment.experiment import Experiment
from experiment.hyper_param_opt import GridSearch
from models.tensorflow.model import Model
from models.tensorflow.tf_train_eval import TfTrainEvalModelFactory
if __name__ == '__main__':
exp = Experiment('density/synthetic/sin_normal')
conf.num_workers = 4
conf.visible_device_list = [0,1]
conf.eval_batch_size = {'0': 10000, '1': 10000}
exp.data_loader = registry.sin_normal_noise()
exp.model_factory = TfTrainEvalModelFactory(Model(name="RNADE_laplace"))
exp.hyper_param_search = GridSearch([
Categorical([1,20,50,100,150,200], name='km'),
Categorical([20,60,100,140,200], name='sh'),
Categorical([128], name='bs'),
Categorical([1], name='rs'),
Categorical(['AdamOptimizer'], name='opt'),
Categorical([1e-4,1e-3,1e-2], name='opt_lr'),
])
exp.early_stopping = EarlyStop(monitor_every_epoch=1, patience=[30])
exp.run()
("preprocessing", preprocessing()),
("estimator", GradientBoostingRegressor()),
]
)
estimator_elastic_net = Pipeline(
steps=[
("preprocessing", preprocessing()),
("estimator", ElasticNet(max_iter=10000)),
]
)
svm_search_space = {
"estimator__C": Real(1e-6, 1e6, prior="log-uniform"),
"estimator__gamma": Real(1e-6, 1e1, prior="log-uniform"),
"estimator__degree": Integer(1, 8),
"estimator__kernel": Categorical(["linear", "poly", "rbf"]),
}
rf_search_space = {
"estimator__max_depth": Integer(low=10, high=50),
"estimator__max_features": Categorical(["log2", "sqrt", None]),
"estimator__min_samples_leaf": Integer(low=2, high=10),
"estimator__min_samples_split": Integer(low=2, high=4),
"estimator__n_estimators": Integer(low=40, high=200),
"estimator__bootstrap": Categorical([True, False]),
}
gb_search_space = {
"estimator__n_estimators": Integer(low=100, high=1000),
"estimator__learning_rate": Real(low=0.025, high=0.5, prior="log-uniform"),
"estimator__max_depth": Integer(low=2, high=15),
"estimator__subsample": Real(low=0.5, high=1),
"y_train : \n\n%s"%y_train
)
print(
"X_test : \n\n%s"%X_test
)
print(
"y_test : \n\n%s"%y_test
)
fbestmodel = "./GPMinModelTank"
dim_first_neuron = Integer(low=3, high=300, name="first_neuron")
dim_n_hidden = Integer(low=1, high=10, name="n_hidden")
dim_magnifier = Real(low=1e-4, high=1.0, name="magnifier", prior="uniform")
dim_dropout = Real(low=1e-3, high=0.5, name="dropout", prior="uniform")
dim_lr = Real(low=3e-5, high=3e-2, name="lr")
dim_first_activation = Categorical(categories=["relu","sigmoid","elu","tanh"],name="first_activation")
dim_hidden_activation = Categorical(categories=["relu","sigmoid","elu","tanh"],name="hidden_activation")
dim_last_activation = Categorical(categories=["relu","sigmoid","elu","tanh"],name="last_activation")
dim_regulariser = Categorical(categories=[reg.l1, reg.l2, reg.l1_l2, None],name="regulariser")
dim_l1 = Real(low=0,high=1e-1,name="l1")
dim_l2 = Real(low=0,high=1e-1,name="l2")
dimensions = [dim_first_neuron, dim_n_hidden, dim_magnifier, dim_dropout, dim_lr, dim_first_activation, dim_hidden_activation, dim_last_activation, dim_regulariser, dim_l1, dim_l2]
best_error = 10000
f = open("./gp_min_best_error.txt","w")
f.write("%s"%best_error)
f.close()
@use_named_args(dimensions=dimensions)
def optimize(self, x, y):
""""
n_estimators int, default=100, The number of trees in the forest. Changed in version 0.22: The default value of n_estimators changed from 10 to 100 in 0.22.
criterion{“gini”, “entropy”}, default=”gini”, The function to measure the quality of a split. Supported criteria are “gini” for the Gini impurity and “entropy” for the information gain. Note: this parameter is tree-specific.
max_depth int, default=None, The maximum depth of the tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples.
min_samples_split int or float, default=2, The minimum number of samples required to split an internal node: If int, then consider min_samples_split as the minimum number. If float, then min_samples_split is a fraction and ceil(min_samples_split * n_samples) are the minimum number of samples for each split. Changed in version 0.18: Added float values for fractions.
min_samples_leaf int or float, default=1, The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least min_samples_leaf training samples in each of the left and right branches. This may have the effect of smoothing the model, especially in regression. If int, then consider min_samples_leaf as the minimum number. If float, then min_samples_leaf is a fraction and ceil(min_samples_leaf * n_samples) are the minimum number of samples for each node. Changed in version 0.18: Added float values for fractions.
min_weight_fraction_leaf float, default=0.0 The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided.
max_leaf_nodes int, default=None, Grow trees with max_leaf_nodes in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes.
min_impurity_decrease float, default=0.0, A node will be split if this split induces a decrease of the impurity greater than or equal to this value. The weighted impurity decrease equation is the following:
min_impurity_split float, default=None, Threshold for early stopping in tree growth. A node will split if its impurity is above the threshold, otherwise it is a leaf.
ccp_alpha non-negative float, default=0.0, Complexity parameter used for Minimal Cost-Complexity Pruning. The subtree with the largest cost complexity that is smaller than ccp_alpha will be chosen. By default, no pruning is performed. See Minimal Cost-Complexity Pruning for details.
New in version 0.22.
max_samples int or float, default=None, If bootstrap is True, the number of samples to draw from X to train each base estimator.
n_jobs int, default=None
The number of jobs to run in parallel. fit, predict, decision_path and apply are all parallelized over the trees. None means 1 unless in a joblib.parallel_backend context. -1 means using all processors. See Glossary for more details.
random_state int or RandomState, default=None
Controls both the randomness of the bootstrapping of the samples used when building trees (if bootstrap=True) and the sampling of the features to consider when looking for the best split at each node (if max_features < n_features). See Glossary for details.
"""
self.iterations = []
space = [
Integer(10, 5000, name='n_estimators'),
Categorical(['gini', 'entropy'], name='criterion'),
Integer(1, 100, name='max_depth'),
Integer(1, 100, name='min_samples_split'),
Integer(1, 100, name='min_samples_leaf'),
Real(1e-8, 1, name='min_weight_fraction_leaf'),
Integer(1, 100, name='max_leaf_nodes'),
Real(1e-8, 1, name='min_impurity_decrease'),
Real(1e-8, 1, name='min_impurity_split')
]
@use_named_args(space)
def objective(
n_estimators,
criterion,
max_depth,
min_samples_split,
min_samples_leaf,
min_weight_fraction_leaf,
max_leaf_nodes,
min_impurity_decrease,
min_impurity_split,
):
try:
scores = []
params = {
'n_estimators': int(round(n_estimators, 0)),
'criterion': criterion,
'max_depth': max_depth,
'min_samples_split': min_samples_split,
'min_samples_leaf': min_samples_leaf,
'min_weight_fraction_leaf': min_weight_fraction_leaf,
'max_leaf_nodes': int(round(max_leaf_nodes, 0)),
'min_impurity_decrease': min_impurity_decrease,
'min_impurity_split': min_impurity_split,
'random_state': self.random_state
}
if isinstance(self.fixed_parameters, dict):
params.update(self.fixed_parameters)
skf = StratifiedKFold(self.n_folds,
shuffle=self.shuffle,
random_state=self.random_state)
for train_index, valid_index in skf.split(x, y):
x_train, y_train = x[train_index, :], y[train_index]
x_valid, y_valid = x[valid_index, :], y[valid_index]
rf = RandomForestClassifier(**params, n_jobs=-1)
rf.fit(x_train, y_train)
y_valid_hat = rf.predict(x_valid)
loss_valid = log_loss(y_valid, y_valid_hat)
scores.append(loss_valid)
result = np.mean(scores)
self.iterations.append((params, result))
return result
except ValueError:
return np.inf
return self.execute_optimization(objective, space)
from models.tensorflow.tf_train_eval import TfTrainEvalModelFactory
if __name__ == '__main__':
exp = Experiment('density/synthetic/inv_sin_normal')
conf.num_workers = 4
conf.visible_device_list = [0, 1]
conf.eval_batch_size = {'0': 10000, '1': 10000}
exp.data_loader = registry.inv_sin_normal()
exp.model_factory = TfTrainEvalModelFactory(
Model(name="MONDE_copula_const_cov"))
exp.hyper_param_search = GridSearch([
Categorical([32, 64, 128], name='hxy_sh'),
Categorical([1, 2, 3], name='hxy_nh'),
Categorical([32, 64, 128], name='x_sh'),
Categorical([1, 2, 3], name='x_nh'),
Categorical([16, 32], name='hxy_x'),
Categorical([0.05, 0.01], name='clr'),
Categorical([128], name='bs'),
Categorical([1], name='rs'),
Categorical(['AdamOptimizer'], name='opt'),
Categorical([1e-4, 1e-3, 1e-2], name='opt_lr'),
])
exp.early_stopping = EarlyStop(monitor_every_epoch=1, patience=[30])
exp.run()
