def train(self):
start_time = time.time()
self.init_matrices()
state_matrix = self.compute_state_matrix(self.x_train)
new_x_train = np.concatenate((self.x_train,state_matrix), axis=2).reshape(
self.N * self.T , self.num_dim+self.N_x)
state_matrix = None
gc.collect()
new_labels = np.repeat(self.y_train,self.T,axis=0)
ridge_classifier = Ridge(alpha=self.lamda)
ridge_classifier.fit(new_x_train,new_labels)
state_matrix = self.compute_state_matrix(self.x_val)
new_x_val = np.concatenate((self.x_val,state_matrix), axis=2).reshape(
self.x_val.shape[0] * self.T , self.num_dim+self.N_x)
y_pred_val = ridge_classifier.predict(new_x_val)
y_pred_val = self.reshape_prediction(y_pred_val,self.x_val.shape[0],self.T)
df_val_metrics = calculate_metrics(np.argmax(self.y_val, axis=1),y_pred_val,0.0)
train_acc = df_val_metrics['accuracy'][0]
state_matrix = self.compute_state_matrix(self.x_test)
new_x_test = np.concatenate((self.x_test,state_matrix), axis=2).reshape(self.x_test.shape[0] * self.T , self.num_dim+self.N_x)
state_matrix = None
gc.collect()
y_pred = ridge_classifier.predict(new_x_test)
y_pred = self.reshape_prediction(y_pred,self.x_test.shape[0],self.T)
duration = time.time() - start_time
df_metrics = calculate_metrics(self.y_true,y_pred,duration)
self.W_out = ridge_classifier.coef_
ridge_classifier = None
gc.collect()
df_metrics.to_csv(self.output_directory+'df_metrics.csv', index=False)
return df_metrics , train_acc
def predict(self,
x_test,
y_true,
x_train,
y_train,
y_test,
return_df_metrics=True):
def loss1(y, yhat):
return KB.sum(KB.abs(yhat - y))
# model = keras.models.load_model(model_path, custom_objects={'custom_loss_function': custom_loss_function})
start_time = time.time()
model_path = self.output_directory + 'best_model.hdf5'
model = keras.models.load_model(model_path,
custom_objects={'loss1': loss1})
y_pred = model.predict(x_test, batch_size=self.batch_size)
if return_df_metrics:
y_pred = np.argmax(y_pred, axis=1)
df_metrics = calculate_metrics(y_true, y_pred, 0.0)
return df_metrics
else:
test_duration = time.time() - start_time
save_test_duration(self.output_directory + 'test_duration.csv',
test_duration)
return y_pred
def predict(self, x_test, y_true, model_path, return_df_metrics=True):
model = tf.keras.models.load_model(model_path)
y_pred = model.predict(x_test)
if return_df_metrics:
y_pred = np.argmax(y_pred, axis=1)
df_metrics = calculate_metrics(y_true, y_pred, 0.0)
return df_metrics
else:
return y_pred
def predict(self, x_test,y_true,x_train,y_train,y_test,return_df_metrics = True):
model_path = self.output_directory + 'best_model.hdf5'
model = keras.models.load_model(model_path)
y_pred = model.predict(x_test)
if return_df_metrics:
y_pred = np.argmax(y_pred, axis=1)
df_metrics = calculate_metrics(y_true, y_pred, 0.0)
return df_metrics
else:
return y_pred
def predict(self, x_test, y_true, return_df_metrics=True):
start_time = time.time()
model = keras.models.load_model(self.model_path)
y_pred = model.predict(x_test, batch_size=self.batch_size)
if return_df_metrics:
y_pred = np.argmax(y_pred, axis=1)
df_metrics = calculate_metrics(y_true, y_pred, 0.0)
return df_metrics
else:
test_duration = time.time() - start_time
save_test_duration(self.output_directory + 'test_duration.csv',
test_duration)
return y_pred
def run_iterations(args, augmentator, augmentator_name, tmp_output_directory,
iterations, datasets_dict, classifier_name, epochs, start):
print('\t\twithout augmentation: ', augmentator_name)
for dataset_name in dataset_names_for_archive[ARCHIVE_NAMES[0]]:
print('\t\t\tdataset_name: ', dataset_name)
upper_dir = tmp_output_directory + augmentator_name + '/' + dataset_name
done = check_dir(upper_dir)
if not done:
#save all the predictions and the corresponding true class
predicted_y = []
expected_y = []
for iter in range(iterations):
print('\t\t\t\titer', iter)
trr = '_itr_' + str(iter)
output_directory = upper_dir + '/' + trr + '/'
#print(output_directory)
create_directory(output_directory)
y_pred, y_true = fit_classifier_aug(args, augmentator,
datasets_dict,
dataset_name,
classifier_name, epochs,
output_directory)
print('\t\t\t\tDONE')
# the creation of this directory means
create_directory(output_directory + '/DONE')
if (y_pred.shape == y_true.shape):
predicted_y.extend(y_pred)
expected_y.extend(y_true)
else:
raise Exception("FALSE: y_pred.shape==y_true.shape.")
totalduration = time.time() - start
df_metrics = calculate_metrics(expected_y, predicted_y,
totalduration)
df_metrics.to_csv(upper_dir + '/avg_metrics.csv', index=False)
create_directory(upper_dir + '/DONE')
print('iterations DONE!')
print(df_metrics)
def eval_model(epoch, is_save=True):
batch_time = AverageMeter()
losses = AverageMeter()
acc_score = AverageMeter()
model.eval()
num_steps = len(eval_loader)
print(f'total batches: {num_steps}')
end = time.time()
eval_criterion = nn.CrossEntropyLoss()
with torch.no_grad():
for i, (XI, label) in enumerate(eval_loader):
x = Variable(XI.cuda(device_id))
# label = Variable(torch.LongTensor(label).cuda(device_id))
label = Variable(label.cuda(device_id))
# Forward pass: Compute predicted y by passing x to the model
output = model(x)
# Compute and print loss
loss = eval_criterion(output, label)
losses.update(loss.data.item(), x.size(0))
# update metrics
output = nn.Softmax(dim=1)(output)
confs, predicts = torch.max(output.detach(), dim=1)
acc_score.update(calculate_metrics(predicts.cpu(), label.cpu()), 1)
lr = optimizer.param_groups[0]['lr']
batch_time.update(time.time() - end)
end = time.time()
if i % LOG_FREQ == 0:
print(f'{epoch} [{i}/{num_steps}]\t'
f'time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
f'loss {losses.val:.4f} ({losses.avg:.4f})\t'
f'acc {acc_score.val:.4f} ({acc_score.avg:.4f})\t'
f'lr {lr:.8f}')
print(f' * Eval loss {losses.avg:.4f}\t'
f'acc({acc_score.avg:.4f})\t'
f'total time {batch_time.sum}')
if is_save:
train_logger.log(phase="eval",
values={
'epoch': epoch,
'loss': format(losses.avg, '.4f'),
'acc': format(acc_score.avg, '.4f'),
'lr': optimizer.param_groups[0]['lr']
})
scheduler.step()
return losses.avg
def train_model(epoch):
batch_time = AverageMeter()
losses = AverageMeter()
acc_score = AverageMeter()
model.train()
num_steps = len(train_loader)
print(f'total batches: {num_steps}')
end = time.time()
for i, (XI, label) in enumerate(train_loader):
x = Variable(XI.cuda(device_id))
# label = Variable(torch.LongTensor(label).cuda(device_id))
label = Variable(label.cuda(device_id))
# Forward pass: Compute predicted y by passing x to the model
output = model(x)
# Compute and print loss
loss = criterion(output, label)
# update metrics
losses.update(loss.data.item(), x.size(0))
confs, predicts = torch.max(output.detach(), dim=1)
acc_score.update(calculate_metrics(predicts.cpu(), label.cpu()), 1)
optimizer.zero_grad()
loss.backward()
optimizer.step()
lr = optimizer.param_groups[0]['lr']
batch_time.update(time.time() - end)
end = time.time()
if i % LOG_FREQ == 0:
print(f'{epoch} [{i}/{num_steps}]\t'
f'time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
f'loss {losses.val:.4f} ({losses.avg:.4f})\t'
f'acc {acc_score.val:.4f} ({acc_score.avg:.4f})\t'
f'lr {lr:.8f}')
print(f' * Train loss {losses.avg:.4f}\t' f'acc({acc_score.avg:.4f})')
train_logger.log(phase="train",
values={
'epoch': epoch,
'loss': format(losses.avg, '.4f'),
'acc': format(acc_score.avg, '.4f'),
'lr': optimizer.param_groups[0]['lr']
})
scheduler.step()
return losses.val
def fit(self, x_train, y_train, x_test, y_test, y_true):
y_pred = np.zeros(shape=y_test.shape)
l = 0
for dataset in datasets_names:
if dataset == self.dataset_name:
continue
curr_dir = self.transfer_directory + dataset + '/' + self.dataset_name + '/'
predictions_file_name = curr_dir + 'y_pred.npy'
if check_if_file_exits(predictions_file_name):
# then load only the predictions from the file
curr_y_pred = np.load(predictions_file_name)
else:
# predict from models saved
model = keras.models.load_model(curr_dir + 'best_model.hdf5')
curr_y_pred = model.predict(x_test)
keras.backend.clear_session()
np.save(predictions_file_name, curr_y_pred)
y_pred = y_pred + curr_y_pred
l += 1
keras.backend.clear_session()
y_pred = y_pred / l
# save predictions
np.save(self.output_directory + 'y_pred.npy', y_pred)
# convert the predicted from binary to integer
y_pred = np.argmax(y_pred, axis=1)
df_metrics = calculate_metrics(y_true, y_pred, 0.0)
df_metrics.to_csv(self.output_directory + 'df_metrics.csv',
index=False)
print(self.dataset_name, df_metrics['accuracy'][0])
gc.collect()
def fit(self, x_train, y_train, x_test, y_test, y_true):
# Reshape x_train and x_test so it fits the classifier
x_train = x_train.reshape((x_train.shape[0], x_train.shape[1]))
x_test = x_test.reshape((x_test.shape[0], x_test.shape[1]))
# Get non-oneHot-encoded labels
y_train = np.argmax(y_train, axis=1)
y_test = np.argmax(y_test, axis=1)
###########################
##Train ROCKET Classifier##
###########################
start_time = time.time()
rocket = Base_Classifier_ROCKET(self.n_kernels, self.kss, self.verbose)
rocket.fit(x_train, y_train)
duration = time.time() - start_time
train_timings = rocket.train_timings_
##########################
##Test ROCKET Classifier##
##########################
acc = rocket.score(x_test, y_test)
test_timings = rocket.test_timings_
#############################
##Predict ROCKET Classifier##
#############################
y_pred = rocket.predict(x_test)
# Save Metrics
df_metrics = calculate_metrics(y_test, y_pred, duration)
if self.verbose:
print(df_metrics)
df_metrics.to_csv(self.output_dir +
'df_metrics.csv', index=False)
# Save train and test timings
train_timings.append(train_timings[0]+train_timings[1])
test_timings.append(test_timings[0]+test_timings[1])
df_timings = pd.DataFrame([train_timings, test_timings], index=[
'train', 'test'], columns=['x_transform', 'ridge_operation', 'total'])
if self.verbose:
print(df_timings)
df_timings.to_csv(self.output_dir + 'df_timings.csv', index=False)
def fit(self, x_test, y_true):
# no training since models are pre-trained
y_pred1 = self.model1.predict(x_test)
y_pred2 = self.model2.predict(x_test)
y_pred = (y_pred1 + y_pred2) / 2
# convert the predicted from binary to integer
y_pred = np.argmax(y_pred, axis=1)
df_metrics = calculate_metrics(y_true, y_pred, 0.0)
df_metrics.to_csv(self.output_directory + 'df_metrics.csv',
index=False)
keras.backend.clear_session()
gc.collect()
def predict(self,
x_test,
y_true,
x_train,
y_train,
y_test,
return_df_metrics=True):
best_model_path = self.get_file_path('best_model.hdf5')
model = keras.models.load_model(best_model_path)
start_time = time.time()
y_pred = model.predict(x_test)
if return_df_metrics:
y_pred = np.argmax(y_pred, axis=1)
df_metrics = utils.calculate_metrics(y_true, y_pred, 0.0)
return df_metrics
else:
test_duration = time.time() - start_time
duration_file = self.get_file_path('test_duration.csv')
utils.save_test_duration(duration_file, test_duration)
return y_pred
def predict(self, x_test, y_true, x_train, y_train, y_test):
batch_size = 256
# limit the number of augmented time series if series too long or too many
if x_train.shape[1] > 500 or x_train.shape[0] > 2000 or x_test.shape[
0] > 2000:
self.warping_ratios = [1]
self.slice_ratio = 0.9
# increase the slice if series too short
if x_train.shape[1] * self.slice_ratio < 8:
self.slice_ratio = 8 / x_train.shape[1]
new_x_train, new_y_train, new_x_test, new_y_test, tot_increase_num = \
self.pre_processing(x_train, y_train, x_test, y_test)
model_path = self.output_directory + 'best_model.hdf5'
model = keras.models.load_model(model_path)
y_pred = model.predict(new_x_test, batch_size=batch_size)
# convert the predicted from binary to integer
y_pred = np.argmax(y_pred, axis=1)
# get the true predictions of the test set
y_predicted = []
test_num_batch = int(new_x_test.shape[0] / tot_increase_num)
for i in range(test_num_batch):
unique_value, sub_ind, correspond_ind, count = np.unique(
y_pred, True, True, True)
idx_max = np.argmax(count)
predicted_label = unique_value[idx_max]
y_predicted.append(predicted_label)
y_pred = np.array(y_predicted)
df_metrics = calculate_metrics(y_true, y_pred, 0.0)
return df_metrics
def train(self):
start_time = time.time()
################
### Training ###
################
# init the matrices
self.init_matrices()
# compute the state matrices which is the new feature space
state_matrix = self.compute_state_matrix(self.x_train)
# add the input to form the new feature space and transform to
# the new feature space to be feeded to the classifier
new_x_train = np.concatenate((self.x_train, state_matrix),
axis=2).reshape(self.N * self.T,
self.num_dim + self.N_x)
# memory free
state_matrix = None
gc.collect()
# transform the corresponding labels
new_labels = np.repeat(self.y_train, self.T, axis=0)
# new model
ridge_classifier = Ridge(alpha=self.lamda)
# fit the new feature space
ridge_classifier.fit(new_x_train, new_labels)
################
## Validation ##
################
# compute state matrix for validation set
state_matrix = self.compute_state_matrix(self.x_val)
# add the input to form the new feature space and transform to
# the new feature space to be feeded to the classifier
new_x_val = np.concatenate(
(self.x_val, state_matrix),
axis=2).reshape(self.x_val.shape[0] * self.T,
self.num_dim + self.N_x)
# get the prediction on the train set
y_pred_val = ridge_classifier.predict(new_x_val)
# reconstruct the training prediction
y_pred_val = self.reshape_prediction(y_pred_val, self.x_val.shape[0],
self.T)
# get the metrics for the train
df_val_metrics = calculate_metrics(np.argmax(self.y_val, axis=1),
y_pred_val, 0.0)
# get the train accuracy
train_acc = df_val_metrics['accuracy'][0]
###############
### Testing ###
###############
# get the predicition on the test set
# transform the test set to the new features
state_matrix = self.compute_state_matrix(self.x_test)
# add the input to form the new feature space and transform to the new feature space to be feeded to the classifier
new_x_test = np.concatenate(
(self.x_test, state_matrix),
axis=2).reshape(self.x_test.shape[0] * self.T,
self.num_dim + self.N_x)
# memory free
state_matrix = None
gc.collect()
# get the prediction on the test set
y_pred = ridge_classifier.predict(new_x_test)
# reconstruct the test predictions
y_pred = self.reshape_prediction(y_pred, self.x_test.shape[0], self.T)
duration = time.time() - start_time
# get the metrics for the test predictions
df_metrics = calculate_metrics(self.y_true, y_pred, duration)
# get the output layer weights
self.W_out = ridge_classifier.coef_
ridge_classifier = None
gc.collect()
# save the model
np.savetxt(self.output_directory + 'W_in.txt', self.W_in)
np.savetxt(self.output_directory + 'W.txt', self.W)
np.savetxt(self.output_directory + 'W_out.txt', self.W_out)
# save the metrics
df_metrics.to_csv(self.output_directory + 'df_metrics.csv',
index=False)
# return the training accuracy and the prediction metrics on the test set
return df_metrics, train_acc
def fit(self, x_train, y_train, x_test, y_test, y_true):
# no training since models are pre-trained
start_time = time.time()
y_pred = np.zeros(shape=y_test.shape)
ll = 0
# loop through all classifiers
for model_name in self.classifiers:
# loop through different initialization of classifiers
for itr in self.iterations_to_take:
if itr == 0:
itr_str = ''
else:
itr_str = '_itr_' + str(itr)
curr_archive_name = self.archive_name + itr_str
curr_dir = self.models_dir.replace('classifier',
model_name).replace(
self.archive_name,
curr_archive_name)
model = self.create_classifier(model_name,
None,
None,
curr_dir,
build=False)
predictions_file_name = curr_dir + 'y_pred.npy'
# check if predictions already made
if check_if_file_exits(predictions_file_name):
# then load only the predictions from the file
curr_y_pred = np.load(predictions_file_name)
else:
# then compute the predictions
curr_y_pred = model.predict(x_test,
y_true,
return_df_metrics=False)
keras.backend.clear_session()
np.save(predictions_file_name, curr_y_pred)
y_pred = y_pred + curr_y_pred
ll += 1
# average predictions
y_pred = y_pred / ll
# save predictions
np.save(self.output_directory + 'y_pred.npy', y_pred)
# convert the predicted from binary to integer
y_pred = np.argmax(y_pred, axis=1)
duration = time.time() - start_time
df_metrics = calculate_metrics(y_true, y_pred, duration)
df_metrics.to_csv(self.output_directory + 'df_metrics.csv',
index=False)
gc.collect()
def fit(self, x_train, y_train, x_val, y_val, y_true):
self.y_true = y_true
self.x_test = x_val
self.y_test = y_val
self.x_train, self.x_val, self.y_train, self.y_val = \
train_test_split(x_train, y_train, test_size=0.2)
# 1. Tune ESN and num_filter
self.x_train, self.y_train, self.x_val, self.x_test, model_init, hist_init, duration_init, acc_init, num_filter = self.tune_esn()
current_acc = acc_init
hist_final = hist_init
model_final = model_init
duration_final = duration_init
ratio_final = [0.1, 0.2]
for ratio in self.ratio[1:]:
# 1. Build Model
input_shape = (self.len_series, self.units, 1)
model = self.build_model(
input_shape, self.nb_classes, self.len_series, ratio, num_filter)
#if(self.verbose == True):
#model.summary()
# 3. Train Model
batch = self.batch
epoch = self.epoch
start_time = time.time()
hist = model.fit(self.x_train, self.y_train, batch_size=batch, epochs=epoch,
verbose=False, validation_data=(self.x_val, self.y_val), callbacks=self.callbacks)
duration = time.time() - start_time
model_loss, model_acc = model.evaluate(
self.x_val, self.y_val, verbose=False)
print('val_loss: {0}, val_acc: {1}'.format(
model_loss, model_acc))
y_pred = model.predict(self.x_test)
# convert the predicted from binary to integer
y_pred = np.argmax(y_pred, axis=1)
df_metrics = calculate_metrics(self.y_true, y_pred, duration)
temp_output_dir = self.output_dir + str(self.it)+'/'
create_directory(temp_output_dir)
df_metrics.to_csv(temp_output_dir +
'df_metrics.csv', index=False)
model.save(temp_output_dir + 'model.hdf5')
params = [self.final_params_selected[0],
self.final_params_selected[1], self.final_params_selected[2], ratio]
param_print = pd.DataFrame(np.array([params], dtype=object), columns=[
'input_scaling', 'connectivity', 'num_filter', 'ratio'])
param_print.to_csv(temp_output_dir + 'df_params.csv', index=False)
if (model_acc > current_acc):
print('New winner')
hist_final = hist
model_final = model
duration_final = duration
ratio_final = ratio
current_acc = model_acc
keras.backend.clear_session()
self.it += 1
print('Final ratio: {0}'.format(ratio_final))
self.final_params_selected.append(ratio_final)
self.model = model_final
self.hist = hist_final
y_pred = self.model.predict(self.x_test)
# convert the predicted from binary to integer
y_pred = np.argmax(y_pred, axis=1)
param_print = pd.DataFrame(np.array([self.final_params_selected], dtype=object), columns=[
'input_scaling', 'connectivity', 'num_filter', 'ratio'])
param_print.to_csv(self.output_dir +
'df_final_params.csv', index=False)
save_logs(self.output_dir, self.hist, y_pred, self.y_true,
duration_final, self.verbose, lr=False)
keras.backend.clear_session()
def train(self):
batch_size = self.batch_size
nb_epochs = self.nb_epochs
curr_loss = 1e10
final_model = None
final_hist = None
final_cell = None
final_dur = None
input_shape = self.x_train.shape[1:]
for cell in self.lstm_cells:
model = self.build_model(input_shape, self.nb_classes, cell)
if self.verbose:
model.summary()
start_time = time.time()
hist = model.fit(self.x_train,
self.y_train,
batch_size=batch_size,
epochs=nb_epochs,
verbose=False,
validation_data=(self.x_test, self.y_test),
callbacks=self.callbacks)
model.load_weights(self.output_dir + 'best_curr_weights.hdf5')
print("Weights loaded from {0}best_curr_weights.hdf5".format(
self.output_dir))
#Best model loaded, now evaluate on train set (No overfitting for test set)
model_loss, model_acc = model.evaluate(self.x_train,
self.y_train,
batch_size=batch_size,
verbose=False)
print('Best weights --> val loss: {0}, val acc: {1}'.format(
model_loss, model_acc))
duration = time.time() - start_time
y_pred = model.predict(self.x_test)
# convert the predicted from binary to integer
y_pred = np.argmax(y_pred, axis=1)
df_metrics = calculate_metrics(self.y_true, y_pred, duration)
temp_output_dir = self.output_dir + 'num_cells_' + str(cell) + '/'
create_directory(temp_output_dir)
df_metrics.to_csv(temp_output_dir + 'df_metrics.csv', index=False)
model.save(temp_output_dir + 'model.hdf5')
if (model_loss < curr_loss):
curr_loss = model_loss
final_cell = cell
#curr_loss = model_loss
final_model = model
final_hist = hist
final_dur = duration
final_model.save(self.output_dir + 'best_model.hdf5')
keras.backend.clear_session()
print('Final Cell Selected:', final_cell)
file_cells = open(self.output_dir + 'best_num_cells.txt', 'w')
file_cells.write(str(final_cell))
file_cells.close()
return final_model, final_hist, final_dur
def fit(self, x_train, y_train, x_test, y_test, y_true):
start_time_tuning = time.time()
#######################
##Grid Search Stage 1##
#######################
param_grid_1 = dict(input_scaling=self.input_scaling,
connectivity=self.connectivity,
num_filter=self.num_filter)
emn_stage_1 = Base_Classifier_EMN(nb_classes=self.nb_classes,
verbose=False)
grid_1 = GridSearchCV(estimator=emn_stage_1,
param_grid=param_grid_1,
cv=3,
verbose=3)
grid_1_result = grid_1.fit(x_train, y_train)
#######################
##Grid Search Stage 2##
#######################
param_grid_2 = dict(ratio=self.ratio)
emn_stage_2 = grid_1_result.best_estimator_
grid_2 = GridSearchCV(estimator=emn_stage_2,
param_grid=param_grid_2,
cv=3,
verbose=3)
grid_2_result = grid_2.fit(x_train, y_train)
duration_tuning = time.time() - start_time_tuning
# Print Tune Duration
print('Tune duration: {0}s'.format(duration_tuning))
f = open(self.output_dir + '/grid_search_duration.txt', 'w')
f.write(str(duration_tuning))
f.close()
#####################################
##Final Training on whole train set##
#####################################
emn_final = grid_2_result.best_estimator_
if self.verbose:
emn_final.verbose = True
# Save Params
df_final_tuned = pd.DataFrame(
[[
emn_final.input_scaling, emn_final.connectivity,
emn_final.num_filter, emn_final.ratio
]],
columns=['input_scaling', 'connectivity', 'num_filter', 'ratio'])
print(df_final_tuned)
df_final_tuned.to_csv(self.output_dir + 'df_final_params.csv',
index=False)
#Three iterations for the best parameters
for i in range(3):
start_time = time.time()
emn_final.fit(x_train, y_train)
y_pred = emn_final.predict(x_test)
duration = time.time() - start_time
# Save Metrics
df_metrics = calculate_metrics(y_true, y_pred, duration)
if self.verbose:
print(df_metrics)
df_metrics.to_csv(self.output_dir + 'df_metrics' + str(i) + '.csv',
index=False)
def fit(self, x_train, y_train, x_val, y_val, y_true):
##################################
##Train n individual classifiers##
##################################
for it in range(self.num_ensemble_it):
if it == 0:
itr_str = 'network'
verbosity = self.verbose
else:
itr_str = 'network'+str(it)
verbosity = False
tmp_output_dir = self.output_dir + itr_str + '/'
create_directory(tmp_output_dir)
inception = Classifier_INCEPTION(
tmp_output_dir, self.input_shape, self.nb_classes, verbose = verbosity)
print('Fitting network {0} out of {1}'.format(
it+1, self.num_ensemble_it))
inception.fit(x_train, y_train, x_val, y_val, y_true)
#######################################
##Ensemble the individual classifiers##
#######################################
start_time = time.time()
y_pred = np.zeros(shape=y_val.shape)
ll = 0
for it in range(self.num_ensemble_it):
if it == 0:
itr_str = 'network'
else:
itr_str = 'network'+str(it)
classifier_dir = self.output_dir + itr_str + '/'
predictions_file_name = classifier_dir + 'y_pred.npy'
curr_y_pred = np.load(predictions_file_name)
y_pred = y_pred + curr_y_pred
ll += 1
# average predictions
y_pred = y_pred / ll
# save predictions
np.save(self.output_dir + 'y_pred.npy', y_pred)
# convert the predicted from binary to integer
y_pred = np.argmax(y_pred, axis=1)
duration = time.time() - start_time
df_metrics = calculate_metrics(y_true, y_pred, duration)
print(df_metrics)
df_metrics.to_csv(self.output_dir + 'df_metrics.csv', index=False)
gc.collect()
def train(self, x_train, y_train, x_test, y_test,y_true, pool_factor=None, filter_size=None,do_train=True):
window_size = 0.2
n_train_batch = 10
n_epochs = 200
max_train_batch_size = 256
# print('Original train shape: ', x_train.shape)
# print('Original test shape: ', x_test.shape)
# split train into validation set with validation_size = 0.2 train_size
x_train,y_train,x_val,y_val = self.split_train(x_train,y_train)
ori_len = x_train.shape[1] # original_length of time series
slice_ratio = 0.9
if do_train == True:
kernel_size = int(ori_len * filter_size)
if do_train == False:
model = keras.models.load_model(self.output_directory+'best_model.hdf5')
# model.summary()
pool_size = model.get_layer('max_pooling1d_1').get_config()['pool_size'][0]
conv_shape = model.get_layer('conv1d_1').output_shape[1]
pool_factor = self.get_pool_factor(conv_shape,pool_size)
#restrict slice ratio when data lenght is too large
if ori_len > 500 :
slice_ratio = slice_ratio if slice_ratio > 0.98 else 0.98
elif ori_len < 16:
slice_ratio = 0.7
increase_num = ori_len - int(ori_len * slice_ratio) + 1 #this can be used as the bath size
train_batch_size = int(x_train.shape[0] * increase_num / n_train_batch)
if train_batch_size > max_train_batch_size :
# limit the train_batch_size
n_train_batch = int(x_train.shape[0] * increase_num / max_train_batch_size)
# data augmentation by slicing the length of the series
x_train,y_train = self.slice_data(x_train,y_train,slice_ratio)
x_val,y_val = self.slice_data(x_val,y_val,slice_ratio)
x_test,y_test = self.slice_data(x_test,y_test,slice_ratio)
train_set_x, train_set_y = x_train,y_train
valid_set_x, valid_set_y = x_val,y_val
test_set_x, _ = x_test,y_test
valid_num = valid_set_x.shape[0]
# print("increase factor is ", increase_num, ', ori len', ori_len)
valid_num_batch = int(valid_num / increase_num)
test_num = test_set_x.shape[0]
test_num_batch = int(test_num / increase_num)
length_train = train_set_x.shape[1] #length after slicing.
window_size = int(length_train * window_size) if window_size < 1 else int(window_size)
#*******set up the ma and ds********#
ma_base,ma_step,ma_num = 5, 6, 1
ds_base,ds_step, ds_num = 2, 1, 4
ds_num_max = length_train / (pool_factor * window_size)
ds_num = int(min(ds_num, ds_num_max))
#*******set up the ma and ds********#
(ma_train, ma_valid, ma_test , ma_lengths) = self.batch_movingavrg(train_set_x,
valid_set_x, test_set_x,
ma_base, ma_step, ma_num)
(ds_train, ds_valid, ds_test , ds_lengths) = self.batch_downsample(train_set_x,
valid_set_x, test_set_x,
ds_base, ds_step, ds_num)
#concatenate directly
data_lengths = [length_train]
#downsample part:
if ds_lengths != []:
data_lengths += ds_lengths
train_set_x = np.concatenate([train_set_x, ds_train], axis = 1)
valid_set_x = np.concatenate([valid_set_x, ds_valid], axis = 1)
test_set_x = np.concatenate([test_set_x, ds_test], axis = 1)
#moving average part
if ma_lengths != []:
data_lengths += ma_lengths
train_set_x = np.concatenate([train_set_x, ma_train], axis = 1)
valid_set_x = np.concatenate([valid_set_x, ma_valid], axis = 1)
test_set_x = np.concatenate([test_set_x, ma_test], axis = 1)
# print("Data length:", data_lengths)
n_train_size = train_set_x.shape[0]
n_valid_size = valid_set_x.shape[0]
n_test_size = test_set_x.shape[0]
batch_size = int(n_train_size / n_train_batch)
n_train_batches = int(n_train_size / batch_size)
data_dim = train_set_x.shape[1]
num_dim = train_set_x.shape[2] # For MTS
nb_classes = train_set_y.shape[1]
# print('train size', n_train_size, ',valid size', n_valid_size, ' test size', n_test_size)
# print('batch size ', batch_size)
# print('n_train_batches is ', n_train_batches)
# print('data dim is ', data_dim)
# print('---------------------------')
######################
# BUILD ACTUAL MODEL #
######################
# print('building the model...')
input_shapes, max_length = self.get_list_of_input_shapes(data_lengths,num_dim)
start_time = time.time()
best_validation_loss = np.inf
if do_train == True:
model = self.build_model(input_shapes, nb_classes, pool_factor, kernel_size)
if (self.verbose==True) :
model.summary()
# print('Training')
# early-stopping parameters
patience = 10000 # look as this many examples regardless
patience_increase = 2 # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
max_before_stopping = 500
best_iter = 0
valid_loss = 0.
epoch = 0
done_looping = False
num_no_update_epoch = 0
epoch_avg_cost = float('inf')
epoch_avg_err = float('inf')
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
epoch_train_err = 0.
epoch_cost = 0.
num_no_update_epoch += 1
if num_no_update_epoch == max_before_stopping:
break
for minibatch_index in range(n_train_batches):
iteration = (epoch - 1) * n_train_batches + minibatch_index
x = train_set_x[minibatch_index*batch_size: (minibatch_index+1)*batch_size]
y = train_set_y[minibatch_index*batch_size: (minibatch_index+1)*batch_size]
x = self.split_input_for_model(x,input_shapes)
cost_ij, accuracy = model.train_on_batch(x,y)
train_err = 1 - accuracy
epoch_train_err = epoch_train_err + train_err
epoch_cost = epoch_cost + cost_ij
if (iteration + 1) % validation_frequency == 0:
valid_losses = []
for i in range(valid_num_batch):
x = valid_set_x[i * (increase_num) : (i + 1) * (increase_num)]
y_pred = model.predict_on_batch(self.split_input_for_model(x,input_shapes))
# convert the predicted from binary to integer
y_pred = np.argmax(y_pred , axis=1)
label = np.argmax(valid_set_y[i * increase_num])
unique_value, sub_ind, correspond_ind, count = np.unique(y_pred, True, True, True)
unique_value = unique_value.tolist()
curr_err = 1.
if label in unique_value:
target_ind = unique_value.index(label)
count = count.tolist()
sorted_count = sorted(count)
if count[target_ind] == sorted_count[-1]:
if len(sorted_count) > 1 and sorted_count[-1] == sorted_count[-2]:
curr_err = 0.5 #tie
else:
curr_err = 0
valid_losses.append(curr_err)
valid_loss = sum(valid_losses) / float(len(valid_losses))
# print('...epoch%i,valid err: %.5f |' % (epoch,valid_loss))
# if we got the best validation score until now
if valid_loss <= best_validation_loss:
num_no_update_epoch = 0
#improve patience if loss improvement is good enough
if valid_loss < best_validation_loss*improvement_threshold:
patience = max(patience,iteration*patience_increase)
# save best validation score and iteration number
best_validation_loss = valid_loss
best_iter = iteration
# save model in h5 format
model.save(self.output_directory+'best_model.hdf5')
model.save(self.output_directory + 'last_model.hdf5')
if patience<= iteration:
done_looping=True
break
epoch_avg_cost = epoch_cost/n_train_batches
epoch_avg_err = epoch_train_err/n_train_batches
# print ('train err %.5f, cost %.4f' %(epoch_avg_err,epoch_avg_cost))
if epoch_avg_cost == 0:
break
# print('Optimization complete.')
# test the model
# print('Testing')
# load best model
model = keras.models.load_model(self.output_directory+'best_model.hdf5')
# get the true predictions of the test set
y_predicted = []
for i in range(test_num_batch):
x = test_set_x[i * (increase_num) : (i + 1) * (increase_num)]
y_pred = model.predict_on_batch(self.split_input_for_model(x,input_shapes))
# convert the predicted from binary to integer
y_pred = np.argmax(y_pred , axis=1)
unique_value, sub_ind, correspond_ind, count = np.unique(y_pred, True, True, True)
idx_max = np.argmax(count)
predicted_label = unique_value[idx_max]
y_predicted.append(predicted_label)
y_pred = np.array(y_predicted)
duration = time.time() - start_time
df_metrics = calculate_metrics(y_true,y_pred, duration)
# print(y_true.shape)
# print(y_pred.shape)
df_metrics.to_csv(self.output_directory+'df_metrics.csv', index=False)
return df_metrics, model , best_validation_loss
# get the synthetic train and labels
syn_x_train, syn_y_train = syn_train_set
# concat the synthetic with the reduced random train and labels
aug_x_train = np.array(x_train.tolist() + syn_x_train.tolist())
aug_y_train = np.array(y_train.tolist() + syn_y_train.tolist())
print(np.unique(y_train, return_counts=True))
print(np.unique(aug_y_train, return_counts=True))
y_pred = classifier.fit(aug_x_train, aug_y_train, x_test,
y_test)
else:
# no data augmentation
y_pred = classifier.fit(x_train, y_train, x_test, y_test)
df_metrics = calculate_metrics(y_test, y_pred, 0.0)
df_metrics.to_csv(output_dir + 'df_metrics.csv', index=False)
print('DONE')
create_directory(output_dir + 'DONE')
else:
# for ensemble you will have to compute both models in order to ensemble them
from ensemble import Classifier_ENSEMBLE
classifier_ensemble = Classifier_ENSEMBLE(output_dir,
x_train.shape[1:],
nb_classes, False)
classifier_ensemble.fit(x_test, y_test)
# plot pairwise once all results are computed for resnet and resnet_augment and ensemble
plot_pairwise(root_deep_learning_dir, root_dir_dataset_archive, 'resnet',
'resnet_augment')
def tune_esn(self):
input_scaling_final = None
connect_final = None
num_filter_final = None
x_train_final = None
y_train_final = None
x_val_final = None
x_test_final = None
duration_final = None
model_final = None
hist_final = None
current_acc = 0
self.it = 0
for input_scaling in self.input_scaling:
for connect in self.connectivity:
x_train, y_train, x_val, x_test = self.ff_esn(input_scaling, connect)
for num_filter in self.num_filter:
ratio = [0.1, 0.2]
# 2. Build Model
input_shape = (self.len_series, self.units, 1)
model = self.build_model(
input_shape, self.nb_classes, self.len_series, ratio, num_filter)
#if(self.verbose == True):
#model.summary()
# 3. Train Model
batch = self.batch
epoch = self.epoch
start_time = time.time()
hist = model.fit(x_train, y_train, batch_size=batch, epochs=epoch,
verbose=False, validation_data=(x_val, self.y_val), callbacks=self.callbacks)
duration = time.time() - start_time
model_loss, model_acc = model.evaluate(
x_val, self.y_val, verbose=False)
print('val_loss: {0}, val_acc: {1}'.format(
model_loss, model_acc))
y_pred = model.predict(x_test)
# convert the predicted from binary to integer
y_pred = np.argmax(y_pred, axis=1)
df_metrics = calculate_metrics(
self.y_true, y_pred, duration)
temp_output_dir = self.output_dir + str(self.it)+'/'
create_directory(temp_output_dir)
df_metrics.to_csv(temp_output_dir +
'df_metrics.csv', index=False)
model.save(temp_output_dir + 'model.hdf5')
params = [input_scaling, connect, num_filter, ratio]
param_print = pd.DataFrame(np.array([params], dtype=object), columns=[
'input_scaling', 'connectivity', 'num_filter', 'ratio'])
param_print.to_csv(temp_output_dir +
'df_params.csv', index=False)
if (model_acc > current_acc):
print('New winner')
input_scaling_final = input_scaling
connect_final = connect
num_filter_final = num_filter
x_train_final = x_train
y_train_final = y_train
x_val_final = x_val
x_test_final = x_test
duration_final = duration
model_final = model
hist_final = hist
current_acc = model_acc
self.it += 1
keras.backend.clear_session()
print('Final input_scaling: {0}; Final connectivity: {1}; Final filter: {2}'.format(
input_scaling_final, connect_final, num_filter_final))
self.final_params_selected.append(input_scaling_final)
self.final_params_selected.append(connect_final)
self.final_params_selected.append(num_filter_final)
return x_train_final, y_train_final, x_val_final, x_test_final, model_final, hist_final, duration_final, current_acc, num_filter_final
