def train_classifier(train_set, dev_set, num_iterations, learning_rate, model,
regularization, model_file):
best_dev_accuracy = 0.0
for epoch in xrange(num_iterations):
np.random.shuffle(train_set)
print '\nbegan doing epoch no.', epoch + 1
total_loss = 0.0 # total loss in this iteration.
avg_loss = 0.0
count = 0.0
batch_size = 10
start_time = time.time()
for batch_index in range(0, len(train_set), batch_size):
loss = model.train_on_mini_batch(
train_set[batch_index:batch_index + batch_size, :-1],
train_set[batch_index:batch_index + batch_size,
-1].astype(dtype=int), learning_rate, regularization)
total_loss += loss
avg_loss += loss
count += batch_size
if count % 5000 == 0:
took = time.time() - start_time
start_time = time.time()
print '\t', (count / len(train_set)) * 100, '% complete.', 'took:', took,\
'seconds. average loss <-', avg_loss / 5000
avg_loss = 0
train_loss = total_loss / len(train_set)
train_accuracy = accuracy_on_dataset(train_set, model)
dev_accuracy = accuracy_on_dataset(dev_set, model)
print 'epoch:', epoch + 1, 'loss:', train_loss, 'train accuracy:', train_accuracy, 'dev accuracy:', dev_accuracy
if dev_accuracy > best_dev_accuracy:
best_dev_accuracy = dev_accuracy
print 'saving model....'
save_model(model, model_file)
def train_classifier(train_set, dev_set, num_iterations, learning_rate, model,
regularization, model_file):
best_dev_accuracy = 0.0
for epoch in xrange(num_iterations):
np.random.shuffle(train_set)
print '\nbegan doing epoch no.', epoch + 1
total_loss = 0.0 # total loss in this iteration.
avg_loss = 0.0
count = 0.0
start_time = time.time()
for example in train_set:
loss = model.train_on_example(example[:-1], example[-1],
learning_rate, regularization)
total_loss += loss
avg_loss += loss
count += 1
if count % 5000 == 0:
took = time.time() - start_time
start_time = time.time()
print '\t', (count / len(train_set)) * 100, '% complete.', 'took:', took,\
'seconds. average loss <-', avg_loss / 5000
model.add_loss_data(avg_loss)
avg_loss = 0
train_loss = total_loss / len(train_set)
train_accuracy = accuracy_on_dataset(train_set, model)
dev_accuracy = accuracy_on_dataset(dev_set, model)
print 'epoch:', epoch + 1, 'loss:', train_loss, 'train accuracy:', train_accuracy, 'dev accuracy:', dev_accuracy
if dev_accuracy > best_dev_accuracy:
best_dev_accuracy = dev_accuracy
print 'saving model....'
save_model(model, model_file)
def main():
if len(sys.argv) == 3:
database_filepath, model_filepath = sys.argv[1:]
print('Loading data...\n DATABASE: {}'.format(database_filepath))
X, Y, category_names = load_data(database_filepath)
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)
print('Building model...')
model = build_model()
print('Training model...')
model.fit(X_train, Y_train)
print('Evaluating model...')
evaluate_model(model, X_test, Y_test, category_names)
print('Saving model...\n MODEL: {}'.format(model_filepath))
save_model(model, model_filepath)
print('Trained model saved!')
else:
print('Please provide the filepath of the disaster messages database '\
'as the first argument and the filepath of the pickle file to '\
'save the model to as the second argument. \n\nExample: python '\
'train_classifier.py ../data/DisasterResponse.db classifier.pkl')
def train_classifier(train_set, dev_set, num_iterations, models, roulette):
length = len(train_set)
population_size = len(models)
blank_models = []
for b in range(population_size - 4):
blank = ga_mdl.Genetic_NNModel([28 * 28, 128, 10])
blank_models.append(blank)
batch_size = 200
population = models
best = None
for epoch in range(num_iterations):
np.random.shuffle(train_set)
avg_loss = 0
print '\nbegan doing epoch no.', epoch + 1
s_time = time.time()
for batch_index in range(0, length, batch_size):
losses = []
batch = train_set[batch_index:batch_index + batch_size]
# s_t = time.time()
calc_loss(population, batch)
# print time.time() - s_t
for model in population:
losses.append([model.batch_loss, model])
losses = sorted(losses)
avg_loss += losses[0][0]
if batch_index % 1000 == 0 and batch_index != 0:
e_time = time.time()
print '\t', batch_index / float(length), '% complete. took:', e_time - s_time,\
'average loss <-', avg_loss / 1000
avg_loss = 0
s_time = time.time()
if batch_index % 5000 == 0:
print '\taccuracy on dev: best:', accuracy_on_dataset(dev_set, losses[0][1]),\
'loss of best:', '%.2f' % (losses[0][0] / batch_size), \
'\n\taccuracy on dev: worst:', accuracy_on_dataset(dev_set, losses[-1][1]), \
'loss of worst:', '%.2f' % (losses[-1][0] / batch_size)
new_population = []
best = losses[0][1]
new_population.append(losses[0][1])
new_population.append(losses[1][1])
new_population.append(losses[2][1])
new_population.append(losses[3][1])
crossovers(losses, roulette, new_population, blank_models,
population_size - 4)
blank_models = []
for j in range(2, population_size, 1):
blank_models.append(losses[j][1])
population = new_population
train_accuracy = accuracy_on_dataset(train_set, best)
dev_accuracy = accuracy_on_dataset(dev_set, best)
print '****** EPOCH SUMMARY', epoch + 1, train_accuracy, dev_accuracy
print '****** saving population...'
count = 0
for m in population:
save_model(m,
'../saved_models/ga200_p2after_m' + str(count) + '.mdl')
count += 1
def train(model, epochs, train_dl, val_dl, fold):
best_score = 0.0
lr0 = args.init_lr_0
iterations = epochs * len(train_dl)
idx = 0
# create optimizer with differential learning rates
optimizer = create_optimizer(MODEL, BASE_OPTIMIZER, args.init_lr_0,
DIFF_LR_FACTORS)
# set up lr schedule based on val loss
lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,
'min',
verbose=True,
patience=args.patience)
for epoch in range(epochs):
total_loss = 0
# training loop
model.train()
for batch_idx, (data, target) in enumerate(train_dl):
data, target = data.cuda().float(), target.cuda().float()
output = model(data)
loss = F.binary_cross_entropy_with_logits(output, target)
total_loss += loss.item()
optimizer.zero_grad()
loss.backward()
optimizer.step()
idx += 1
# unfreeze deeper layers sequentially
if idx == int(0.1 * iterations):
model.unfreeze(1)
logger.info("Iteration %d: Unfreezing group 1" % idx)
if idx == int(0.2 * iterations):
model.unfreeze(0)
logger.info("Iteration %d: Unfreezing group 0" % idx)
if batch_idx % 100 == 0:
logger.info("Epoch %d (Batch %d / %d)\t Train loss: %.3f" % \
(epoch+1, batch_idx, len(train_dl), loss.item()))
# train loss
train_loss = total_loss / len(train_dl)
logger.info("Epoch %d\t Train loss: %.3f" % (epoch + 1, train_loss))
mlflow.log_metric('train_loss', train_loss, step=epoch)
# validation scores
val_f2_score, val_loss = validate(model, val_dl, 0.2)
# lr monitoring val_loss
lr_scheduler.step(val_loss)
logger.info("Epoch %d \t Validation loss: %.3f, F2 score: %.3f" % \
(epoch+1, val_loss, val_f2_score))
mlflow.log_metric('val_loss', val_loss, step=epoch)
mlflow.log_metric('val_f2_score', val_f2_score, step=epoch)
# model saving
if val_f2_score > best_score:
best_score = val_f2_score
best_model_path = os.path.join(MODEL_DIR, 'fold_%s' % fold, 'model_VGG19_%d.pth' % \
(100*val_f2_score))
logger.info("Saving model to %s" % best_model_path)
save_model(model, best_model_path)
def train(model, epochs, train_dl, val_dl):
best_score = 0.0
# create optimizer with differential learning rates
optimizer = create_optimizer(model, BASE_OPTIMIZER, args.init_lr_0,
DIFF_LR_FACTORS)
iterations = epochs * len(train_dl)
idx = 0
for epoch in range(epochs):
lr0 = lr_scheduler(epoch, args.lr_decay_factor, args.init_lr_0,
args.lr_decay_epoch) # set base lr for this epoch
optimizer = create_optimizer(model, BASE_OPTIMIZER, lr0,
DIFF_LR_FACTORS)
total_loss = 0
# training loop
for batch_idx, (data, target) in enumerate(train_dl):
data, target = data.cuda().float(), target.cuda().float()
output = model(data)
loss = F.binary_cross_entropy_with_logits(output, target)
total_loss += loss.item()
optimizer.zero_grad()
loss.backward()
optimizer.step()
idx += 1
# unfreeze deeper layers sequentially
if idx == int(0.1 * iterations):
model.unfreeze(1)
logger.info("Iteration %d: Unfreezing group 1" % idx)
if idx == int(0.2 * iterations):
model.unfreeze(0)
logger.info("Iteration %d: Unfreezing group 0" % idx)
if batch_idx % 100 == 0:
logger.info("Epoch %d (Batch %d / %d)\t Train loss: %.3f" % \
(epoch+1, batch_idx, len(train_dl), loss.item()))
# train loss
train_loss = total_loss / len(train_dl)
logger.info("Epoch %d\t Train loss: %.3f" % (epoch + 1, train_loss))
mlflow.log_metric('train_loss', train_loss, step=epoch)
# validation scores
val_f2_score, val_loss = validate(model, val_dl, 0.2)
logger.info("Epoch %d \t Validation loss: %.3f, F2 score: %.3f" % \
(epoch+1, val_loss, val_f2_score))
mlflow.log_metric('val_loss', val_loss, step=epoch)
mlflow.log_metric('val_f2_score', val_f2_score, step=epoch)
# model saving
if val_f2_score > best_score:
best_score = val_f2_score
best_model_path = os.path.join(MODEL_DIR, 'model_resnet34_%d.pth' % \
(100*val_f2_score))
logger.info("Saving model to %s" % best_model_path)
save_model(model, best_model_path)
bar.finish()
result = model.evaluate_generator(
(current_queue.get() for _ in range(validation_batches)),
steps=validation_batches)
process = processes.pop(0)
process.terminate()
processes.append(
mp.Process(target=get_datasets,
args=(current_output % threads, output_queues)))
processes[-1].start()
current_output += 1
end = timer()
print(
'train_loss: {:.4f} | train_acc: {:.4f} | test_loss: {:.4f} | test_acc: {:.4f} | epoch_time: {:.1f}'
.format(train_loss / train_batches, train_acc / train_batches,
result[0], result[1], end - start))
formatted_directory_name = 'models/{}/'.format(model_directory)
formatted_model_name = '{}-{}-{:.4f}-{:.4f}'.format(
model_name, e, train_acc / train_batches, result[1])
save_model(model, formatted_directory_name, formatted_model_name)
for process in processes:
process.terminate()
Y_train = np_utils.to_categorical(encoded_Y_train)
# coverting testing output to one-hot representation
encoded_Y_test = encoder.transform(np.ravel(Y_test))
Y_test = np_utils.to_categorical(encoded_Y_test)
# defining the deep learning model model
model = Sequential()
model.add(Dense(8, input_dim=3, activation='relu'))
model.add(Dense(3, activation='softmax'))
# compile the model
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# fit the model and save the history
hist = model.fit(X_train, Y_train, epochs=1000, verbose=0)
# test the performance of the model
test_model(Y_test, X_test, model)
# print the prediction
# print(predict_category(model, encoder, 5.6, 3, 4.5))
# plot the loss of the training proccess
plot_history(hist)
# save the model and weights
save_model(model)
def save_best(self, name='best'):
hf.save_model(self.model, self.name + '/' + name, folder=output_folder)
cls_loss_decoder_outputs_i_j_i)
rec_loss.backward()
cls_loss.backward()
encoder_optimizer.step()
decoder_optimizer.step()
classifier_optimizer.step()
total_loss = rec_loss.item() + cls_loss.item()
# print('-', total_loss)
c = write_to_board(
writer, total_loss, rec_loss, cls_loss,
(acc_i_i, acc_i_j, acc_i_j_i),
[recall_tuple_i_i, recall_tuple_i_j, recall_tuple_i_j_i], c)
if batch_idx % show_every == 0: # and batch_idx != 0:
show_reconstructions(input_text=input_text,
reconstruction_i_i=reconstruction_i_i,
reconstruction_i_j=reconstruction_i_j,
reconstruction_i_j_i=reconstruction_i_j_i,
first_few=1)
# Save the model
if min_loss > total_loss:
save_model(enc, dec, epoch_number)
min_loss = total_loss
writer.close()
create_gif(cfg.losses_dir)
for val_loss in cb.val_losses:
val_losses.append(val_loss)
fn = output_folder + model.name + '/lr_log.txt'
try:
os.remove(fn)
except Exception:
pass
thefile = open(fn, 'w')
for lr in lrs:
thefile.write("%s\n" % str(lr))
thefile.close()
hf.save_model(model, model.name)
hf.save_training_plot(losses,
val_losses,
lrs,
model_name=model.name,
plot_folder=output_folder)
# except Exception as e:
# import yagmail
# yag = yagmail.SMTP('*****@*****.**', 'testchpc1')
# body = 'Training CNN failed! \n \n Args: ' + str(args) + ' \n Exception: ' + str(e)
# yag.send('*****@*****.**', 'CNN Training failed for some reason', body)
##
# EoF #
def save_best(self, name='best'):
hf.save_model(self.model, self.name + '/' + name, folder=output_folder)
self.model_simple.set_weights(self.model.get_weights())
hf.save_model(self.model_simple,
self.name + '/' + name + '_simple',
folder=output_folder)
def train_classifier(train_set,
dev_set,
num_iterations,
models,
roulette,
elitism=8):
length = len(train_set)
population_size = len(models)
batch_size = 100
population = models
for epoch in range(num_iterations):
np.random.shuffle(train_set)
avg_loss = 0
print '\nbegan doing epoch no.', epoch + 1
s_time = time.time()
for batch_index in range(0, length, batch_size):
batch_x = train_set[batch_index:batch_index + batch_size, :-1]
batch_y = train_set[batch_index:batch_index + batch_size,
-1].astype(dtype=int)
for m in population:
m.loss_mini_batch(batch_x, batch_y)
population.sort(key=operator.attrgetter('batch_loss'))
avg_loss += population[0].batch_loss
if batch_index % 1000 == 0 and batch_index != 0:
e_time = time.time()
print '\t', batch_index / float(length), '% complete. took:', e_time - s_time,\
'average loss <-', avg_loss / 1000
avg_loss = 0
s_time = time.time()
if batch_index % 5000 == 0 and batch_index != 0:
print '\taccuracy on dev: best:', accuracy_on_dataset(dev_set, population[0]),\
'loss of best:', '%.2f' % (population[0].batch_loss / batch_size), \
'\n\taccuracy on dev: worst:', accuracy_on_dataset(dev_set, population[-1]), \
'loss of worst:', '%.2f' % (population[-1].batch_loss / batch_size)
new_population = []
for e in range(elitism):
new_population.append(population[e][1])
for c in range(0, population_size - elitism, 2):
p1_index = rand.choice(roulette)
p2_index = rand.choice(roulette)
parent1 = population[p1_index][1]
parent2 = population[p2_index][1]
child1, child2 = parent1.crossover(parent2)
if rand.randint(0, 2) == 1:
child1.mutate()
if rand.randint(0, 2) == 1:
child2.mutate()
new_population.append(child1)
new_population.append(child2)
population = new_population
train_accuracy = accuracy_on_dataset(train_set, population[0])
dev_accuracy = accuracy_on_dataset(dev_set, population[0])
print '****** EPOCH SUMMARY', epoch + 1, train_accuracy, dev_accuracy
print '****** saving population...'
count = 0
for m in population:
save_model(m, '../saved_models/ga200' + str(count) + '.mdl')
count += 1
nb_filters[n]) + '_' + str(
nb_epoch[o]) + '_' + str(DA[p])
print('Starting : ' + name)
model.name = name
model.compile(loss=loss, optimizer='adadelta')
val_loss = hf.run_model(model,
X_train,
Y_train,
X_test,
Y_test,
nb_epoch=nb_epoch[o],
drops=drops,
batch_size=batch_size,
DA=DA[p])
hf.save_model(model, name=name)
outputs = hf.predict_outputs(model,
X_test,
target_dim=(256,
256,
1))
hf.display_distribution_of_outputs_regression(
outputs,
Y_test,
name,
display=display,
output_folder=name)
if attempt < 5:
model2 = cg.simple_CNN1(
nb_dense=nb_dense[i],
final_relu=final_relu[j],
data_directory = parsed_results.data_directory
checkpoint = parsed_results.save_path
pretrained_model = parsed_results.pretrained_model
lr = float(parsed_results.lr)
hidden_units = int(parsed_results.hidden_units)
epochs = int(parsed_results.num_epochs)
device_flag = bool(parsed_results.use_gpu)
# Load datasets
train_data,test_data,validationdata,trainloader,testloader,validationloader = data_load(data_directory)
# Load pre-trained model
model = getattr(models,pretrained_model)(pretrained = True)
# Modify classifier based on current dataset
model = modify_classifier(model,model.classifier[0].in_features,device_flag)
# Initialize loss and optimizer
criterion = nn.NLLLoss()
optimizer = optim.Adam(model.classifier.parameters(), lr)
# Train the model on the dataset
model, optimizer = train_model(3,50,model,trainloader,criterion,optimizer,validationloader,device_flag)
# Use the test dataset for accuracy
test_accuracy(model,testloader,device_flag)
# Save the model onto disk
save_model(model,checkpoint,train_data,optimizer)