# print('x_tr.shape =', x_tr.shape)
# print('y_tr.shape =', y_tr.shape)
# print('x_te.shape =', x_te.shape)
# print('y_te.shape =', y_te.shape)
#x_tr = x_tr.reshape(x_tr.shape[0], x_tr.shape[1],1,1)
#x_te = x_te.reshape(x_te.shape[0], x_te.shape[1],1,1)
filepath="./model_files/raim-improvement-{epoch:02d}-{val_acc:.2f}.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
callbacks_list = [checkpoint]
# if the accuracy does not increase over 10 epochs, we reduce the learning rate by half.
reduce_lr = ReduceLROnPlateau(monitor='val_acc', factor=0.5, patience=10, min_lr=0.0001, verbose=1)
metrics_history = MetricsHistory()
batch_size = 128
model.fit(x= [x_tr1,x_tr2, x_tr3, x_tr4, x_tr5, x_tr6 ],
y=y_tr,
batch_size=batch_size,
epochs=200,
verbose=1,
shuffle=True,
validation_data=([x_te1, x_te2, x_te3, x_te4, x_te5, x_te6], y_te),
callbacks=[metrics_history, reduce_lr,checkpoint])
model.save('./model_files/m5_run1.h5')
pdb.set_trace()
file_logger.close()
def train(args):
"""Train the neural network. Write out model every several iterations.
Args:
workspace: str, path of workspace.
tr_snr: float, training SNR.
te_snr: float, testing SNR.
lr: float, learning rate.
"""
class MetricsHistory(Callback):
def on_epoch_end(self, epoch, logs={}):
file_logger.write([str(epoch),
str(logs['loss']),
str(logs['val_loss'])
])
print(args)
workspace = args.workspace
#tr_snr = args.tr_snr
#te_snr = args.te_snr
lr = args.lr
#TF = args.TF
model_name = args.model_name
#model_save_dir = os.path.join(args.workspace, 'saved_models')
# Load data
t1 = time.time()
print("Loading the train and vallidation dataset")
tr_hdf5_path = os.path.join(workspace, "packed_features", "train", "mag.h5")
te_hdf5_path = os.path.join(workspace, "packed_features", "val", "mag.h5")
(tr_x, tr_y) = pp_data.load_hdf5(tr_hdf5_path)
(te_x, te_y) = pp_data.load_hdf5(te_hdf5_path)
print('train_x shape:')
print(tr_x.shape, tr_y.shape)
print('test_x shape:')
print(te_x.shape, te_y.shape)
print("Load data time: %f s" % (time.time() - t1))
print('\n')
# Scale data
if True:
print("Scaling train and test dataset. This will take some time, please wait patiently...")
t1 = time.time()
scaler_path = os.path.join(workspace, "packed_features", "train", "mag_scaler.p")
scaler = pickle.load(open(scaler_path, 'rb'))
tr_x = pp_data.scale_on_3d(tr_x, scaler)
tr_y = pp_data.scale_on_2d(tr_y, scaler)
te_x = pp_data.scale_on_3d(te_x, scaler)
te_y = pp_data.scale_on_2d(te_y, scaler)
print("Scale data time: %f s" % (time.time() - t1))
# Debug plot.
if False:
plt.matshow(tr_x[0 : 1000, 0, :].T, origin='lower', aspect='auto', cmap='jet')
plt.show()
#time.sleep(secs)
os.system("pause")
# Build model
batch_size = 150
epoch = 100
print("The neural networks you have chosed is %s" % model_name)
print("The training batch is set to %d and the %s will be training for at most %d epoches" % (batch_size, model_name.upper(), epoch))
print("======iteration of one epoch======" )
iter_each_epoch = int(tr_x.shape[0] / batch_size)
#val_each_epoch = int(te_x.shape[0] / batch_size)
#print("There are %d iterations / epoch" % int(tr_x.shape[0] / batch_size))
print("There are %d iterations / epoch" % iter_each_epoch)
log_save_dir = os.path.join(workspace, 'log')
if not os.path.isdir(log_save_dir):
os.makedirs(log_save_dir)
log_path = os.path.join(log_save_dir, 'out_{}.csv'.format(model_name))
#log_path = os.path.join(log_save_dir, 'out_%ddb_%s.csv' %(int(snr[0]), model_name))
file_logger = FileLogger(log_path, ['epoch', 'train_loss', 'val_loss'])
(_, n_concat, n_freq) = tr_x.shape
#temp_tr_x = tr_x[:, 3, :][:, np.newaxis, :]
#print(temp_tr_x.shape)
#np.axis
n_hid = 2048
#data_gen = DataGenerator(batch_size=batch_size, type='train')
#tr_gen = data_gen.generate(xs=[tr_x], ys=[tr_y])
#te_gen = data_gen.generate(xs=[te_x], ys=[te_y])
#temp_tr_x = tr_gen[:, 3, :][:, np.newaxis, :]
'''
model = Sequential()
model.add(Flatten(input_shape=(n_concat, n_freq)))
model.add(BatchNormalization())
model.add(Dense(n_hid, activation='relu', kernel_regularizer=regularizers.l2(l=0.0001)))
model.add(Dropout(0.2))
model.add(BatchNormalization())
model.add(Dense(n_hid, activation='relu', kernel_regularizer=regularizers.l2(l=0.0001)))
model.add(Dropout(0.2))
model.add(BatchNormalization())
model.add(Dense(n_hid, activation='relu', kernel_regularizer=regularizers.l2(l=0.0001)))
model.add(Dropout(0.2))
model.add(Dense(n_freq, activation='linear'))
#model.summary()
'''
print('Model selected:', model_name.lower())
if model_name == 'dnn':
model = dnn(n_hid, n_concat, n_freq)
elif model_name == 'sdnn1':
model = sdnn1(n_hid, n_concat, n_freq)
elif model_name == 'sdnn2':
model = sdnn2(n_hid, n_concat, n_freq)
elif model_name == 'sdnn3':
model = sdnn3(n_hid, n_concat, n_freq)
elif model_name == 'fcn':
model = fcn(n_concat, n_freq)
elif model_name == 'fcn1':
model = fcn1(n_concat, n_freq)
elif model_name == 'fcn1':
model = fcn1_re(n_concat, n_freq)
elif model_name == 'fcn2':
model = fcn2(n_concat, n_freq)
elif model_name == 'fcn3':
model = fcn3(n_concat, n_freq)
elif model_name == 'fcn4':
model = fcn4(n_concat, n_freq)
elif model_name == 'm_vgg':
model = m_vgg(n_concat, n_freq)
elif model_name == 'm_vgg1':
model = m_vgg1(n_concat, n_freq)
elif model_name == 'm_vgg2':
model = m_vgg2(n_concat, n_freq)
elif model_name == 'm_vgg3':
model = m_vgg3(n_concat, n_freq)
elif model_name == 'm_vgg4':
model = m_vgg3(n_concat, n_freq)
elif model_name == 'CapsNet':
model = CapsNet(n_concat, n_freq, 3)
elif model_name == 'brnn' :
recur_layers = 7
unit = 256
output_dim = n_freq
model = brnn(n_concat, n_freq, unit, recur_layers, output_dim)
elif model_name == 'rnn' :
output_dim = n_freq
model = rnn(n_concat, n_freq, output_dim)
elif model_name == 'tcn' :
input_dim = n_freq
model = tcn(n_concat, input_dim)
if model is None:
exit('Please choose a valid model: [dnn, sdnn, sdnn1, cnn, scnn1]')
#mean_squared_error
model.compile(loss = 'mean_squared_error',
optimizer=Adam(lr=lr))
print(model.summary())
#plot model
#plot_model(model, to_file=args.save_dir+'/model.png', show_shapes=True)
#plot_model(model, to_file='%s/%s_model.png' % (log_save_dir, model_name), show_shapes=True)
# Save model and weights
model_save_dir = os.path.join(workspace, 'saved_models', "%s" % model_name)
model_save_name = "weights-checkpoint-{epoch:02d}-{val_loss:.2f}.h5"
if not os.path.isdir(model_save_dir):
os.makedirs(model_save_dir)
model_path = os.path.join(model_save_dir, model_save_name)
checkpoint = ModelCheckpoint(model_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min')
print('Saved trained model at %s' % model_save_dir)
#reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4, min_lr=0.00001, verbose=1)
lr_decay = LearningRateScheduler(schedule=lambda epoch: lr * (0.9 ** epoch))
metrics_history = MetricsHistory()
hist = model.fit(x=tr_x,
y=tr_y,
batch_size=batch_size,
epochs=epoch,
verbose=1,
shuffle=True,
validation_data=(te_x, te_y),
#validation_split=0.1,
callbacks=[metrics_history, checkpoint, lr_decay])
'''
hist = model.fit_generator(tr_gen,
steps_per_epoch=iter_each_epoch,
epochs=epoch,
verbose=1,
validation_data=te_gen,
validation_steps=val_each_epoch,
callbacks=[metrics_history, checkpoint, reduce_lr])
'''
print(hist.history.keys())
# list all data in history
#print(hist.history.keys())
'''
# summarize history for accuracy
plt.plot(hist.history['acc'])
plt.plot(hist.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
'''
# summarize history for loss
model_png = "train_test_loss"
loss_fig_dir = os.path.join(log_save_dir, '%s_%s.png' % (model_name, model_png))
plt.plot(hist.history['loss'])
plt.plot(hist.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper right')
plt.savefig(loss_fig_dir)
#plt.show()
'''
fig = plt.gcf()
plt.show()
fig.savefig('tessstttyyy.png', dpi=100)
'''
file_logger.close()
'''
# Data generator.
tr_gen = DataGenerator(batch_size=batch_size, type='train')
eval_te_gen = DataGenerator(batch_size=batch_size, type='test', te_max_iter=100)
eval_tr_gen = DataGenerator(batch_size=batch_size, type='test', te_max_iter=100)
# Directories for saving models and training stats
model_dir = os.path.join(workspace, "models", "%ddb" % int(tr_snr))
pp_data.create_folder(model_dir)
stats_dir = os.path.join(workspace, "training_stats", "%ddb" % int(tr_snr))
pp_data.create_folder(stats_dir)
# Print loss before training.
iter = 0
tr_loss = eval(model, eval_tr_gen, tr_x, tr_y)
te_loss = eval(model, eval_te_gen, te_x, te_y)
print("Iteration: %d, tr_loss: %f, te_loss: %f" % (iter, tr_loss, te_loss))
# Save out training stats.
stat_dict = {'iter': iter,
'tr_loss': tr_loss,
'te_loss': te_loss, }
stat_path = os.path.join(stats_dir, "%diters.p" % iter)
cPickle.dump(stat_dict, open(stat_path, 'wb'), protocol=cPickle.HIGHEST_PROTOCOL)
# Train.
t1 = time.time()
for (batch_x, batch_y) in tr_gen.generate(xs=[tr_x], ys=[tr_y]):
#loss = model.train_on_batch(batch_x, batch_y)
if iter % 2000 == 0:
lr *= 0.1
model.train_on_batch(batch_x, batch_y)
iter += 1
# Validate and save training stats.
if iter % 1000 == 0:
tr_loss = eval(model, eval_tr_gen, tr_x, tr_y)
te_loss = eval(model, eval_te_gen, te_x, te_y)
print("Iteration: %d, tr_loss: %f, te_loss: %f" % (iter, tr_loss, te_loss))
# Save out training stats.
stat_dict = {'iter': iter,
'tr_loss': tr_loss,
'te_loss': te_loss, }
stat_path = os.path.join(stats_dir, "%diters.p" % iter)
cPickle.dump(stat_dict, open(stat_path, 'wb'), protocol=cPickle.HIGHEST_PROTOCOL)
# Save model.
if iter % 5000 == 0:
model_path = os.path.join(model_dir, "md_%diters.h5" % iter)
model.save(model_path)
print("Saved model to %s" % model_path)
if iter == 10001:
break
'''
print("Training time: %s s" % (time.time() - t1,))
