# print('x_tr.shape =', x_tr.shape) # print('y_tr.shape =', y_tr.shape) # print('x_te.shape =', x_te.shape) # print('y_te.shape =', y_te.shape) #x_tr = x_tr.reshape(x_tr.shape[0], x_tr.shape[1],1,1) #x_te = x_te.reshape(x_te.shape[0], x_te.shape[1],1,1) filepath="./model_files/raim-improvement-{epoch:02d}-{val_acc:.2f}.hdf5" checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max') callbacks_list = [checkpoint] # if the accuracy does not increase over 10 epochs, we reduce the learning rate by half. reduce_lr = ReduceLROnPlateau(monitor='val_acc', factor=0.5, patience=10, min_lr=0.0001, verbose=1) metrics_history = MetricsHistory() batch_size = 128 model.fit(x= [x_tr1,x_tr2, x_tr3, x_tr4, x_tr5, x_tr6 ], y=y_tr, batch_size=batch_size, epochs=200, verbose=1, shuffle=True, validation_data=([x_te1, x_te2, x_te3, x_te4, x_te5, x_te6], y_te), callbacks=[metrics_history, reduce_lr,checkpoint]) model.save('./model_files/m5_run1.h5') pdb.set_trace() file_logger.close()
def train(args): """Train the neural network. Write out model every several iterations. Args: workspace: str, path of workspace. tr_snr: float, training SNR. te_snr: float, testing SNR. lr: float, learning rate. """ class MetricsHistory(Callback): def on_epoch_end(self, epoch, logs={}): file_logger.write([str(epoch), str(logs['loss']), str(logs['val_loss']) ]) print(args) workspace = args.workspace #tr_snr = args.tr_snr #te_snr = args.te_snr lr = args.lr #TF = args.TF model_name = args.model_name #model_save_dir = os.path.join(args.workspace, 'saved_models') # Load data t1 = time.time() print("Loading the train and vallidation dataset") tr_hdf5_path = os.path.join(workspace, "packed_features", "train", "mag.h5") te_hdf5_path = os.path.join(workspace, "packed_features", "val", "mag.h5") (tr_x, tr_y) = pp_data.load_hdf5(tr_hdf5_path) (te_x, te_y) = pp_data.load_hdf5(te_hdf5_path) print('train_x shape:') print(tr_x.shape, tr_y.shape) print('test_x shape:') print(te_x.shape, te_y.shape) print("Load data time: %f s" % (time.time() - t1)) print('\n') # Scale data if True: print("Scaling train and test dataset. This will take some time, please wait patiently...") t1 = time.time() scaler_path = os.path.join(workspace, "packed_features", "train", "mag_scaler.p") scaler = pickle.load(open(scaler_path, 'rb')) tr_x = pp_data.scale_on_3d(tr_x, scaler) tr_y = pp_data.scale_on_2d(tr_y, scaler) te_x = pp_data.scale_on_3d(te_x, scaler) te_y = pp_data.scale_on_2d(te_y, scaler) print("Scale data time: %f s" % (time.time() - t1)) # Debug plot. if False: plt.matshow(tr_x[0 : 1000, 0, :].T, origin='lower', aspect='auto', cmap='jet') plt.show() #time.sleep(secs) os.system("pause") # Build model batch_size = 150 epoch = 100 print("The neural networks you have chosed is %s" % model_name) print("The training batch is set to %d and the %s will be training for at most %d epoches" % (batch_size, model_name.upper(), epoch)) print("======iteration of one epoch======" ) iter_each_epoch = int(tr_x.shape[0] / batch_size) #val_each_epoch = int(te_x.shape[0] / batch_size) #print("There are %d iterations / epoch" % int(tr_x.shape[0] / batch_size)) print("There are %d iterations / epoch" % iter_each_epoch) log_save_dir = os.path.join(workspace, 'log') if not os.path.isdir(log_save_dir): os.makedirs(log_save_dir) log_path = os.path.join(log_save_dir, 'out_{}.csv'.format(model_name)) #log_path = os.path.join(log_save_dir, 'out_%ddb_%s.csv' %(int(snr[0]), model_name)) file_logger = FileLogger(log_path, ['epoch', 'train_loss', 'val_loss']) (_, n_concat, n_freq) = tr_x.shape #temp_tr_x = tr_x[:, 3, :][:, np.newaxis, :] #print(temp_tr_x.shape) #np.axis n_hid = 2048 #data_gen = DataGenerator(batch_size=batch_size, type='train') #tr_gen = data_gen.generate(xs=[tr_x], ys=[tr_y]) #te_gen = data_gen.generate(xs=[te_x], ys=[te_y]) #temp_tr_x = tr_gen[:, 3, :][:, np.newaxis, :] ''' model = Sequential() model.add(Flatten(input_shape=(n_concat, n_freq))) model.add(BatchNormalization()) model.add(Dense(n_hid, activation='relu', kernel_regularizer=regularizers.l2(l=0.0001))) model.add(Dropout(0.2)) model.add(BatchNormalization()) model.add(Dense(n_hid, activation='relu', kernel_regularizer=regularizers.l2(l=0.0001))) model.add(Dropout(0.2)) model.add(BatchNormalization()) model.add(Dense(n_hid, activation='relu', kernel_regularizer=regularizers.l2(l=0.0001))) model.add(Dropout(0.2)) model.add(Dense(n_freq, activation='linear')) #model.summary() ''' print('Model selected:', model_name.lower()) if model_name == 'dnn': model = dnn(n_hid, n_concat, n_freq) elif model_name == 'sdnn1': model = sdnn1(n_hid, n_concat, n_freq) elif model_name == 'sdnn2': model = sdnn2(n_hid, n_concat, n_freq) elif model_name == 'sdnn3': model = sdnn3(n_hid, n_concat, n_freq) elif model_name == 'fcn': model = fcn(n_concat, n_freq) elif model_name == 'fcn1': model = fcn1(n_concat, n_freq) elif model_name == 'fcn1': model = fcn1_re(n_concat, n_freq) elif model_name == 'fcn2': model = fcn2(n_concat, n_freq) elif model_name == 'fcn3': model = fcn3(n_concat, n_freq) elif model_name == 'fcn4': model = fcn4(n_concat, n_freq) elif model_name == 'm_vgg': model = m_vgg(n_concat, n_freq) elif model_name == 'm_vgg1': model = m_vgg1(n_concat, n_freq) elif model_name == 'm_vgg2': model = m_vgg2(n_concat, n_freq) elif model_name == 'm_vgg3': model = m_vgg3(n_concat, n_freq) elif model_name == 'm_vgg4': model = m_vgg3(n_concat, n_freq) elif model_name == 'CapsNet': model = CapsNet(n_concat, n_freq, 3) elif model_name == 'brnn' : recur_layers = 7 unit = 256 output_dim = n_freq model = brnn(n_concat, n_freq, unit, recur_layers, output_dim) elif model_name == 'rnn' : output_dim = n_freq model = rnn(n_concat, n_freq, output_dim) elif model_name == 'tcn' : input_dim = n_freq model = tcn(n_concat, input_dim) if model is None: exit('Please choose a valid model: [dnn, sdnn, sdnn1, cnn, scnn1]') #mean_squared_error model.compile(loss = 'mean_squared_error', optimizer=Adam(lr=lr)) print(model.summary()) #plot model #plot_model(model, to_file=args.save_dir+'/model.png', show_shapes=True) #plot_model(model, to_file='%s/%s_model.png' % (log_save_dir, model_name), show_shapes=True) # Save model and weights model_save_dir = os.path.join(workspace, 'saved_models', "%s" % model_name) model_save_name = "weights-checkpoint-{epoch:02d}-{val_loss:.2f}.h5" if not os.path.isdir(model_save_dir): os.makedirs(model_save_dir) model_path = os.path.join(model_save_dir, model_save_name) checkpoint = ModelCheckpoint(model_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min') print('Saved trained model at %s' % model_save_dir) #reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4, min_lr=0.00001, verbose=1) lr_decay = LearningRateScheduler(schedule=lambda epoch: lr * (0.9 ** epoch)) metrics_history = MetricsHistory() hist = model.fit(x=tr_x, y=tr_y, batch_size=batch_size, epochs=epoch, verbose=1, shuffle=True, validation_data=(te_x, te_y), #validation_split=0.1, callbacks=[metrics_history, checkpoint, lr_decay]) ''' hist = model.fit_generator(tr_gen, steps_per_epoch=iter_each_epoch, epochs=epoch, verbose=1, validation_data=te_gen, validation_steps=val_each_epoch, callbacks=[metrics_history, checkpoint, reduce_lr]) ''' print(hist.history.keys()) # list all data in history #print(hist.history.keys()) ''' # summarize history for accuracy plt.plot(hist.history['acc']) plt.plot(hist.history['val_acc']) plt.title('model accuracy') plt.ylabel('accuracy') plt.xlabel('epoch') plt.legend(['train', 'test'], loc='upper left') plt.show() ''' # summarize history for loss model_png = "train_test_loss" loss_fig_dir = os.path.join(log_save_dir, '%s_%s.png' % (model_name, model_png)) plt.plot(hist.history['loss']) plt.plot(hist.history['val_loss']) plt.title('model loss') plt.ylabel('loss') plt.xlabel('epoch') plt.legend(['train', 'val'], loc='upper right') plt.savefig(loss_fig_dir) #plt.show() ''' fig = plt.gcf() plt.show() fig.savefig('tessstttyyy.png', dpi=100) ''' file_logger.close() ''' # Data generator. tr_gen = DataGenerator(batch_size=batch_size, type='train') eval_te_gen = DataGenerator(batch_size=batch_size, type='test', te_max_iter=100) eval_tr_gen = DataGenerator(batch_size=batch_size, type='test', te_max_iter=100) # Directories for saving models and training stats model_dir = os.path.join(workspace, "models", "%ddb" % int(tr_snr)) pp_data.create_folder(model_dir) stats_dir = os.path.join(workspace, "training_stats", "%ddb" % int(tr_snr)) pp_data.create_folder(stats_dir) # Print loss before training. iter = 0 tr_loss = eval(model, eval_tr_gen, tr_x, tr_y) te_loss = eval(model, eval_te_gen, te_x, te_y) print("Iteration: %d, tr_loss: %f, te_loss: %f" % (iter, tr_loss, te_loss)) # Save out training stats. stat_dict = {'iter': iter, 'tr_loss': tr_loss, 'te_loss': te_loss, } stat_path = os.path.join(stats_dir, "%diters.p" % iter) cPickle.dump(stat_dict, open(stat_path, 'wb'), protocol=cPickle.HIGHEST_PROTOCOL) # Train. t1 = time.time() for (batch_x, batch_y) in tr_gen.generate(xs=[tr_x], ys=[tr_y]): #loss = model.train_on_batch(batch_x, batch_y) if iter % 2000 == 0: lr *= 0.1 model.train_on_batch(batch_x, batch_y) iter += 1 # Validate and save training stats. if iter % 1000 == 0: tr_loss = eval(model, eval_tr_gen, tr_x, tr_y) te_loss = eval(model, eval_te_gen, te_x, te_y) print("Iteration: %d, tr_loss: %f, te_loss: %f" % (iter, tr_loss, te_loss)) # Save out training stats. stat_dict = {'iter': iter, 'tr_loss': tr_loss, 'te_loss': te_loss, } stat_path = os.path.join(stats_dir, "%diters.p" % iter) cPickle.dump(stat_dict, open(stat_path, 'wb'), protocol=cPickle.HIGHEST_PROTOCOL) # Save model. if iter % 5000 == 0: model_path = os.path.join(model_dir, "md_%diters.h5" % iter) model.save(model_path) print("Saved model to %s" % model_path) if iter == 10001: break ''' print("Training time: %s s" % (time.time() - t1,))