def evaluate(eval_data, num_problems, num_concepts, model): problem_seqs, concept_seqs, answer_seqs = eval_data # process the input data response_data, concept_data, seq_lengths = process_data( eval_data, num_problems, num_concepts, device=model.device) # get y_next_true + y_cur_true next_answer_seqs = copy.deepcopy(answer_seqs) for n_ans_seq in next_answer_seqs: del n_ans_seq[0] y_next_true = list(itertools.chain.from_iterable(next_answer_seqs)) y_next_true = torch.tensor(y_next_true, dtype=torch.float, device=model.device) y_cur_true = list(itertools.chain.from_iterable(answer_seqs)) y_cur_true = torch.tensor(y_cur_true, dtype=torch.float, device=model.device) # forward + y_next_pred + y_cur_pred y_next_pred, y_cur_pred, batch_future_pred, batch_concept_mastery = model( response_data, concept_data, seq_lengths) y_next_pred_label = np.where(y_next_pred.cpu().numpy() <= 0.5, 0, 1) auc_n = roc_auc_score(y_next_true.cpu(), y_next_pred.cpu()) auc_c = roc_auc_score(y_cur_true.cpu(), y_cur_pred.cpu()) acc = accuracy_score(y_next_true.cpu(), y_next_pred_label) rmse = np.sqrt(mean_squared_error(y_next_true.cpu(), y_next_pred.cpu())) mae = mean_absolute_error(y_next_true.cpu(), y_next_pred.cpu()) return auc_n, auc_c, acc, rmse, mae
def fit(self, X, y=None): print('\nProcessing data...') self.data_train = data_utils.process_data(X, y, test_size=0) if self.config.plot: self.data_plot = self.data_train self.config.num_batches = self.data_train.num_batches( self.config.batch_size) if not self.config.isBuilt: self.config.restore = True self.build_model(self.data_train.height, self.data_train.width, self.data_train.num_channels) else: assert (self.config.height == self.data_train.height) and (self.config.width == self.data_train.width) and \ (self.config.num_channels == self.data_train.num_channels), \ 'Wrong dimension of data. Expected shape {}, and got {}'.format((self.config.height,self.config.width, \ self.config.num_channels), \ (self.data_train.height, self.data_train.width, \ self.data_train.num_channels) \ ) ''' ------------------------------------------------------------------------------- TRAIN THE MODEL ------------------------------------------------------------------------------------- ''' print('\nTraining a model...') with tf.Session(graph=self.graph) as session: tf.set_random_seed(self.config.seeds) self.session = session logger = Logger(self.session, self.config.log_dir) saver = tf.train.Saver() early_stopper = EarlyStopping(name='total loss', decay_fn=self.decay_fn) if (self.config.restore and self.load(self.session, saver)): load_config = file_utils.load_args(self.config.model_name, self.config.config_dir) self.config.update(load_config) num_epochs_trained = self.model_graph.cur_epoch_tensor.eval( self.session) print('EPOCHS trained: ', num_epochs_trained) else: print('Initializing Variables ...') tf.global_variables_initializer().run() for cur_epoch in range( self.model_graph.cur_epoch_tensor.eval(self.session), self.config.epochs + 1, 1): print('EPOCH: ', cur_epoch) self.current_epoch = cur_epoch losses_tr = self._train(self.data_train, self.session, logger) if np.isnan(losses_tr[0]): print( 'Encountered NaN, stopping training. Please check the learning_rate settings and the momentum.' ) for lname, lval in zip(self.model_graph.losses, losses_tr): print(lname, lval) sys.exit() train_msg = 'TRAIN: \n' for lname, lval in zip(self.model_graph.losses, losses_tr): train_msg += str(lname) + ': ' + str(lval) + ' | ' print(train_msg) print() if (cur_epoch == 1) or ((cur_epoch % self.config.save_epoch == 0) and (cur_epoch != 0)): gc.collect() self.save( self.session, saver, self.model_graph.global_step_tensor.eval(self.session)) if self.config.plot: self.plot_latent(cur_epoch) self.session.run(self.model_graph.increment_cur_epoch_tensor) # Early stopping if (self.config.early_stopping and early_stopper.stop(losses_tr[0])): print('Early Stopping!') break if cur_epoch % self.config.colab_save == 0: if self.config.colab: self.push_colab() self.save(self.session, saver, self.model_graph.global_step_tensor.eval(self.session)) if self.config.plot: self.plot_latent(cur_epoch) if self.config.colab: self.push_colab() return
EEG_stream_name = 'BioSemi' pre_stimulus_time = -.5 post_stimulus_time = 1. EEG_stream_preset = json.load( open( 'D:\PycharmProjects\RealityNavigation\LSLPresets\BioSemiActiveTwo.json' )) notch_f0 = 60. notch_band_demoninator = 200. EEG_fresample = 50 if __name__ == '__main__': # for windows all mp must be guarded by the main condition freeze_support() target_label = [3] epoched_EEG_timevector, epoched_EEG_average_trial_chan, epoched_EEG_max_trial_chan, epoched_EEG_min_trial_chan = process_data( files, EM_stream_name, EEG_stream_name, target_label, pre_stimulus_time, post_stimulus_time, EEG_stream_preset) plt.plot(epoched_EEG_timevector, epoched_EEG_average_trial_chan, c='b') target_label = [2, 4, 5, 6] epoched_EEG_timevector, epoched_EEG_average_trial_chan, epoched_EEG_max_trial_chan, epoched_EEG_min_trial_chan = process_data( files, EM_stream_name, EEG_stream_name, target_label, pre_stimulus_time, post_stimulus_time, EEG_stream_preset) plt.plot(epoched_EEG_timevector, epoched_EEG_average_trial_chan, c='r') # plt.fill_between(epoched_EEG_timevector, epoched_EEG_min_trial_chan, epoched_EEG_max_trial_chan, alpha=0.5) plt.xlabel('Time (sec)') plt.ylabel('Amplitude (\u03BCV)') plt.title('All (baselined)') plt.show()
def train(args: dict): model_name = args['--model'] batch_size = int(args['--batch-size']) max_patience = int(args['--max-patience']) max_num_trial = int(args['--max-num-trial']) lr_decay = float(args['--lr-decay']) train_ratio = float(args['--train-ratio']) model_save_to = args['--model-save-to'] max_epoch = int(args['--max-epoch']) model_path = args['model-path'] train_data_path = args['train-data-path'] pid2pidx_path = args['pid2pidx-path'] cid2cidx_path = args['cid2cidx-path'] pidx2cidx_path = args['pidx2cidx-path'] cidx2cname_path = args['cidx2cname-path'] raw_data, problem_map = read_data_from_file(train_data_path, pid2pidx_path, cid2cidx_path, pidx2cidx_path, cidx2cname_path) num_problems = len(problem_map) num_concepts = problem_map.num_concepts train_size = int(train_ratio * len(raw_data)) val_size = len(raw_data) - train_size train_dataset, val_dataset = random_split(raw_data, [train_size, val_size]) unpacked_train_dataset = unpack_data(train_dataset) unpacked_val_dataset = unpack_data(val_dataset) if model_name == 'DKT': model = DKT(num_problems=num_problems, hidden_size=int(args['--hidden-size']), dropout_rate=float(args['--dropout'])) criterion = torch.nn.BCELoss(reduction='mean') optimizer = optim.Adam(model.parameters(), lr=float(args['--lr'])) elif model_name == "DKTEnhanced": model = DKTEnhanced(problem_map=problem_map, hidden_size=int(args['--hidden-size']), dropout_rate=float(args['--dropout'])) criterion = PredictionConsistentBCELoss( lambda_r=float(args['--lambda-r'])) optimizer = optim.Adam(model.parameters(), lr=float(args['--lr'])) else: raise ValueError("wrong value of model_name[{}]".format(model_name)) if args['--gpu'] is not None: device = torch.device( "cuda:" + args['--gpu'] if torch.cuda.is_available() else "cpu") else: device = torch.device("cpu") logging.info("Using {}".format(device)) model.to(device) # for p in model.parameters(): # p.data.uniform_(-0.1, 0.1) train_iter = 0 patience = 0 num_trial = 0 hist_valid_scores = [] # print statistic infomation with torch.no_grad(): model.eval() t_auc_n, t_auc_c, t_acc, t_rmse, t_mae = evaluate( unpacked_train_dataset, num_problems, num_concepts, model) v_auc_n, v_auc_c, v_acc, v_rmse, v_mae = evaluate( unpacked_val_dataset, num_problems, num_concepts, model) print("Init:") print("valid_auc(n) %.6f\ttrain_auc(n) %.6f\t" % (v_auc_n, t_auc_n)) print("valid_auc(c) %.6f\ttrain_auc(c) %.6f\t" % (v_auc_c, t_auc_c)) print("valid_acc %.6f\ttrain_acc %.6f\t" % (v_acc, t_acc)) print("valid_rmse %.6f\ttrain_rmse %.6f\t" % (v_rmse, t_rmse)) print("valid_mae %.6f\ttrain_mae %.6f\t" % (v_mae, t_mae)) batch_num = math.ceil(len(train_dataset) / batch_size) for epoch in range(max_epoch): # loop over the dataset multiple times for batch_data in tqdm.tqdm(batch_iter(train_dataset, batch_size=batch_size), desc="Epoch[{}]".format(epoch + 1), total=batch_num): train_iter += 1 model.train() problem_seqs, concept_seqs, answer_seqs = batch_data # get y_next_true ans y_cur_true next_answer_seqs = copy.deepcopy(answer_seqs) for n_ans_seq in next_answer_seqs: del n_ans_seq[0] y_next_true = list(itertools.chain.from_iterable(next_answer_seqs)) y_next_true = torch.tensor(y_next_true, dtype=torch.float, device=model.device) y_cur_true = list(itertools.chain.from_iterable(answer_seqs)) y_cur_true = torch.tensor(y_cur_true, dtype=torch.float, device=model.device) # process the input data response_data, concept_data, seq_lengths = process_data( batch_data, num_problems, num_concepts, device=model.device) # forward + backward + optimize y_next_pred, y_cur_pred, batch_future_pred, batch_concept_mastery = model( response_data, concept_data, seq_lengths) if model_name == 'DKT': loss = criterion(y_next_pred, y_next_true) elif model_name == 'DKTEnhanced': loss = criterion(y_next_pred, y_next_true, y_cur_pred, y_cur_true) else: raise ValueError loss.backward() # clip gradient nn.utils.clip_grad_norm_(model.parameters(), max_norm=float(args['--clip-grad'])) optimizer.step() # zero the parameter gradients optimizer.zero_grad() # print statistic information with torch.no_grad(): model.eval() t_auc_n, t_auc_c, t_acc, t_rmse, t_mae = evaluate( unpacked_train_dataset, num_problems, num_concepts, model) v_auc_n, v_auc_c, v_acc, v_rmse, v_mae = evaluate( unpacked_val_dataset, num_problems, num_concepts, model) print("valid_auc(n) %.6f\ttrain_auc(n) %.6f\t" % (v_auc_n, t_auc_n)) print("valid_auc(c) %.6f\ttrain_auc(c) %.6f\t" % (v_auc_c, t_auc_c)) print("valid_acc %.6f\ttrain_acc %.6f\t" % (v_acc, t_acc)) print("valid_rmse %.6f\ttrain_rmse %.6f\t" % (v_rmse, t_rmse)) print("valid_mae %.6f\ttrain_mae %.6f\t" % (v_mae, t_mae)) # check for early stop is_better = len( hist_valid_scores) == 0 or v_auc_n > max(hist_valid_scores) hist_valid_scores.append(v_auc_n) if is_better: patience = 0 if not os.path.exists(model_save_to): os.makedirs(model_save_to) model.save(model_path) # also save the optimizers' state torch.save(optimizer.state_dict(), model_path + '.optim') elif patience < int(max_patience): patience += 1 print('hit patience %d' % patience) if patience == int(max_patience): num_trial += 1 print('hit #%d trial' % num_trial) if num_trial == int(max_num_trial): print('early stop!') break # decay lr, and restore from previously best checkpoint lr = optimizer.param_groups[0]['lr'] * float(lr_decay) # load model params = torch.load(model_path, map_location=lambda storage, loc: storage) model.load_state_dict(params['state_dict']) model = model.to(device) optimizer.load_state_dict(torch.load(model_path + '.optim')) # set new lr for param_group in optimizer.param_groups: param_group['lr'] = lr # reset patience patience = 0 print('Finished Training')
def test_model(test_path, model_name, trained_model, save_path, traineddataset, testdataset, opt, lr, pretrain, num_classes, test_split=False): state = defaultdict() test_loader = defaultdict() test_transform = trn.Compose([ trn.ToTensor(), trn.Normalize([0.566, 0.496, 0.469], [0.266, 0.256, 0.258]) ]) img_pixels = (224, 224) # Load test data test_set_X, test_set_y, _ = data_utils.process_data(test_path) test_loader = data_utils.make_dataloader_iter( test_set_X, test_set_y, img_size=img_pixels, batch_size=10, transform_test=test_transform) # Load model if model_name == 'alexnet': net = Generalmodels.alexnet(num_classes, pretrain, trained_model, if_test=True) elif model_name == 'VGG': net = Generalmodels.VGG16(num_classes, pretrain, trained_model, if_test=True) elif model_name == 'densenet121': net = Generalmodels.densenet121(num_classes, pretrain, trained_model, if_test=True) elif model_name == 'resnet50': net = Generalmodels.resnet50(num_classes, pretrain, trained_model, if_test=True) device = torch.device("cuda") net.load_state_dict(torch.load(trained_model)) net = net.to(device) # If test general performance if not test_split: if not os.path.exists(save_path): os.makedirs(save_path) test(net, test_loader, state) with open( os.path.join( save_path, '{}_{}_{}_{}_{}_results.csv'.format( traineddataset, model_name, opt, lr, testdataset)), 'a') as f: f.write('%s,%0.6f\n' % ( '(test)', state['test_accuracy'], )) # Test performance per age if test_split: new_path = os.path.join(save_path, 'split') if not os.path.exists(new_path): os.makedirs(new_path) if testdataset == 'FineTuneData': testdataset = 'testing' gt,tp,mae,prediction,classpredicts,regrepredicts,labels=\ test_range(net,test_loader,state,num_classes) # write predictions f = open( os.path.join( new_path, '{}_{}_{}_{}_{}_allpredictions.csv'.format( traineddataset, model_name, opt, lr, testdataset)), 'w') f.write('%s,%s,%s\n' % ( 'label', 'classpred', 'regreepred', )) for i in range(len(classpredicts)): f.write('%d,%d,%0.2f\n' % (labels[i], classpredicts[i], regrepredicts[i])) f.close() # Write performance f = open( os.path.join( new_path, '{}_{}_{}_{}_{}_results.csv'.format(traineddataset, model_name, opt, lr, testdataset)), 'w') f.write('%s,%s,%s,%s,%s\n' % ('age', 'number', 'accuracy', 'mae', 'perceived')) for i in range(len(gt)): f.write('%d,%d,%0.6f,%0.3f,%0.3f\n' % (i, gt[i], tp[i] / gt[i], mae[i] / gt[i], prediction[i] / gt[i])) f.close()