def evaluate(eval_data, num_problems, num_concepts, model):
problem_seqs, concept_seqs, answer_seqs = eval_data
# process the input data
response_data, concept_data, seq_lengths = process_data(
eval_data, num_problems, num_concepts, device=model.device)
# get y_next_true + y_cur_true
next_answer_seqs = copy.deepcopy(answer_seqs)
for n_ans_seq in next_answer_seqs:
del n_ans_seq[0]
y_next_true = list(itertools.chain.from_iterable(next_answer_seqs))
y_next_true = torch.tensor(y_next_true,
dtype=torch.float,
device=model.device)
y_cur_true = list(itertools.chain.from_iterable(answer_seqs))
y_cur_true = torch.tensor(y_cur_true,
dtype=torch.float,
device=model.device)
# forward + y_next_pred + y_cur_pred
y_next_pred, y_cur_pred, batch_future_pred, batch_concept_mastery = model(
response_data, concept_data, seq_lengths)
y_next_pred_label = np.where(y_next_pred.cpu().numpy() <= 0.5, 0, 1)
auc_n = roc_auc_score(y_next_true.cpu(), y_next_pred.cpu())
auc_c = roc_auc_score(y_cur_true.cpu(), y_cur_pred.cpu())
acc = accuracy_score(y_next_true.cpu(), y_next_pred_label)
rmse = np.sqrt(mean_squared_error(y_next_true.cpu(), y_next_pred.cpu()))
mae = mean_absolute_error(y_next_true.cpu(), y_next_pred.cpu())
return auc_n, auc_c, acc, rmse, mae
def fit(self, X, y=None):
print('\nProcessing data...')
self.data_train = data_utils.process_data(X, y, test_size=0)
if self.config.plot:
self.data_plot = self.data_train
self.config.num_batches = self.data_train.num_batches(
self.config.batch_size)
if not self.config.isBuilt:
self.config.restore = True
self.build_model(self.data_train.height, self.data_train.width,
self.data_train.num_channels)
else:
assert (self.config.height == self.data_train.height) and (self.config.width == self.data_train.width) and \
(self.config.num_channels == self.data_train.num_channels), \
'Wrong dimension of data. Expected shape {}, and got {}'.format((self.config.height,self.config.width, \
self.config.num_channels), \
(self.data_train.height,
self.data_train.width, \
self.data_train.num_channels) \
)
'''
-------------------------------------------------------------------------------
TRAIN THE MODEL
-------------------------------------------------------------------------------------
'''
print('\nTraining a model...')
with tf.Session(graph=self.graph) as session:
tf.set_random_seed(self.config.seeds)
self.session = session
logger = Logger(self.session, self.config.log_dir)
saver = tf.train.Saver()
early_stopper = EarlyStopping(name='total loss',
decay_fn=self.decay_fn)
if (self.config.restore and self.load(self.session, saver)):
load_config = file_utils.load_args(self.config.model_name,
self.config.config_dir)
self.config.update(load_config)
num_epochs_trained = self.model_graph.cur_epoch_tensor.eval(
self.session)
print('EPOCHS trained: ', num_epochs_trained)
else:
print('Initializing Variables ...')
tf.global_variables_initializer().run()
for cur_epoch in range(
self.model_graph.cur_epoch_tensor.eval(self.session),
self.config.epochs + 1, 1):
print('EPOCH: ', cur_epoch)
self.current_epoch = cur_epoch
losses_tr = self._train(self.data_train, self.session, logger)
if np.isnan(losses_tr[0]):
print(
'Encountered NaN, stopping training. Please check the learning_rate settings and the momentum.'
)
for lname, lval in zip(self.model_graph.losses, losses_tr):
print(lname, lval)
sys.exit()
train_msg = 'TRAIN: \n'
for lname, lval in zip(self.model_graph.losses, losses_tr):
train_msg += str(lname) + ': ' + str(lval) + ' | '
print(train_msg)
print()
if (cur_epoch
== 1) or ((cur_epoch % self.config.save_epoch == 0) and
(cur_epoch != 0)):
gc.collect()
self.save(
self.session, saver,
self.model_graph.global_step_tensor.eval(self.session))
if self.config.plot:
self.plot_latent(cur_epoch)
self.session.run(self.model_graph.increment_cur_epoch_tensor)
# Early stopping
if (self.config.early_stopping
and early_stopper.stop(losses_tr[0])):
print('Early Stopping!')
break
if cur_epoch % self.config.colab_save == 0:
if self.config.colab:
self.push_colab()
self.save(self.session, saver,
self.model_graph.global_step_tensor.eval(self.session))
if self.config.plot:
self.plot_latent(cur_epoch)
if self.config.colab:
self.push_colab()
return
EEG_stream_name = 'BioSemi'
pre_stimulus_time = -.5
post_stimulus_time = 1.
EEG_stream_preset = json.load(
open(
'D:\PycharmProjects\RealityNavigation\LSLPresets\BioSemiActiveTwo.json'
))
notch_f0 = 60.
notch_band_demoninator = 200.
EEG_fresample = 50
if __name__ == '__main__': # for windows all mp must be guarded by the main condition
freeze_support()
target_label = [3]
epoched_EEG_timevector, epoched_EEG_average_trial_chan, epoched_EEG_max_trial_chan, epoched_EEG_min_trial_chan = process_data(
files, EM_stream_name, EEG_stream_name, target_label,
pre_stimulus_time, post_stimulus_time, EEG_stream_preset)
plt.plot(epoched_EEG_timevector, epoched_EEG_average_trial_chan, c='b')
target_label = [2, 4, 5, 6]
epoched_EEG_timevector, epoched_EEG_average_trial_chan, epoched_EEG_max_trial_chan, epoched_EEG_min_trial_chan = process_data(
files, EM_stream_name, EEG_stream_name, target_label,
pre_stimulus_time, post_stimulus_time, EEG_stream_preset)
plt.plot(epoched_EEG_timevector, epoched_EEG_average_trial_chan, c='r')
# plt.fill_between(epoched_EEG_timevector, epoched_EEG_min_trial_chan, epoched_EEG_max_trial_chan, alpha=0.5)
plt.xlabel('Time (sec)')
plt.ylabel('Amplitude (\u03BCV)')
plt.title('All (baselined)')
plt.show()
def train(args: dict):
model_name = args['--model']
batch_size = int(args['--batch-size'])
max_patience = int(args['--max-patience'])
max_num_trial = int(args['--max-num-trial'])
lr_decay = float(args['--lr-decay'])
train_ratio = float(args['--train-ratio'])
model_save_to = args['--model-save-to']
max_epoch = int(args['--max-epoch'])
model_path = args['model-path']
train_data_path = args['train-data-path']
pid2pidx_path = args['pid2pidx-path']
cid2cidx_path = args['cid2cidx-path']
pidx2cidx_path = args['pidx2cidx-path']
cidx2cname_path = args['cidx2cname-path']
raw_data, problem_map = read_data_from_file(train_data_path, pid2pidx_path,
cid2cidx_path, pidx2cidx_path,
cidx2cname_path)
num_problems = len(problem_map)
num_concepts = problem_map.num_concepts
train_size = int(train_ratio * len(raw_data))
val_size = len(raw_data) - train_size
train_dataset, val_dataset = random_split(raw_data, [train_size, val_size])
unpacked_train_dataset = unpack_data(train_dataset)
unpacked_val_dataset = unpack_data(val_dataset)
if model_name == 'DKT':
model = DKT(num_problems=num_problems,
hidden_size=int(args['--hidden-size']),
dropout_rate=float(args['--dropout']))
criterion = torch.nn.BCELoss(reduction='mean')
optimizer = optim.Adam(model.parameters(), lr=float(args['--lr']))
elif model_name == "DKTEnhanced":
model = DKTEnhanced(problem_map=problem_map,
hidden_size=int(args['--hidden-size']),
dropout_rate=float(args['--dropout']))
criterion = PredictionConsistentBCELoss(
lambda_r=float(args['--lambda-r']))
optimizer = optim.Adam(model.parameters(), lr=float(args['--lr']))
else:
raise ValueError("wrong value of model_name[{}]".format(model_name))
if args['--gpu'] is not None:
device = torch.device(
"cuda:" + args['--gpu'] if torch.cuda.is_available() else "cpu")
else:
device = torch.device("cpu")
logging.info("Using {}".format(device))
model.to(device)
# for p in model.parameters():
# p.data.uniform_(-0.1, 0.1)
train_iter = 0
patience = 0
num_trial = 0
hist_valid_scores = []
# print statistic infomation
with torch.no_grad():
model.eval()
t_auc_n, t_auc_c, t_acc, t_rmse, t_mae = evaluate(
unpacked_train_dataset, num_problems, num_concepts, model)
v_auc_n, v_auc_c, v_acc, v_rmse, v_mae = evaluate(
unpacked_val_dataset, num_problems, num_concepts, model)
print("Init:")
print("valid_auc(n) %.6f\ttrain_auc(n) %.6f\t" % (v_auc_n, t_auc_n))
print("valid_auc(c) %.6f\ttrain_auc(c) %.6f\t" % (v_auc_c, t_auc_c))
print("valid_acc %.6f\ttrain_acc %.6f\t" % (v_acc, t_acc))
print("valid_rmse %.6f\ttrain_rmse %.6f\t" % (v_rmse, t_rmse))
print("valid_mae %.6f\ttrain_mae %.6f\t" % (v_mae, t_mae))
batch_num = math.ceil(len(train_dataset) / batch_size)
for epoch in range(max_epoch): # loop over the dataset multiple times
for batch_data in tqdm.tqdm(batch_iter(train_dataset,
batch_size=batch_size),
desc="Epoch[{}]".format(epoch + 1),
total=batch_num):
train_iter += 1
model.train()
problem_seqs, concept_seqs, answer_seqs = batch_data
# get y_next_true ans y_cur_true
next_answer_seqs = copy.deepcopy(answer_seqs)
for n_ans_seq in next_answer_seqs:
del n_ans_seq[0]
y_next_true = list(itertools.chain.from_iterable(next_answer_seqs))
y_next_true = torch.tensor(y_next_true,
dtype=torch.float,
device=model.device)
y_cur_true = list(itertools.chain.from_iterable(answer_seqs))
y_cur_true = torch.tensor(y_cur_true,
dtype=torch.float,
device=model.device)
# process the input data
response_data, concept_data, seq_lengths = process_data(
batch_data, num_problems, num_concepts, device=model.device)
# forward + backward + optimize
y_next_pred, y_cur_pred, batch_future_pred, batch_concept_mastery = model(
response_data, concept_data, seq_lengths)
if model_name == 'DKT':
loss = criterion(y_next_pred, y_next_true)
elif model_name == 'DKTEnhanced':
loss = criterion(y_next_pred, y_next_true, y_cur_pred,
y_cur_true)
else:
raise ValueError
loss.backward()
# clip gradient
nn.utils.clip_grad_norm_(model.parameters(),
max_norm=float(args['--clip-grad']))
optimizer.step()
# zero the parameter gradients
optimizer.zero_grad()
# print statistic information
with torch.no_grad():
model.eval()
t_auc_n, t_auc_c, t_acc, t_rmse, t_mae = evaluate(
unpacked_train_dataset, num_problems, num_concepts, model)
v_auc_n, v_auc_c, v_acc, v_rmse, v_mae = evaluate(
unpacked_val_dataset, num_problems, num_concepts, model)
print("valid_auc(n) %.6f\ttrain_auc(n) %.6f\t" %
(v_auc_n, t_auc_n))
print("valid_auc(c) %.6f\ttrain_auc(c) %.6f\t" %
(v_auc_c, t_auc_c))
print("valid_acc %.6f\ttrain_acc %.6f\t" % (v_acc, t_acc))
print("valid_rmse %.6f\ttrain_rmse %.6f\t" % (v_rmse, t_rmse))
print("valid_mae %.6f\ttrain_mae %.6f\t" % (v_mae, t_mae))
# check for early stop
is_better = len(
hist_valid_scores) == 0 or v_auc_n > max(hist_valid_scores)
hist_valid_scores.append(v_auc_n)
if is_better:
patience = 0
if not os.path.exists(model_save_to):
os.makedirs(model_save_to)
model.save(model_path)
# also save the optimizers' state
torch.save(optimizer.state_dict(), model_path + '.optim')
elif patience < int(max_patience):
patience += 1
print('hit patience %d' % patience)
if patience == int(max_patience):
num_trial += 1
print('hit #%d trial' % num_trial)
if num_trial == int(max_num_trial):
print('early stop!')
break
# decay lr, and restore from previously best checkpoint
lr = optimizer.param_groups[0]['lr'] * float(lr_decay)
# load model
params = torch.load(model_path,
map_location=lambda storage, loc: storage)
model.load_state_dict(params['state_dict'])
model = model.to(device)
optimizer.load_state_dict(torch.load(model_path + '.optim'))
# set new lr
for param_group in optimizer.param_groups:
param_group['lr'] = lr
# reset patience
patience = 0
print('Finished Training')
def test_model(test_path,
model_name,
trained_model,
save_path,
traineddataset,
testdataset,
opt,
lr,
pretrain,
num_classes,
test_split=False):
state = defaultdict()
test_loader = defaultdict()
test_transform = trn.Compose([
trn.ToTensor(),
trn.Normalize([0.566, 0.496, 0.469], [0.266, 0.256, 0.258])
])
img_pixels = (224, 224)
# Load test data
test_set_X, test_set_y, _ = data_utils.process_data(test_path)
test_loader = data_utils.make_dataloader_iter(
test_set_X,
test_set_y,
img_size=img_pixels,
batch_size=10,
transform_test=test_transform)
# Load model
if model_name == 'alexnet':
net = Generalmodels.alexnet(num_classes,
pretrain,
trained_model,
if_test=True)
elif model_name == 'VGG':
net = Generalmodels.VGG16(num_classes,
pretrain,
trained_model,
if_test=True)
elif model_name == 'densenet121':
net = Generalmodels.densenet121(num_classes,
pretrain,
trained_model,
if_test=True)
elif model_name == 'resnet50':
net = Generalmodels.resnet50(num_classes,
pretrain,
trained_model,
if_test=True)
device = torch.device("cuda")
net.load_state_dict(torch.load(trained_model))
net = net.to(device)
# If test general performance
if not test_split:
if not os.path.exists(save_path):
os.makedirs(save_path)
test(net, test_loader, state)
with open(
os.path.join(
save_path, '{}_{}_{}_{}_{}_results.csv'.format(
traineddataset, model_name, opt, lr, testdataset)),
'a') as f:
f.write('%s,%0.6f\n' % (
'(test)',
state['test_accuracy'],
))
# Test performance per age
if test_split:
new_path = os.path.join(save_path, 'split')
if not os.path.exists(new_path):
os.makedirs(new_path)
if testdataset == 'FineTuneData':
testdataset = 'testing'
gt,tp,mae,prediction,classpredicts,regrepredicts,labels=\
test_range(net,test_loader,state,num_classes)
# write predictions
f = open(
os.path.join(
new_path, '{}_{}_{}_{}_{}_allpredictions.csv'.format(
traineddataset, model_name, opt, lr, testdataset)), 'w')
f.write('%s,%s,%s\n' % (
'label',
'classpred',
'regreepred',
))
for i in range(len(classpredicts)):
f.write('%d,%d,%0.2f\n' %
(labels[i], classpredicts[i], regrepredicts[i]))
f.close()
# Write performance
f = open(
os.path.join(
new_path,
'{}_{}_{}_{}_{}_results.csv'.format(traineddataset, model_name,
opt, lr, testdataset)),
'w')
f.write('%s,%s,%s,%s,%s\n' %
('age', 'number', 'accuracy', 'mae', 'perceived'))
for i in range(len(gt)):
f.write('%d,%d,%0.6f,%0.3f,%0.3f\n' %
(i, gt[i], tp[i] / gt[i], mae[i] / gt[i],
prediction[i] / gt[i]))
f.close()