def train_model(model, train_loader, epoch, num_epochs, optimizer, writer, current_lr, log_every=1, n_classes=3): metric = torch.nn.CrossEntropyLoss() y_probs = np.zeros((0, n_classes), np.float) losses, y_trues = [], [] model.train() for m in model.modules(): if isinstance(m, nn.BatchNorm2d): m.train() m.weight.requires_grad = False m.bias.requires_grad = False for i, (image, label, case_id) in enumerate(train_loader): print(f'Starting with batch {i}, case id {case_id}') # if i == 18: # pdb.set_trace() optimizer.zero_grad() if torch.cuda.is_available(): image = image.cuda() label = label.cuda() image = torch.squeeze(image, dim=0) prediction = model.forward(image.float()) loss = metric(prediction, label.long()) loss.backward() optimizer.step() print(f'Done with batch {i}') loss_value = loss.item() losses.append(loss_value) y_prob = F.softmax(prediction, dim=1) y_probs = np.concatenate([y_probs, y_prob.detach().cpu().numpy()]) y_trues.append(label.item()) metric_collects = utils.calc_multi_cls_measures(y_probs, y_trues) n_iter = epoch * len(train_loader) + i writer.add_scalar('Train/Loss', loss_value, n_iter) if (i % log_every == 0) and i > 0: utils.print_progress(epoch + 1, num_epochs, i, len(train_loader), np.mean(losses), current_lr, metric_collects) train_loss_epoch = np.round(np.mean(losses), 4) return train_loss_epoch, metric_collects
def validate(model, val_loader, data_param, train_param, _log): with th.no_grad(): criterion = nn.BCEWithLogitsLoss(pos_weight=th.Tensor([train_param['pos_weight']]).cuda()) val_iter = iter(val_loader) running_loss = 0 prune = data_param['prune'] for _ in range(len(val_iter)): batch_sample = val_iter.next() data = batch_sample['data'].cuda() gt = batch_sample['gt'].cuda() prds = model.forward(data)[:, :, prune:-prune, prune:-prune] indices = gt>=0 loss = criterion(prds[indices], gt[indices]) running_loss += loss.item() del data, gt, prds, indices return running_loss/len(val_iter)
def train(trainloader, model, optimization, start_epoch, stop_epoch, params, config): if optimization == 'Adam': optimizer = torch.optim.Adam(model.parameters()) else: raise ValueError('Unknown optimization, please define by yourself') loss_fn = nn.CrossEntropyLoss() pgd = attack.AttackPGD(config) for epoch in range(start_epoch, stop_epoch): model.train() print_freq = 50 avg_loss = 0 correct, total = 0, 0 for i, (x, y, _) in enumerate(trainloader): noise = torch.zeros_like(x).uniform_(-8 / 255., 8 / 255.) x = torch.clamp(x + noise, 0., 1.) x, y = x.cuda(), y.cuda() optimizer.zero_grad() scores, _ = model.forward(x) predicted = torch.argmax(scores, 1) correct += (predicted == y).sum().item() total += predicted.size(0) loss = loss_fn(scores, y) loss.backward() optimizer.step() avg_loss = avg_loss + loss.data.item() if i % print_freq == 0: print( 'Epoch {:d} | Batch {:d}/{:d} | Loss {:f} | Train Acc {:f}' .format(epoch, i, len(trainloader), avg_loss / float(i + 1), 100. * correct / total)) if not os.path.isdir(params.checkpoint_dir): os.makedirs(params.checkpoint_dir) if (epoch % params.save_freq == 0) or (epoch == stop_epoch - 1): outfile = os.path.join(params.checkpoint_dir, '{:d}.tar'.format(epoch)) state_dict = {} state_dict['epoch'] = epoch state_dict['feature'] = model.feature.state_dict() state_dict['classifier'] = model.classifier.state_dict() torch.save(state_dict, outfile) return model
def train(): # Turn on training mode which enables dropout. model.train() total_loss = 0 start_time = time.time() nlabels = args.labels hidden = model.init_hidden(args.batch_size) #for batch, load in enumerate(range(0,args.batch_size,len(corpus.trainid))): for batch, load in enumerate(range(0, len(corpus.trainid), args.batch_size)): #print("TRAIN CORPUS LEN",len(corpus.trainid)) if load + args.batch_size > len(corpus.trainid): continue data, targets, lens = get_batch(corpus.trainid, corpus.trainlab, corpus.trainlen, load, args.batch_size) # Starting each batch, we detach the hidden state from how it was previously produced. # If we didn't, the model would try backpropagating all the way to start of the dataset. hidden = repackage_hidden(hidden) model.zero_grad() #print(data) output, hidden, perm_index = model.forward(data, hidden, lens) targets = targets[:, perm_index] #print(output.view(-1, nlabels)) #print(torch.t(targets)) loss = criterion(output.view(-1, nlabels), torch.t(targets)) loss.backward() # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs. torch.nn.utils.clip_grad_norm(model.parameters(), args.clip) for p in model.parameters(): p.data.add_(-lr, p.grad.data) total_loss += loss.data if batch % args.log_interval == 0 and batch > 0: cur_loss = total_loss[0] / args.log_interval elapsed = time.time() - start_time print( '| epoch {:3d} | {:5d}/{:5d} batches | lr {:02.2f} | ms/batch {:5.2f} | ' 'loss {:5.2f} | ppl {:8.2f}'.format( epoch, batch, len(corpus.trainid) // args.bptt, lr, elapsed * 1000 / args.log_interval, cur_loss, math.exp(cur_loss))) total_loss = 0 start_time = time.time()
def evaluate_model(model, val_loader, epoch, num_epochs, writer, current_lr, log_every=20): n_classes = model.n_classes metric = torch.nn.CrossEntropyLoss() model.eval() for m in model.modules(): if isinstance(m, nn.BatchNorm2d): m.train() m.weight.requires_grad = False m.bias.requires_grad = False y_probs = np.zeros((0, n_classes), np.float) losses, y_trues = [], [] for i, (image, label, case_id) in enumerate(val_loader): # if torch.cuda.is_available(): # image = image.cuda() # label = label.cuda() prediction = model.forward(image.float()) loss = metric(prediction, label.long()) loss_value = loss.item() losses.append(loss_value) y_prob = F.softmax(prediction, dim=1) y_probs = np.concatenate([y_probs, y_prob.detach().cpu().numpy()]) y_trues.append(label.item()) metric_collects = utils.calc_multi_cls_measures(y_probs, y_trues) n_iter = epoch * len(val_loader) + i writer.add_scalar('Val/Loss', loss_value, n_iter) if (i % log_every == 0) & (i > 0): prefix = '*Val|' utils.print_progress(epoch + 1, num_epochs, i, len(val_loader), np.mean(losses), current_lr, metric_collects, prefix=prefix) val_loss_epoch = np.round(np.mean(losses), 4) return val_loss_epoch, metric_collects
def train(model, training_data, loss_fn, optimiser, batch_size=1): for ( train, target, ) in training_data: # i in range(0, len(training_data) - batch_size, batch_size): # train = torch.tensor([x[0] for x in training_data[i:batch_size]]) # target = torch.tensor([x[0] for x in training_data[i:batch_size]]) model.zero_grad() pred = model.forward(train) loss = loss_fn(pred[-1], target[-1]) # print("Target", target, "Loss", loss.item()) optimiser.zero_grad() loss.backward() optimiser.step() return loss.item()
def evaluate_1step_pred(args, model, test_dataset): # turn on evaluation mode which disables dropout model.eval() total_loss = 0 with torch.no_grad(): hidden = model.init_hidden(args.eval_batch_size) for nbatch, i in enumerate( range(0, test_dataset.size(0) - 1, args.bptt)): inputSeq, targetSeq = get_batch(args, test_dataset, i) outSeq, hidden = model.forward(inputSeq, hidden) loss = criterion(outSeq.view(args.batch_size, -1), targetSeq.view(args.batch_size, -1)) hidden = model.repackage_hidden(hidden) total_loss += loss.item() return total_loss / nbatch
def train_model(model, train_loader, epoch, num_epochs, optimizer, current_lr, log_every=100): n_classes = model.n_classes metric = torch.nn.CrossEntropyLoss() y_probs = np.zeros((0, n_classes), np.float) losses, y_trues = [], [] model.train() for m in model.modules(): if isinstance(m, nn.BatchNorm2d): m.train() m.weight.requires_grad = False m.bias.requires_grad = False for i, (image, label) in enumerate(train_loader): optimizer.zero_grad() if torch.cuda.is_available(): image = image.cuda() label = label.cuda() prediction = model.forward(image.float()) loss = metric(prediction, label.long()) loss.backward() optimizer.step() loss_value = loss.item() losses.append(loss_value) y_prob = F.softmax(prediction, dim=1) y_probs = np.concatenate([y_probs, y_prob.detach().cpu().numpy()]) label = np.array(label) y_trues = np.append(y_trues, label) y_trues = np.array(y_trues) metric_collects = utils.calc_multi_cls_measures(y_probs, y_trues) train_loss_epoch = np.round(np.mean(losses), 4) return train_loss_epoch, metric_collects
def save_testdata_statistics(model, data_loader, cuda_mode): for batch in data_loader: x, y = batch x = torch.autograd.Variable(x) out = model.forward(x).data.cpu().numpy() try: np.concatenate([logits, out], 0) except NameError: logits = out m = logits.mean(0) C = np.cov(logits, rowvar=False) pfile = open('./test_data_statistics.p', "wb") pickle.dump({'m': m, 'C': C}, pfile) pfile.close()
def evaluate(data_source, test=False): # Turn on evaluation mode which disables dropout. model.eval() total_loss = 0 nlabels = args.labels hidden = model.init_hidden(eval_batch_size) #for i in range(0, data_source.size(0) - 1, args.bptt): #for batch, load in enumerate(range(0,eval_batch_size,len(corpus.validid))): for batch, load in enumerate(range(0, len(corpus.validid), eval_batch_size)): if test: if load + eval_batch_size > len(corpus.testid): continue data, targets, lens = get_batch(corpus.testid, corpus.testlab, corpus.testlen, load, eval_batch_size, evaluation=True) else: if load + eval_batch_size > len(corpus.validid): continue data, targets, lens = get_batch(corpus.validid, corpus.validlab, corpus.validlen, load, eval_batch_size, evaluation=True) output, hidden, perm_index = model.forward(data, hidden, lens) targets = targets[:, perm_index] output_flat = output.view(-1, nlabels) #print(output_flat, torch.t(targets)) total_loss += criterion(output_flat, torch.t(targets)).data #print(total_loss[0]) hidden = repackage_hidden(hidden) if test: return total_loss[0] else: return total_loss[0]
def validation(test_loader, model): correct = 0 total_test = 0 cnt = 0 cross_entropy = 0 model.eval() with torch.no_grad(): for sample_batch in test_loader: images, labels = sample_batch if params.useGPU: images, labels = Variable(images.cuda()), Variable( labels.cuda()) out = model.forward(images) loss = torch.nn.CrossEntropyLoss()(out, labels) _, pred = torch.max(out, 1) correct += (pred == labels).sum().item() cross_entropy += loss total_test += labels.size(0) cnt += 1 return correct / total_test, cross_entropy / cnt
def evaluate(conf, params, X_data, Y_data): """Evaluate a trained model on X_data. Args: conf: Configuration dictionary params: Dictionary with parameters X_data: numpy array of floats with shape [input dimension, number of examples] Y_data: numpy array of integers with shape [output dimension, number of examples] Returns: num_correct_total: Integer num_examples_evaluated: Integer """ num_examples = X_data.shape[1] num_examples_evaluated = 0 num_correct_total = 0 start_ind = 0 end_ind = conf['batch_size'] while True: X_batch = X_data[:, start_ind:end_ind] Y_batch = model.one_hot(Y_data[start_ind:end_ind], conf['output_dimension']) Y_proposal, _ = model.forward(conf, X_batch, params, is_training=False) _, num_correct = model.cross_entropy_cost(Y_proposal, Y_batch) num_correct_total += num_correct num_examples_evaluated += end_ind - start_ind start_ind += conf['batch_size'] end_ind += conf['batch_size'] if end_ind >= num_examples: end_ind = num_examples if start_ind >= num_examples: break return num_correct_total, num_examples_evaluated
def test(test_data, ground_truth): with tf.Graph().as_default(): x = tf.placeholder(tf.float32, [None, IMG_SIZE[0], IMG_SIZE[1], CH]) logits = model.forward(x, False, with_dropout=False, keep_prob=0.5, batch_size=1) logits_reshape = tf.reshape(logits, [-1, model.Num_Classes]) prob = tf.nn.softmax(logits_reshape, -1) prediction = tf.argmax(prob, -1) y = tf.reshape(prediction, IMG_SIZE) saver = tf.train.Saver() with tf.Session() as sess: ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) global_step = ckpt.model_checkpoint_path.split('/')[-1].split( '-')[-1] print(global_step) Num = test_data.shape[0] ImgOut = np.zeros([Num, IMG_SIZE[0], IMG_SIZE[1]], dtype=np.int64) for i in range(Num): img = sess.run(y, feed_dict={x: test_data[i:i + 1, :, :, :]}) ImgOut[i, :, :] = np.array(img) scipy.io.savemat('results.mat', {'mydata': ImgOut}) else: print("No checkpoint is found.") return
def train_model(model, train_loader, epoch, num_epochs, optimizer, writer, current_lr, log_every=100): _ = model.train() if torch.cuda.is_available(): model.cuda() y_preds = [] y_trues = [] losses = [] a = np.zeros([1, 1]) for i, (image, label, weight) in enumerate(train_loader): optimizer.zero_grad() if torch.cuda.is_available(): image = image.cuda() label = label.cuda() weight = weight.cuda() label = label[0] weight = weight[0] prediction = model.forward(image.float()).squeeze(0) loss = torch.nn.BCEWithLogitsLoss(weight=weight)(prediction, label) loss.backward() optimizer.step() loss_value = loss.item() losses.append(loss_value) probas = torch.sigmoid(prediction) y_trues.append(int(label[0])) y_preds.append(probas[0].item()) try: auc = metrics.roc_auc_score(y_trues, y_preds) except: auc = 0.5 writer.add_scalar('Train/Loss', loss_value, epoch * len(train_loader) + i) writer.add_scalar('Train/AUC', auc, epoch * len(train_loader) + i) if (i % log_every == 0) & (i > 0): print( '''[Epoch: {0} / {1} |Single batch number : {2} / {3} ]| avg train loss {4} | train auc : {5} | lr : {6}''' .format(epoch + 1, num_epochs, i, len(train_loader), np.round(np.mean(losses), 4), np.round(auc, 4), current_lr)) writer.add_scalar('Train/AUC_epoch', auc, epoch + i) train_loss_epoch = np.round(np.mean(losses), 4) train_auc_epoch = np.round(auc, 4) return train_loss_epoch, train_auc_epoch
def fit(hp, model, train_data, test_data, dev_data): logs = [ 'Epoch', 'LR', 'Loss', 'Time(m)', 'DevWS', 'DevPOS', 'TestWS', 'TestPOS' ] print(hp) print('\t'.join(logs)) utils.dump_data(hp, hp['HYPERPARAMS']) decay_counter = 0 best_dev_score = 0. indice = [i for i in range(len(train_data.ws_data))] for e in range(hp['EPOCH']): t = time.time() losses = 0. random.shuffle(indice) for i in indice: # Word Segmentation X = train_data.ws_data[i][0] Y = train_data.ws_data[i][1] obs = model.encode_ws(X, train=True) gold_score = model.score_sentence(obs, Y) forward_score = model.forward(obs) loss = forward_score - gold_score # Update loss.backward() model.trainer.update() losses += loss.value() # POS-tagging X = train_data.pos_data[i][0] Y = train_data.pos_data[i][1] loss = model.get_POStagging_loss(X, Y) losses += loss.value() # Update loss.backward() model.trainer.update() model.model.save(hp['EPOCH_MODEL']) dev_ws_f, dev_pos_f = evaluation(hp, fn_model=hp['EPOCH_MODEL'], data=dev_data) if dev_ws_f > best_dev_score: best_dev_score = dev_ws_f decay_counter = 0 model.model.save(hp['MODEL']) test_ws_f, test_pos_f = evaluation(hp, fn_model=hp['MODEL'], data=test_data) else: decay_counter += 1 if decay_counter >= hp['DECAY']: model.trainer.learning_rate = model.trainer.learning_rate / 2 decay_counter = 0 logs = [ e, model.trainer.learning_rate, losses, (time.time() - t) / 60, dev_ws_f, dev_pos_f, test_ws_f, test_pos_f ] print('\t'.join([log[:5] for log in map(str, logs)]))
def test_model(): x = tf.placeholder(tf.float32, [batch_size,input_dim,1,1]) y_ = tf.placeholder(tf.int32, [batch_size, n_class]) y = model.forward(x, two_ch, 'adxd', False) global_step = tf.Variable(0, trainable=False) with tf.name_scope('loss'): loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=y)) tf.summary.scalar('loss', loss) correct_predition = tf.equal(tf.argmax(y,1),tf.argmax(y_,1)) accuracy = tf.reduce_mean(tf.cast(correct_predition,tf.float32)) saver = tf.train.Saver() merged = tf.summary.merge_all() with tf.Session() as sess: coord=tf.train.Coordinator() threads= tf.train.start_queue_runners(coord=coord) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) sess.run(init_op) #第一次训练模型注释掉以下两行 model_path=os.path.join(MODEL_SAVE_PATH, MODEL_NAME) tf.train.Saver().restore(sess,model_path) ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) k = 0 m = 0 writer = tf.summary.FileWriter("./train", sess.graph) print('#######################################################') doubel_file_name = get_doubel_file_from_txt('doubel_name_test.txt') d=0 for doubel_file in doubel_file_name: print('第%d个人的数据:' % d) d = d+1 file_1 = doubel_file[0] file_2 = doubel_file[1] EEG_1, EEG_2, onset, duration, stage_code = get_data(file_1, file_2, data_dir) EEG1_sequence, EEG2_sequence, stage_sequence = split_data_to_30s(EEG_1 , EEG_2, onset, duration, stage_code) if k!=0: EEG1_sequence, EEG2_sequence, stage_sequence = K_EEG_together(EEG1_sequence, EEG2_sequence, stage_sequence, k) batch_time = len(EEG1_sequence)/batch_size print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!', d) for i in range(int(batch_time)): if two_ch: batch_data, batch_label = two_ch_batch(EEG1_sequence, EEG2_sequence, stage_sequence, batch_size) #最后返回 batch_data, batch_label else: batch_data, batch_label = one_ch_batch(EEG1_sequence, stage_sequence, batch_size) summary, loss_value,acc, y_pre = sess.run([merged, loss, accuracy, y], feed_dict= {x:batch_data, y_:batch_label}) writer.add_summary(summary, m) writer.close() m = m+1 print('loss is %g......acc is %f' % (loss_value,acc))
# # classes = jsonfile.read_json_file(os.path.join(DATA_ROOT, 'classes.json')) f = open(os.path.join(out_folder, 'result.csv'), 'w') f.write('file_id,{}\n'.format(','.join(classes))) for track in test_list: track_id = track[0] track_data = samples.get(track_id)[()].astype(np.float32) track_data = track_data[:TRACK_LENGTH] if track_data.size < TRACK_LENGTH: track_data = np.pad(track_data, (0, TRACK_LENGTH - track_data.size), mode='constant', constant_values=0) track_data = track_data.reshape(-1, 44100) input_var = autograd.Variable(torch.from_numpy(track_data)) if not args.no_cuda: input_var = input_var.cuda() prediction = model.forward(input_var).data.cpu().numpy() avg = prediction.mean(axis=0) f.write('{},{}\n'.format(track_id, ','.join(str(x) for x in avg))) f.close()
# Training the model epochs = results.epochs steps = 0 running_loss = 0 training_losses, validation_losses = [], [] for e in range(epochs): for inputs, targets in dataloaders['train']: steps += 1 inputs, targets = inputs.to(device), targets.to(device) optimizer.zero_grad() logps = model.forward(inputs) loss = criterion(logps, targets) loss.backward() optimizer.step() running_loss += loss.item() else: valid_loss = 0 accuracy = 0 model.eval() with torch.no_grad(): for inputs, labels in dataloaders['valid']: inputs, labels = inputs.to(device), labels.to(device) logps = model.forward(inputs) batch_loss = criterion(logps, labels)
def evaluate_model(model, val_loader, epoch, num_epochs, writer, current_lr, log_every=20): _ = model.eval() if torch.cuda.is_available(): model.cuda() y_trues = [] y_preds = [] losses = [] a = np.zeros([1, 1]) for i, (image, label, weight) in enumerate(val_loader): if torch.cuda.is_available(): image = image.cuda() label = label.cuda() weight = weight.cuda() label = label[0] weight = weight[0] prediction = model.forward(image.float()) b = label.numpy() c = pd.concat([pd.DataFrame(a), pd.DataFrame(b)], axis=1) c.columns = ['dud', 'target'] c['dud'] = 1 - c['target'] print(c) label = torch.from_numpy(np.asarray(c)) loss = torch.nn.BCEWithLogitsLoss(weight=weight)(prediction, label) loss_value = loss.item() losses.append(loss_value) probas = torch.sigmoid(prediction) y_trues.append(int(label[0][1])) y_preds.append(probas[0][1].item()) try: auc = metrics.roc_auc_score(y_trues, y_preds) except: auc = 0.5 writer.add_scalar('Val/Loss', loss_value, epoch * len(val_loader) + i) writer.add_scalar('Val/AUC', auc, epoch * len(val_loader) + i) if (i % log_every == 0) & (i > 0): print( '''[Epoch: {0} / {1} |Single batch number : {2} / {3} ] | avg val loss {4} | val auc : {5} | lr : {6}''' .format(epoch + 1, num_epochs, i, len(val_loader), np.round(np.mean(losses), 4), np.round(auc, 4), current_lr)) writer.add_scalar('Val/AUC_epoch', auc, epoch + i) val_loss_epoch = np.round(np.mean(losses), 4) val_auc_epoch = np.round(auc, 4) return val_loss_epoch, val_auc_epoch
def evaluate_model(model, val_loader, epoch, num_epochs, writer, current_lr, log_every=20): _ = model.eval() if torch.cuda.is_available(): model.cuda() y_trues = [] y_preds = [] losses = [] for i, (image, label, weight) in enumerate(val_loader): if torch.cuda.is_available(): image = image.cuda() label = label.cuda() weight = weight.cuda() label = label[0] weight = weight[0] prediction = model.forward(image.float()) loss = torch.nn.BCEWithLogitsLoss(weight=weight)(prediction, label) loss_value = loss.item() losses.append(loss_value) probas = torch.sigmoid(prediction) y_trues.append(int(label[0][1])) y_preds.append(probas[0][1].item()) #confusion_matrix(y_trues, y_preds) try: auc = metrics.roc_auc_score(y_trues, y_preds) except: auc = 0.5 #writer.add_scalar('Val/Loss', loss_value, epoch * len(val_loader) + i) #writer.add_scalar('Val/AUC', auc, epoch * len(val_loader) + i) if (i % log_every == 0) & (i > 0): print( '''[Epoch: {0} / {1} |Single batch number : {2} / {3} ] | avg val loss {4} | val auc : {5} | lr : {6}''' .format(epoch + 1, num_epochs, i, len(val_loader), np.round(np.mean(losses), 4), np.round(auc, 4), current_lr)) writer.add_scalar('Val/AUC_epoch', auc, epoch + i) val_loss = np.round(np.mean(losses), 4) val_auc = np.round(auc, 4) precision = my_precision(y_trues, y_preds) recall = my_recall(y_trues, y_preds) f1_score = my_f1_score(y_trues, y_preds) accuracy = my_accuracy(y_trues, y_preds) writer.add_scalar('Val/precision_epoch', precision, epoch + i) writer.add_scalar('Val/recall_epoch', recall, epoch + i) writer.add_scalar('Val/f1_score_epoch', f1_score, epoch + i) writer.add_scalar('Val/accuracy_epoch', accuracy, epoch + i) return precision, recall, f1_score, accuracy, val_loss, val_auc
if os.path.exists(model_path) == False: print("Error: Model not found!") exit(1) train_dataset = dataset.Image2SteeringDataset(dataset_path) dataloader = DataLoader(train_dataset, batch_size=4, shuffle=True, num_workers=2) model = torch.load(model_path, map_location=device) model.eval() while True: image_np = random.choice(train_dataset)['image'] image = np.expand_dims(image_np.copy(), axis=0) image = torch.from_numpy(image) image = image.to(device) output = model.forward(image.float()) print(output) image_np = np.reshape( image_np, (image_np.shape[1], image_np.shape[2], image_np.shape[0])) cv2.imshow('image', image_np) cv2.waitKey(0)
def evaluate_model(model, val_loader, valid_loader, epoch, num_epochs, writer, current_lr, log_every=20): _ = model.eval() if torch.cuda.is_available(): model.cuda() y_trues = [] y_preds = [] losses = [] a = np.zeros([1, 1]) for i, (image, label, weight) in enumerate(val_loader): if torch.cuda.is_available(): image = image.cuda() label = label.cuda() weight = weight.cuda() label = label[0] weight = weight[0] prediction = model.forward(image.float()).squeeze(0) loss = torch.nn.BCEWithLogitsLoss(weight=weight)(prediction, label) loss_value = loss.item() losses.append(loss_value) probas = torch.sigmoid(prediction) y_trues.append(int(label[0])) y_preds.append(probas[0].item()) try: auc = metrics.roc_auc_score(y_trues, y_preds) except: auc = 0.5 writer.add_scalar('Val/Loss', loss_value, epoch * len(val_loader) + i) writer.add_scalar('Val/AUC', auc, epoch * len(val_loader) + i) if ((i % log_every == 0) & (i > 0)) | (i == 281): print( '''[Epoch: {0} / {1} |Single batch number : {2} / {3} ] | avg val loss {4} | val auc : {5} | lr : {6}''' .format(epoch + 1, num_epochs, i, len(val_loader), np.round(np.mean(losses), 4), np.round(auc, 4), current_lr)) writer.add_scalar('Val/AUC_epoch', auc, epoch + i) val_loss_epoch = np.round(np.mean(losses), 4) val_auc_epoch = np.round(auc, 4) vle = val_loss_epoch vae = val_auc_epoch y_trues = [] y_preds = [] losses = [] a = np.zeros([1, 1]) for i, (image, label, weight) in enumerate(valid_loader): if torch.cuda.is_available(): image = image.cuda() label = label.cuda() weight = weight.cuda() label = label[0] weight = weight[0] prediction = model.forward(image.float()).squeeze(0) probas = torch.sigmoid(prediction) y_trues.append(int(label[0])) y_preds.append(probas[0].item()) try: auc = metrics.roc_auc_score(y_trues, y_preds) except: auc = 0.5 if ((i % log_every == 0) & (i > 0)) | (i == 119): print( '''[Epoch: {0} / {1} |Single batch number : {2} / {3} ] | val auc : {4} | lr : {5}''' .format(epoch + 1, num_epochs, i, len(valid_loader), np.round(auc, 4), current_lr)) return vle, vae, auc
gt_labels_ids_length=[] for i in range(len(test_data)): gt_labels_ids.append([total_node_list.index(label) for label in test_data[i]['label']]) gt_labels_ids_length.append(len(test_data[i]['label'])) for epoch in range(1): for batch_idx, (context,entity,ids,left_right,lens,gt_labels) in enumerate(test_data_loader): gt_labels_id = [total_node_list.index(i) for i in gt_labels[0]] parent_list=[parent_of[total_node_list.index(label)] for label in gt_labels[0]] if eval_on_leaf_only == True: is_leaf=is_only_leaf(gt_labels[0]) if is_leaf != 0: continue predict_quant_embd,id_list = model.forward(context.to(device), lens.to(device),ids,left_right) predict_quant_embd=predict_quant_embd.data.cpu().numpy() mask_matrix=np.multiply(quantam_concept_embds, predict_quant_embd) normalizing_constant=np.sum(np.abs(predict_quant_embd)**2,axis=-1) score=(np.sum(np.abs(mask_matrix) ** 2, axis=-1))/normalizing_constant predicted_labels_id,best_level_1_node,best_level_1_score,best_level_2_node,best_level_2_score=predict_labels(score,threshold_level_wise,constant_to_divide) predicted_labels=[total_node_list[i] for i in predicted_labels_id] temp_dict = {} temp_dict['gt_label'] = gt_labels[0] temp_dict['predicted_label'] = predicted_labels temp_dict['best_level_1_node']=best_level_1_node temp_dict['best_level_2_node'] = best_level_2_node temp_dict['best_level_1_score']=best_level_1_score temp_dict['best_level_2_score']=best_level_2_score
dataset_testing = dataset.Create(ds, testing_count, batch_size, device="cuda") input_shape = dataset_training.get_input_shape() output_shape = dataset_training.get_output_shape() model = model.Create(input_shape[1], output_shape[1]) optimizer = torch.optim.Adam(model.parameters(), lr=0.001) epoch_count = 100 for epoch in range(epoch_count): for i in range(training_count // batch_size): target_output, input = dataset_training.get_random_batch() predicted_output = model.forward(input) optimizer.zero_grad() loss = ((target_output - predicted_output)**2).mean() loss.backward() optimizer.step() target_output, input = dataset_testing.get_random_batch() predicted_output = model.forward(input) loss = ((target_output - predicted_output)**2).mean() print("EPOCH = ", epoch) for i in range(batch_size): print("target = ", target_output[i].detach().to("cpu").numpy())
for index, sample in enumerate(dataloader): img = Variable(sample['image'].float()).cuda() label = Variable(sample['label'].float()).cuda() label = label.unsqueeze(dim=1) #print(label.shape) #onehot = torch.FloatTensor(opts.batch_size, 2).cuda() #onehot.zero_() #onehot.scatter_(1,label,1) #one_hot = label.scatter_(1,label,1) #print(onehot) #one_hot = torch.FloatTensor(opts.batch_size, 2, n).zero_() prediction_prob = model.forward(img) #print(prediction_prob) #print(label) #print(prediction) loss = criterion(prediction_prob, label) loss.backward() optimizer.step() optimizer.zero_grad() if index % 500 == 0: print('prediction prob {}'.format(prediction_prob)) prediction = prediction_prob > 0.5 # print('Ground truth {}'.format(label))
def train(conf, X_train, Y_train, X_devel, Y_devel): """Run training Args: conf: Configuration dictionary X_train: numpy array of floats with shape [input dimension, number of train examples] Y_train: numpy array of integers with shape [output dimension, number of train examples] X_devel: numpy array of floats with shape [input dimension, number of devel examples] Y_devel: numpy array of integers with shape [output dimension, number of devel examples] Returns: params: Dictionary with trained parameters train_progress: Dictionary with progress data, to be used in visualization. devel_progress: Dictionary with progress data, to be used in visualization. """ print("Run training") # Preparation num_examples_in_epoch = X_train.shape[1] example_indices = np.arange(0, num_examples_in_epoch) np.random.shuffle(example_indices) # Initialisation params = model.initialization(conf) # For displaying training progress train_steps = [] train_ccr = [] train_cost = [] devel_steps = [] devel_ccr = [] # Start training step = 0 epoch = 0 num_correct_since_last_check = 0 batch_start_index = 0 batch_end_index = conf['batch_size'] print("Number of training examples in one epoch: ", num_examples_in_epoch) print("Start training") while True: start_time = time.time() batch_indices = get_batch_indices(example_indices, batch_start_index, batch_end_index) X_batch = X_train[:, batch_indices] Y_batch = model.one_hot(Y_train[batch_indices], conf['output_dimension']) Y_proposal, features = model.forward(conf, X_batch, params, is_training=True) print("Finish Foward") cost_value, num_correct = model.cross_entropy_cost(Y_proposal, Y_batch) #print("Finish Cross Entropy") grad_params = model.backward(conf, Y_proposal, Y_batch, params, features) print("Finish Backward") params = model.gradient_descent_update(conf, params, grad_params) print("Finish Gradient Update") print("Finish Training Number" + repr(step)) num_correct_since_last_check += num_correct batch_start_index += conf['batch_size'] batch_end_index += conf['batch_size'] if batch_start_index >= num_examples_in_epoch: epoch += 1 np.random.shuffle(example_indices) batch_start_index = 0 batch_end_index = conf['batch_size'] step += 1 if np.isnan(cost_value): print("ERROR: nan encountered") break if step % conf['train_progress'] == 0: elapsed_time = time.time() - start_time sec_per_batch = elapsed_time / conf['train_progress'] examples_per_sec = conf['batch_size'] * conf[ 'train_progress'] / elapsed_time ccr = num_correct / conf['batch_size'] running_ccr = (num_correct_since_last_check / conf['train_progress'] / conf['batch_size']) num_correct_since_last_check = 0 train_steps.append(step) train_ccr.append(running_ccr) train_cost.append(cost_value) if conf['verbose']: print( "S: {0:>7}, E: {1:>4}, cost: {2:>7.4f}, CCR: {3:>7.4f} ({4:>6.4f}), " "ex/sec: {5:>7.3e}, sec/batch: {6:>7.3e}".format( step, epoch, cost_value, ccr, running_ccr, examples_per_sec, sec_per_batch)) if step % conf['devel_progress'] == 0: num_correct, num_evaluated = evaluate(conf, params, X_devel, Y_devel) devel_steps.append(step) devel_ccr.append(num_correct / num_evaluated) if conf['verbose']: print( "S: {0:>7}, Test on development set. CCR: {1:>5} / {2:>5} = {3:>6.4f}" .format(step, num_correct, num_evaluated, num_correct / num_evaluated)) if step >= conf['max_steps']: print("Terminating training after {} steps".format(step)) break train_progress = { 'steps': train_steps, 'ccr': train_ccr, 'cost': train_cost } devel_progress = {'steps': devel_steps, 'ccr': devel_ccr} return params, train_progress, devel_progress
def bow_evaluate(x, y, model, cost,test_word_to_ix): predict = model.forward(x, model.lookuptable,test_word_to_ix) right, total = rightness(predict, y) # 返回值为(正确样例数,总样本数) loss = cost(predict, y) return right, total, loss
def backward(train_data,train_labels,train_num): with tf.Graph().as_default() as g: with tf.name_scope('input'): x = tf.placeholder(dtype=tf.float32,shape=[BATCH_SIZE,IMG_SIZE[0],IMG_SIZE[1],IMG_CHANNEL]) y_ = tf.placeholder(dtype=tf.float32,shape=[BATCH_SIZE,IMG_SIZE[0],IMG_SIZE[1],IMG_CHANNEL]) # forward y,output_Stokes,output_Stokes_GT,grad_output,grad_output_GT = model.forward(x,y_,True) # learning rate global_step = tf.Variable(0,trainable=False) learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,global_step, train_num//BATCH_SIZE, LEARNING_RATE_DECAY, staircase=True) #loss function with tf.name_scope('loss'): mse = (1.0/BATCH_SIZE)*tf.nn.l2_loss(tf.subtract(y,y_)) mse_stokes = (1.0/BATCH_SIZE)*tf.nn.l2_loss(tf.subtract(output_Stokes, output_Stokes_GT)) mse_grad = (1.0/BATCH_SIZE)*tf.nn.l2_loss(tf.subtract(grad_output, grad_output_GT)) loss_init = mse + mse_stokes loss = mse + mse_stokes + 0.1*mse_grad #Optimizer # GradientDescent with tf.name_scope('train'): # Adam optimizer = tf.train.AdamOptimizer(learning_rate) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): train_op_init = optimizer.minimize(loss_init, global_step=global_step) train_op = optimizer.minimize(loss,global_step=global_step) # Save model variables = tf.contrib.framework.get_variables_to_restore() variables_to_resotre = [v for v in variables if v.name.split('/')[0] != 'Gradient'] saver = tf.train.Saver(variables_to_resotre,max_to_keep=50) epoch = 0 config = tf.ConfigProto() with tf.Session(config=config) as sess: tf.global_variables_initializer().run() ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess,ckpt.model_checkpoint_path) epoch = int(ckpt.model_checkpoint_path.split('/')[-1].split('_')[-1].split('-')[-2]) while epoch<MAX_EPOCH: max_step = train_num//BATCH_SIZE listtmp = np.random.permutation(train_num) j = 0 for i in range(max_step): file =open("loss.txt",'a') ind = listtmp[j:j+BATCH_SIZE] j = j + BATCH_SIZE xs = train_data[ind,:,:,:] ys = train_labels[ind,:,:,:] mode = np.random.permutation(8) xs = DA.data_augmentation(xs,mode[0]) ys = DA.data_augmentation(ys,mode[0]) if epoch <50: _, loss_v, step = sess.run([train_op_init, loss_init, global_step], feed_dict={x: xs, y_: ys}) file.write("Epoch: %d Step is: %d After [ %d / %d ] training, the batch loss is %g.\n" % (epoch + 1, step, i + 1, max_step, loss_v)) file.close() else: _,loss_v,step = sess.run([train_op,loss,global_step],feed_dict={x:xs, y_:ys}) file.write("Epoch: %d Step is: %d After [ %d / %d ] training, the batch loss is %g.\n" % (epoch + 1, step, i + 1, max_step, loss_v)) file.close() #print("Epoch: %d After [ %d / %d ] training, the batch loss is %g." % (epoch + 1, i + 1, max_step, loss_v)) saver.save(sess,os.path.join(MODEL_SAVE_PATH,MODEL_NAME+'_epoch_'+str(epoch+1)),global_step = global_step) epoch +=1
def evaluate_specific(data, model, suffix=""): # Evaluate if args.threshold is None: print("direct predict task") else: print("threshold task,threshold={}".format(args.threshold)) if not args.logits: print("use softmax") correct = 0 correct2 = 0 correct3 = 0 correct4 = 0 total = 0 total2 = 0 mrr = 0 nota_fp = nota_fp2 = nota_tp = nota_tp2 = nota_fn = nota_fn2 = nota_tn = nota_tn2 = 0 prob_nota = [] prob_other = [] num_batches = math.ceil(len(data) / args.batch_size) for i in range(num_batches): batch_rows = data[i * args.batch_size:(i + 1) * args.batch_size] is_nota = [e[1][0] == '_nota' for e in batch_rows] if len(batch_rows) == 0: continue if args.threshold is not None: #remove _nota for threshold experiments batch_rows[:] = [ list(filter((["_nota"]).__ne__, b)) for b in batch_rows ] ctx_seq, ctx_lens, resp_seq, resp_lens = model.prep_batch(batch_rows) ctx_seq = ctx_seq.to(device) resp_seq = resp_seq.to(device) # Forward pass proba = model.forward(ctx_seq, ctx_lens, resp_seq, resp_lens) if not args.logits: proba = F.softmax(proba.squeeze(1), dim=-1) proba = proba.squeeze(1) # Compute metrics alter_best = [] for b in range(len(batch_rows)): if args.threshold is None: nota_pred = batch_rows[b][proba[b].argmax() + 1][0] == "_nota" nota_pred2 = batch_rows[b][proba[b, :2].argmax() + 1][0] == "_nota" else: nota_pred = proba[b].max().item() < args.threshold nota_pred2 = proba[b, :2].max().item() < args.threshold if is_nota[b]: prob_nota.append(proba[b].cpu().detach()) correct3 += int(nota_pred) correct4 += int(nota_pred2) nota_tp += int(nota_pred) nota_tp2 += int(nota_pred2) nota_fn += int(not nota_pred) nota_fn2 += int(not nota_pred2) else: prob_other.append(proba[b].cpu().detach()) correct += int(proba[b].argmax() == 0 and not nota_pred) correct2 += int(proba[b, :2].argmax() == 0 and not nota_pred2) correct3 += int(not nota_pred) correct4 += int(not nota_pred2) nota_fp += int(nota_pred) nota_fp2 += int(nota_pred2) nota_tn += int(not nota_pred) nota_tn2 += int(not nota_pred2) total += 1 total2 += proba.size(0) with open( args.model_path + "{}_{}_distri{}.pkl".format( "logits" if args.logits else "prob", 99, suffix), "wb") as f: pickle.dump(prob_nota, f) pickle.dump(prob_other, f) #print("max non-NOTA",max(alter_best)) print('Current Test R@1/100:{:.4f}'.format(correct / total)) #print('Current Test R@1/2:{:.4f}'.format(correct2/total)) print('Current Test NOTA@100 ACC:{:.4f}'.format(correct3 / total2)) #print('Current Test NOTA@2 ACC:{:.4f}'.format(correct4/total2)) #print('NOTA percentage:'.format(total2/total)) f1_nota, f1_other = calculate_f1_nota(nota_tp, nota_fp, nota_fn, nota_tn) #f1_nota2,f1_other2 = calculate_f1_nota(nota_tp2,nota_fp2,nota_fn2,nota_tn2) print('NOTA@100 F1:{:.4f}'.format(f1_nota)) #print('NOTA@2 F1:{:.4f}'.format(f1_nota2)) print('non-NOTA@100 F1:{:.4f}'.format(f1_other)) #print('non-NOTA@2 F1:{:.4f}'.format(f1_other2)) #print("weighted F1:{:.4f}".format((f1_nota+f1_nota2+f1_other+f1_other2)/4)) return correct / total
def backward(): x = tf.placeholder(tf.float32, [batch_size, input_dim, 1, 2]) y_ = tf.placeholder(tf.int32, [batch_size, n_class]) y = model.forward(x, two_ch, 'add', False) global_step = tf.Variable(0, trainable=False) loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)) train_step = tf.train.AdamOptimizer(learning_rate=lr, beta1=0.9, beta2=0.999, epsilon=1e-8, name="Adam").minimize(loss) correct_predition = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_predition, tf.float32)) saver = tf.train.Saver() with tf.Session() as sess: coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) sess.run(init_op) #第一次训练模型注释掉以下两行 #model_path=os.path.join(MODEL_SAVE_PATH, MODEL_NAME) #tf.train.Saver().restore(sess,model_path) ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) k = 0 epochs = 10 for epoch in range(epochs): print('#######################################################') doubel_file_name = get_doubel_file(data_dir) d = 0 for doubel_file in doubel_file_name: print('第%d轮训练数据,第%d个人的数据:' % (epoch, d)) d = d + 1 file_1 = doubel_file[0] file_2 = doubel_file[1] EEG_1, EEG_2, onset, duration, stage_code = get_data( file_1, file_2, data_dir) EEG1_sequence, EEG2_sequence, stage_sequence = split_data_to_30s( EEG_1, EEG_2, onset, duration, stage_code) if k != 0: EEG1_sequence, EEG2_sequence, stage_sequence = K_EEG_together( EEG1_sequence, EEG2_sequence, stage_sequence, k) batch_time = len(EEG1_sequence) / batch_size print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!', d) for i in range(int(batch_time)): if two_ch: batch_data, batch_label = two_ch_batch( EEG1_sequence, EEG2_sequence, stage_sequence, batch_size) #最后返回 batch_data, batch_label else: batch_data, batch_label = one_ch_batch( EEG1_sequence, stage_sequence, batch_size) _, loss_value, acc, y_pre = sess.run( [train_step, loss, accuracy, y], feed_dict={ x: batch_data, y_: batch_label }) print('loss is %g......acc is %f' % (loss_value, acc)) saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME))