def test(data_matrix, saved_model_path, title): tf.reset_default_graph() pred, input_placeholder, labels_placeholder, _, loss_op = build_model(data_matrix, train=False) saver = tf.train.Saver() loss_list = [] label_list= [] pred_list = [] with tf.Session() as sess: sess.run(tf.global_variables_initializer()) saver.restore(sess, saved_model_path) print("Model restored.") minibatches = util.get_minibatches_lm(data_matrix, batch_size) for tup in minibatches: #label_data = np.zeros((batch_size, util.vocab_size)) #label_data[np.arange(batch_size), tup[1]] = 1 pred_temp, loss, labels_temp = sess.run([pred, loss_op, labels_placeholder], feed_dict={input_placeholder: tup[0], labels_placeholder: tup[1]}) for i, row in enumerate(pred_temp): pred_list.append(np.where(row == max(row))[0][0]) for i, row in enumerate(labels_temp): label_list.append(np.where(row == max(row))[0][0]) loss_list.append(loss) print "Loss: " + str(np.mean(loss_list)) + "\n" #util.outputConfusionMatrix(pred_list, label_list, "confusion_matrix " + title + " " + today) util.get_accuracy(pred_list, label_list)
def test(data_matrix, data_labels, saved_model_path, batch_size=1000): tf.reset_default_graph() pred, input_placeholder, labels_placeholder, _, loss_op = build_model( data_matrix, data_labels) saver = tf.train.Saver() loss_list = [] label_list = [] pred_list = [] with tf.Session() as sess: sess.run(tf.global_variables_initializer()) saver.restore(sess, saved_model_path) print("Model restored.") minibatches = util.get_minibatches(data_matrix, data_labels, batch_size) for tup in minibatches: pred_temp, loss, labels_temp = sess.run( [pred, loss_op, labels_placeholder], feed_dict={ input_placeholder: tup[0], labels_placeholder: tup[1] }) for i, row in enumerate(pred_temp): pred_list.append(np.where(row == max(row))[0][0]) for i, row in enumerate(labels_temp): label_list.append(np.where(row == max(row))[0][0]) loss_list.append(loss) print "Loss: " + str(np.mean(loss_list)) + "\n" util.outputConfusionMatrix(pred_list, label_list, "confusion_matrix_baseline_test") util.get_accuracy(pred_list, label_list)
def validate(validation_loader, model, criterion_is_noise, criterion_is_next): model.eval() losses = [] accs_is_noise = [] accs_is_next = [] start = time.time() for i, (noise_type, continuity_type, origin_sentence, enc_sentence, mask) in enumerate(validation_loader): # enc_sentence: (batch_size, len_sentence, len_morpheme, len_phoneme) # print(num_morpheme, origin_sentence, enc_sentence.size()) enc_sentence = enc_sentence.to(device) mask = mask.to(device) outputs_is_noise, outputs_is_next = model(enc_sentence, mask) if noise: noise_type = np.array(noise_type) noise_type[noise_type == 'no'] = 0 noise_type[noise_type != '0'] = 1 target_is_noise = torch.FloatTensor(noise_type.astype(float)).to(device) ce_is_noise = criterion_is_noise(outputs_is_noise, target_is_noise) acc_is_noise = get_accuracy(outputs_is_noise, target_is_noise) accs_is_noise.append(acc_is_noise) if continuous: continuity_type = np.array(continuity_type) continuity_type[continuity_type == 'no'] = 0 continuity_type[continuity_type != '0'] = 1 target_is_next = torch.FloatTensor(continuity_type.astype(float)).to(device) ce_is_next = criterion_is_next(outputs_is_next, target_is_next) acc_is_next = get_accuracy(outputs_is_next, target_is_next) accs_is_next.append(acc_is_next) if 'ce_is_noise' in locals() and 'ce_is_next' in locals(): loss = ce_is_noise + ce_is_next elif 'ce_is_noise' in locals(): loss = ce_is_noise elif 'ce_is_next' in locals(): loss = ce_is_next else: raise ValueError('There is no loss') losses.append(loss) trace_validation = '\nValidation Loss {loss_avg:.4f}\t'.format(loss_avg=sum(losses)/len(losses)) if 'acc_is_noise' in locals(): trace_validation += 'Noise Accuracy {acc_is_noise_avg:.4f}\t'.format( acc_is_noise_avg=sum(accs_is_noise)/len(accs_is_noise)) if 'acc_is_next' in locals(): trace_validation += 'Continuity Accuracy {acc_is_next_avg:.4f}\t'.format( acc_is_next_avg=sum(accs_is_next)/len(accs_is_next)) trace_validation += '\n' print(trace_validation) return sum(losses)/len(losses)
def test_classifier(data_matrix, data_labels, saved_model_path, title, max_sequence_length=70, batch_size=60): tf.reset_default_graph() pred, input_placeholder, labels_placeholder, train_op, loss_op = build_classifier( data_matrix, data_labels, max_sequence_length) saver = tf.train.Saver() loss_list = [] label_list = [] pred_list = [] with tf.Session() as sess: sess.run(tf.global_variables_initializer()) saver.restore(sess, saved_model_path) print("Model restored.") minibatches = util.get_minibatches_seq_test(data_matrix, data_labels, batch_size, max_sequence_length) for tup in minibatches: pred_temp, loss, labels_temp = sess.run( [pred, loss_op, labels_placeholder], feed_dict={ input_placeholder: np.transpose(tup[0]), labels_placeholder: tup[1] }) for i, row in enumerate(pred_temp): pred_list.append(np.where(row == max(row))[0][0]) for i, row in enumerate(labels_temp): label_list.append(np.where(row == max(row))[0][0]) loss_list.append(loss) count = 0 for i in range(len(pred_list)): if pred_list[i] != label_list[i]: count += 1 print "sentence: ", reconstruct_sentence( (tup[0][i:i + 1, :max_sequence_length]).tolist()) print "predicted label: ", pred_list[i] print "correct label: ", label_list[i] if count > 4: break break print "Loss: " + str(np.mean(loss_list)) + "\n" util.outputConfusionMatrix(pred_list, label_list, "confusion_matrix " + title + " " + today) util.get_accuracy(pred_list, label_list)
def do_ref(num_features, X, y, X_test, y_test): s = [i for i in range(num_features)] r = [] while len(s) is not 0: X_new = np.c_[X[:, s], X[:, -1]] w = solve_svm(X_new, y) w = np.asarray(w).reshape(-1) print(len(s)) accuracy = get_accuracy(w, np.c_[X_test[:, s], X_test[:, -1]], y_test) print("Node Id: " + str(node_id) + ", Accuracy: " + str(accuracy)) w = w[:-1] c = np.array([w_i**2 for w_i in w]) c_list = list(c) idx = c.argmin() # Remove similar values across other places min_c = c[idx] min_real_indices = [] min_indices = np.array(np.where(c == min_c)).reshape(-1) for ind in sorted(min_indices, reverse=True): s_removed = s.pop(ind) min_real_indices.append(s_removed) c_list.pop(ind) # s_f = s.pop(idx) # c_list.pop(idx) r = np.append(min_real_indices, r) return np.array(r, dtype=int)
def test_dummy_baseline(alg): n = 3 m = 2 true_labels = [0, 30, 57] data = np.zeros((n, m)) alg.train(data, true_labels) labels = alg.evaluate(data) acc = util.get_accuracy(labels, true_labels) print("accuracy on dummy data with " + str(alg) + ": " + str(acc))
def run_train(): input_features, labels, optional = define_placeholder(use_img_feat=True) # 입력값 정의 # model = baseline.Baseline(input_features, labels, optional) # 모델 구축 model = model_mgr.get_model('umsoimg_v2')(input_features, labels, optional) # 최적화 알고리즘 정의 optimizer = tf.train.AdamOptimizer(config.learning_rate) grads_and_vars = optimizer.compute_gradients(model.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=model.global_step, name="train_op") _dataset = DatasetAll(target=None) sess = tf.InteractiveSession() init_op = tf.global_variables_initializer() saver = tf.train.Saver(max_to_keep=20) sess.run(init_op) for i in range(config.epochs): for j, _dict in enumerate(_dataset.batch_loader(config.batch_size)): _label_weight = util.make_label_weights(_dict, b_weight=1., m_weight=1.2, s_weight=1.3, d_weight=1.4) # label 별 weight 생성 feed_dict = { input_features['product']: _dict['product'], input_features['img_feat']: _dict['img_feat'], labels['b']: _dict['bcateid'], labels['m']: _dict['mcateid'], labels['s']: _dict['scateid'], labels['d']: _dict['dcateid'], optional['label_weight']: _label_weight, optional['keep_prob']: 0.7 } _, step, _loss, _prediction = sess.run([train_op, model.global_step, model.loss, model.prediction], feed_dict) _b, _m, _s, _d = _prediction if j % 100 == 0: result = util.get_accuracy(_dict, _b, _m, _s, _d) print("epoch : {} [t:{}], global_step: {}, Acc: {}, loss: {}".format( i + 1, _dataset.now_dataset, step, [str(result[0])[:6], str(result[1])[:6], str(result[2])[:6], str(result[3])[:6]], str(_loss)[:6])) model_name = '{}_e{}.ckpt'.format(model.__class__.__name__, str(i) if i >= 10 else ('0' + str(i))) print("save model... {}".format(model_name)) # save_path = saver.save(sess, "../tmp/{}".format(model_name)) save_path = saver.save(sess, os.path.join(config.result_model_dir, model_name))
def test_model(keys=1): model = tf.keras.models.load_model('model.h5') scaler = joblib.load('scaler.pkl') trainer = get_pyspectre(keys).getTrainerStr() secret = get_pyspectre(keys).getSecretStr() repetitions = 5 samples = 3 print("Testing model with keysize " + str(keys) + ". " + str(repetitions) + " repetitions of " + str(samples) + " samples each.") total_acc = 0 total_runtime = 0 for s in range(repetitions): start_time = time.time() guessed_secrets = [] weights = np.zeros((len(secret), len(trainer))) for _ in range(samples): # print("Sampling...") X = np.zeros((len(secret), len(trainer))) for i in range(len(secret)): X[i] = np.take(np.array(get_pyspectre(keys).readMemoryByte(i, False)), [ord(x) for x in trainer]) X = scaler.transform(X) prediction = model.predict(X) guessed_chars = np.argmax(prediction, axis=1) guessed_secret = "".join([trainer[x] for x in guessed_chars]) weights += prediction guessed_secrets.append(guessed_secret) weight_to_char_map = {char: index for index, char in enumerate(trainer)} majority_guess = util.majority_vote(guessed_secrets, weights, weight_to_char_map) acc = util.get_accuracy(majority_guess, secret) total_acc += acc elapsed = time.time()-start_time total_runtime += elapsed print(str(total_acc / repetitions * 100) + ",\t" + str(total_runtime / repetitions))
inference_logits, { input_data: source_batch, source_sequence_length: sources_lengths, target_sequence_length: targets_lengths, keep_prob: 1.0 }) batch_valid_logits = sess.run( inference_logits, { input_data: valid_sources_batch, source_sequence_length: valid_sources_lengths, target_sequence_length: valid_targets_lengths, keep_prob: 1.0 }) train_acc = get_accuracy(target_batch, batch_train_logits) valid_acc = get_accuracy(valid_targets_batch, batch_valid_logits) print( 'Epoch {:>3} Batch {:>4}/{} - Train Accuracy: {:>6.4f}, Validation Accuracy: {:>6.4f}, Loss: {:>6.4f}' .format(epoch_i, batch_i, len(en.text_as_ids) // batch_size, train_acc, valid_acc, loss)) saver = tf.train.Saver() saver.save(sess, 'dev') print('Model Trained and Saved') with open('preprocess.p', mode='rb') as in_file:
def get_baseline_predictions(train, test, x_cols, y_col): avg = train[y_col].sum() / len(train) prediction = [avg] * len(test) return prediction with open("train_data.csv", 'r') as csvfile: train = pandas.read_csv(csvfile) csvfile.close() with open("validation_data.csv", 'r') as csvfile: validate = pandas.read_csv(csvfile) csvfile.close() print('Training set size = ' + str(len(train))) print('Validation set size = ' + str(len(validate))) # print(data.columns.values) #list of all column names x_cols = train.columns.values[3:] y_col = train.columns.values[0] actual = validate[y_col] prediction = get_baseline_predictions(train, validate, x_cols, y_col) acc = util.get_accuracy(prediction, actual) util.make_accuracy_plot(prediction, actual) util.write_accuracy_data(prediction, actual, 'baseline_results.csv') print(acc)
y_train = np.array(df_training[str(num_features)]) X_train = np.array(X_train_int) X_train = np.c_[X_train, np.ones(len(y_train))] X_test_int = df_test.drop(columns=[str(num_features)]) y_test = np.array(df_test[str(num_features)]) X_test = np.array(X_test_int) X_test = np.c_[X_test, np.ones(len(y_test))] rankings = do_ref(num_features, X_train, y_train, X_test, y_test) feature_counter = -1 accuracies = [] while feature_counter > (-1) * (len(rankings) + 1): sub_rankings = rankings[feature_counter:] w = solve_svm(np.c_[X_train[:, sub_rankings], X_train[:, -1]], y_train) acc = get_accuracy(w, np.c_[X_test[:, sub_rankings], X_test[:, -1]], y_test) accuracies.append(acc) feature_counter -= 1 print(-1 * feature_counter) # w = solve_svm(X_train, y_train) x = [i + 1 for i in range(len(rankings))] y = accuracies print(len(x), len(y)) plt.plot(x, y, c=np.random.rand(3, )) plt.xlabel("Num of features") plt.ylabel("Accuracy - %") plt.show()
sigma, K, W, bound_, SLK_option, C_init, bound_lambda=lmbda, bound_iterations=200) if ts: trivial[count] = 1 continue # Evaluate the performance on validation set current_nmi = nmi(gnd_val, l[val_ind]) acc, _ = get_accuracy(gnd_val, l[val_ind]) print('lambda = ', lmbda, ' : NMI= %0.4f' % current_nmi) print('accuracy %0.4f' % acc) if current_nmi > bestnmi: bestnmi = current_nmi best_lambda_nmi = lmbda if acc > bestacc: bestacc = acc best_lambda_acc = lmbda best_C_init = C_init.copy() print('Best result: NMI= %0.4f' % bestnmi, '|NMI lambda = ', best_lambda_nmi)
def test(test_loader, model, criterion_is_noise, criterion_is_next): model.eval() # if not self.noise: # noise_threshold = 1 # if noise_threshold < 0.05: # noise_type = 'removing_phoneme' # elif noise_threshold < 0.1: # noise_type = 'replacing_phoneme' # elif noise_threshold < 0.15: # noise_type = 'removing_syllable' # elif noise_threshold < 0.2: # noise_type = 'replacing_syllable' # elif noise_threshold < 0.25: # noise_type = 'removing_morpheme' # elif noise_threshold < 0.3: # noise_type = 'replacing_morpheme' # elif noise_threshold < 0.35: # noise_type = 'removing_word_phrase' # elif noise_threshold < 0.4: # noise_type = 'replacing_word_phrase' # else: # noise_type = 'no' # # continuity_threshold = np.random.uniform(0, 1, 1) # if continuity_threshold < 0.5: # continuity_type = 'no' # else: # continuity_type = 'yes' losses = [] accs_is_noise = [] accs_is_next = [] confusions_is_noise = {'True_no': 0, 'False_no': 0, 'True_removing_phoneme': 0, 'False_removing_phoneme': 0, 'True_replacing_phoneme': 0, 'False_replacing_phoneme': 0, 'True_removing_syllable': 0, 'False_removing_syllable': 0, 'True_replacing_syllable': 0, 'False_replacing_syllable': 0, 'True_removing_morpheme': 0, 'False_removing_morpheme': 0, 'True_replacing_morpheme': 0, 'False_replacing_morpheme': 0, 'True_removing_word_phrase': 0, 'False_removing_word_phrase': 0, 'True_replacing_word_phrase': 0, 'False_replacing_word_phrase': 0, } confusions_is_next = [[0, 0], [0, 0]] data_size = len(test_loader.dataset) start = time.time() for i, (noise_type, continuity_type, origin_sentence, enc_sentence, mask) in enumerate(test_loader): if i % print_freq == 0: print('PROGESS: %.2f' % (i*batch_size/data_size)) # enc_sentence: (batch_size, len_sentence, len_morpheme, len_phoneme) # print(num_morpheme, origin_sentence, enc_sentence.size()) enc_sentence = enc_sentence.to(device) mask = mask.to(device) outputs_is_noise, outputs_is_next = model(enc_sentence, mask) if noise: noise_type = np.array(noise_type) noise_type_copy = noise_type.copy() noise_type[noise_type == 'no'] = 0 noise_type[noise_type != '0'] = 1 target_is_noise = torch.FloatTensor(noise_type.astype(float)).to(device) ce_is_noise = criterion_is_noise(outputs_is_noise, target_is_noise) acc_is_noise = get_accuracy(outputs_is_noise, target_is_noise) accs_is_noise.append(acc_is_noise) t = Variable(torch.Tensor([0.5])).to(device) binary_outputs = (outputs_is_noise > t).float() * 1 for j, (y_actual, y_predict) in enumerate(zip(noise_type_copy, binary_outputs)): if 'no' in y_actual: if y_predict: confusions_is_noise['False_no'] += 1 else: confusions_is_noise['True_no'] += 1 else: if 'phoneme' in y_actual: if 'removing' in y_actual: if y_predict: confusions_is_noise['True_removing_phoneme'] += 1 else: confusions_is_noise['False_removing_phoneme'] += 1 else: if y_predict: confusions_is_noise['True_replacing_phoneme'] += 1 else: confusions_is_noise['False_replacing_phoneme'] += 1 elif 'syllable' in y_actual: if 'removing' in y_actual: if y_predict: confusions_is_noise['True_removing_syllable'] += 1 else: confusions_is_noise['False_removing_syllable'] += 1 else: if y_predict: confusions_is_noise['True_replacing_syllable'] += 1 else: confusions_is_noise['False_replacing_syllable'] += 1 elif 'morpheme' in y_actual: if 'removing' in y_actual: if y_predict: confusions_is_noise['True_removing_morpheme'] += 1 else: confusions_is_noise['False_removing_morpheme'] += 1 else: if y_predict: confusions_is_noise['True_replacing_morpheme'] += 1 else: confusions_is_noise['False_replacing_morpheme'] += 1 else: if 'removing' in y_actual: if y_predict: confusions_is_noise['True_removing_word_phrase'] += 1 else: confusions_is_noise['False_removing_word_phrase'] += 1 else: if y_predict: confusions_is_noise['True_replacing_word_phrase'] += 1 else: confusions_is_noise['False_replacing_word_phrase'] += 1 if continuous: continuity_type = np.array(continuity_type) continuity_type[continuity_type == 'no'] = 0 continuity_type[continuity_type != '0'] = 1 target_is_next = torch.FloatTensor(continuity_type.astype(float)).to(device) ce_is_next = criterion_is_next(outputs_is_next, target_is_next) acc_is_next = get_accuracy(outputs_is_next, target_is_next) accs_is_next.append(acc_is_next) t = Variable(torch.Tensor([0.5])).to(device) binary_outputs = (outputs_is_next > t).float() * 1 confusions_is_next += confusion_matrix(continuity_type.astype(int), np.array(binary_outputs.to("cpu")).astype(int)) if 'ce_is_noise' in locals() and 'ce_is_next' in locals(): loss = ce_is_noise + ce_is_next elif 'ce_is_noise' in locals(): loss = ce_is_noise elif 'ce_is_next' in locals(): loss = ce_is_next else: raise ValueError('There is no loss') losses.append(loss) # TODO: 각각 최종 accuracy, confusion matrix, trace_test = '\nTest Loss {loss_avg:.4f}\t'.format(loss_avg=sum(losses)/len(losses)) if 'acc_is_noise' in locals(): trace_test += 'Noise Accuracy {acc_is_noise_avg:.4f}\t'.format( acc_is_noise_avg=sum(accs_is_noise)/len(accs_is_noise)) print(confusions_is_noise) if 'acc_is_next' in locals(): trace_test += 'Continuity Accuracy {acc_is_next_avg:.4f}\t'.format( acc_is_next_avg=sum(accs_is_next)/len(accs_is_next)) print(confusions_is_next) trace_test += '\n' print(trace_test) return sum(losses)/len(losses)
def train_val(config): device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') train_loader = get_dataloader(img_dir=config.train_img_dir, mask_dir=config.train_mask_dir, mode="train", batch_size=config.batch_size, num_workers=config.num_workers) val_loader = get_dataloader(img_dir=config.val_img_dir, mask_dir=config.val_mask_dir, mode="val", batch_size=config.batch_size, num_workers=config.num_workers) writer = SummaryWriter( comment="LR_%f_BS_%d_MODEL_%s_DATA_%s" % (config.lr, config.batch_size, config.model_type, config.data_type)) if config.model_type not in [ 'UNet', 'R2UNet', 'AUNet', 'R2AUNet', 'SEUNet', 'SEUNet++', 'UNet++', 'DAUNet', 'DANet', 'AUNetR', 'RendDANet', "BASNet" ]: print('ERROR!! model_type should be selected in supported models') print('Choose model %s' % config.model_type) return if config.model_type == "UNet": model = UNet() elif config.model_type == "AUNet": model = AUNet() elif config.model_type == "R2UNet": model = R2UNet() elif config.model_type == "SEUNet": model = SEUNet(useCSE=False, useSSE=False, useCSSE=True) elif config.model_type == "UNet++": model = UNetPP() elif config.model_type == "DANet": model = DANet(backbone='resnet101', nclass=1) elif config.model_type == "AUNetR": model = AUNet_R16(n_classes=1, learned_bilinear=True) elif config.model_type == "RendDANet": model = RendDANet(backbone='resnet101', nclass=1) elif config.model_type == "BASNet": model = BASNet(n_channels=3, n_classes=1) else: model = UNet() if torch.cuda.device_count() > 1: print("Let's use", torch.cuda.device_count(), "GPUs!") model = nn.DataParallel(model) model = model.to(device, dtype=torch.float) if config.optimizer == "sgd": optimizer = SGD(model.parameters(), lr=config.lr, weight_decay=1e-6, momentum=0.9) else: optimizer = torch.optim.Adam(model.parameters(), lr=config.lr) if config.loss == "dice": criterion = DiceLoss() elif config.loss == "bce": criterion = nn.BCELoss() elif config.loss == "bas": criterion = BasLoss() else: criterion = MixLoss() scheduler = lr_scheduler.StepLR(optimizer, step_size=40, gamma=0.1) global_step = 0 best_dice = 0.0 for epoch in range(config.num_epochs): epoch_loss = 0.0 with tqdm(total=config.num_train, desc="Epoch %d / %d" % (epoch + 1, config.num_epochs), unit='img') as train_pbar: model.train() for image, mask in train_loader: image = image.to(device, dtype=torch.float) mask = mask.to(device, dtype=torch.float) d0, d1, d2, d3, d4, d5, d6, d7 = model(image) loss = criterion(d0, d1, d2, d3, d4, d5, d6, d7, mask) epoch_loss += loss.item() writer.add_scalar('Loss/train', loss.item(), global_step) train_pbar.set_postfix(**{'loss (batch)': loss.item()}) optimizer.zero_grad() loss.backward() optimizer.step() train_pbar.update(image.shape[0]) global_step += 1 # if global_step % 100 == 0: # writer.add_images('masks/true', mask, global_step) # writer.add_images('masks/pred', d0 > 0.5, global_step) scheduler.step() epoch_dice = 0.0 epoch_acc = 0.0 epoch_sen = 0.0 epoch_spe = 0.0 epoch_pre = 0.0 current_num = 0 with tqdm(total=config.num_val, desc="Epoch %d / %d validation round" % (epoch + 1, config.num_epochs), unit='img') as val_pbar: model.eval() locker = 0 for image, mask in val_loader: current_num += image.shape[0] image = image.to(device, dtype=torch.float) mask = mask.to(device, dtype=torch.float) d0, d1, d2, d3, d4, d5, d6, d7 = model(image) batch_dice = dice_coeff(mask, d0).item() epoch_dice += batch_dice * image.shape[0] epoch_acc += get_accuracy(pred=d0, true=mask) * image.shape[0] epoch_sen += get_sensitivity(pred=d0, true=mask) * image.shape[0] epoch_spe += get_specificity(pred=d0, true=mask) * image.shape[0] epoch_pre += get_precision(pred=d0, true=mask) * image.shape[0] if locker == 200: writer.add_images('masks/true', mask, epoch + 1) writer.add_images('masks/pred', d0 > 0.5, epoch + 1) val_pbar.set_postfix(**{'dice (batch)': batch_dice}) val_pbar.update(image.shape[0]) locker += 1 epoch_dice /= float(current_num) epoch_acc /= float(current_num) epoch_sen /= float(current_num) epoch_spe /= float(current_num) epoch_pre /= float(current_num) epoch_f1 = get_F1(SE=epoch_sen, PR=epoch_pre) if epoch_dice > best_dice: best_dice = epoch_dice writer.add_scalar('Best Dice/test', best_dice, epoch + 1) torch.save( model, config.result_path + "/%s_%s_%d.pth" % (config.model_type, str(epoch_dice), epoch + 1)) logging.info('Validation Dice Coeff: {}'.format(epoch_dice)) print("epoch dice: " + str(epoch_dice)) writer.add_scalar('Dice/test', epoch_dice, epoch + 1) writer.add_scalar('Acc/test', epoch_acc, epoch + 1) writer.add_scalar('Sen/test', epoch_sen, epoch + 1) writer.add_scalar('Spe/test', epoch_spe, epoch + 1) writer.add_scalar('Pre/test', epoch_pre, epoch + 1) writer.add_scalar('F1/test', epoch_f1, epoch + 1) writer.close() print("Training finished")
# Q4.4.2 Majority Baseline ################################################################################ print( "\n************************************************************************" ) print( "*************************** Majority Label *****************************") print( "************************************************************************") (values, counts) = np.unique(label_tr, return_counts=True) majority_label = values[np.argmax(counts)] prediction = np.ones(label_dv.shape) * majority_label print("Accuracy for majority label on dev set = %.2f" % (util.get_accuracy(label_dv, prediction))) prediction = np.ones(label_te.shape) * majority_label print("Accuracy for majority label on test set = %.2f" % (util.get_accuracy(label_te, prediction))) ################################################################################ # Simple Perceptron with fixed learning rate ################################################################################ print( "\n************************************************************************" ) print( "************* Simple Perceptron (Fixed learning rate) ******************") print( "************************************************************************")
############## # ONE VS ONE # ############## print "[One VS One]" t0 = time.time() clf_1v1 = OneVsOneClassifier( svm.LinearSVC(random_state=42, class_weight='balanced')) clf_1v1.fit(X_train_svd, newsgroup_train.target) y_pred = clf_1v1.predict(X_test_svd) util.print_time_in_miliseconds(t0, time.time()) y_true = newsgroup_test.target print "Accuracy (overall): %.6f" % util.get_accuracy(y_pred, y_true) print metrics.classification_report(y_true, y_pred, target_names=categories) print "Confusion Matrix:" print metrics.confusion_matrix(newsgroup_test.target, y_pred) print "" #-------------------------------- ################### # ONE VS THE REST # ################### print "[One Vs Rest]" clf_1vr = OneVsRestClassifier(svm.LinearSVC(random_state=42))
def train(train_loader, model, optimizer, criterion_is_noise, criterion_is_next, epoch): model.train() losses = [] accs_is_noise = [] accs_is_next = [] start = time.time() for i, (noise_type, continuity_type, origin_sentence, enc_sentence, mask) in enumerate(train_loader): # enc_sentence: (batch_size, len_sentence, len_morpheme, len_phoneme) # print(num_morpheme, origin_sentence, enc_sentence.size()) enc_sentence = enc_sentence.to(device) mask = mask.to(device) outputs_is_noise, outputs_is_next = model(enc_sentence, mask) if noise: noise_type = np.array(noise_type) noise_type[noise_type == 'no'] = 0 noise_type[noise_type != '0'] = 1 target_is_noise = torch.FloatTensor(noise_type.astype(float)).to(device) ce_is_noise = criterion_is_noise(outputs_is_noise, target_is_noise) acc_is_noise = get_accuracy(outputs_is_noise, target_is_noise) accs_is_noise.append(acc_is_noise) if continuous: continuity_type = np.array(continuity_type) continuity_type[continuity_type == 'no'] = 1 continuity_type[continuity_type != '1'] = 0 target_is_next = torch.FloatTensor(continuity_type.astype(float)).to(device) ce_is_next = criterion_is_next(outputs_is_next, target_is_next) acc_is_next = get_accuracy(outputs_is_next, target_is_next) accs_is_next.append(acc_is_next) if 'ce_is_noise' in locals() and 'ce_is_next' in locals(): loss = ce_is_noise + ce_is_next elif 'ce_is_noise' in locals(): loss = ce_is_noise elif 'ce_is_next' in locals(): loss = ce_is_next else: raise ValueError('There is no loss') a = list(model.parameters())[0].clone() optimizer.zero_grad() loss.backward() clip_gradient(optimizer, grad_clip) optimizer.step() b = list(model.parameters())[0].clone() losses.append(loss) if i % print_freq == 0: trace_training = 'Epoch: [{0}][{1}/{2}]\t' \ 'Loss {loss:.4f} ({loss_avg:.4f})\t'.format(epoch, i, len(train_loader), loss=loss, loss_avg=sum(losses)/len(losses)) if 'acc_is_noise' in locals(): trace_training += 'Noise Accuracy {acc_is_noise:.4f} ({acc_is_noise_avg:.4f})\t'.format( acc_is_noise=acc_is_noise, acc_is_noise_avg=sum(accs_is_noise)/len(accs_is_noise)) if 'acc_is_next' in locals(): trace_training += 'Continuity Accuracy {acc_is_next:.4f} ({acc_is_next_avg:.4f})\t'.format( acc_is_next=acc_is_next, acc_is_next_avg=sum(accs_is_next)/len(accs_is_next)) # TODO: 값 추정하기 print(trace_training, torch.equal(a, b))