def main(): print("Testing...") args = parse_args() model, config = load_from_path(args.save_path, return_config=True) if args.dataset_path is not None: config.defrost() config.DATASET.ROOT = args.dataset_path config.freeze() evaluate(model, config)
def attention_test(args): chosen_params = dict(params) results = [] for attention in [False, True]: chosen_params['attention'] = attention runs = 5 acc_d = {} f1_d = {} for i in tqdm(range(runs)): train_dataset, valid_dataset, test_dataset = data.load_dataset( args.train_batch_size, args.test_batch_size, min_freq=chosen_params['min_freq']) embedding = data.generate_embedding_matrix( train_dataset.dataset.text_vocab, rand=chosen_params['rand_emb'], freeze=chosen_params['freeze']) model = RNN(embedding, chosen_params) criterion = nn.BCEWithLogitsLoss() optimizer = torch.optim.Adam(model.parameters(), lr=chosen_params['lr']) for epoch in range(args.epochs): print(f'******* epoch: {epoch+1} *******') train(model, train_dataset, optimizer, criterion, args) evaluate(model, valid_dataset, criterion, 'Validation') acc, f1 = evaluate(model, test_dataset, criterion, 'Test') acc_d['acc_' + 'run' + str(i)] = acc f1_d['f1_' + 'run' + str(i)] = f1 mean = np.mean(list(acc_d.values())) std = np.std(list(acc_d.values())) acc_d['mean'] = mean acc_d['std'] = std mean = np.mean(list(f1_d.values())) std = np.std(list(f1_d.values())) f1_d['mean'] = mean f1_d['std'] = std results.append((acc_d, f1_d)) with open(os.path.join(SAVE_DIR, 'attention.txt'), 'a') as f: print(f'', file=f) for idx, (acc, f1) in enumerate(results): print('[no attention]' if idx == 0 else '[attention]', file=f) print(acc, file=f) print(f1, file=f)
def validate(self, lr, batch_size, epochs, input_classes, embedding_dimensions=[4, 20, 10], hidden_nodes=64, save_labels_predictions=False, weighted='unweighted'): """ Performs k-fold cross-validation on the objects data set with a given model on given levels of the data set with given training parameters. Args: n_classes = the number of classes is the dataset hidden_nodes = the number of hidden nodes in the hidden layers of the LSTM model number_of_layers= the number of layers of the LSTM model lr = learning rate, default is 5e-3 batch_size = the batch size epochs = the number of epochs Returns: - A tuple containing the average (precision, recall, F1-measure). """ scores = np.empty(self.k) total_labels_predictions = pd.DataFrame() for i in range(self.k): # Initialises model. model = LSTM(input_dimensions=[2, 13, 4], embedding_dimensions=embedding_dimensions, hidden_nodes=hidden_nodes, n_layers=1, n_classes=13, input_classes=input_classes).to(device) # Trains the model on the training set belonging to the iteration of the k-fold. self.data.set_dialogue_ids(self.train_ids[i]) train(model, self.data, lr, batch_size, epochs, weighted) # Tests the model on the test set belonging to the iteration of the k-fold. self.data.set_dialogue_ids(self.test_ids[i]) if save_labels_predictions: labels_predictions_fold, scores[i] = evaluate( model, self.data, save_labels_predictions) labels_predictions_fold = pd.DataFrame( labels_predictions_fold.reshape(-1, 3)) total_labels_predictions = pd.concat( [total_labels_predictions, labels_predictions_fold]) else: scores[i] = evaluate(model, self.data, save_labels_predictions) if save_labels_predictions: return total_labels_predictions, scores return scores
def translate(): data = request.get_json(force=True) message = data['src'] message_normalized = unicode_to_ascii(message) message_normalized = normalize_string(message) if data['direction'] == 0: # en-fi translation = evaluate(encoder_en_fi, attn_decoder_en_fi, message_normalized, en_lang, fi_lang) else: translation = evaluate(encoder_fi_en, attn_decoder_fi_en, message_normalized, fi_lang, en_lang) return jsonify({"src": message, 'tgt': ' '.join(translation[0][:-1])})
def test_model_score(loader, output_lang, encoder, decoder, targetlang): encoder.eval() decoder.eval() total = 0 total_loss = 0 predict_file = 'predict_temp' predict_lines = open(predict_file, 'w') for i, (input_batches, input_lengths, target_batches, target_lengths) in enumerate(loader): input_batches, input_lengths, target_batches, target_lengths = input_batches.to( device), input_lengths.to(device), target_batches.to( device), target_lengths.to(device) decoded_words, all_decoder_outputs = evaluate( input_batches, input_lengths, output_lang, target_batches, target_lengths, encoder, decoder) loss = masked_cross_entropy(all_decoder_outputs.contiguous(), target_batches.contiguous(), target_lengths) total_loss += loss total += 1 predict_lines.write(''.join(decoded_words) + '\n') predict_lines.close() if targetlang == 'zh': target_file = '../iwslt-zh-en-processed/dev.tok.en' else: target_file = '../iwslt-vi-en-processed/dev.tok.en' result = subprocess.run('cat {} | sacrebleu {}'.format( predict_file, target_file), shell=True, stdout=subprocess.PIPE) score = get_blue_score(str(result)) return (total_loss / total), score
def main(args): workdir = Path(args.workdir) logger = get_logger() logger.info(f'config:\n{args}') state = Saver.load_best_from_folder(workdir, map_location='cpu') device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') logger.info(f'use device: {device}') num_points = len(args.data.symmetry) model = models.__dict__[args.model.name](num_points) model.load_state_dict(state['model']) model.to(device) # datasets valtransform = Transform(args.dsize, args.padding, args.data.meanshape, args.data.meanbbox) valdata = datasets.__dict__[args.data.name](**args.data.val) valdata.transform = valtransform valloader = DataLoader(valdata, args.batch_size, False, num_workers=args.num_workers, pin_memory=False) score_fn = IbugScore(args.left_eye, args.right_eye) evalmeter = evaluate(model, valloader, score_fn, device) logger.info(evalmeter)
def eval_MH(x_dev, mask_dev, y_dev): if FLAGS.use_pre_vec: glove_vec = load_glove(FLAGS.vocab_file, FLAGS.vec_file) else: glove_vec = None model = Text_attention2(glove_vec, FLAGS.seq_length, FLAGS.num_classes, FLAGS.embedding_size, FLAGS.vocab_size, FLAGS.batch_size, FLAGS.hidden_size, FLAGS.num_attention_heads, FLAGS.num_hidden_layers, FLAGS.sent_attention_size, FLAGS.initializer_range, FLAGS.base_learning_rate, FLAGS.decay_rate) print("Loading test data...") start_time = time.time() # x_test, y_test = load_polarity_data_and_labels(FLAGS.positive_data_file, FLAGS.negative_data_file) # x_test, mask, y = process_file(x_test, y_test, FLAGS.vocab_file, FLAGS.seq_length) session = tf.Session() session.run(tf.global_variables_initializer()) saver = tf.train.Saver() saver.restore(sess=session, save_path=save_path) # 读取保存的模型 print('Testing...') loss_test, acc_test = evaluate(session, model, x_dev, mask_dev, y_dev, FLAGS.batch_size) msg = 'Test Loss: {0:>6.2}, Test Acc: {1:>7.2%}' print(msg.format(loss_test, acc_test)) """
def test_inference(args, model, testloader): """ Returns the test accuracy and loss. """ device = 'cuda' if torch.cuda.is_available( ) and not args.cpu_only else 'cpu' confmat = evaluate(model, testloader, device, args.num_classes) return confmat.acc_global, confmat.iou_mean, str(confmat)
def main(opts): train_dataset = ColonCancerDataset('dataset/colon_cancer', train=True) train_loader = DataLoader(train_dataset, batch_size=opts.batch_size, shuffle=True, num_workers=opts.num_workers, drop_last=False) attention_model = AttentionModelColonCancer(squeeze_channels=True, softmax_smoothing=0) feature_model = FeatureModelColonCancer(in_channels=3, out_channels=500) classification_head = ClassificationHead(in_channels=500, num_classes=len(train_dataset.CLASSES)) ats_model = ATSModel(attention_model, feature_model, classification_head, n_patches=opts.n_patches, patch_size=opts.patch_size) ats_model = ats_model.to(opts.device) optimizer = optim.Adam(ats_model.parameters(), lr=opts.lr) logger = AttentionSaverTrafficSigns(opts.output_dir, ats_model, train_dataset, opts) criterion = nn.CrossEntropyLoss() entropy_loss_func = MultinomialEntropy(opts.regularizer_strength) for epoch in range(opts.epochs): train_loss, train_metrics = train(ats_model, optimizer, train_loader, criterion, entropy_loss_func, opts) with torch.no_grad(): test_loss, test_metrics = evaluate(ats_model, train_loader, criterion, entropy_loss_func, opts) logger(epoch, (train_loss, test_loss), (train_metrics, test_metrics))
def test(): data = np.load(config.test_dir, allow_pickle=True) word_test = data["words"] label_test = data["labels"] test_dataset = NERDataset(word_test, label_test, config) logging.info("--------Dataset Build!--------") # build data_loader test_loader = DataLoader(test_dataset, batch_size=config.batch_size, shuffle=False, collate_fn=test_dataset.collate_fn) logging.info("--------Get Data-loader!--------") # Prepare model if config.model_dir is not None: model = BertNER.from_pretrained(config.model_dir) model.to(config.device) logging.info("--------Load model from {}--------".format( config.model_dir)) else: logging.info("--------No model to test !--------") return val_metrics = evaluate(test_loader, model, mode='test') val_f1 = val_metrics['f1'] logging.info("test loss: {}, f1 score: {}".format(val_metrics['loss'], val_f1)) val_f1_labels = val_metrics['f1_labels'] for label in config.labels: logging.info("f1 score of {}: {}".format(label, val_f1_labels[label]))
def extract_sentiment_words(): # create vocabulary using wikitext2 train_txt, _, _ = torchtext.datasets.WikiText2.splits(TEXT) TEXT.build_vocab(train_txt) start = time.time() x_train, y_train, x_val, y_val, rtrain, rtest = preprocess() end = time.time() print("PREPROCESSING TIME: {}".format(end - start)) ntokens = len(TEXT.vocab.stoi) # the size of vocabulary # FIXME set up batched examples for better generality # batch_size = 20 # eval_batch_size = 10 # configs emsize = 200 # embedding dimension nhid = 200 # feedforward dimension nlayers = 2 # n encoders nhead = 2 # multiattention heads dropout = 0.2 # the dropout value # initialize main torch vars model = TransformerModel(ntokens, emsize, nhead, nhid, nlayers, dropout).to(device) criterion = nn.CrossEntropyLoss().to(device) lr = 0.05 # learning rate optimizer = torch.optim.SGD(model.parameters(), lr=lr) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95) best_val_loss = float("inf") epochs = 50 best_model = None for epoch in range(1, epochs + 1): epoch_start_time = time.time() train_model(x_train, y_train, model, criterion, optimizer, scheduler, epoch) val_loss = evaluate(x_val, y_val,rtest, model,criterion) print('-' * 89) print('| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | ' 'valid ppl {:8.2f}'.format(epoch, (time.time() - epoch_start_time), val_loss, math.exp(val_loss))) print('-' * 89) if val_loss < best_val_loss: best_val_loss = val_loss best_model = model scheduler.step() # test_loss = evaluate(best_model, criterion, test_data) # print('=' * 89) # print('| End of training | test loss {:5.2f} | test ppl {:8.2f}'.format( # test_loss, math.exp(test_loss))) # print('=' * 89) return best_model
def main(): args = parse_config() cfg = load_config(args.config) # create folder save_path = cfg.base.save_path log_path = cfg.base.log_path if os.path.exists(save_path): warning = 'Save path {} exists.\nDo you want to overwrite it? (y/n)\n'.format( save_path) if not (args.overwrite or input(warning) == 'y'): sys.exit(0) else: os.makedirs(save_path) logger = SummaryWriter(log_path) copy_config(args.config, save_path) # print configuration if args.print_config: print_config({ 'BASE CONFIG': cfg.base, 'DATA CONFIG': cfg.data, 'TRAIN CONFIG': cfg.train }) else: print_msg('LOADING CONFIG FILE: {}'.format(args.config)) # train set_random_seed(cfg.base.random_seed) model = generate_model(cfg) train_dataset, test_dataset, val_dataset = generate_dataset(cfg) estimator = Estimator(cfg.train.criterion, cfg.data.num_classes) train(cfg=cfg, model=model, train_dataset=train_dataset, val_dataset=val_dataset, estimator=estimator, logger=logger) # test print('This is the performance of the best validation model:') checkpoint = os.path.join(save_path, 'best_validation_weights.pt') evaluate(cfg, model, checkpoint, test_dataset, estimator) print('This is the performance of the final model:') checkpoint = os.path.join(save_path, 'final_weights.pt') evaluate(cfg, model, checkpoint, test_dataset, estimator)
def hyperparam_optim_test(args): var_params = { 'cell_name': ['lstm'], 'hidden_size': [50, 150, 300], 'num_layers': [2, 4, 5], 'min_freq': [0, 100, 500], 'lr': [1e-3, 1e-4, 1e-7], 'dropout': [0, 0.4, 0.7], 'freeze': [False, True], 'rand_emb': [False, True], 'attention': [False] } results = [] for i in tqdm(range(10)): chosen_params = {k: random.choice(v) for (k, v) in var_params.items()} train_dataset, valid_dataset, test_dataset = data.load_dataset( args.train_batch_size, args.test_batch_size, min_freq=chosen_params['min_freq']) embedding = data.generate_embedding_matrix( train_dataset.dataset.text_vocab, rand=chosen_params['rand_emb'], freeze=chosen_params['freeze']) model = RNN(embedding, chosen_params) criterion = nn.BCEWithLogitsLoss() optimizer = torch.optim.Adam(model.parameters(), lr=chosen_params['lr']) for epoch in range(args.epochs): print(f'******* epoch: {epoch+1} *******') train(model, train_dataset, optimizer, criterion, args) evaluate(model, valid_dataset, criterion, 'Validation') acc, f1 = evaluate(model, test_dataset, criterion, 'Test') result = dict(chosen_params) result['acc'] = acc result['f1'] = f1 results.append(result) with open(os.path.join(SAVE_DIR, 'params_search.txt'), 'a') as f: for result in results: print(result, file=f)
def cell_comparison_test(args): train_dataset, valid_dataset, test_dataset = data.load_dataset( args.train_batch_size, args.test_batch_size) embedding = data.generate_embedding_matrix( train_dataset.dataset.text_vocab) var_params = { 'hidden_size': [50, 150, 300], 'num_layers': [1, 2, 4], 'dropout': [0.1, 0.4, 0.7], 'bidirectional': [True, False], 'attention': [False] } for idx, (key, values) in enumerate(var_params.items()): fig, ax = plt.subplots(nrows=1, ncols=1) ax.set_title('Variable ' + key) for cell_name in tqdm(['rnn', 'lstm', 'gru']): results = [] for i in range(len(values)): current_params = { k: v[i] if k == key else v[1] for (k, v) in var_params.items() } current_params['cell_name'] = cell_name model = RNN(embedding, current_params) model criterion = nn.BCEWithLogitsLoss() optimizer = torch.optim.Adam(model.parameters(), lr=1e-4) for epoch in range(args.epochs): print(f'******* epoch: {epoch+1} *******') train(model, train_dataset, optimizer, criterion, args) evaluate(model, valid_dataset, criterion, 'Validation') result, _ = evaluate(model, test_dataset, criterion, 'Test') results.append(result) ax.plot(values, results, marker='o', label=cell_name) plt.legend(loc='best') plt.xlabel(key) plt.ylabel('accuracy') fig.savefig(os.path.join(SAVE_DIR, key + '.png')) plt.close(fig)
def main(): parser = argparse.ArgumentParser() parser.add_argument( "--data_path", default="./data/binary_memory_50/binary_data_memory_test.pkl", type=str, help="test data path", ) parser.add_argument("--model_path", required=True, type=str, help="model path") parser.add_argument("--args_path", required=True, type=str, help="args path") args_ = parser.parse_args() args = torch.load(args_.args_path) args.eval_data_file = args_.data_path args.eval_output_dir = os.path.join(args.output_dir, "test_result/") os.makedirs(args.eval_output_dir, exist_ok=True) # Setup logging logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.DEBUG if args.debug else logging.INFO, ) n_pitcher = [*[2] * args.n_pitcher_disc, *[5] * args.n_pitcher_cont] n_batter = [*[2] * args.n_batter_disc, *[5] * args.n_batter_cont] n_state = [4, 4, 5, 8, 9, 10, 16, 22, 23] model = BaseballTransformer( n_pitcher, n_batter, n_state, args.n_memory_layer, args.n_encoder_layer, memory_len=args.memory_len, d_model=args.d_model, n_head=args.n_head, dropout=args.dropout, attention_type=args.attention_type, ).to(args.device) if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel): model = torch.nn.DataParallel(model) model.load_state_dict(torch.load(args_.model_path), strict=False) model.eval() result, _, f1_log, cm = evaluate(args, args.eval_data_file, model) result_dir = os.path.join(args.eval_output_dir, "test_results.txt") utils.print_result(result_dir, result, f1_log, cm) logger.info(" Results are saved to, %s", result_dir) draw_cm(cm, args.eval_output_dir)
def run(): args = parse_opts() os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152 os.environ["CUDA_VISIBLE_DEVICES"] = "9" # GLOBAL VARS # MODE = args.mode CLASS_WEIGHT = False N_EP = 20 FLATTEN = args.flatten RNN = args.rnn BATCH_SIZE = args.batch_size #### datasets, dataloaders = init_dataset( BATCH_SIZE, single_channel=args.single_channel) print('[Train] class counts', np.unique( datasets['train'].target_vals, return_counts=True)) print('[Test] class counts', np.unique( datasets['test'].target_vals, return_counts=True)) n_ch = 1 if args.single_channel else 3 if MODE == 'min': in_channels = datasets['train'].min_depth*n_ch elif MODE == 'max': in_channels = datasets['train'].max_depth*n_ch torch.manual_seed(0) # init net net = init_net(opt=args.model_idx, in_channels=in_channels) class_weight = None if CLASS_WEIGHT: cnts = Counter(datasets['train'].target_vals) n = len(datasets['train']) class_weight = [max(cnts.values())/cnts['0'], max(cnts.values())/cnts['1']] class_weight = torch.FloatTensor(class_weight) cross_entrp_loss = nn.CrossEntropyLoss(weight=class_weight).cuda() focal_loss = FocalLoss().cuda() optimizer = optim.Adam(net.parameters(), lr=0.000027) criterion = cross_entrp_loss # scheduler = optim.lr_scheduler.ReduceLROnPlateau( # optimizer, 'min', verbose=True, patience=7) for ep in range(N_EP): train_epoch(net, dataloaders['train'], optimizer, criterion, ep, scheduler=None, flatten=FLATTEN, MODE=MODE, rnn=RNN) valid_loss = evaluate(net, dataloaders['test'], criterion, ep, flatten=FLATTEN, MODE=MODE, rnn=RNN)
def autoencoder(self, sentence): encoder = torch.load('test_encoder_imdb10000_0.0037651115254803288', map_location={'cuda:0': 'cpu'}) decoder = torch.load('test_decoder_imdb10000_0.0037651115254803288', map_location={'cuda:0': 'cpu'}) output_words, _, loss = evaluate(encoder, decoder, sentence, self.input_lang, self.output_lang) return output_words, loss
def main(args): chosen_params = dict(params) chosen_params['attention'] = True train_dataset, valid_dataset, test_dataset = data.load_dataset( args.train_batch_size, args.test_batch_size) embedding = data.generate_embedding_matrix( train_dataset.dataset.text_vocab, freeze=False) model = RNN(embedding, chosen_params) criterion = nn.BCEWithLogitsLoss() optimizer = torch.optim.Adam(model.parameters(), lr=1e-4) for epoch in range(args.epochs): print(f'******* epoch: {epoch} *******') train(model, train_dataset, optimizer, criterion, args) evaluate(model, valid_dataset, criterion, 'Validation') evaluate(model, test_dataset, criterion, 'Test')
def test(config, model, test_data): """ test function """ test_acc, test_loss, test_report, test_confusion = evaluate(config, model, test_data, test=True) msg = '\nTest Loss: {0:>5.2}, Test Acc: {1:>6.2%}' print(msg.format(test_loss, test_acc)) print("Precision, Recall and F1-Score...") print(test_report) print("Confusion Matrix...") print(test_confusion)
def build_sensitivity_profile(model, criterion, data_loader_test): quant_layer_names = [] for name, module in model.named_modules(): if name.endswith("_quantizer"): module.disable() layer_name = name.replace("._input_quantizer", "").replace("._weight_quantizer", "") if layer_name not in quant_layer_names: quant_layer_names.append(layer_name) for i, quant_layer in enumerate(quant_layer_names): print("Enable", quant_layer) for name, module in model.named_modules(): if name.endswith("_quantizer") and quant_layer in name: module.enable() print(F"{name:40}: {module}") with torch.no_grad(): evaluate(model, criterion, data_loader_test, device="cuda") for name, module in model.named_modules(): if name.endswith("_quantizer") and quant_layer in name: module.disable() print(F"{name:40}: {module}")
def embedding_baseline_test(args): chosen_params = dict(params) results = [] for rand_emb in [True, False]: chosen_params['rand_emb'] = rand_emb train_dataset, valid_dataset, test_dataset = data.load_dataset( args.train_batch_size, args.test_batch_size, min_freq=chosen_params['min_freq']) embedding = data.generate_embedding_matrix( train_dataset.dataset.text_vocab, rand=chosen_params['rand_emb'], freeze=chosen_params['freeze']) result = {} for m in ['baseline', 'rnn']: if m == 'rnn': model = RNN(embedding, chosen_params) else: model = Baseline(embedding) criterion = nn.BCEWithLogitsLoss() optimizer = torch.optim.Adam(model.parameters(), lr=1e-4) for epoch in range(args.epochs): print(f'******* epoch: {epoch} *******') train(model, train_dataset, optimizer, criterion, args) evaluate(model, valid_dataset, criterion, 'Validation') acc, f1 = evaluate(model, test_dataset, criterion, 'Test') result[m + '_acc_rand_emb' + str(rand_emb)] = acc result[m + '_f1_rand_emb' + str(rand_emb)] = f1 results.append(result) with open(os.path.join(SAVE_DIR, 'embedding_baseline.txt'), 'a') as f: for res in results: print(res, file=f)
def test_on_fer_test_set(fer_path, model_type="CustomVGG"): start_time = time() fer = pd.read_csv(fer_path) if "attribution" not in fer: raise Exception( "Fer not split between train/val/test. Please run split_fer script." ) fer_test = fer[fer["attribution"] == "test"].reset_index() model = load_model(model_type=model_type) print("Loaded fer test set and model in {}s".format( round(time() - start_time, 2))) start_time = time() def preprocess_batch(pixelstring_batch, emotions_batch, DEVICE): if model_type == "CustomVGG": return preprocess_batch_custom_vgg(pixelstring_batch, emotions_batch, DEVICE, False, config["loss_mode"]) elif model_type == "DenseSIFTHybrid": return preprocess_batch_dense_sift_hybrid(pixelstring_batch, emotions_batch, DEVICE, False, config["loss_mode"]) elif model_type == "SIFTHybrid": return preprocess_batch_sift_hybrid(pixelstring_batch, emotions_batch, DEVICE, False, config["loss_mode"]) use_descriptors = (model_type == "DenseSIFTHybrid" or model_type == "SIFTHybrid") dummy_weights = torch.FloatTensor([1] * len(config["catslist"])).to( DEVICE) # we don't care about the test loss value here. proba, _, acc, cm1, cm2, acc_fact = evaluate( model, fer_test, preprocess_batch, dummy_weights, DEVICE, compute_cm=True, use_descriptors=use_descriptors) print("FINAL ACCURACY: {}".format(acc)) print("Average predicted proba for right class: {}".format(proba)) print("Duration on {} test faces: {}s".format( len(fer_test), round(time() - start_time, 2))) print("Accuracy with grouped classes : {}".format(acc_fact)) print("Close the confusion matrices to end the script.") plot_confusion_matrix(cm1, config["catslist"]) plot_confusion_matrix(cm2, ["bad", "good", "surprise", "neutral"])
def get_scores(self, dev_or_test, data_dir=DATA_DIR, models_dir=MODELS_BASE_DIR, device=DEVICE): if dev_or_test not in ("dev", "test"): raise ValueError # BioBERT_en id2scores_m1 = bert_predict(os.path.join(models_dir, "biobert-en"), test_or_dev=dev_or_test, use_data="en", max_seq_length=256, batch_size=16, data_dir=data_dir, device=device, return_logits=True) test_loader, dev_loader, V, Tv, C, T = get_test_data("clstm", "en", max_seq_len=256, batch_size=64, data_dir=data_dir) vocab_size = V titles_vocab_size = Tv num_classes = C T = T.to(device) model = models.ICDCodeAttentionLSTM(vocab_size, 400, num_classes, T, Tv=titles_vocab_size, h=300, bidirectional=True) model.load_state_dict( torch.load( os.path.join(MODELS_BASE_DIR, "clstm-pubmed-en", "model.pt"))) model.to(device) model.eval() if dev_or_test == "dev": data_loader = dev_loader else: data_loader = test_loader _, (scores_m2, _, _, scores_m2_ids, _) = evaluate(data_loader, model, device, no_labels=True) id2scores_m2 = { val: scores_m2[i] for i, val in enumerate(scores_m2_ids) } return id2scores_m1, id2scores_m2, num_classes
def main(args): weights_file = (args.weightsFile) positiveImagePath = (args.positiveTestImages) negativeImagePath = (args.negativeTestImages) os.system("python3 createTrainingData.py {} {} {}".format( positiveImagePath, negativeImagePath, 1)) X_LL, X_LH, X_HL, X_HH, X_index, Y, imageCount = readWaveletData( positiveImagePath, negativeImagePath, positiveTestImagePath, positiveTestImagePath) X_LL = np.array(X_LL) X_LL = X_LL.reshape((imageCount, height, width, depth)) X_LH = np.array(X_LH) X_LH = X_LH.reshape((imageCount, height, width, depth)) X_HL = np.array(X_HL) X_HL = X_HL.reshape((imageCount, height, width, depth)) X_HH = np.array(X_HH) X_HH = X_HH.reshape((imageCount, height, width, depth)) CNN_model = createModel(height, width, depth, num_classes) CNN_model.load_weights(weights_file) evaluate(CNN_model, X_LL, X_LH, X_HL, X_HH, Y)
def main(): holdout_test = True fine_tune_task = config["task"] logger.info("init model...") pretrained_name = get_pretrained_name(config["model_name"]) if fine_tune_task == "paws": num_labels = 2 train_loader, dev_loader = load_paws(config["PAWS-QQP"], balanced=config["balanced"]) elif fine_tune_task == "standsent": num_labels = 5 train_loader, dev_loader, test_loader = load_stanford( config["STANFORD_LOC"], phrase_len=config["phrase_len"], reserve_test=holdout_test) else: logger.error("unsupport fine tune task: {}".format(fine_tune_task)) model, tokenizer = load_model(pretrained_name=pretrained_name, load_tuned=False, num_labels=num_labels) optimizer = AdamW(model.parameters(), lr=1e-5) logger.info("training...") model.train() init_prec, init_loss, best_prec, best_loss, best_model = train_iter( model, tokenizer, optimizer, train_loader, dev_loader, task=fine_tune_task, early_stopping=True, max_epochs=config["n_epochs"], print_every=config["print_every"], evaluate_every=config["evaluate_every"]) logger.info("done training.") training_info_str = \ """ training summary: training loss {} -> {} test precision {} -> {} """.format(init_loss, best_loss, init_prec, best_prec) logger.info(training_info_str) if holdout_test and (fine_tune_task == "standsent"): # evaluate on holdout test set (with phrases of various lengths) test_prec = evaluate(best_model, tokenizer, test_loader, fine_tune_task) logger.info("precision on holdout test: {}".format(test_prec))
def train(self, args, model_class: nn.Module, train) -> nn.Module: """Trains a model locally using Hogwild! Args: model_class (nn.Module): The class of a PyTorch model. train: A function that trains the model instance locally. It must also encapsulate the dataloading. """ torch.manual_seed(args.seed) model = model_class() model.share_memory( ) # gradients are allocated lazily, so they are not shared here processes = [] if args.capture_results: start = timer() for rank in range(args.num_processes): p = mp.Process(target=train, args=(rank, args, model)) p.start() processes.append(p) for p in processes: p.join() acc, loss = evaluate(args, model) if args.capture_results: end = timer() acc, loss = evaluate(args, model) t = end - start print('{} {} {}'.format(acc, loss, t)) return model
def main(): # reproducibility seed = CONFIG['RANDOM_SEED'] set_random_seed(seed) # load dataset train_dataset, test_dataset, val_dataset = generate_data( CONFIG['TRAIN_PATH'], CONFIG['TEST_PATH'], CONFIG['TEST_PATH'], CONFIG['INPUT_SIZE']) save_dir = os.path.split(CONFIG['SAVE_PATH'])[0] if not os.path.exists(save_dir): os.makedirs(save_dir) model = MyModel(bbr_net.resnet_bbr, CONFIG['BOTTLENECK_SIZE'], CONFIG['NUM_CLASS'], pretrained=False).cuda() # train model, record_epochs, val_ious, losses = train( model=model, train_dataset=train_dataset, val_dataset=val_dataset, epochs=CONFIG['EPOCHS'], learning_rate=CONFIG['LEARNING_RATE'], batch_size=CONFIG['BATCH_SIZE'], save_path=CONFIG['SAVE_PATH']) pickle.dump((record_epochs, val_ious, losses), open(CONFIG['RECORD_PATH'], 'wb')) # test visualized_images_path = CONFIG['VISUALIZED_IMAGES_PATH'] if not os.path.exists(visualized_images_path): os.makedirs(visualized_images_path) evaluate(CONFIG['SAVE_PATH'], test_dataset, visualized_images_path)
def evaluate_hw2(cfg: DictConfig) -> float: main_utils.init(cfg) load_v2() # Load dataset path_image_train = '/datashare/train2014/COCO_train2014_' path_question_train = '/datashare/v2_OpenEnded_mscoco_train2014_questions.json' train_dataset = VQADataset(path_answers=cfg['main']['paths']['train'], path_image=path_image_train, path_questions=path_question_train) path_image_val = '/datashare/val2014/COCO_val2014_' path_question_train = '/datashare/v2_OpenEnded_mscoco_val2014_questions.json' val_dataset = VQADataset(path_answers=cfg['main']['paths']['validation'], path_image=path_image_val, path_questions=path_question_train, word_dict=train_dataset.word_dict) eval_loader = DataLoader(val_dataset, cfg['train']['batch_size'], shuffle=True, num_workers=cfg['main']['num_workers']) image_dim = train_dataset.pic_size output_dim = 2410 model = VQAModel(batch_size=cfg['train']['batch_size'], word_vocab_size=train_dataset.vocab_size, lstm_hidden=cfg['train']['num_hid'], output_dim=output_dim, dropout=cfg['train']['dropout'], word_embedding_dim=cfg['train']['word_embedding_dim'], question_output_dim=cfg['train']['question_output_dim'], image_dim=image_dim, last_hidden_fc_dim=cfg['train']['last_hidden_fc_dim']) if torch.cuda.is_available(): model = model.cuda() model.load_state_dict( torch.load('model.pkl', map_location=lambda storage, loc: storage)['model_state']) model.train(False) eval_score, eval_loss = evaluate(model, eval_loader) print(f"The evaluation score is {eval_score}") return eval_score
def main(opts): train_dataset = MNIST('dataset/mega_mnist', train=True) train_loader = DataLoader(train_dataset, batch_size=opts.batch_size, shuffle=True, num_workers=1) test_dataset = MNIST('dataset/mega_mnist', train=False) test_loader = DataLoader(test_dataset, shuffle=False, batch_size=opts.batch_size, num_workers=1) attention_model = AttentionModelMNIST(squeeze_channels=True, softmax_smoothing=1e-4) feature_model = FeatureModelMNIST(in_channels=1) classification_head = ClassificationHead(in_channels=32, num_classes=10) ats_model = ATSModel(attention_model, feature_model, classification_head, n_patches=opts.n_patches, patch_size=opts.patch_size) ats_model = ats_model.to(opts.device) optimizer = optim.Adam(ats_model.parameters(), lr=opts.lr) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=opts.decrease_lr_at, gamma=0.1) logger = AttentionSaverMNIST(opts.output_dir, ats_model, test_dataset, opts) criterion = nn.CrossEntropyLoss() entropy_loss_func = MultinomialEntropy(opts.regularizer_strength) for epoch in range(opts.epochs): train_loss, train_metrics = train(ats_model, optimizer, train_loader, criterion, entropy_loss_func, opts) with torch.no_grad(): test_loss, test_metrics = evaluate(ats_model, test_loader, criterion, entropy_loss_func, opts) logger(epoch, (train_loss, test_loss), (train_metrics, test_metrics)) scheduler.step()
def test_model_loss(loader, output_lang, encoder, decoder): encoder.eval() decoder.eval() total = 0 total_loss = 0 for i, (input_batches, input_lengths, target_batches, target_lengths) in enumerate(loader): input_batches, input_lengths, target_batches, target_lengths = input_batches.to(device), input_lengths.to(device), target_batches.to(device), target_lengths.to(device) decoded_words, all_decoder_outputs = evaluate(input_batches, input_lengths, output_lang, target_batches, target_lengths, encoder, decoder) #print(all_decoder_outputs.shape) loss = masked_cross_entropy( all_decoder_outputs.contiguous(), target_batches.contiguous(), target_lengths) total_loss += loss.item() total += 1 return (total_loss / total)
q_net.cuda() agent = dqn.DQNAgent(q_net, args.double_dqn, train_env.num_actions) if args.noisy_net: train_policy = GreedyEpsilonPolicy(0, agent) else: train_policy = LinearDecayGreedyEpsilonPolicy( args.train_start_eps, args.train_final_eps, args.train_eps_num_steps, agent) eval_policy = GreedyEpsilonPolicy(args.eval_eps, agent) replay_memory = ReplayMemory(args.replay_buffer_size) replay_memory.burn_in(train_env, agent, args.burn_in_frames) evaluator = lambda logger: evaluate(eval_env, eval_policy, 10, logger) train(agent, train_env, train_policy, replay_memory, args.gamma, args.batch_size, args.num_iters, args.frames_per_update, args.frames_per_sync, args.frames_per_eval, evaluator, args.output)