def train_results(model, train_test_data): results = pd.DataFrame( {}, columns=[ 'train_size', 'train_support', 'train_accuracy', 'train_precision', 'train_recall', 'train_f1', 'train_roc', 'TrueNeg', 'FalsePos', 'FalseNeg', 'TruePos', 'size', 'support', 'accuracy', 'precision', 'recall', 'f1', 'roc', 'top_ft_1', 'top_inf_1', 'top_ft_2', 'top_inf_2', 'top_ft_3', 'top_inf_3' ]) (x_train, y_train), (x_test, y_test) = train_test_data y_train_pred = model_predict(model, x_train) y_test_pred = model_predict(model, x_test) for i, vul in enumerate(ALL_VULS): try: train_score_ = trainer.evaluate(y_train[:, i], y_train_pred[:, i]) test_score_ = trainer.evaluate(y_test[:, i], y_test_pred[:, i]) confusion_matrix_ = confusion_matrix(y_test[:, i], y_test_pred[:, i]) influence_ = np.array([['', 0], ['', 0], ['', 0]]) result = [ *train_score_, *confusion_matrix_.reshape(-1), *test_score_, *influence_.reshape(6) ] results.loc[vul] = result except Exception as ex: print(vul, '\terror: %s' % ex) return results
def run(model, val_loader, clusters, prob_thresh, nms_thresh, predictions_file, multiscale=False): if osp.exists(predictions_file): os.remove(predictions_file) if multiscale: trainer.evaluate_multiscale(model, val_loader, clusters, prob_thresh=prob_thresh, nms_thresh=nms_thresh) else: trainer.evaluate(model, val_loader, clusters, prob_thresh=prob_thresh, nms_thresh=nms_thresh)
def evaluate_single(run_name, model_name): # Get model path cur_dir = os.getcwd() model_path = "%s\\Runs\\%s\\Models\\%s.h5" % (cur_dir, run_name, model_name) assert os.path.exists(model_path), "Model does not exist." # Load Test Data print("Loading test data...", end="", flush=True) test_ids, test_images_raw = fileutils.read_test_data_raw('./Data/test.csv') print("done.") # Normalize print("Normalizing data...", end="", flush=True) reshaped_images = processing.reshape_images(test_images_raw) test_images = processing.normalize_images(reshaped_images) print("done.") # Load best print("Evaluating test set...") _, test_labels = trainer.evaluate(test_images, model_path) print("Generating classification CSV...", end="", flush=True) fileutils.generate_classification(test_ids, test_labels, run_name) print("done.")
def main(config): logger = prepare_logger(config) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # get loaders if not config.is_train_source: target_loader = get_loader(type="MNIST", train=False, batch_size=config.batch_size) source_train_loader = get_loader(type="SVHN", train=True, batch_size=config.batch_size) source_test_loader = get_loader(type="SVHN", train=False, batch_size=config.batch_size) # build source classifier model_src = LeNet(config.num_gpus).to(device) if (not config.is_train_source) or config.is_finetune: model_src.load_state_dict(torch.load(config.model_dir)) # train source classifier if config.is_train_source: logger.info("train source classifier..") train_source(model_src, source_train_loader, source_test_loader, config, logger) logger.info("evaluate source classifier..") logger.info("test accurracy in source domain: %f\n" % (evaluate(model_src, source_test_loader))) else: # initialize target classifer with source classifer model_trg = torch.load(open("./pretrained/lenet-source.pth", "rb")) # build discriminator D = Discriminator(config.num_gpus) # adaptation process logger.info("start adaptation process..") adapt_target_domain(D, model_src, model_trg, source_train_loader, target_loader, config) logger.info("evaluate target classifier..") logger.info("accurracy in target domain: %f\n" % (evaluate(model_trg, target_loader)))
def main(args): model = RCNN(vocab_size=args.vocab_size, embedding_dim=args.embedding_dim, hidden_size=args.hidden_size, hidden_size_linear=args.hidden_size_linear, class_num=args.class_num, dropout=args.dropout).to(args.device) if args.n_gpu > 1: model = torch.nn.DataParallel(model, dim=0) train_texts, train_labels = read_file(args.train_file_path) word2idx = build_dictionary(train_texts, vocab_size=args.vocab_size) logger.info('Dictionary Finished!') full_dataset = CustomTextDataset(train_texts, train_labels, word2idx) num_train_data = len(full_dataset) - args.num_val_data train_dataset, val_dataset = random_split( full_dataset, [num_train_data, args.num_val_data]) train_dataloader = DataLoader(dataset=train_dataset, collate_fn=lambda x: collate_fn(x, args), batch_size=args.batch_size, shuffle=True) valid_dataloader = DataLoader(dataset=val_dataset, collate_fn=lambda x: collate_fn(x, args), batch_size=args.batch_size, shuffle=True) optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) train(model, optimizer, train_dataloader, valid_dataloader, args) logger.info('******************** Train Finished ********************') # Test if args.test_set: test_texts, test_labels = read_file(args.test_file_path) test_dataset = CustomTextDataset(test_texts, test_labels, word2idx) test_dataloader = DataLoader(dataset=test_dataset, collate_fn=lambda x: collate_fn(x, args), batch_size=args.batch_size, shuffle=True) model.load_state_dict( torch.load(os.path.join(args.model_save_path, "best.pt"))) _, accuracy, precision, recall, f1, cm = evaluate( model, test_dataloader, args) logger.info('-' * 50) logger.info( f'|* TEST SET *| |ACC| {accuracy:>.4f} |PRECISION| {precision:>.4f} |RECALL| {recall:>.4f} |F1| {f1:>.4f}' ) logger.info('-' * 50) logger.info('---------------- CONFUSION MATRIX ----------------') for i in range(len(cm)): logger.info(cm[i]) logger.info('--------------------------------------------------')
def run_baseline(output_dir, num_models=3, max_train_samples=None, epochs=3, finetune=False, uncertainty_strategy='best'): started = time.time() model_dirs = [] for i in range(num_models): model_dir = os.path.join(output_dir, f'm{i + 1}') model_dirs.append(model_dir) for model_dir in model_dirs: print("=== Training model", model_dir, "===") trainer = Trainer(max_train_samples=max_train_samples, epochs=epochs, finetune=finetune, uncertainty_strategy=uncertainty_strategy, output_path=model_dir, arch="resnet", layers=18) trainer.train() y_true, y_pred = trainer.evaluate() plot_roc_auc(y_true, y_pred, save_to_file=True, output_path=model_dir) print("=== Completed training of", model_dir, "===") display_elapsed_time(started, "Total elapsed") print() ensemble = load_ensemble_from_dirs(model_dirs) results = evaluate(model=ensemble, dataloader=get_val_loader(), device=get_device()) labels = results['labels'] preds = results['predictions'] final_auc = mt.roc_auc_score(labels, preds) print("Ensemble Validation AUC Score", final_auc) plot_roc_auc(labels, preds, save_to_file=True, output_path=output_dir) display_elapsed_time(started, "Total time taken")
started = time.time() model_dirs = args.output_path.split(",") for model_dir in model_dirs: print("=== Training model", model_dir, "===") trainer = Trainer(max_train_samples=args.max_train_samples, epochs=args.epochs, finetune=args.finetune, uncertainty_strategy=args.uncertainty_strategy, output_path=model_dir) trainer.train() y_true, y_pred = trainer.evaluate() plot_roc_auc(y_true, y_pred, save_to_file=True, output_path=model_dir) print("=== Completed training of", model_dir, "===") display_elapsed_time(started, "Total elapsed") print() ensemble = load_ensemble_from_dirs(model_dirs) results = evaluate(model=ensemble, dataloader=get_val_loader(), device=get_device()) labels = results['labels'] preds = results['predictions'] final_auc = mt.roc_auc_score(labels, preds) print("Ensemble Validation AUC Score", final_auc) plot_roc_auc(labels, preds, save_to_file=True, output_path="./models/baseline") display_elapsed_time(started, "Total time taken")
def run(model, val_loader, templates, prob_thresh, nms_thresh, device): dets = trainer.evaluate(model, val_loader, templates, prob_thresh, nms_thresh, device) return dets
dev_dataloader = DataLoader(utils.MyDataset(dev_features), batch_size=args.eval_batch_size, shuffle=False, collate_fn=utils.TextPairCollate()) optimizer = optim.Adam(model.parameters(), lr=args.lr,) best_model, best_val_result = trainer.train( model, args.num_train_epochs, train_dataloader, dev_dataloader, loss_fn, optimizer, utils.acc_p_r_f1, device) utils.save_experiment(config, best_model, best_val_result, args.output_dir) tokenizer.save_vocab(args.output_dir) if args.do_eval: if args.do_train: args.model_dir = args.output_dir test_examples = processor.get_test_examples() test_features = utils.lcqmc_examples_to_features( test_examples, label2id, tokenizer, max_len=args.max_len, verbose=True) test_dataloader = DataLoader(utils.MyDataset(test_features), batch_size=args.eval_batch_size, shuffle=False, collate_fn=utils.TextPairCollate()) model.load_state_dict(torch.load( os.path.join(args.model_dir, 'pytorch_model.bin'))) test_result = trainer.evaluate(model, test_dataloader, loss_fn, utils.acc_p_r_f1, device, ) logger.info('*** Test result ***') for k, v in test_result.items(): logger.info('{}: {}'.format(k, v))
def main(args): acc_list = [] f1_score_list = [] prec_list = [] recall_list = [] for i in range(10): setup_data() model = RCNN(vocab_size=args.vocab_size, embedding_dim=args.embedding_dim, hidden_size=args.hidden_size, hidden_size_linear=args.hidden_size_linear, class_num=args.class_num, dropout=args.dropout).to(args.device) if args.n_gpu > 1: model = torch.nn.DataParallel(model, dim=0) train_texts, train_labels = read_file(args.train_file_path) word2idx, embedding = build_dictionary(train_texts, args.vocab_size, args.lexical, args.syntactic, args.semantic) logger.info('Dictionary Finished!') full_dataset = CustomTextDataset(train_texts, train_labels, word2idx, args) num_train_data = len(full_dataset) - args.num_val_data train_dataset, val_dataset = random_split( full_dataset, [num_train_data, args.num_val_data]) train_dataloader = DataLoader(dataset=train_dataset, collate_fn=lambda x: collate_fn(x, args), batch_size=args.batch_size, shuffle=True) valid_dataloader = DataLoader(dataset=val_dataset, collate_fn=lambda x: collate_fn(x, args), batch_size=args.batch_size, shuffle=True) optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) train(model, optimizer, train_dataloader, valid_dataloader, embedding, args) logger.info('******************** Train Finished ********************') # Test if args.test_set: test_texts, test_labels = read_file(args.test_file_path) test_dataset = CustomTextDataset(test_texts, test_labels, word2idx, args) test_dataloader = DataLoader( dataset=test_dataset, collate_fn=lambda x: collate_fn(x, args), batch_size=args.batch_size, shuffle=True) model.load_state_dict( torch.load(os.path.join(args.model_save_path, "best.pt"))) _, accuracy, precision, recall, f1, cm = evaluate( model, test_dataloader, embedding, args) logger.info('-' * 50) logger.info( f'|* TEST SET *| |ACC| {accuracy:>.4f} |PRECISION| {precision:>.4f} |RECALL| {recall:>.4f} |F1| {f1:>.4f}' ) logger.info('-' * 50) logger.info('---------------- CONFUSION MATRIX ----------------') for i in range(len(cm)): logger.info(cm[i]) logger.info('--------------------------------------------------') acc_list.append(accuracy / 100) prec_list.append(precision) recall_list.append(recall) f1_score_list.append(f1) avg_acc = sum(acc_list) / len(acc_list) avg_prec = sum(prec_list) / len(prec_list) avg_recall = sum(recall_list) / len(recall_list) avg_f1_score = sum(f1_score_list) / len(f1_score_list) logger.info('--------------------------------------------------') logger.info( f'|* TEST SET *| |Avg ACC| {avg_acc:>.4f} |Avg PRECISION| {avg_prec:>.4f} |Avg RECALL| {avg_recall:>.4f} |Avg F1| {avg_f1_score:>.4f}' ) logger.info('--------------------------------------------------') plot_df = pd.DataFrame({ 'x_values': range(10), 'avg_acc': acc_list, 'avg_prec': prec_list, 'avg_recall': recall_list, 'avg_f1_score': f1_score_list }) plt.plot('x_values', 'avg_acc', data=plot_df, marker='o', markerfacecolor='blue', markersize=12, color='skyblue', linewidth=4) plt.plot('x_values', 'avg_prec', data=plot_df, marker='', color='olive', linewidth=2) plt.plot('x_values', 'avg_recall', data=plot_df, marker='', color='olive', linewidth=2, linestyle='dashed') plt.plot('x_values', 'avg_f1_score', data=plot_df, marker='', color='olive', linewidth=2, linestyle='dashed') plt.legend() fname = 'lexical-semantic-syntactic.png' if args.lexical and args.semantic and args.syntactic \ else 'semantic-syntactic.png' if args.semantic and args.syntactic \ else 'lexical-semantic.png' if args.lexical and args.semantic \ else 'lexical-syntactic.png'if args.lexical and args.syntactic \ else 'lexical.png' if args.lexical \ else 'syntactic.png' if args.syntactic \ else 'semantic.png' if args.semantic \ else 'plain.png' if not (path.exists('./images')): mkdir('./images') plt.savefig(path.join('./images', fname))
def main(): args = parse_args() # Set the GPU to use torch.cuda.set_device(args.gpu) annotations = osp.expanduser(args.annotations) questions = osp.expanduser(args.questions) vqa_loader = dataset.get_train_dataloader(annotations, questions, args.images, args) # We always use the vocab from the training set vocab = vqa_loader.dataset.vocab maps = { "word_to_wid": vqa_loader.dataset.word_to_wid, "wid_to_word": vqa_loader.dataset.wid_to_word, "ans_to_aid": vqa_loader.dataset.ans_to_aid, "aid_to_ans": vqa_loader.dataset.aid_to_ans, } val_loader = dataset.get_val_dataloader(osp.expanduser( args.val_annotations), osp.expanduser(args.val_questions), args.val_images, args, maps=maps, vocab=vocab, shuffle=False) arch = Models[args.arch].value model = arch(len(vocab), output_dim=args.top_answer_limit) if torch.cuda.is_available(): model.cuda() criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=tuple(args.betas), weight_decay=args.weight_decay) scheduler = lr_scheduler.StepLR(optimizer, step_size=args.decay_interval, gamma=args.lr_decay) vis = visualize.Visualizer(args.port) print("Beginning training") print("#" * 80) for epoch in range(args.start_epoch, args.epochs): scheduler.step() trainer.train(model, vqa_loader, criterion, optimizer, epoch, args, vis=vis) trainer.evaluate(model, val_loader, criterion, epoch, args, vis=vis) print("Training complete!")
def evaluate_multiple(run_names, model_names, categories, backup_model): runs_len = len(run_names) models_len = len(model_names) categories_len = len(categories) assert runs_len == models_len and models_len == categories_len, "Runs, models, and categories must be the same length." # Get model paths cur_dir = os.getcwd() model_paths = list() for i in range(runs_len): run_name = run_names[i] model_name = model_names[i] model_path = "%s\\Runs\\%s\\Models\\%s.h5" % (cur_dir, run_name, model_name) assert os.path.exists( model_path), "Model for %s does not exist." % run_name model_paths.append(model_path) # Load Test Data print("Loading test data...", end="", flush=True) test_ids, test_images_raw = fileutils.read_test_data_raw('./Data/test.csv') print("done.") # Normalize print("Normalizing data...", end="", flush=True) test_images = processing.normalize_images(test_images_raw) print("done.") # Load best print("Creating evaluations...") eval_labels = list() for i in range(runs_len): predictions, _ = trainer.evaluate(test_images, model_paths[i]) print("Creating evaluations... (%s/%s)" % (i + 1, runs_len)) eval_labels.append(predictions) print("Creating backup evaluations...") backup_confidence, backup_predictions = trainer.evaluate( test_images, backup_model) print("Generating labels...") test_predictions = [0] * test_ids.shape[0] for j in range(len(test_predictions)): # Check each prediction and get highest confidence, also count conflicts conflicts = 0 highest_confidence = -1 highest_confidence_label = -1 for k in range(runs_len): cur_conf = eval_labels[k][j][0] cur_not_conf = eval_labels[k][j][1] if cur_conf > cur_not_conf and cur_conf > highest_confidence: highest_confidence = cur_conf highest_confidence_label = categories[k] if highest_confidence_label != -1: conflicts = conflicts + 1 # Maybe do something with predictions based off of conflicts if highest_confidence_label == -1: # None of the invidual models think its that classification, refer to backup model highest_confidence_label = backup_predictions[j] highest_confidence = backup_confidence[k][highest_confidence_label] test_predictions[j] = highest_confidence_label print("Generating classification CSV...", end="", flush=True) fileutils.generate_classification(test_ids, test_predictions, run_names[0]) print("done.")
def run_train(seed, train_dataset, valid_dataset, test_dataset, param_dict): reset_seed(seed) training_param_dict = param_dict['training_param'] train_dataloader = get_dataloader(train_dataset, shuffle=True, param_dict=param_dict) valid_dataloader = get_dataloader(valid_dataset, shuffle=False, param_dict=param_dict) test_dataloader = get_dataloader(test_dataset, shuffle=False, param_dict=param_dict) model = MloClassifier(param_dict) print(model) model = model.cuda() criterion = torch.nn.BCEWithLogitsLoss() criterion = criterion.cuda() optimizer = torch.optim.AdamW( model.parameters(), lr=training_param_dict['learning_rate'], weight_decay=training_param_dict['weight_decay']) scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=1) optimizer.zero_grad() max_auc = -1 max_model = None for epoch in tqdm(range(training_param_dict['num_epochs'])): model, optimizer, scheduler, train_loss = train( model, optimizer, scheduler, criterion, train_dataloader) #print(f'{epoch}th Epoch Train Loss: {train_loss}') eval_loss, pred_auc, prob_auc = evaluate(model, criterion, valid_dataloader, test=False) #print(f'{epoch}th Epoch Valid Loss: {eval_loss}, Pred AUC: {pred_auc}, Prob AUC: {prob_auc}') if pred_auc > max_auc: max_auc = pred_auc max_model = copy.deepcopy(model) eval_loss, pred_auc, prob_auc = evaluate(max_model, criterion, test_dataloader, test=True) print( f'Evaluation on Test Dataset: Pred AUC: {pred_auc:.04f}, Prob AUC: {prob_auc:.04f}' ) rd = param_dict['rep_dim'] mp = param_dict['model_param'] tp = param_dict['training_param'] save_str = 'model' save_str += f'_feature_dim_{rd["feature"]}' save_str += f'_electra_dim_{rd["electra"]}' save_str += f'_liwc_dim_{rd["liwc"]}' save_str += f'_vader_dim{rd["vader"]}' save_str += f'_liwc_leaves_dim_{rd["liwc_leaves"]}' save_str += f'_vader_leaves_dim_{rd["vader_leaves"]}' save_str += f'_final_dim_{rd["final"]}' save_str += f'_use_attention_{mp["use_attention"]}' save_str += f'_dropout_{mp["dropout"]}' save_str += f'_seed_{tp["seed"]}' save_str += f'_learning_rate_{tp["learning_rate"]}' save_str += f'_batch_size_{tp["batch_size"]}' save_str += f'_weight_decay_{tp["weight_decay"]}' save_str += f'_num_epochs_{tp["num_epochs"]}' torch.save(model.state_dict(), f'./saved_models/{save_str}') return pred_auc, prob_auc, max_model
def main(config): logger = config.get_logger('test') # setup data_loader instances batch_size = 1 if config['data_loader_val']['args']['max_dataset_size'] == 'inf': max_dataset_size = float('inf') else: max_dataset_size = config['data_loader_val']['args'][ 'max_dataset_size'] data_loader = getattr(module_data, config['data_loader_val']['type'])( img_dir=config['data_loader_val']['args']['img_dir'], label_dir=config['data_loader_val']['args']['label_dir'], batch_size=batch_size, years=config['data_loader_val']['args']['years'], max_dataset_size=max_dataset_size, shuffle=False, num_workers=1, ) landsat_mean, landsat_std = (0.3326, 0.3570, 0.2224), (0.1059, 0.1086, 0.1283) # build model architecture model = config.initialize('arch', module_arch) logger.info(model) # get function handles of loss and metrics loss_fn = config.initialize('loss', module_loss) # loss_fn = getattr(module_loss, config['loss']) metric_fns = [getattr(module_metric, met) for met in config['metrics']] logger.info('Loading checkpoint: {} ...'.format(config.resume)) checkpoint = torch.load(config.resume) state_dict = checkpoint['state_dict'] if config['n_gpu'] > 1: model = torch.nn.DataParallel(model) model.load_state_dict(state_dict) # prepare model for testing device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') model = model.to(device) model.eval() total_loss = 0.0 total_metrics = torch.zeros(len(metric_fns)) pred_dir = '/'.join(str(config.resume.absolute()).split('/')[:-1]) out_dir = os.path.join( pred_dir, get_output_dir(config['data_loader_val']['args']['img_dir'])) if not os.path.isdir(pred_dir): os.makedirs(pred_dir) if not os.path.isdir(out_dir): os.makedirs(out_dir) hist = np.zeros((2, 2)) with torch.no_grad(): for i, batch in enumerate(tqdm(data_loader)): # for i, (data, target) in enumerate(tqdm(data_loader)): init_time = time.time() loss = None data, target = batch udata = normalize_inverse(data, landsat_mean, landsat_std) data, target = data.to(device, dtype=torch.float), target.to( device, dtype=torch.float) output = model(data) output_probs = F.sigmoid(output) binary_target = threshold_outputs( target.data.cpu().numpy().flatten()) output_binary = threshold_outputs( output_probs.data.cpu().numpy().flatten()) np.save('planet_prediction_{}.npy'.format(i), output_binary.reshape(-1, 1, 256, 256)[0, 0, :, :]) # print(output_binary.shape, 'SHAPEEE') hist += fast_hist(output_binary, binary_target) images = { 'img': udata.cpu().numpy(), 'gt': target.cpu().numpy(), 'pred': output_binary.reshape(-1, 1, 256, 256), } # Save single images (C=3) or double images (C=6) if images['img'].shape[1] == 3: save_simple_images(3, images, out_dir, i * batch_size) else: save_double_images(3, images, out_dir, i * batch_size) # computing loss, metrics on test set loss = loss_fn(output, target) batch_size = data.shape[0] total_loss += loss.item() * batch_size # Update binary segmentation metrics acc, acc_cls, mean_iu, fwavacc, precision, recall, f1_score = \ evaluate(hist=hist) n_samples = len(data_loader.sampler) log = { 'loss': total_loss / n_samples, 'acc': acc, 'mean_iu': mean_iu, 'fwavacc': fwavacc, 'precision': precision, 'recall': recall, 'f1_score': f1_score } logger.info(log)
def main(): args = parse_args() # Set the GPU to use torch.cuda.set_device(args.gpu) transform = transforms.Compose([ transforms.Resize(256), transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) vqa_loader = dataset.get_train_dataloader(osp.expanduser(args.annotations), osp.expanduser(args.questions), args.images, args, raw_images=args.raw_images, transforms=transform) # We always use the vocab from the training set vocab = vqa_loader.dataset.vocab maps = { "vocab": vocab, "word_to_wid": vqa_loader.dataset.word_to_wid, "wid_to_word": vqa_loader.dataset.wid_to_word, "ans_to_aid": vqa_loader.dataset.ans_to_aid, "aid_to_ans": vqa_loader.dataset.aid_to_ans, } val_transform = transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) val_loader = dataset.get_val_dataloader(osp.expanduser( args.val_annotations), osp.expanduser(args.val_questions), args.val_images, args, raw_images=args.raw_images, maps=maps, vocab=vocab, shuffle=False, transforms=val_transform) arch = Models[args.arch].value model = arch(len(vocab), output_dim=args.top_answer_limit, raw_images=args.raw_images) if args.resume: state = torch.load(args.resume) model.load_state_dict(state["model"]) model.cuda() criterion = nn.CrossEntropyLoss().cuda() optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=tuple(args.betas), weight_decay=args.weight_decay) scheduler = lr_scheduler.StepLR(optimizer, step_size=args.decay_interval, gamma=args.lr_decay) if args.visualize: vis = visualize.Visualizer(args.port) else: vis = None print("Beginning training") print("#" * 80) for epoch in range(args.start_epoch, args.epochs): scheduler.step() trainer.train(model, vqa_loader, criterion, optimizer, epoch, args, vis=vis) trainer.evaluate(model, val_loader, criterion, epoch, args, vis=vis) print("Training complete!")
logger.info("***** Running predictions *****") logger.info(" Num orig examples = %d", len(examples)) logger.info(" Num split examples = %d", len(features)) logger.info(" Batch size = %d", args.batch_size) # Run prediction for full data dataloader = DataLoader(dataset, batch_size=args.batch_size, num_workers=os.cpu_count()) model.frozen_bert = False metrics = evaluate( args, model, tqdm(dataloader, desc="Prediction"), output_composer=output_composer, sequence_metrics=SequenceMetrics([]), # Empty metrics reset=True, ) # Get predictions for all examples all_y_pred_raw = output_composer.get_outputs() # Filter invalid predictions all_y_pred = [ tag_encoder.decode_valid(y_pred) for y_pred in all_y_pred_raw ] # Write predictions to output file if args.output_format == 'conll': write_conll_prediction_file(args.output_file, examples, all_y_pred)
# embed_size=len(dct), # device=device) # model = Bi_RNN_ATTN(class_num=len(CATEGIRY_LIST), # embed_size=len(dct), # embed_dim=64, # device=device) lr = 0.001 optimizer = optim.Adam(model.parameters(), lr=lr) # train logger.info('training...') history = trainer.train(model, optimizer, train_dl, valid_dl, device=device, epochs=5) # evaluate loss, acc = trainer.evaluate(model, valid_dl, device=device) # predict logger.info('predicting...') y_pred = trainer.predict(model, test_dl, device=device) y_true = test_ds.labels test_acc = (y_true == y_pred).sum() / y_pred.shape[0] logger.info('test - acc: {}'.format(test_acc))
import time # criterion = cal_loss criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters()) train_losses = [] test_losses = [] for epoch in range(1, N_EPOCHS + 1): train_iter = gen_bptt_iter(train_data, BATCH_SIZE, SEQ_LEN, DEVICE) valid_iter = gen_bptt_iter(valid_data, BATCH_SIZE, SEQ_LEN, DEVICE) start_time = time.time() train_loss = train(model, train_iter, optimizer, criterion, CLIP) valid_loss = evaluate(model, valid_iter, criterion) end_time = time.time() epoch_mins, epoch_secs = epoch_time(start_time, end_time) print(f'Epoch: {epoch:02} | Time: {epoch_mins}m {epoch_secs}s') print( f'\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}' ) print( f'\t Val. Loss: {valid_loss:.3f} | Val. PPL: {math.exp(valid_loss):7.3f}' ) train_losses.append(train_loss) test_losses.append(valid_loss) test_iter = gen_bptt_iter(test_data, BATCH_SIZE, SEQ_LEN, DEVICE) test_loss = evaluate(model, test_iter, criterion)
def main(config): logger = config.get_logger('test') # setup data_loader instances batch_size = 1 if config['data_loader_val']['args']['max_dataset_size'] == 'inf': max_dataset_size = float('inf') else: max_dataset_size = config['data_loader_val']['args'][ 'max_dataset_size'] data_loader = getattr(module_data, config['data_loader_val']['type'])( img_dir=config['data_loader_val']['args']['img_dir'], label_dir=config['data_loader_val']['args']['label_dir'], video_dir=config['data_loader_val']['args']['video_dir'], batch_size=batch_size, max_dataset_size=max_dataset_size, shuffle=False, num_workers=1) landsat_mean, landsat_std = (0.3326, 0.3570, 0.2224), (0.1059, 0.1086, 0.1283) # build model architecture model = config.initialize('arch', module_arch) logger.info(model) # get function handles of loss and metrics loss_fn = config.initialize('loss', module_loss) # loss_fn = getattr(module_loss, config['loss']) metric_fns = [getattr(module_metric, met) for met in config['metrics']] logger.info('Loading checkpoint: {} ...'.format(config.resume)) checkpoint = torch.load(config.resume) state_dict = checkpoint['state_dict'] if config['n_gpu'] > 1: model = torch.nn.DataParallel(model) model.load_state_dict(state_dict) # prepare model for testing device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') model = model.to(device) model.eval() total_loss = 0.0 total_metrics = torch.zeros(len(metric_fns)) pred_dir = '/'.join(str(config.resume.absolute()).split('/')[:-1]) # pred_dir = os.path.join(pred_dir, 'predictions') # out_dir = os.path.join(pred_dir, 'video_loss_last_three') out_dir = os.path.join(pred_dir, 'rm') if not os.path.isdir(pred_dir): os.makedirs(pred_dir) if not os.path.isdir(out_dir): os.makedirs(out_dir) # out_dir = '/'.join(str(config.resume.absolute()).split('/')[:-1]) # out_dir = os.path.join(out_dir, 'predictions') hist = np.zeros((2, 2)) hist2013 = np.zeros((2, 2)) hist2014 = np.zeros((2, 2)) hist2015 = np.zeros((2, 2)) hist2016 = np.zeros((2, 2)) hist2017 = np.zeros((2, 2)) # This script only supports batch=1 with torch.no_grad(): for i, batch in enumerate(tqdm(data_loader)): if i not in [0, 84, 55]: continue # if i not in [18, 43, 51, 61, 73, 84, 85, 88, 116, 124, 198, 201, 214, 245, 325, 330]: # if i not in [84, 85, 88, 116, 124, 198, 201, 214, 245, 325]: # continue # for i, (data, target) in enumerate(tqdm(data_loader)): init_time = time.time() loss = None img_arr2013, mask_arr2013 = batch['2013']['img_arr'], batch[ '2013']['mask_arr'] img_arr2014, mask_arr2014 = batch['2014']['img_arr'], batch[ '2014']['mask_arr'] img_arr2015, mask_arr2015 = batch['2015']['img_arr'], batch[ '2015']['mask_arr'] img_arr2016, mask_arr2016 = batch['2016']['img_arr'], batch[ '2016']['mask_arr'] img_arr2017, mask_arr2017 = batch['2017']['img_arr'], batch[ '2017']['mask_arr'] img_arr2015p, _ = batch['2015p']['img_arr'], batch['2015'][ 'mask_arr'] img_arr2016p, _ = batch['2016p']['img_arr'], batch['2016'][ 'mask_arr'] img_arr2017p, _ = batch['2017p']['img_arr'], batch['2017'][ 'mask_arr'] uimg_arr2013, uimg_arr2014, uimg_arr2015, uimg_arr2016, uimg_arr2017 = \ normalize_inverse(img_arr2013, landsat_mean, landsat_std), \ normalize_inverse(img_arr2014, landsat_mean, landsat_std), \ normalize_inverse(img_arr2015, landsat_mean, landsat_std), \ normalize_inverse(img_arr2016, landsat_mean, landsat_std), \ normalize_inverse(img_arr2017, landsat_mean, landsat_std) uimg_arr2015p, uimg_arr2016p, uimg_arr2017p = normalize_inverse(img_arr2015p, landsat_mean, landsat_std), \ normalize_inverse(img_arr2016p, landsat_mean, landsat_std), \ normalize_inverse(img_arr2017p, landsat_mean, landsat_std), \ pred2013 = update_individual_hists(img_arr2013, mask_arr2013, hist2013, device, model) pred2014 = update_individual_hists(img_arr2014, mask_arr2014, hist2014, device, model) pred2015 = update_individual_hists(img_arr2015, mask_arr2015, hist2015, device, model) pred2016 = update_individual_hists(img_arr2016, mask_arr2016, hist2016, device, model) pred2017 = update_individual_hists(img_arr2017, mask_arr2017, hist2017, device, model) pred2015p = update_individual_hists(img_arr2015p, mask_arr2015, hist2015, device, model) pred2016p = update_individual_hists(img_arr2016p, mask_arr2016, hist2016, device, model) pred2017p = update_individual_hists(img_arr2017p, mask_arr2017, hist2017, device, model) images = { '2013': { 'img': uimg_arr2013.cpu().numpy(), 'gt': mask_arr2013.cpu().numpy(), 'pred': pred2013 }, '2014': { 'img': uimg_arr2014.cpu().numpy(), 'gt': mask_arr2014.cpu().numpy(), 'pred': pred2014 }, '2015': { 'img': uimg_arr2015.cpu().numpy(), 'gt': mask_arr2015.cpu().numpy(), 'pred': pred2015 }, '2016': { 'img': uimg_arr2016.cpu().numpy(), 'gt': mask_arr2016.cpu().numpy(), 'pred': pred2016 }, '2017': { 'img': uimg_arr2017.cpu().numpy(), 'gt': mask_arr2017.cpu().numpy(), 'pred': pred2017 }, '2015p': { 'img': uimg_arr2015p.cpu().numpy(), 'gt': mask_arr2015.cpu().numpy(), 'pred': pred2015p }, '2016p': { 'img': uimg_arr2016p.cpu().numpy(), 'gt': mask_arr2016.cpu().numpy(), 'pred': pred2016p }, '2017p': { 'img': uimg_arr2017p.cpu().numpy(), 'gt': mask_arr2017.cpu().numpy(), 'pred': pred2017p }, } save_video_images256(images, out_dir, i * batch_size) acc2013, acc_cls2013, mean_iu2013, fwavacc2013, precision2013, recall2013, f1_score2013 = \ evaluate(hist=hist2013) acc2014, acc_cls2014, mean_iu2014, fwavacc2014, precision2014, recall2014, f1_score2014 = \ evaluate(hist=hist2014) acc2015, acc_cls2015, mean_iu2015, fwavacc2015, precision2015, recall2015, f1_score2015 = \ evaluate(hist=hist2015) acc2016, acc_cls2016, mean_iu2016, fwavacc2016, precision2016, recall2016, f1_score2016 = \ evaluate(hist=hist2016) acc2017, acc_cls2017, mean_iu2017, fwavacc2017, precision2017, recall2017, f1_score2017 = \ evaluate(hist=hist2017) n_samples = len(data_loader.sampler) log2013 = { 'loss2013': -1, 'acc': acc2013, 'mean_iu': mean_iu2013, 'fwavacc': fwavacc2013, 'precision': precision2013, 'recall': recall2013, 'f1_score': f1_score2013 } log2014 = { 'loss2014': -1, 'acc': acc2014, 'mean_iu': mean_iu2014, 'fwavacc': fwavacc2014, 'precision': precision2014, 'recall': recall2014, 'f1_score': f1_score2014 } log2015 = { 'loss2015': -1, 'acc': acc2015, 'mean_iu': mean_iu2015, 'fwavacc': fwavacc2015, 'precision': precision2015, 'recall': recall2015, 'f1_score': f1_score2015 } log2016 = { 'loss2016': -1, 'acc': acc2016, 'mean_iu': mean_iu2016, 'fwavacc': fwavacc2016, 'precision': precision2016, 'recall': recall2016, 'f1_score': f1_score2016 } log2017 = { 'loss2017': -1, 'acc': acc2017, 'mean_iu': mean_iu2017, 'fwavacc': fwavacc2017, 'precision': precision2017, 'recall': recall2017, 'f1_score': f1_score2017 } logger.info(log2013) logger.info(log2014) logger.info(log2015) logger.info(log2016) logger.info(log2017)
def main(): global args, best_prec1 args = opts() current_epoch = 0 # define base model model = resnet(args) # define multi-GPU model = torch.nn.DataParallel(model).cuda() # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() criterion_afem = AdaptiveFilteringEMLossForTarget(eps=args.eps).cuda() np.random.seed(args.seed) random.seed(args.seed) torch.manual_seed(args.seed) if torch.cuda.device_count() > 1: torch.cuda.manual_seed_all(args.seed) # apply different learning rates to different layers lr_fe = args.lr * 0.1 if args.pretrained else args.lr if args.arch.find('resnet') != -1: params_list = [ { 'params': model.module.conv1.parameters(), 'lr': lr_fe }, { 'params': model.module.bn1.parameters(), 'lr': lr_fe }, { 'params': model.module.layer1.parameters(), 'lr': lr_fe }, { 'params': model.module.layer2.parameters(), 'lr': lr_fe }, { 'params': model.module.layer3.parameters(), 'lr': lr_fe }, { 'params': model.module.layer4.parameters(), 'lr': lr_fe }, { 'params': model.module.fc1.parameters() }, { 'params': model.module.fc2.parameters() }, ] if args.optimizer == 'sgd': optimizer = torch.optim.SGD(params_list, lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay, nesterov=args.nesterov) if args.lr_scheduler == 'dann': lr_lambda = lambda epoch: 1 / pow( (1 + 10 * epoch / args.epochs), 0.75) scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda, last_epoch=-1) elif args.lr_scheduler == 'cosine': scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, T_max=args.epochs, eta_min=0, last_epoch=-1) elif args.lr_scheduler == 'step': lr_lambda = lambda epoch: args.gamma**( epoch + 1 > args.decay_epoch[ 1] and 2 or epoch + 1 > args.decay_epoch[0] and 1 or 0) scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda, last_epoch=-1) else: raise ValueError('Unavailable model architecture!!!') if args.resume: print("==> loading checkpoints '{}'".format(args.resume)) checkpoint = torch.load(args.resume) current_epoch = checkpoint['epoch'] best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['model_state_dict']) optimizer.load_state_dict(checkpoint['optimizer_state_dict']) scheduler.load_state_dict(checkpoint['scheduler_state_dict']) print("==> loaded checkpoint '{}'(epoch {})".format( args.resume, checkpoint['epoch'])) if not os.path.isdir(args.log): os.makedirs(args.log) log = open(os.path.join(args.log, 'log.txt'), 'a') state = {k: v for k, v in args._get_kwargs()} log.write(json.dumps(state) + '\n') log.close() # start time log = open(os.path.join(args.log, 'log.txt'), 'a') log.write('\n-------------------------------------------\n') log.write(time.asctime(time.localtime(time.time()))) log.write('\n-------------------------------------------') log.close() cudnn.benchmark = True # process data and prepare dataloaders train_loader_source, train_loader_target, val_loader_target, val_loader_source = generate_dataloader( args) if args.eval_only: prec1 = evaluate(val_loader_target, model, criterion, -1, args) print(' * Eval acc@1: {:.3f}'.format(prec1)) return print('begin training') train_loader_source_batch = enumerate(train_loader_source) train_loader_target_batch = enumerate(train_loader_target) batch_number = count_epoch_on_large_dataset(train_loader_target, train_loader_source) num_itern_total = args.epochs * batch_number test_freq = int(num_itern_total / 200) print('test_freq: ', test_freq) args.start_epoch = current_epoch cs_1 = Variable( torch.cuda.FloatTensor(args.num_classes, model.module.feat1_dim).fill_(0)) ct_1 = Variable( torch.cuda.FloatTensor(args.num_classes, model.module.feat1_dim).fill_(0)) cs_2 = Variable( torch.cuda.FloatTensor(args.num_classes, model.module.feat2_dim).fill_(0)) ct_2 = Variable( torch.cuda.FloatTensor(args.num_classes, model.module.feat2_dim).fill_(0)) for itern in range(args.start_epoch * batch_number, num_itern_total): # train for one iteration train_loader_source_batch, train_loader_target_batch, cs_1, ct_1, cs_2, ct_2 = train_compute_class_mean( train_loader_source, train_loader_source_batch, train_loader_target, train_loader_target_batch, model, criterion, criterion_afem, optimizer, itern, current_epoch, cs_1, ct_1, cs_2, ct_2, args) # evaluate on target if (itern + 1) % batch_number == 0 or (itern + 1) % test_freq == 0: prec1 = evaluate(val_loader_target, model, criterion, current_epoch, args) # record the best prec1 is_best = prec1 > best_prec1 if is_best: best_prec1 = prec1 log = open(os.path.join(args.log, 'log.txt'), 'a') log.write( '\n best acc: %3f' % (best_prec1)) log.close() # update learning rate if (itern + 1) % batch_number == 0: scheduler.step() current_epoch += 1 # save checkpoint save_checkpoint( { 'epoch': current_epoch, 'arch': args.arch, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'scheduler_state_dict': scheduler.state_dict(), 'best_prec1': best_prec1, }, is_best, args) if current_epoch > args.stop_epoch: break # end time log = open(os.path.join(args.log, 'log.txt'), 'a') log.write('\n * best acc: %3f' % best_prec1) log.write('\n-------------------------------------------\n') log.write(time.asctime(time.localtime(time.time()))) log.write('\n-------------------------------------------\n') log.close()
import argparse from trainer import train, evaluate, predict if __name__ == '__main__': parser = argparse.ArgumentParser(description='Process some integers.') parser.add_argument('mode', help='train, eval or predict') args = parser.parse_args() if args.mode == "train": train() if args.mode == "eval": evaluate() if args.mode == "predict": predict()
def main(config): if config.mod_split == 'computation_split' and config.dataset in ['boston', 'wine', 'kin8nm', 'naval', 'protein']: config.hc_threshold = 0.5 elif config.mod_split == 'computation_split' and config.dataset in ['cement', 'energy_efficiency', 'power_plant', 'yacht']: config.hc_threshold = 0.75 data = dataset.load_dataset(config) n_feature_sets = len(data.keys()) - 1 X = [np.array(data['{}'.format(i)]) for i in range(n_feature_sets)] y = np.array(data['y']) config.n_feature_sets = n_feature_sets config.feature_split_lengths = [i.shape[1] for i in X] print('Dataset used ', config.dataset) print('Number of feature sets ', n_feature_sets) [print('Shape of feature set {} {}'.format(e, np.array(i).shape)) for e,i in enumerate(X)] utils.make_model_dir(config.model_dir) if config.build_model == 'mc_dropout': config.n_models = 1 if config.dataset in ['boston', 'cement', 'power_plant', 'wine', 'yacht', 'kin8nm', 'energy_efficiency', 'naval']: config.units = 50 elif config.dataset in ['msd', 'protein', 'toy']: config.units = 100 elif config.dataset in ['alzheimers', 'alzheimers_test']: config.units = 100 config.feature_split_lengths[-1] = 21 # COMPARE features after PCA config.n_folds = 5 if config.build_model == 'combined_pog' and config.dataset in ['cement', 'protein', 'yacht', 'power_plant']: config.y_scaling = 1 if config.dataset == 'protein': config.n_folds = 5 if config.dataset == 'msd': config.n_models = 2 if config.mod_split == 'none': config.n_feature_sets = 1 if config.task == 'train': print('Training..') trainer.train(X, y, config) elif config.task == 'evaluate': print('Evaluating..') trainer.evaluate(X, y, config) elif config.task == 'experiment': config.plot_name = os.path.join(config.plot_path, '{}_{}.png'.format(config.dataset, config.exp_name)) if config.exp_name == 'defer_simulation': print('Plotting Calibration..') experiments.plot_defer_simulation(X, y, config) elif config.exp_name == 'toy_regression': print('Toy regression ..') experiments.plot_toy_regression(config) elif config.exp_name == 'clusterwise_ood': print('Plotting OOD..') experiments.plot_ood(X, y, config) elif config.exp_name == 'kl_mode': print('Plotting KL..') experiments.plot_kl(X, y, config) elif config.exp_name == 'show_summary': print('Showing..') experiments.show_model_summary(X, y, config) elif config.exp_name == 'empirical_rule_test': print('Emprical rule tests..') experiments.empirical_rule_test(X, y, config)
name = "hidden_512_embed_150" hidden_size = 512 num_tags = len(TAG_MAP) embed_dim = 300 embed_size = len(dct) dropout = 0.5 device = device condtraints = condtraints model = BiLSTM_CRF(Config()) model = model.to(device) lr = 0.001 optimizer = optim.Adam(model.parameters(), lr=lr) # train model train(model, optimizer, train_dl, val_dl, device=device, epochs=20, early_stop=True, save_every_n_epochs=3) # evaluate test_corpus = read_corpus(test_corpus_path) test_ds = NER_DataSet(test_corpus, dct) test_dl = DataLoader(test_ds, batch_size=64) metric = evaluate(model, test_dl, device) print(metric.report())
def main(): # logging configuration logging.basicConfig(level=logging.INFO, format="[%(asctime)s]: %(message)s") # parse command line input opt = utils.parse_arg() # Set GPU opt.cuda = opt.gpuid >= 0 if opt.cuda: torch.cuda.set_device(opt.gpuid) else: # please use GPU for training, CPU version is not supported for now. raise NotImplementedError #logging.info("GPU acceleration is disabled.") # prepare training and validation dataset db = data_prepare.prepare_db(opt) # sanity check for FG-NET dataset, not used for now # assertion: the total images in the eval set lists should be 1002 total_eval_imgs = sum([len(db['eval'][i]) for i in range(len(db['eval']))]) print(total_eval_imgs) if db['train'][0].name == 'FGNET': assert total_eval_imgs == 1002, 'The preparation of the evalset is incorrect.' # training if opt.train: best_MAEs = [] last_MAEs = [] # record the current time opt.save_dir += time.asctime(time.localtime(time.time())) # for FG-NET, do training multiple times for leave-one-out validation # for CACD, do training just once for exp_id in range(len(db['train'])): # initialize the model model_train = model.prepare_model(opt) #print("model shape:") # print(db['train'].head) #print( np.array(db['eval']).shape) # configurate the optimizer and learning rate scheduler optim, sche = optimizer.prepare_optim(model_train, opt) # train the model and record mean average error (MAE) model_train, MAE, last_MAE = trainer.train(model_train, optim, sche, db, opt, exp_id) best_MAEs += MAE last_MAEs.append(last_MAE.data.item()) # remove the trained model for leave-one-out validation if exp_id != len(db['train']) - 1: del model_train #np.save('./MAE.npy', np.array(best_MAEs)) #np.save('./Last_MAE.npy', np.array(last_MAEs)) # save the final trained model #utils.save_model(model_train, opt) # testing a pre-trained model elif opt.evaluate: # path to the pre-trained model save_dir = opt.test_model_path #example: save_dir = '../model/CACD_MAE_4.59.pth' model_loaded = torch.load(save_dir) # test the model on the evaluation set # the last subject is the test set (compatible with FG-NET) trainer.evaluate(model_loaded, db['eval'][-1], opt) return
def main(config): logger = config.get_logger('train', config['trainer']['verbosity']) '''===== Data Loader =====''' logger.info('preparing data loader') # setup data_loader instances data_loader = config.initialize('data_loader', module_data) valid_data_loader = data_loader.split_validation() '''===== Generator =====''' logger.info('preparing Generator') # build model architecture, then print to console model = config.initialize('arch', model_arch, 'generator') logger.info(model) # get function handles of loss and metrics loss_fn = getattr(module_loss, config['generator']['loss']) metric_fns = [ getattr(module_metric, met) for met in config['generator']['metrics'] ] # build optimizer, learning rate scheduler. delete every lines containing lr_scheduler for disabling scheduler trainable_params = filter(lambda p: p.requires_grad, model.parameters()) optimizer = config.initialize('optimizer', torch.optim, 'generator', trainable_params) lr_scheduler = config.initialize('lr_scheduler', torch.optim.lr_scheduler, 'generator', optimizer) generator = { 'model': model, 'loss_fn': loss_fn, 'metric_fns': metric_fns, 'optimizer': optimizer, 'lr_scheduler': lr_scheduler } '''===== Discriminator =====''' logger.info('preparing Discriminator') # build model architecture, then print to console model = config.initialize('arch', model_arch, 'discriminator') logger.info(model) # get function handles of loss and metrics loss_fn = getattr(module_loss, config['discriminator']['loss']) metric_fns = [ getattr(module_metric, met) for met in config['discriminator']['metrics'] ] # build optimizer, learning rate scheduler. delete every lines containing lr_scheduler for disabling scheduler trainable_params = filter(lambda p: p.requires_grad, model.parameters()) optimizer = config.initialize('optimizer', torch.optim, 'discriminator', trainable_params) lr_scheduler = config.initialize('lr_scheduler', torch.optim.lr_scheduler, 'discriminator', optimizer) discriminator = { 'model': model, 'loss_fn': loss_fn, 'metric_fns': metric_fns, 'optimizer': optimizer, 'lr_scheduler': lr_scheduler } gif_generator = GifGenerator(str(config.output_dir / "progress.gif")) '''===== Training =====''' trainer = Trainer(config, logger, generator, discriminator, gif_generator, data_loader, valid_data_loader) trainer.train() '''===== Testing =====''' log = evaluate(generator, discriminator, config, valid_data_loader) log_msg = '< Evaluation >\n' log_msg += ' Generator :\n' for key, value in log['generator'].items(): if isinstance(value, float): value = round(value, 6) log_msg += ' {:15s}: {}'.format(str(key), value) + '\n' log_msg += ' Discriminator :\n' for key, value in log['discriminator'].items(): if isinstance(value, float): value = round(value, 6) log_msg += ' {:15s}: {}'.format(str(key), value) + '\n' logger.info(log_msg) '''===== Generate samples =====''' num_samples = 16 if trainer.default_z: z = trainer.default_z else: if num_samples != trainer.num_samples and trainer.num_samples > 0: num_samples = trainer.num_samples z = Variable(torch.randn(num_samples, trainer.z_size)) z = z.to(trainer.device) generator['model'].eval() samples = generator['model'](z).detach().cpu() output_file = str(config.output_dir / "final.png") img = save_generated_images(samples.detach(), output_file) logger.info("saved generated images at {}".format(output_file)) gif_generator.add_image(img) gif_generator.save() logger.info("saved progress as a gif at {}".format(gif_generator.gif_name))
arg_parser.add_argument('--epochs', type=str, default='200') arg_parser.add_argument('--generator', type=str, default='gan') arg_parser.add_argument('--optim', type=str, default='adam') arg_parser.add_argument('--proxy_dataset', type=str, default='cifar10') arg_parser.add_argument('--sample_optimization', type=str, default='class') arg_parser.add_argument('--samples', type=str, default='optimized') arg_parser.add_argument('--size', type=int, default=32) arg_parser.add_argument('--student', type=str, default='half_lenet') arg_parser.add_argument('--teacher', type=str, default='lenet') arg_parser.add_argument('--true_dataset', type=str, default='split_fmnist') env = arg_parser.parse_args() teacher, teacher_dataset, student = setup.prepare_teacher_student(env) trainer.evaluate(teacher, teacher_dataset) generator = setup.prepare_generator(env) student_dataset = setup.prepare_student_dataset(env, teacher, teacher_dataset, student, generator) if env.optim == 'sgd': trainer.train_or_restore_predictor(student, student_dataset, loss_type='binary', n_epochs=int(env.epochs)) else: trainer.train_or_restore_predictor_adam(student, student_dataset, loss_type='binary',
def main(config): logger = config.get_logger('test') '''===== Data Loader =====''' logger.info('preparing data loader') # setup data_loader instances data_loader = getattr(module_data, config['data_loader']['type'])( config['data_loader']['args']['data_dir'], batch_size=512, shuffle=False, validation_split=0.0, training=False, num_workers=2) '''===== Generator =====''' logger.info('preparing Generator') # build model architecture, then print to console model = config.initialize('arch', model_arch, 'generator') logger.info('Loading checkpoint for Generator: {} ...'.format( config.resume['generator'])) checkpoint = torch.load(config.resume['generator']) state_dict = checkpoint['state_dict'] if config['n_gpu'] > 1: model = torch.nn.DataParallel(model) model.load_state_dict(state_dict) logger.info(model) # get function handles of loss and metrics loss_fn = getattr(module_loss, config['generator']['loss']) metric_fns = [ getattr(module_metric, met) for met in config['generator']['metrics'] ] generator = {'model': model, 'loss_fn': loss_fn, 'metric_fns': metric_fns} '''===== Discriminator =====''' logger.info('preparing Discriminator') # build model architecture, then print to console model = config.initialize('arch', model_arch, 'discriminator') logger.info('Loading checkpoint for Discriminator: {} ...'.format( config.resume['discriminator'])) checkpoint = torch.load(config.resume['discriminator']) state_dict = checkpoint['state_dict'] if config['n_gpu'] > 1: model = torch.nn.DataParallel(model) model.load_state_dict(state_dict) logger.info(model) # get function handles of loss and metrics loss_fn = getattr(module_loss, config['discriminator']['loss']) metric_fns = [ getattr(module_metric, met) for met in config['discriminator']['metrics'] ] discriminator = { 'model': model, 'loss_fn': loss_fn, 'metric_fns': metric_fns } '''===== Testing =====''' log = evaluate(generator, discriminator, config, data_loader) log_msg = '< Evaluation >\n' log_msg += ' Generator :\n' for key, value in log['generator'].items(): if isinstance(value, float): value = round(value, 6) log_msg += ' {:15s}: {}'.format(str(key), value) + '\n' log_msg += ' Discriminator :\n' for key, value in log['discriminator'].items(): if isinstance(value, float): value = round(value, 6) log_msg += ' {:15s}: {}'.format(str(key), value) + '\n' logger.info(log_msg)