def main(): viz = Visdom(server='http://192.168.1.108', port=8097, env='cifar100') assert viz.check_connection() train_set = ds.CIFAR100('./data', train=True, transform=transform, download=True) train_loader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True, num_workers=NUM_OF_WORKERS) test_set = ds.CIFAR100('./data', train=False, transform=transform, download=True) test_loader = DataLoader(test_set, batch_size=BATCH_SIZE * 2, shuffle=True, num_workers=NUM_OF_WORKERS) net = getDefaultAlexNet(INPUT_CHANNEL, NUM_CLASS, "dcm").cuda() criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(net.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY) # ------- Save training data ------------------------------------------------------------- train_file = open(os.path.join(SAVE_DIR, 'train.csv'), 'w') train_file.write('Epoch,Loss,Training Accuracy\n') test_file = open(os.path.join(SAVE_DIR, 'test.csv'), 'w') test_file.write('Epoch,Loss,Test Accuracy\n ') # Starting training process: train_loss_plot, test_loss_plot = np.array([]), np.array([]) train_acc_plot, test_acc_plot = np.array([]), np.array([]) scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=STEP_SIZE, gamma=0.1) for i in range(1, NUM_OF_EPOCHS + 1): print("---------------------------------------------------------------------------------------") train_loss, train_acc = train(i, net, optimizer, criterion, train_loader, train_file) train_loss_plot = np.append(train_loss_plot, train_loss) train_acc_plot = np.append(train_acc_plot, train_acc) test_loss, test_acc = test(i, net, criterion, test_loader, test_file) test_acc_plot = np.append(test_acc_plot, test_acc) test_loss_plot = np.append(test_loss_plot, test_loss) scheduler.step(epoch=i) print("---------------------------------------------------------------------------------------") if i % VISUAL_EVERY_EPOCH == 0: x_axis = range(1, i + 1) viz.line( Y=np.column_stack((train_loss_plot, test_loss_plot)), X=np.column_stack((x_axis, x_axis)), opts=dict( title="loss plot at epoch (%d)" % i, linecolor=np.row_stack((np.array(RGB_CYAN), np.array(RGB_MAGENTA))), legend=["train loss", "test loss"] ) ) viz.line( Y=np.column_stack((train_acc_plot, test_acc_plot)), X=np.column_stack((x_axis, x_axis)), opts=dict( title="accuracy plot at epoch (%d)" % i, linecolor=np.row_stack((np.array(RGB_DODGERBLUE), np.array(RGB_FIREBRICK))), legend=["train accuracy", "test accuracy"] ) )
def cv(dataset, fvs, images_to_indices, b, Winit, debug=False, verbose=False): if debug: W, _ = optimization.ssgd(Winit, dataset, fvs=fvs, image_to_index=images_to_indices, verbose=verbose, b=b, iterations=0, cache=False) else: W, _ = optimization.ssgd(Winit, dataset, fvs=fvs, image_to_index=images_to_indices, verbose=verbose, b=b, cache=False) tp, fp, fn, tn = train_and_test.test(dataset, W, b, 'Test', verbose=verbose) acc = (tp + tn) / float(tp + fp + fn + tn) return acc
pretrained=config.pretrained, img_size=config.img_size, prototype_shape=config.prototype_shape, num_classes=config.num_classes, prototype_activation_function=config.prototype_activation_function, add_on_layers_type=config.add_on_layers_type, batch_norm_features=config.batch_norm_features) print('load model from ' + path_to_model_with_max_push_acc) load_model_from_train_state(path_to_model_with_max_push_acc, ppnet_test) ppnet_test = ppnet_test.cuda() accu = test(model=ppnet_test, dataloader=test_loader, config=config, log_writer=log_writer, step=step, weighting_attention=args.weighting_attention) epoch_for_ppnet_test = path_to_model_with_max_push_acc.split(".")[-7].split( "/")[-1] pos_matrix, neg_matrix = generate_prototype_activation_matrices( ppnet_test, test_push_loader, train_push_loader, epoch_for_ppnet_test, model_dir, torch.device('cuda')) log_writer.add_figure('test_prototype_analysis/positive', pos_matrix, global_step=step) log_writer.add_figure('test_prototype_analysis/negative', neg_matrix, global_step=step)
optimizer=warm_optimizer, class_specific=class_specific, coefs=coefs, log=log) else: tnt.joint(model=ppnet_multi, log=log) joint_lr_scheduler.step() _ = tnt.train(model=ppnet_multi, dataloader=train_loader, optimizer=joint_optimizer, class_specific=class_specific, coefs=coefs, log=log) accu = tnt.test(model=ppnet_multi, dataloader=test_loader, class_specific=class_specific, log=log) save.save_model_w_condition(model=ppnet, model_dir=model_dir, model_name=str(epoch) + 'nopush', accu=accu, target_accu=0.70, log=log) if epoch >= push_start and epoch in push_epochs: push.push_prototypes( train_push_loader, # pytorch dataloader (must be unnormalized in [0,1]) prototype_network_parallel= ppnet_multi, # pytorch network with prototype_vectors class_specific=class_specific, preprocess_input_function=
CLASS_NAMES = ["dog", "cat"] NUM_TRAIN_EPOCHS = 10 STOP_IF_NOT_IMPROVE = 0 LOAD_MODEL = True LOAD_MODEL_PATH = 'models/model_1574546050.9673572.pkl' device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") if __name__ == '__main__': torch.multiprocessing.freeze_support() if REORGANIZED_DATA: d = DataOrganization() d.organized_data(IMG_SIZE, VAL_DIR_NAME, TRAIN_DIR_NAME, CLASS_NAMES, DATA_ROOT_DIR) data_loaders = create_dataloders(DATA_ROOT_DIR) net, epoch = get_model(LOAD_MODEL, LOAD_MODEL_PATH, IMG_SIZE, len(CLASS_NAMES)) net, best_acc, best_loss, accs_train, losses_train, accs_val, losses_val = \ train_and_eval(net, data_loaders, NUM_TRAIN_EPOCHS, STOP_IF_NOT_IMPROVE) save_model(net, best_acc, epoch + len(accs_val)) # plot_train_results(best_acc, best_loss, accs_train, losses_train, accs_val, losses_val) test(net, os.path.join(VAL_DIR_NAME, CLASS_NAMES[0]), data_loaders['train'].dataset.class_to_idx, 10) test(net, os.path.join(VAL_DIR_NAME, CLASS_NAMES[1]), data_loaders['train'].dataset.class_to_idx, 10)
def run(cfg): data_path = cfg.root_path.path model_cfg = cfg.model.cfg model_path = cfg.model.path os.environ['CUDA_VISIBLE_DEVICES'] = '1' root_dir = '/home/zhuminqin/Code/DisentangleCNN' model_dir = os.path.join(root_dir, 'saved_model' + str(int(time.time()))) makedir(model_dir) shutil.copy(os.path.join(root_dir, 'dis_model.py'), os.path.join(model_dir, 'dis_model.py')) shutil.copy(os.path.join(root_dir, './train_and_test.py'), os.path.join(model_dir, 'train_and_test.py')) shutil.copy(os.path.join(root_dir, './main.py'), os.path.join(model_dir, 'main.py')) log = SummaryWriter(log_dir=os.path.join(model_dir, 'log')) trans = transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) data = CUB_dataset(data_path, trans=trans) train_data, test_data = data_split( data, os.path.join(data.root, 'CUB_200_2011/train_test_split.txt')) train_loader = DataLoader(train_data, batch_size=32, shuffle=False, collate_fn=case_collete, num_workers=4) test_loader = DataLoader(test_data, batch_size=32, shuffle=False, collate_fn=case_collete) # a visualize demo dataiter = iter(train_loader) _, images, labels = dataiter.next() # create grid of images img_grid = make_grid(images) # write to tensorboard log.add_image('four_fashion_mnist_images', img_grid) net = Dis_features(model_cfg, model_path, pretrained=False, batch_norm=True) net = net.cuda() log.add_graph(net, images.cuda()) optimizer = torch.optim.Adam(params=net.parameters(), lr=0.0001, betas=(0.9, 0.999), eps=1e-8, weight_decay=0) for epoch in range(50): print('epoch: \t{0}'.format(epoch)) acc = tnt.train(model=net, loader=train_loader, optimizer=optimizer, log=log, epoch=epoch) log.add_scalar('pretrain_' + 'train' + '/epoch_acc', acc * 100, epoch) acc = tnt.test(model=net, loader=test_loader, log=log, epoch=epoch) log.add_scalar('pretrain_' + 'test' + '/epoch_acc', acc * 100, epoch) torch.save( net.state_dict(), os.path.join(model_dir, 'cub' + str(epoch) + '_dis_adam' '.pth'))
def main(args): with open("./configs/{}/{}_{}_{}.yaml".format(args.net, args.dataset, args.backbone, args.mode)) as fp: cfg = yaml.safe_load(fp) NET_ARGS = cfg['NET_ARGS'] DATA_ARGS = cfg['DATA_ARGS'] EXP_ARGS = cfg['EXP_ARGS'] os.environ['CUDA_VISIBLE_DEVICES'] = '0,1' model_dir = os.path.join('./saved_models/', args.dataset, args.backbone, args.net, args.mode) makedir(model_dir) log, logclose = create_logger(log_filename=os.path.join( model_dir, 'train_logger_{}.txt'.format( datetime.datetime.now().strftime("%H:%M:%S")))) img_dir = os.path.join(model_dir, 'img') makedir(img_dir) weight_matrix_filename = 'outputL_weights' prototype_img_filename_prefix = 'prototype-img' prototype_self_act_filename_prefix = 'prototype-self-act' proto_bound_boxes_filename_prefix = 'bb' log(pformat(cfg)) # ---------------------------------------- Get DataLoaders ---------------------------------------------- normalize = transforms.Normalize(mean=NET_ARGS['mean'], std=NET_ARGS['std']) train_transforms = transforms.Compose([ transforms.Resize(size=(DATA_ARGS['img_size'], DATA_ARGS['img_size'])), transforms.ToTensor(), normalize, ]) train_dataset = datasets.ImageFolder(DATA_ARGS['train_dir'], train_transforms) train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=EXP_ARGS['train_batch_size'], shuffle=True, num_workers=4, pin_memory=False) train_push_dataset = datasets.ImageFolder( DATA_ARGS['train_push_dir'], transforms.Compose([ transforms.Resize(size=(DATA_ARGS['img_size'], DATA_ARGS['img_size'])), transforms.ToTensor(), ])) train_push_loader = torch.utils.data.DataLoader( train_push_dataset, batch_size=EXP_ARGS['train_push_batch_size'], shuffle=False, num_workers=4, pin_memory=False) test_dataset = datasets.ImageFolder( DATA_ARGS['test_dir'], transforms.Compose([ transforms.Resize(size=(DATA_ARGS['img_size'], DATA_ARGS['img_size'])), transforms.ToTensor(), normalize, ])) test_loader = torch.utils.data.DataLoader( test_dataset, batch_size=EXP_ARGS['test_batch_size'], shuffle=False, num_workers=4, pin_memory=False) log('training set size: {0}'.format(len(train_loader.dataset))) log('push set size: {0}'.format(len(train_push_loader.dataset))) log('test set size: {0}'.format(len(test_loader.dataset))) log('batch size: {0}'.format(EXP_ARGS['train_batch_size'])) # ------------------------------------ Model and Optimizer ---------------------------------------------- ppnet = model_AttProto.construct_PPNet( base_architecture=NET_ARGS['base_architecture'], pretrained=True, img_size=DATA_ARGS['img_size'], prototype_shape=NET_ARGS['prototype_shape'], num_classes=DATA_ARGS['num_classes'], prototype_activation_function=NET_ARGS[ 'prototype_activation_function'], add_on_layers_type=NET_ARGS['add_on_layers_type'], att_version=NET_ARGS['ATT_VERSION']) ppnet = ppnet.cuda() ppnet_multi = torch.nn.DataParallel(ppnet) class_specific = True if EXP_ARGS['RESUME']['iS_RESUME']: ppnet = torch.load(EXP_ARGS['RESUME']['PATH']) log(" Resumed from model: {}".format(EXP_ARGS['RESUME']['PATH'])) ppnet_multi = torch.nn.DataParallel(ppnet) accu = tnt.test(model=ppnet_multi, dataloader=test_loader, class_specific=True, log=log, EXP_ARGS=EXP_ARGS) log("\nInit Accuracy {:.2f} \n\n".format(accu)) ppnet_multi = torch.nn.DataParallel(ppnet) warm_optimizer_lrs = EXP_ARGS['OPTIMIZER']['warm_optimizer_lrs'] warm_optimizer_specs = [ { 'params': ppnet.add_on_layers.parameters(), 'lr': warm_optimizer_lrs['add_on_layers'], 'weight_decay': 1e-3 }, { 'params': ppnet.prototype_vectors, 'lr': warm_optimizer_lrs['prototype_vectors'] }, { 'params': ppnet.att_layer.parameters(), 'lr': warm_optimizer_lrs['att_layer'], 'weight_decay': 1e-3 }, ] warm_optimizer = torch.optim.Adam(warm_optimizer_specs) joint_optimizer_lrs = EXP_ARGS['OPTIMIZER']['joint_optimizer_lrs'] joint_optimizer_specs = [{ 'params': ppnet.features.parameters(), 'lr': joint_optimizer_lrs['features'], 'weight_decay': 1e-3 }, { 'params': ppnet.add_on_layers.parameters(), 'lr': joint_optimizer_lrs['add_on_layers'], 'weight_decay': 1e-3 }, { 'params': ppnet.prototype_vectors, 'lr': joint_optimizer_lrs['prototype_vectors'] }, { 'params': ppnet.att_layer.parameters(), 'lr': joint_optimizer_lrs['att_layer'], 'weight_decay': 1e-3 }] joint_optimizer = torch.optim.Adam(joint_optimizer_specs) joint_lr_scheduler = torch.optim.lr_scheduler.StepLR( joint_optimizer, step_size=int(joint_optimizer_lrs['joint_lr_step_size']), gamma=0.1) push_epochs = [ i for i in range(EXP_ARGS['num_train_epochs']) if i % 10 == 0 ] log('\n\n------------------------ Start Training ----------------------------\n\n' ) max_acc = 0.0 max_acc_epoch = 0 max_acc_iter = 0 target_accu = 0.1 for epoch in range(EXP_ARGS['start_epoch'], EXP_ARGS['num_train_epochs']): log('------------------------- Epoch: {} -------------------------------------' .format(epoch)) if epoch < EXP_ARGS['num_warm_epochs']: tnt.warm_only(model=ppnet_multi, log=log) _ = tnt.train(model=ppnet_multi, dataloader=train_loader, optimizer=warm_optimizer, class_specific=class_specific, coefs=EXP_ARGS['LOSS']['loss_coefs_warm'], log=log, EXP_ARGS=EXP_ARGS) else: tnt.joint(model=ppnet_multi, log=log) joint_lr_scheduler.step() _ = tnt.train(model=ppnet_multi, dataloader=train_loader, optimizer=joint_optimizer, class_specific=class_specific, coefs=EXP_ARGS['LOSS']['loss_coefs_joint'], log=log, EXP_ARGS=EXP_ARGS) accu = tnt.test(model=ppnet_multi, dataloader=test_loader, class_specific=class_specific, log=log, EXP_ARGS=EXP_ARGS) if accu > max_acc: max_acc = accu max_acc_iter = 0 max_acc_epoch = epoch save.save_model_w_condition(model=ppnet, model_dir=model_dir, model_name='', accu=accu, target_accu=target_accu, log=log, best=True, stage='prepush_{}'.format(epoch)) log("\nBest Accuracy {:.2f} at epoch {} and iter {}\n\n".format( max_acc, max_acc_epoch, max_acc_iter)) if epoch >= EXP_ARGS['push_start'] and epoch in push_epochs: save.save_model_w_condition(model=ppnet, model_dir=model_dir, model_name='', accu=accu, target_accu=target_accu, log=log, best=True, stage='prepushfinal_{}'.format(epoch)) log('\n------------------------- Push Prototypes -----------------------------' ) push.push_prototypes( train_push_loader, prototype_network_parallel=ppnet_multi, class_specific=class_specific, preprocess_input_function=preprocess_input_function, prototype_layer_stride=1, root_dir_for_saving_prototypes=img_dir, epoch_number=epoch, prototype_img_filename_prefix=prototype_img_filename_prefix, prototype_self_act_filename_prefix= prototype_self_act_filename_prefix, proto_bound_boxes_filename_prefix= proto_bound_boxes_filename_prefix, save_prototype_class_identity=True, log=log) accu = tnt.test(model=ppnet_multi, dataloader=test_loader, class_specific=class_specific, log=log, EXP_ARGS=EXP_ARGS) save.save_model_w_condition(model=ppnet, model_dir=model_dir, model_name='', accu=accu, target_accu=target_accu, log=log, best=True, stage='push_{}'.format(epoch)) last_layer_optimizer_specs = [{ 'params': ppnet.last_layer.parameters(), 'lr': EXP_ARGS['OPTIMIZER']['last_layer_optimizer_lrs'] ['last_layer_optimizer_lr'] }] last_layer_optimizer = torch.optim.Adam(last_layer_optimizer_specs) last_lr_lr_scheduler = torch.optim.lr_scheduler.StepLR( last_layer_optimizer, step_size=EXP_ARGS['OPTIMIZER']['last_layer_optimizer_lrs'] ['last_lr_step_size'], gamma=0.1) log('\n------------------------- Last Layer Training -----------------------------------' ) if NET_ARGS['prototype_activation_function'] != 'linear': tnt.last_only(model=ppnet_multi, log=log) max_acc_post, max_acc_post_iter, max_acc_post_epoch = 0, 0, epoch for i in range( EXP_ARGS['OPTIMIZER']['last_layer_optimizer_lrs'] ['last_layer_optimizer_iters']): log('Last layer optimization, Iteration: {0}'.format(i)) _ = tnt.train(model=ppnet_multi, dataloader=train_loader, optimizer=last_layer_optimizer, class_specific=class_specific, coefs=EXP_ARGS['LOSS']['loss_coefs_joint'], log=log, EXP_ARGS=EXP_ARGS) last_lr_lr_scheduler.step() accu = tnt.test(model=ppnet_multi, dataloader=test_loader, class_specific=class_specific, log=log, EXP_ARGS=EXP_ARGS) if accu > max_acc_post: max_acc_post = accu max_acc_post_iter = i max_acc_post_epoch = epoch save.save_model_w_condition( model=ppnet, model_dir=model_dir, model_name='', accu=accu, target_accu=0.70, log=log, best=True, stage='postpush_{}'.format(epoch)) log("Best Accuracy - PostPush {:.2f} at epoch {} and iter {}" .format(max_acc_post, max_acc_post_epoch, max_acc_post_iter)) save.save_model_w_condition( model=ppnet, model_dir=model_dir, model_name='', accu=accu, target_accu=0.70, log=log, best=True, stage='postpushfinal_{}'.format(epoch)) logclose()
step += 1 prune.prune_prototypes( dataloader=train_push_loader, ppnet=ppnet, k=k, prune_threshold=prune_threshold, preprocess_input_function=preprocess_input_function, # normalize original_model_dir=original_model_dir, epoch_number=epoch, # model_name=None, log=print, copy_prototype_imgs=True, find_threshold_prune_n_patches=find_threshold_prune_n_patches, only_n_most_activated=only_n_most_activated) accu = test(model=ppnet, dataloader=test_loader, config=config, weighting_attention=args.weighting_attention) print('New best score: {}, saving snapshot'.format(accu)) save_train_state( os.path.join(model_dir, original_model_name.split('push')[0] + 'prune'), ppnet, {}, step, mode, epoch, iteration, experiment_run_name, best_accu, accu, accu, config) current_push_best_accu = accu # last layer optimization if optimize_last_layer: last_layer_optimizer_specs = [{ 'params': ppnet.last_layer.parameters(), 'lr':
in_vocab = load_vocabulary(os.path.join(config.vocab_path, 'in_vocab')) slot_vocab = load_vocabulary(os.path.join(config.vocab_path, 'slot_vocab')) intent_vocab = load_vocabulary(os.path.join(config.vocab_path, 'intent_vocab')) train_data, dev_data, test_data = build_dataset(in_vocab['vocab'], slot_vocab['vocab'], intent_vocab['vocab']) train_iter = build_iterator(train_data) dev_iter = build_iterator(dev_data) test_iter = build_iterator(test_data) time_dif = get_time_dif(start_time) print('time usage:', time_dif) config.n_vocab = len(in_vocab['vocab']) x = import_module(model_name) encoder = x.Encoder(config).to(config.device) decoder = x.Decoder(config).to(config.device) init_network(encoder) init_network(decoder) print(encoder.parameters) print(decoder.parameters) # train(config, encoder, decoder, train_iter, dev_iter, test_iter) test(config, encoder, decoder, test_iter)