def test_get_models(self): with tempfile.TemporaryDirectory() as models_dir: model1 = '000013-model.meta' model2 = '000017-model.meta' f1 = open(os.path.join(models_dir, model1), 'w') f1.close() f2 = open(os.path.join(models_dir, model2), 'w') f2.close() model_nums_names = utils.get_models(models_dir) self.assertEqual(len(model_nums_names), 2) self.assertEqual(model_nums_names[0], (13, '000013-model')) self.assertEqual(model_nums_names[1], (17, '000017-model'))
def validate(trained_models_dir, holdout_dir, estimator_model_dir, params): """Validate the latest model on the holdout dataset. Args: trained_models_dir: Directories where the completed generations/models are. holdout_dir: Directories where holdout data are. estimator_model_dir: tf.estimator model directory. params: A MiniGoParams instance of hyperparameters for the model. """ model_num, _ = utils.get_latest_model(trained_models_dir) # Get the holdout game data nums_names = utils.get_models(trained_models_dir) # Model N was trained on games up through model N-1, so the validation set # should only be for models through N-1 as well, thus the (model_num) term. models = [num_name for num_name in nums_names if num_name[0] < model_num] # pair is a tuple of (model_num, model_name), like (13, 000013-modelname) holdout_dirs = [os.path.join(holdout_dir, pair[1]) for pair in models[-params.holdout_generation:]] tf_records = [] with utils.logged_timer('Building lists of holdout files'): for record_dir in holdout_dirs: if os.path.exists(record_dir): # make sure holdout dir exists tf_records.extend( tf.gfile.Glob(os.path.join(record_dir, '*'+_TF_RECORD_SUFFIX))) if not tf_records: print('No holdout dataset for validation! ' 'Please check your holdout directory: {}'.format(holdout_dir)) return print('The length of tf_records is {}.'.format(len(tf_records))) first_tf_record = os.path.basename(tf_records[0]) last_tf_record = os.path.basename(tf_records[-1]) with utils.logged_timer('Validating from {} to {}'.format( first_tf_record, last_tf_record)): dualnet.validate(estimator_model_dir, tf_records, params)
z_dim = args.z_dim beta = args.beta dataset_name = args.dataset_name model_name = args.model_name experiment_name = args.experiment_name pylib.mkdir('./output/%s' % experiment_name) with open('./output/%s/setting.txt' % experiment_name, 'w') as f: f.write(json.dumps(vars(args), indent=4, separators=(',', ':'))) # dataset and models Dataset, img_shape, get_imgs = utils.get_dataset(dataset_name) dataset = Dataset(batch_size=batch_size) dataset_val = Dataset(batch_size=100) Enc, Dec = utils.get_models(model_name) Enc = partial(Enc, z_dim=z_dim) Dec = partial(Dec, channels=img_shape[2]) # ============================================================================== # = graph = # ============================================================================== def enc_dec(img, is_training=True): # encode z_mu, z_log_sigma_sq = Enc(img, is_training=is_training) # sample epsilon = tf.random_normal(tf.shape(z_mu)) if is_training:
def main(): # get args args = get_args() # set up gpus os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu assert torch.cuda.is_available() # load models if 'gal' in args.model_file: leaky_relu = True else: leaky_relu = False ensemble = utils.get_models(args, train=False, as_ensemble=True, model_file=args.model_file, leaky_relu=leaky_relu) models = ensemble.models train_seed = args.model_file.split('/')[-3] train_alg = args.model_file.split('/')[-4] # get data loaders testloader = utils.get_testloader(args, batch_size=args.batch_size) # pick out samples that are correctly classified by all submodels correct = [] for m in models: correct_m = [] for (x, y) in testloader: x, y = x.cuda(), y.cuda() outputs = m(x) _, pred = outputs.max(1) correct_m.append(pred.eq(y)) correct_m = torch.cat(correct_m) correct.append(correct_m) correct = torch.stack(correct, dim=-1).all(-1) correct_idx = correct.nonzero().squeeze(-1) random.seed(0) subset_idx = correct_idx[random.sample(range(correct_idx.size(0)), args.subset_num)].cpu() subset_loader = utils.get_testloader(args, batch_size=args.batch_size, shuffle=False, subset_idx=subset_idx) # PGD eps_list = [0.03] random_start = args.random_start steps = args.steps rob = {} rob['random_start'] = args.random_start rob['steps'] = args.steps for eps in tqdm(eps_list, desc='PGD eps', leave=False, position=0): correct_or_not_rs = torch.zeros((len(models), len(models)+1, args.subset_num, random_start), dtype=torch.bool) for rs in tqdm(range(random_start), desc='Random Start', leave=False, position=1): torch.manual_seed(rs) test_iter = tqdm(subset_loader, desc='Batch', leave=False, position=2) total = 0 for (x, y) in test_iter: x, y = x.cuda(), y.cuda() adv_list = [] for i, m in enumerate(models): adversary = LinfPGDAttack( m, loss_fn=nn.CrossEntropyLoss(reduction="sum"), eps=eps, nb_iter=steps, eps_iter=eps/5, rand_init=True, clip_min=0., clip_max=1., targeted=False) adv = adversary.perturb(x, y) adv_list.append(adv) for i, adv in enumerate(adv_list): for j, m in enumerate(models): if j == i: outputs = m(x) _, pred = outputs.max(1) assert pred.eq(y).all() outputs = m(adv) _, pred = outputs.max(1) correct_or_not_rs[i, j, total:total+x.size(0), rs] = pred.eq(y) outputs = ensemble(adv) _, pred = outputs.max(1) correct_or_not_rs[i, len(models), total:total+x.size(0), rs] = pred.eq(y) total += x.size(0) correct_or_not_rs = torch.all(correct_or_not_rs, dim=-1) asr = np.zeros((len(models), len(models)+1)) tqdm.write("eps: {:.2f}".format(eps)) for i in range(len(models)): message = '' for j in range(len(models)+1): message += '\t{}: {:.2%}'.format(j, 1-correct_or_not_rs[i, j, :].sum().item()/args.subset_num) asr[i, j] = 1-correct_or_not_rs[i, j, :].sum().item()/args.subset_num tqdm.write(message) rob[str(eps)] = asr # save to file if args.save_to_file: output_root = os.path.join('results', 'transferability', train_alg, train_seed) if not os.path.exists(output_root): os.makedirs(output_root) output_filename = args.model_file.split('/')[-2] output = os.path.join(output_root, '.'.join((output_filename, 'pkl'))) with open(output, 'wb') as f: pickle.dump(rob, f, pickle.HIGHEST_PROTOCOL)
# ============================================================================== # = datasets and models = # ============================================================================== source_train_data, target_train_data, source_test_data, target_test_data, \ min_n, preprocessor = utils.get_data(data_path, data_type, use_test) source_train_dataset = utils.make_dataset(source_train_data, batch_size = batch_size) target_train_dataset = utils.make_dataset(target_train_data, batch_size = batch_size) s_iterator = source_train_dataset.make_one_shot_iterator() s_next_element = s_iterator.get_next() t_iterator = target_train_dataset.make_one_shot_iterator() t_next_element = t_iterator.get_next() input_dim = source_train_data.shape[1] Enc, Dec_a, Dec_b, Disc = utils.get_models(model_name) Enc = partial(Enc, code_dim=code_dim) Dec_a = partial(Dec_a, output_dim=input_dim) Dec_b = partial(Dec_b, output_dim=input_dim) # ============================================================================== # = graph = # ============================================================================== def enc_dec(input, AE_type, is_training=True): # encode if AE_type == "standard": c, _ = Enc(input, is_training=is_training) if AE_type == "VAE": c_mu, c_log_sigma_sq = Enc(input, is_training=is_training)
from cornac.datasets import movielens from eval_methods.stratified_evaluation import StratifiedEvaluation from experiment.experiment import STExperiment from utils import get_models, get_metrics import sys sys.stdout = open('log_ml_large_1M.txt', 'w', 1) dims = [e for e in range(10, 110, 10)] # load the movielens dataset ml = movielens.load_feedback(variant="1M") # propensity-based stratified evaluation stra_eval_method = StratifiedEvaluation(data=ml, n_strata=2, rating_threshold=4.0, verbose=True) # run the experiment exp_stra = STExperiment(eval_method=stra_eval_method, models=get_models(variant='large', dims=dims), metrics=get_metrics(variant='small'), verbose=True) exp_stra.run() with open('../data/exp_stra_ml.pkl', 'wb') as exp_file: pickle.dump(exp_stra.result, exp_file)
if 'retrieval' not in args.tasks: # don't want to load code data twice data_to_load.append(('code', code_vocab, args.code_max_length)) if 'language_model_docstring' in args.tasks: if 'retrieval' not in args.tasks: # don't want to load docstring data twice data_to_load.append( ('docstring', docstring_vocab, args.docstring_max_length)) train_data = utils.load_data(args.data + '_train.jsonl', data_to_load) valid_data = utils.load_data(args.data + '_valid.jsonl', data_to_load) test_data = utils.load_data(args.data + '_test.jsonl', data_to_load) print(f'training examples: {utils.get_num_examples(train_data)}') print(f'validation examples: {utils.get_num_examples(valid_data)}') print(f'testing examples: {utils.get_num_examples(test_data)}') print('Creating iterators...') train_iterators = utils.get_iterators(train_data, args.batch_size, shuffle=True) valid_iterators = utils.get_iterators(valid_data, args.batch_size, shuffle=False) test_iterators = utils.get_iterators(test_data, args.batch_size, shuffle=False) print('Creating models...') models = utils.get_models(args.config, data_to_load, args.tasks) for k, v in models.items(): print(k, v)
def main(): # get args args = get_args() # set up gpus os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu assert torch.cuda.is_available() # load models if 'gal' in args.model_file: leaky_relu = True else: leaky_relu = False ensemble = utils.get_models(args, train=False, as_ensemble=True, model_file=args.model_file, leaky_relu=leaky_relu) train_seed = args.model_file.split('/')[-3] train_alg = args.model_file.split('/')[-4] eps_list = [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07] loss_fn_list = ['xent', 'cw'] surrogate_model_list = [ '{:s}{:d}'.format(args.which_ensemble, i) for i in [3, 5, 8] ] method_list = [ 'mdi2_0.5_steps_{:d}'.format(args.steps), 'sgm_0.2_steps_{:d}'.format(args.steps) ] index_list = [ '{:s}_{:s}_mpgd'.format(a, b) for a in surrogate_model_list for b in loss_fn_list ] index_list += [ '{:s}_{:s}_{:s}'.format(a, b, c) for a in surrogate_model_list for b in loss_fn_list for c in method_list ] index_list.append('all') random_start = 3 input_list = [ 'from_{:s}_{:s}_mpgd_steps_{:d}'.format(a, b, args.steps) for a in surrogate_model_list for b in loss_fn_list ] input_list += [ 'from_{:s}_{:s}_{:s}'.format(a, b, c) for a in surrogate_model_list for b in loss_fn_list for c in method_list ] rob = {} rob['source'] = index_list acc_list = [[] for _ in range(len(eps_list))] data_root = os.path.join(args.data_dir, args.folder) # clean acc input_folder = os.path.join(data_root, 'clean') clean_acc = test(ensemble, input_folder, return_acc=True) clean_acc_list = [clean_acc for _ in range(len(input_list) + 1)] rob['clean'] = clean_acc_list # transfer attacks for i, eps in enumerate(tqdm(eps_list, desc='eps', leave=True, position=0)): input_folder = os.path.join(data_root, 'eps_{:.2f}'.format(eps)) correct_over_input = [] for input_adv in tqdm(input_list, desc='source', leave=False, position=1): if 'mpgd' in input_adv: correct_over_rs = [] for rs in tqdm(range(random_start), desc='Random Start', leave=False, position=2): datafolder = os.path.join(input_folder, '_'.join( (input_adv, str(rs)))) correct_over_rs.append(test(ensemble, datafolder)) correct_over_rs = torch.stack(correct_over_rs, dim=-1).all(dim=-1) acc_list[i].append(100. * correct_over_rs.sum().item() / len(correct_over_rs)) correct_over_input.append(correct_over_rs) tqdm.write( 'Clean acc: {:.2f}%, eps: {:.2f}, transfer from {:s} acc: {:.2f}%' .format( clean_acc, eps, input_adv, 100. * correct_over_rs.sum().item() / len(correct_over_rs))) else: datafolder = os.path.join(input_folder, input_adv) correct = test(ensemble, datafolder) acc_list[i].append(100. * correct.sum().item() / len(correct)) correct_over_input.append(correct) tqdm.write( 'Clean acc: {:.2f}%, eps: {:.2f}, transfer from {:s} acc: {:.2f}%' .format(clean_acc, eps, input_adv, 100. * correct.sum().item() / len(correct))) correct_over_input = torch.stack(correct_over_input, dim=-1).all(dim=-1) acc_list[i].append(100. * correct_over_input.sum().item() / len(correct_over_input)) tqdm.write( 'Clean acc: {:.2f}%, eps: {:.2f}, transfer acc: {:.2f}%'.format( clean_acc, eps, 100. * correct_over_input.sum().item() / len(correct_over_input))) rob[str(eps)] = acc_list[i] # save to file if args.save_to_csv: output_root = os.path.join('results', 'bbox', train_alg, train_seed) if not os.path.exists(output_root): os.makedirs(output_root) output_filename = args.model_file.split('/')[-2] output = os.path.join(output_root, '.'.join((output_filename, 'csv'))) df = pd.DataFrame(rob) if args.append_out and os.path.isfile(output): with open(output, 'a') as f: f.write('\n') df.to_csv(output, sep=',', mode='a', header=False, index=False, float_format='%.2f') else: df.to_csv(output, sep=',', index=False, float_format='%.2f')
for a in args.accelerator_ids ] assert cfg.SEARCH.CHILDREN % cfg.SEARCH.PARENTS == 0, \ 'Number of children must be divisible by number of parents.' assert cfg.SEARCH.CHILDREN % len(accelerators) == 0, \ 'Number of children must be divisible by number of accelerators.' print('Using accelerators: {}'.format(accelerators)) cfg.SEARCH.DIR = 'searches/' + args.cfg.split('.')[0] os.makedirs(cfg.SEARCH.DIR, exist_ok=True) # EVOLUTION while 1: # each loop is a generation, run until manually stopped meta_files, models, genotypes = get_models(cfg.SEARCH.DIR) if len(models) == 0: # train initial genotype train_cfg = update_cfg( copy.deepcopy(cfg), accelerator=accelerators[0], gen=0, model_id=0, genotype=genotype_from_blocks_args(DEFAULT_BLOCKS_ARGS), epochs=cfg.SEARCH.GEN0_EPOCHS, parent=None, disp=True) train_wrapper(train_cfg) else: # train next generation
parser.add_argument('--use_scorer', type=int, default=1, choices=[0, 1], help='Whether to calculate FID score') args = parser.parse_args() device = torch.device(f"cuda:{args.gpu}" if ( torch.cuda.is_available()) else "cpu") print(f'Using device: {device}') # Load scorer scorer = fid_scorer(device) if args.use_scorer else None # Load params from text file models = get_models(args.hparams, device) print('Entering Hyperparameter Loop') for h, gen, discr in models: with (SummaryWriter(comment=f'_{h.name}') if args.use_writer else nullcontext) as writer: print(f'Run name: {h.name}') if writer: writer.add_hparams(vars(h), {}) if scorer: scorer.set_params(h.dataroot, h.name, writer)
host = 'localhost' port = 8000 max_stop_time = 10 # in second max_wall_time = 10 # in hour frame = [350, 205 + 20] mean = [0.35739973, 0.35751262, 0.36058474] std = [0.19338858, 0.19159749, 0.2047393] transform = transforms.Compose( [transforms.ToTensor(), transforms.Normalize(mean, std)]) print('Loading model...') encoder, decoder, init_y = get_models() y_bin = pickle.load(open('./data/y_bin_info.pickle', 'rb')) client = Client(ip=host, port=port) scenario = Scenario(weather='EXTRASUNNY', vehicle='voltic', time=[12, 0], drivingMode=-1, location=[-2500, 3250]) client.sendMessage(Start(scenario=scenario)) with torch.no_grad(): # encoder.eval() # decoder.eval()
def main(): # get args args = get_args() # set up gpus os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu assert torch.cuda.is_available() # set up writer, logger, and save directory for models save_root = os.path.join('checkpoints', 'transfer', 'seed_{:d}'.format(args.seed), '{:s}{:d}'.format(args.arch, args.depth)) save_root += "%.2f" % (args.transfer_coeff) if not os.path.exists(save_root): os.makedirs(save_root) else: print('*********************************') print('* The checkpoint already exists *') print('*********************************') writer = SummaryWriter(save_root.replace('checkpoints', 'runs')) # dump configurations for potential future references with open(os.path.join(save_root, 'cfg.json'), 'w') as fp: json.dump(vars(args), fp, indent=4) with open( os.path.join(save_root.replace('checkpoints', 'runs'), 'cfg.json'), 'w') as fp: json.dump(vars(args), fp, indent=4) # set up random seed torch.manual_seed(args.seed) # initialize models models = utils.get_models(args, train=True, as_ensemble=False, model_file="/sync_transfer/CIFAR/epoch_200.pth", dataset="CIFAR-10") # get data loaders source_trainloader, source_testloader = utils.get_loaders( args, dataset="CIFAR-10") target_trainloader, target_testloader = utils.get_loaders(args, dataset="STL-10") # get optimizers and schedulers optimizers = utils.get_optimizers(args, models) schedulers = utils.get_schedulers(args, optimizers) surrogate = utils.get_models(args, train=False, as_ensemble=False, model_file="/sync_transfer/STL/epoch_200.pth", dataset="STL-10") trainer = Transfer_Trainer(models, optimizers, schedulers, source_trainloader, source_testloader, target_trainloader, surrogate, writer, save_root, **vars(args)) trainer.run()
def init_tests(): """Get the data required for the tests.""" get_cifar10_dataset() convert_cifar2imagenet_format() download_images() get_models()
def init_tests(): """Get the data required for the tests.""" get_cifar10_dataset() get_models()
validation_log.close() train_log.close() # When done, ask the threads to stop. coord.request_stop() # Wait for threads to finish. coord.join(threads) return best_validation_accuracy if __name__ == '__main__': # CLI arguments PARSER = argparse.ArgumentParser(description="Train the model") # Required arguments PARSER.add_argument("--model", required=True, choices=utils.get_models()) PARSER.add_argument("--dataset", required=True, choices=utils.get_datasets()) # Restart train or continue PARSER.add_argument("--restart", action='store_true') # Learning rate decay arguments PARSER.add_argument("--lr_decay", action="store_true") PARSER.add_argument("--lr_decay_epochs", type=int, default=25) PARSER.add_argument("--lr_decay_factor", type=float, default=0.1) # L2 regularization arguments PARSER.add_argument("--l2_penalty", type=float, default=0.0)
def main(): # get args args = get_args() # set up gpus os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu assert torch.cuda.is_available() # load models if 'gal' in args.model_file: leaky_relu = True else: leaky_relu = False ensemble = utils.get_models(args, train=False, as_ensemble=True, model_file=args.model_file, leaky_relu=leaky_relu) models = ensemble.models train_seed = args.model_file.split('/')[-3] train_alg = args.model_file.split('/')[-4] # get data loaders testloader = utils.get_testloader(args, batch_size=100) # pick out samples that are correctly classified by all submodels correct = [] for m in models: correct_m = [] for (x, y) in testloader: x, y = x.cuda(), y.cuda() outputs = m(x) _, pred = outputs.max(1) correct_m.append(pred.eq(y)) correct_m = torch.cat(correct_m) correct.append(correct_m) correct = torch.stack(correct, dim=-1).all(-1) correct_idx = correct.nonzero().squeeze(-1) random.seed(0) subset_idx = correct_idx[random.sample(range(correct_idx.size(0)), args.subset_num)].cpu() # use a very small batch size so that we can sample different layers multiple times subset_loader = utils.get_testloader(args, batch_size=10, shuffle=True, subset_idx=subset_idx) eps_list = [0.07] steps = 10 rob = {} rob['steps'] = steps criterion = nn.CrossEntropyLoss(reduction='none') for eps in tqdm(eps_list, desc='eps', leave=False, position=0): losses = torch.zeros((len(models), len(models), args.subset_num)) torch.manual_seed(0) loader = utils.DistillationLoader(subset_loader, subset_loader) test_iter = tqdm(loader, desc='Batch', leave=False, position=1) random.seed(0) total = 0 for batch_idx, (si, sl, ti, tl) in enumerate(test_iter): si, sl = si.cuda(), sl.cuda() ti, tl = ti.cuda(), tl.cuda() layer = random.randint(1, args.depth) adv_list = [] for i, m in enumerate(models): adv = Linf_distillation(m, si, ti, eps, eps / steps, steps, layer, before_relu=True, mu=1, momentum=True, rand_start=False) adv_list.append(adv) with torch.no_grad(): for i, adv in enumerate(adv_list): for j, m in enumerate(models): if j == i: outputs = m(si) _, pred = outputs.max(1) assert pred.eq(sl).all() outputs = m(adv) loss = criterion(outputs, tl) losses[i, j, total:total + si.size(0)] = loss total += si.size(0) losses_np = torch.mean(losses, dim=-1).numpy() tqdm.write("eps: {:.2f}".format(eps)) for i in range(len(models)): message = '' for j in range(len(models)): message += '\t{}: {:.3f}'.format(j, losses_np[i, j]) tqdm.write(message) rob[str(eps)] = losses_np # save to file if args.save_to_file: output_root = os.path.join('results', 'diversity', train_alg, train_seed) if not os.path.exists(output_root): os.makedirs(output_root) output_filename = args.model_file.split('/')[-2] output = os.path.join(output_root, '.'.join((output_filename, 'pkl'))) with open(output, 'wb') as f: pickle.dump(rob, f, pickle.HIGHEST_PROTOCOL)
type=str, help='Run Configuration', default='run_config.yaml') parser.add_argument('--runs', type=int, help='Number of Runs for each model', default=1) #parser.add_argument('--early_stop', type=bool, help='no improvement during 10 epoch -> stop', default=True) args = parser.parse_args() # Load configuration file for pipeline with open(args.run_config) as f: config = yaml.load(f, Loader=yaml.Loader) # Load Available models models = utils.get_models() logger.info('Available models %s' % list(models.keys())) # Load and normalize data input_data = pd.read_csv(args.train_data) input_data.iloc[:, 1:] = MinMaxScaler().fit_transform(input_data.iloc[:, 1:]) train, validate, test = np.split( input_data.sample(frac=1), [int(.6 * len(input_data)), int(.8 * len(input_data))]) train_Y = train.iloc[:, 0].values train_X = train.iloc[:, 1:].values validation_Y = validate.iloc[:, 0].values validation_X = validate.iloc[:, 1:].values
def main(): # get args args = get_args() # set up gpus os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu assert torch.cuda.is_available() # load models if 'gal' in args.model_file: leaky_relu = True else: leaky_relu = False ensemble = utils.get_models(args, train=False, as_ensemble=True, model_file=args.model_file, leaky_relu=leaky_relu) # get data loaders total_sample_num = 10000 if args.subset_num: random.seed(0) subset_idx = random.sample(range(total_sample_num), args.subset_num) testloader = utils.get_testloader(args, batch_size=200, shuffle=False, subset_idx=subset_idx) else: testloader = utils.get_testloader(args, batch_size=200, shuffle=False) loss_fn = nn.CrossEntropyLoss() if args.loss_fn == 'xent' else CarliniWagnerLoss(conf=args.cw_conf) rob = {} rob['sample_num'] = args.subset_num if args.subset_num else total_sample_num rob['loss_fn'] = 'xent' if args.loss_fn == 'xent' else 'cw_{:.1f}'.format(args.cw_conf) train_seed = args.model_file.split('/')[-3] train_alg = args.model_file.split('/')[-4] if args.convergence_check: eps = 0.01 steps_list = [50, 500, 1000] random_start = 1 rob['random_start'] = random_start rob['eps'] = eps # FGSM test_iter = tqdm(testloader, desc='FGSM', leave=False, position=0) adversary = GradientSignAttack( ensemble, loss_fn=nn.CrossEntropyLoss(), eps=eps, clip_min=0., clip_max=1., targeted=False) _, label, pred, advpred = attack_whole_dataset(adversary, test_iter, device="cuda") print("Accuracy: {:.2f}%, FGSM Accuracy: {:.2f}%".format( 100. * (label == pred).sum().item() / len(label), 100. * (label == advpred).sum().item() / len(label))) rob['clean'] = 100. * (label == pred).sum().item() / len(label) rob['fgsm'] = 100. * (label == advpred).sum().item() / len(label) for steps in tqdm(steps_list, desc='PGD steps', leave=False, position=0): correct_or_not = [] for i in tqdm(range(random_start), desc='Random Start', leave=False, position=1): torch.manual_seed(i) test_iter = tqdm(testloader, desc='Batch', leave=False, position=2) adversary = LinfPGDAttack( ensemble, loss_fn=loss_fn, eps=eps, nb_iter=steps, eps_iter=eps/5, rand_init=True, clip_min=0., clip_max=1., targeted=False) _, label, pred, advpred = attack_whole_dataset(adversary, test_iter, device="cuda") correct_or_not.append(label == advpred) correct_or_not = torch.stack(correct_or_not, dim=-1).all(dim=-1) tqdm.write("Accuracy: {:.2f}%, steps: {:d}, PGD Accuracy: {:.2f}%".format( 100. * (label == pred).sum().item() / len(label), steps, 100. * correct_or_not.sum().item() / len(label))) rob[str(steps)] = 100. * correct_or_not.sum().item() / len(label) # save to file if args.save_to_csv: output_root = os.path.join('results', 'wbox', train_alg, train_seed, 'convergence_check') if not os.path.exists(output_root): os.makedirs(output_root) output_filename = args.model_file.split('/')[-2] output = os.path.join(output_root, '.'.join((output_filename, 'csv'))) df = pd.DataFrame(rob, index=[0]) if args.append_out and os.path.isfile(output): with open(output, 'a') as f: f.write('\n') df.to_csv(output, sep=',', mode='a', header=False, index=False, float_format='%.2f') else: df.to_csv(output, sep=',', index=False, float_format='%.2f') else: eps_list = [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07] rob['random_start'] = args.random_start rob['steps'] = args.steps for eps in tqdm(eps_list, desc='PGD eps', leave=True, position=0): correct_or_not = [] for i in tqdm(range(args.random_start), desc='Random Start', leave=False, position=1): torch.manual_seed(i) test_iter = tqdm(testloader, desc='Batch', leave=False, position=2) adversary = LinfPGDAttack( ensemble, loss_fn=loss_fn, eps=eps, nb_iter=args.steps, eps_iter=eps/5, rand_init=True, clip_min=0., clip_max=1., targeted=False) _, label, pred, advpred = attack_whole_dataset(adversary, test_iter, device="cuda") correct_or_not.append(label == advpred) correct_or_not = torch.stack(correct_or_not, dim=-1).all(dim=-1) tqdm.write("Accuracy: {:.2f}%, eps: {:.2f}, PGD Accuracy: {:.2f}%".format( 100. * (label == pred).sum().item() / len(label), eps, 100. * correct_or_not.sum().item() / len(label))) rob['clean'] = 100. * (label == pred).sum().item() / len(label) rob[str(eps)] = 100. * correct_or_not.sum().item() / len(label) # save to file if args.save_to_csv: output_root = os.path.join('results', 'wbox', train_alg, train_seed) if not os.path.exists(output_root): os.makedirs(output_root) output_filename = args.model_file.split('/')[-2] output = os.path.join(output_root, '.'.join((output_filename, 'csv'))) df = pd.DataFrame(rob, index=[0]) if args.append_out and os.path.isfile(output): with open(output, 'a') as f: f.write('\n') df.to_csv(output, sep=',', mode='a', header=False, index=False, float_format='%.2f') else: df.to_csv(output, sep=',', index=False, float_format='%.2f')
# Set device cuda = torch.cuda.is_available() device = torch.device(f"cuda:{args.gpu}" if cuda else "cpu") print(f'Using device: {device}') print('Loading inception model') block_idx = InceptionV3.BLOCK_INDEX_BY_DIM[args.dims] inception = InceptionV3([block_idx]).to(device) print('Finished loading inception') with torch.cuda.device(device) if cuda else nullcontext: # Load params from text file models = get_models(args.hparams, device, load_discr=False) print('Entering Hyperparameter Loop') for h, gen, _ in models: fid = get_fid_score(gen, h.name, h.dataroot, inception, num_samples=args.samples, cuda=cuda) print(fid)
def get_args(): parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('--dexp', help='root experiment folder', default='exp') parser.add_argument('--model', help='which model to use', default='glad', choices=get_models()) parser.add_argument('--epoch', help='max epoch to run for', default=150, type=int) parser.add_argument('--demb', help='word embedding size', default=400, type=int) parser.add_argument('--dhid', help='hidden state size', default=200, type=int) parser.add_argument('--batch_size', help='batch size', default=50, type=int) parser.add_argument('--lr', help='learning rate', default=1e-3, type=float) parser.add_argument('--eps', help='adam eplison', default=1e-8, type=float) parser.add_argument('--stop', help='slot to early stop on', default='joint_goal') parser.add_argument('--resume', help='save directory to resume from', type=str)#action='store_true') parser.add_argument('-n', '--nick', help='nickname for model', default='default') parser.add_argument('--seed', default=42, help='random seed', type=int) parser.add_argument('--test', action='store_true', help='run in evaluation only mode') parser.add_argument('--gpu', type=int, help='which GPU to use') parser.add_argument('--emb_dropout', type=float, default=0.2, help='embedding dropout') parser.add_argument('--local_dropout', type=float, default=0.2, help='local dropout') parser.add_argument('--global_dropout', type=float, default=0.2, help='global dropout') parser.add_argument('--selfattn_dropout', type=float, default=0.0, help='self attention dropout') #parser.add_argument('--dropout', nargs='*', help='dropout rates', default=['emb=0.2', 'local=0.2', 'global=0.2']) parser.add_argument("--word_dropout", help='rate at which word embedding is set to 0', default=0.3, type=float) parser.add_argument('--dataset', type=str, help='dataset to use: woz or dstc', default='woz') parser.add_argument('--threshold', help='sigmoid threshold', default=0.5, type=float) parser.add_argument('--infer_with_asr', action='store_true', help='use asr for inference') parser.add_argument('--infer_with_confnet', action='store_true', help='use confnet for inference') parser.add_argument('--asr_number', type=int, default=1, help='number of asr utterance used during inference') parser.add_argument('--asr_average_method', type=str, default='sum', help='method to accumulate ASR utterances: sum, wrighted_sum, mean') parser.add_argument('--train_using', type=str, default='transcript', help='train using [asr, transcript, confnet, aug_confnet, aug_asr]') parser.add_argument('--max_par_arc', type=int, default=5, help='max number of parallel arcs in the confnet') parser.add_argument('--infer_with_confnet_best_pass', action='store_true', help='use confnet for inference') parser.add_argument('--visualize_attention', action='store_true', help='Visualize the attention weights during eval') parser.add_argument('--joint_training', action='store_true', help='Train asr or confnet embeddings jointly with transcript') parser.add_argument('--ver1', action='store_true', help="use pi instead of alpha in eq 4") parser.add_argument('--ver2', action='store_true', help="summation of p") parser.add_argument('--ver3', action='store_true', help='remove pi from eq 1') parser.add_argument('--ver4', action='store_true', help="pi*tanh(w1*Emb)") parser.add_argument('--forward_pass_time', action='store_true', help='forward pass time') args = parser.parse_args() args.dout = os.path.join(args.dexp, args.model, args.nick) if args.resume: args.resume = os.path.join(args.resume) #args.dropout = {d.split('=')[0]: float(d.split('=')[1]) for d in args.dropout} if not os.path.isdir(args.dout): os.makedirs(args.dout) return args