def test_get_models(self):
with tempfile.TemporaryDirectory() as models_dir:
model1 = '000013-model.meta'
model2 = '000017-model.meta'
f1 = open(os.path.join(models_dir, model1), 'w')
f1.close()
f2 = open(os.path.join(models_dir, model2), 'w')
f2.close()
model_nums_names = utils.get_models(models_dir)
self.assertEqual(len(model_nums_names), 2)
self.assertEqual(model_nums_names[0], (13, '000013-model'))
self.assertEqual(model_nums_names[1], (17, '000017-model'))
def validate(trained_models_dir, holdout_dir, estimator_model_dir, params):
"""Validate the latest model on the holdout dataset.
Args:
trained_models_dir: Directories where the completed generations/models are.
holdout_dir: Directories where holdout data are.
estimator_model_dir: tf.estimator model directory.
params: A MiniGoParams instance of hyperparameters for the model.
"""
model_num, _ = utils.get_latest_model(trained_models_dir)
# Get the holdout game data
nums_names = utils.get_models(trained_models_dir)
# Model N was trained on games up through model N-1, so the validation set
# should only be for models through N-1 as well, thus the (model_num) term.
models = [num_name for num_name in nums_names if num_name[0] < model_num]
# pair is a tuple of (model_num, model_name), like (13, 000013-modelname)
holdout_dirs = [os.path.join(holdout_dir, pair[1])
for pair in models[-params.holdout_generation:]]
tf_records = []
with utils.logged_timer('Building lists of holdout files'):
for record_dir in holdout_dirs:
if os.path.exists(record_dir): # make sure holdout dir exists
tf_records.extend(
tf.gfile.Glob(os.path.join(record_dir, '*'+_TF_RECORD_SUFFIX)))
if not tf_records:
print('No holdout dataset for validation! '
'Please check your holdout directory: {}'.format(holdout_dir))
return
print('The length of tf_records is {}.'.format(len(tf_records)))
first_tf_record = os.path.basename(tf_records[0])
last_tf_record = os.path.basename(tf_records[-1])
with utils.logged_timer('Validating from {} to {}'.format(
first_tf_record, last_tf_record)):
dualnet.validate(estimator_model_dir, tf_records, params)
z_dim = args.z_dim
beta = args.beta
dataset_name = args.dataset_name
model_name = args.model_name
experiment_name = args.experiment_name
pylib.mkdir('./output/%s' % experiment_name)
with open('./output/%s/setting.txt' % experiment_name, 'w') as f:
f.write(json.dumps(vars(args), indent=4, separators=(',', ':')))
# dataset and models
Dataset, img_shape, get_imgs = utils.get_dataset(dataset_name)
dataset = Dataset(batch_size=batch_size)
dataset_val = Dataset(batch_size=100)
Enc, Dec = utils.get_models(model_name)
Enc = partial(Enc, z_dim=z_dim)
Dec = partial(Dec, channels=img_shape[2])
# ==============================================================================
# = graph =
# ==============================================================================
def enc_dec(img, is_training=True):
# encode
z_mu, z_log_sigma_sq = Enc(img, is_training=is_training)
# sample
epsilon = tf.random_normal(tf.shape(z_mu))
if is_training:
def main():
# get args
args = get_args()
# set up gpus
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
assert torch.cuda.is_available()
# load models
if 'gal' in args.model_file:
leaky_relu = True
else:
leaky_relu = False
ensemble = utils.get_models(args, train=False, as_ensemble=True, model_file=args.model_file, leaky_relu=leaky_relu)
models = ensemble.models
train_seed = args.model_file.split('/')[-3]
train_alg = args.model_file.split('/')[-4]
# get data loaders
testloader = utils.get_testloader(args, batch_size=args.batch_size)
# pick out samples that are correctly classified by all submodels
correct = []
for m in models:
correct_m = []
for (x, y) in testloader:
x, y = x.cuda(), y.cuda()
outputs = m(x)
_, pred = outputs.max(1)
correct_m.append(pred.eq(y))
correct_m = torch.cat(correct_m)
correct.append(correct_m)
correct = torch.stack(correct, dim=-1).all(-1)
correct_idx = correct.nonzero().squeeze(-1)
random.seed(0)
subset_idx = correct_idx[random.sample(range(correct_idx.size(0)), args.subset_num)].cpu()
subset_loader = utils.get_testloader(args, batch_size=args.batch_size, shuffle=False, subset_idx=subset_idx)
# PGD
eps_list = [0.03]
random_start = args.random_start
steps = args.steps
rob = {}
rob['random_start'] = args.random_start
rob['steps'] = args.steps
for eps in tqdm(eps_list, desc='PGD eps', leave=False, position=0):
correct_or_not_rs = torch.zeros((len(models), len(models)+1, args.subset_num, random_start), dtype=torch.bool)
for rs in tqdm(range(random_start), desc='Random Start', leave=False, position=1):
torch.manual_seed(rs)
test_iter = tqdm(subset_loader, desc='Batch', leave=False, position=2)
total = 0
for (x, y) in test_iter:
x, y = x.cuda(), y.cuda()
adv_list = []
for i, m in enumerate(models):
adversary = LinfPGDAttack(
m, loss_fn=nn.CrossEntropyLoss(reduction="sum"), eps=eps,
nb_iter=steps, eps_iter=eps/5, rand_init=True, clip_min=0., clip_max=1.,
targeted=False)
adv = adversary.perturb(x, y)
adv_list.append(adv)
for i, adv in enumerate(adv_list):
for j, m in enumerate(models):
if j == i:
outputs = m(x)
_, pred = outputs.max(1)
assert pred.eq(y).all()
outputs = m(adv)
_, pred = outputs.max(1)
correct_or_not_rs[i, j, total:total+x.size(0), rs] = pred.eq(y)
outputs = ensemble(adv)
_, pred = outputs.max(1)
correct_or_not_rs[i, len(models), total:total+x.size(0), rs] = pred.eq(y)
total += x.size(0)
correct_or_not_rs = torch.all(correct_or_not_rs, dim=-1)
asr = np.zeros((len(models), len(models)+1))
tqdm.write("eps: {:.2f}".format(eps))
for i in range(len(models)):
message = ''
for j in range(len(models)+1):
message += '\t{}: {:.2%}'.format(j, 1-correct_or_not_rs[i, j, :].sum().item()/args.subset_num)
asr[i, j] = 1-correct_or_not_rs[i, j, :].sum().item()/args.subset_num
tqdm.write(message)
rob[str(eps)] = asr
# save to file
if args.save_to_file:
output_root = os.path.join('results', 'transferability', train_alg, train_seed)
if not os.path.exists(output_root):
os.makedirs(output_root)
output_filename = args.model_file.split('/')[-2]
output = os.path.join(output_root, '.'.join((output_filename, 'pkl')))
with open(output, 'wb') as f:
pickle.dump(rob, f, pickle.HIGHEST_PROTOCOL)
# ==============================================================================
# = datasets and models =
# ==============================================================================
source_train_data, target_train_data, source_test_data, target_test_data, \
min_n, preprocessor = utils.get_data(data_path, data_type, use_test)
source_train_dataset = utils.make_dataset(source_train_data, batch_size = batch_size)
target_train_dataset = utils.make_dataset(target_train_data, batch_size = batch_size)
s_iterator = source_train_dataset.make_one_shot_iterator()
s_next_element = s_iterator.get_next()
t_iterator = target_train_dataset.make_one_shot_iterator()
t_next_element = t_iterator.get_next()
input_dim = source_train_data.shape[1]
Enc, Dec_a, Dec_b, Disc = utils.get_models(model_name)
Enc = partial(Enc, code_dim=code_dim)
Dec_a = partial(Dec_a, output_dim=input_dim)
Dec_b = partial(Dec_b, output_dim=input_dim)
# ==============================================================================
# = graph =
# ==============================================================================
def enc_dec(input, AE_type, is_training=True):
# encode
if AE_type == "standard":
c, _ = Enc(input, is_training=is_training)
if AE_type == "VAE":
c_mu, c_log_sigma_sq = Enc(input, is_training=is_training)
from cornac.datasets import movielens
from eval_methods.stratified_evaluation import StratifiedEvaluation
from experiment.experiment import STExperiment
from utils import get_models, get_metrics
import sys
sys.stdout = open('log_ml_large_1M.txt', 'w', 1)
dims = [e for e in range(10, 110, 10)]
# load the movielens dataset
ml = movielens.load_feedback(variant="1M")
# propensity-based stratified evaluation
stra_eval_method = StratifiedEvaluation(data=ml,
n_strata=2,
rating_threshold=4.0,
verbose=True)
# run the experiment
exp_stra = STExperiment(eval_method=stra_eval_method,
models=get_models(variant='large', dims=dims),
metrics=get_metrics(variant='small'),
verbose=True)
exp_stra.run()
with open('../data/exp_stra_ml.pkl', 'wb') as exp_file:
pickle.dump(exp_stra.result, exp_file)
if 'retrieval' not in args.tasks: # don't want to load code data twice
data_to_load.append(('code', code_vocab, args.code_max_length))
if 'language_model_docstring' in args.tasks:
if 'retrieval' not in args.tasks: # don't want to load docstring data twice
data_to_load.append(
('docstring', docstring_vocab, args.docstring_max_length))
train_data = utils.load_data(args.data + '_train.jsonl', data_to_load)
valid_data = utils.load_data(args.data + '_valid.jsonl', data_to_load)
test_data = utils.load_data(args.data + '_test.jsonl', data_to_load)
print(f'training examples: {utils.get_num_examples(train_data)}')
print(f'validation examples: {utils.get_num_examples(valid_data)}')
print(f'testing examples: {utils.get_num_examples(test_data)}')
print('Creating iterators...')
train_iterators = utils.get_iterators(train_data,
args.batch_size,
shuffle=True)
valid_iterators = utils.get_iterators(valid_data,
args.batch_size,
shuffle=False)
test_iterators = utils.get_iterators(test_data, args.batch_size, shuffle=False)
print('Creating models...')
models = utils.get_models(args.config, data_to_load, args.tasks)
for k, v in models.items():
print(k, v)
def main():
# get args
args = get_args()
# set up gpus
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
assert torch.cuda.is_available()
# load models
if 'gal' in args.model_file:
leaky_relu = True
else:
leaky_relu = False
ensemble = utils.get_models(args,
train=False,
as_ensemble=True,
model_file=args.model_file,
leaky_relu=leaky_relu)
train_seed = args.model_file.split('/')[-3]
train_alg = args.model_file.split('/')[-4]
eps_list = [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07]
loss_fn_list = ['xent', 'cw']
surrogate_model_list = [
'{:s}{:d}'.format(args.which_ensemble, i) for i in [3, 5, 8]
]
method_list = [
'mdi2_0.5_steps_{:d}'.format(args.steps),
'sgm_0.2_steps_{:d}'.format(args.steps)
]
index_list = [
'{:s}_{:s}_mpgd'.format(a, b) for a in surrogate_model_list
for b in loss_fn_list
]
index_list += [
'{:s}_{:s}_{:s}'.format(a, b, c) for a in surrogate_model_list
for b in loss_fn_list for c in method_list
]
index_list.append('all')
random_start = 3
input_list = [
'from_{:s}_{:s}_mpgd_steps_{:d}'.format(a, b, args.steps)
for a in surrogate_model_list for b in loss_fn_list
]
input_list += [
'from_{:s}_{:s}_{:s}'.format(a, b, c) for a in surrogate_model_list
for b in loss_fn_list for c in method_list
]
rob = {}
rob['source'] = index_list
acc_list = [[] for _ in range(len(eps_list))]
data_root = os.path.join(args.data_dir, args.folder)
# clean acc
input_folder = os.path.join(data_root, 'clean')
clean_acc = test(ensemble, input_folder, return_acc=True)
clean_acc_list = [clean_acc for _ in range(len(input_list) + 1)]
rob['clean'] = clean_acc_list
# transfer attacks
for i, eps in enumerate(tqdm(eps_list, desc='eps', leave=True,
position=0)):
input_folder = os.path.join(data_root, 'eps_{:.2f}'.format(eps))
correct_over_input = []
for input_adv in tqdm(input_list,
desc='source',
leave=False,
position=1):
if 'mpgd' in input_adv:
correct_over_rs = []
for rs in tqdm(range(random_start),
desc='Random Start',
leave=False,
position=2):
datafolder = os.path.join(input_folder, '_'.join(
(input_adv, str(rs))))
correct_over_rs.append(test(ensemble, datafolder))
correct_over_rs = torch.stack(correct_over_rs,
dim=-1).all(dim=-1)
acc_list[i].append(100. * correct_over_rs.sum().item() /
len(correct_over_rs))
correct_over_input.append(correct_over_rs)
tqdm.write(
'Clean acc: {:.2f}%, eps: {:.2f}, transfer from {:s} acc: {:.2f}%'
.format(
clean_acc, eps, input_adv, 100. *
correct_over_rs.sum().item() / len(correct_over_rs)))
else:
datafolder = os.path.join(input_folder, input_adv)
correct = test(ensemble, datafolder)
acc_list[i].append(100. * correct.sum().item() / len(correct))
correct_over_input.append(correct)
tqdm.write(
'Clean acc: {:.2f}%, eps: {:.2f}, transfer from {:s} acc: {:.2f}%'
.format(clean_acc, eps, input_adv,
100. * correct.sum().item() / len(correct)))
correct_over_input = torch.stack(correct_over_input,
dim=-1).all(dim=-1)
acc_list[i].append(100. * correct_over_input.sum().item() /
len(correct_over_input))
tqdm.write(
'Clean acc: {:.2f}%, eps: {:.2f}, transfer acc: {:.2f}%'.format(
clean_acc, eps, 100. * correct_over_input.sum().item() /
len(correct_over_input)))
rob[str(eps)] = acc_list[i]
# save to file
if args.save_to_csv:
output_root = os.path.join('results', 'bbox', train_alg, train_seed)
if not os.path.exists(output_root):
os.makedirs(output_root)
output_filename = args.model_file.split('/')[-2]
output = os.path.join(output_root, '.'.join((output_filename, 'csv')))
df = pd.DataFrame(rob)
if args.append_out and os.path.isfile(output):
with open(output, 'a') as f:
f.write('\n')
df.to_csv(output,
sep=',',
mode='a',
header=False,
index=False,
float_format='%.2f')
else:
df.to_csv(output, sep=',', index=False, float_format='%.2f')
for a in args.accelerator_ids
]
assert cfg.SEARCH.CHILDREN % cfg.SEARCH.PARENTS == 0, \
'Number of children must be divisible by number of parents.'
assert cfg.SEARCH.CHILDREN % len(accelerators) == 0, \
'Number of children must be divisible by number of accelerators.'
print('Using accelerators: {}'.format(accelerators))
cfg.SEARCH.DIR = 'searches/' + args.cfg.split('.')[0]
os.makedirs(cfg.SEARCH.DIR, exist_ok=True)
# EVOLUTION
while 1: # each loop is a generation, run until manually stopped
meta_files, models, genotypes = get_models(cfg.SEARCH.DIR)
if len(models) == 0:
# train initial genotype
train_cfg = update_cfg(
copy.deepcopy(cfg),
accelerator=accelerators[0],
gen=0,
model_id=0,
genotype=genotype_from_blocks_args(DEFAULT_BLOCKS_ARGS),
epochs=cfg.SEARCH.GEN0_EPOCHS,
parent=None,
disp=True)
train_wrapper(train_cfg)
else:
# train next generation
parser.add_argument('--use_scorer',
type=int,
default=1,
choices=[0, 1],
help='Whether to calculate FID score')
args = parser.parse_args()
device = torch.device(f"cuda:{args.gpu}" if (
torch.cuda.is_available()) else "cpu")
print(f'Using device: {device}')
# Load scorer
scorer = fid_scorer(device) if args.use_scorer else None
# Load params from text file
models = get_models(args.hparams, device)
print('Entering Hyperparameter Loop')
for h, gen, discr in models:
with (SummaryWriter(comment=f'_{h.name}')
if args.use_writer else nullcontext) as writer:
print(f'Run name: {h.name}')
if writer:
writer.add_hparams(vars(h), {})
if scorer:
scorer.set_params(h.dataroot, h.name, writer)
host = 'localhost'
port = 8000
max_stop_time = 10 # in second
max_wall_time = 10 # in hour
frame = [350, 205 + 20]
mean = [0.35739973, 0.35751262, 0.36058474]
std = [0.19338858, 0.19159749, 0.2047393]
transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(mean, std)])
print('Loading model...')
encoder, decoder, init_y = get_models()
y_bin = pickle.load(open('./data/y_bin_info.pickle', 'rb'))
client = Client(ip=host, port=port)
scenario = Scenario(weather='EXTRASUNNY',
vehicle='voltic',
time=[12, 0],
drivingMode=-1,
location=[-2500, 3250])
client.sendMessage(Start(scenario=scenario))
with torch.no_grad():
# encoder.eval()
# decoder.eval()
def main():
# get args
args = get_args()
# set up gpus
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
assert torch.cuda.is_available()
# set up writer, logger, and save directory for models
save_root = os.path.join('checkpoints', 'transfer',
'seed_{:d}'.format(args.seed),
'{:s}{:d}'.format(args.arch, args.depth))
save_root += "%.2f" % (args.transfer_coeff)
if not os.path.exists(save_root):
os.makedirs(save_root)
else:
print('*********************************')
print('* The checkpoint already exists *')
print('*********************************')
writer = SummaryWriter(save_root.replace('checkpoints', 'runs'))
# dump configurations for potential future references
with open(os.path.join(save_root, 'cfg.json'), 'w') as fp:
json.dump(vars(args), fp, indent=4)
with open(
os.path.join(save_root.replace('checkpoints', 'runs'), 'cfg.json'),
'w') as fp:
json.dump(vars(args), fp, indent=4)
# set up random seed
torch.manual_seed(args.seed)
# initialize models
models = utils.get_models(args,
train=True,
as_ensemble=False,
model_file="/sync_transfer/CIFAR/epoch_200.pth",
dataset="CIFAR-10")
# get data loaders
source_trainloader, source_testloader = utils.get_loaders(
args, dataset="CIFAR-10")
target_trainloader, target_testloader = utils.get_loaders(args,
dataset="STL-10")
# get optimizers and schedulers
optimizers = utils.get_optimizers(args, models)
schedulers = utils.get_schedulers(args, optimizers)
surrogate = utils.get_models(args,
train=False,
as_ensemble=False,
model_file="/sync_transfer/STL/epoch_200.pth",
dataset="STL-10")
trainer = Transfer_Trainer(models, optimizers, schedulers,
source_trainloader, source_testloader,
target_trainloader, surrogate, writer,
save_root, **vars(args))
trainer.run()
def init_tests():
"""Get the data required for the tests."""
get_cifar10_dataset()
convert_cifar2imagenet_format()
download_images()
get_models()
def init_tests():
"""Get the data required for the tests."""
get_cifar10_dataset()
get_models()
validation_log.close()
train_log.close()
# When done, ask the threads to stop.
coord.request_stop()
# Wait for threads to finish.
coord.join(threads)
return best_validation_accuracy
if __name__ == '__main__':
# CLI arguments
PARSER = argparse.ArgumentParser(description="Train the model")
# Required arguments
PARSER.add_argument("--model", required=True, choices=utils.get_models())
PARSER.add_argument("--dataset",
required=True,
choices=utils.get_datasets())
# Restart train or continue
PARSER.add_argument("--restart", action='store_true')
# Learning rate decay arguments
PARSER.add_argument("--lr_decay", action="store_true")
PARSER.add_argument("--lr_decay_epochs", type=int, default=25)
PARSER.add_argument("--lr_decay_factor", type=float, default=0.1)
# L2 regularization arguments
PARSER.add_argument("--l2_penalty", type=float, default=0.0)
def main():
# get args
args = get_args()
# set up gpus
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
assert torch.cuda.is_available()
# load models
if 'gal' in args.model_file:
leaky_relu = True
else:
leaky_relu = False
ensemble = utils.get_models(args,
train=False,
as_ensemble=True,
model_file=args.model_file,
leaky_relu=leaky_relu)
models = ensemble.models
train_seed = args.model_file.split('/')[-3]
train_alg = args.model_file.split('/')[-4]
# get data loaders
testloader = utils.get_testloader(args, batch_size=100)
# pick out samples that are correctly classified by all submodels
correct = []
for m in models:
correct_m = []
for (x, y) in testloader:
x, y = x.cuda(), y.cuda()
outputs = m(x)
_, pred = outputs.max(1)
correct_m.append(pred.eq(y))
correct_m = torch.cat(correct_m)
correct.append(correct_m)
correct = torch.stack(correct, dim=-1).all(-1)
correct_idx = correct.nonzero().squeeze(-1)
random.seed(0)
subset_idx = correct_idx[random.sample(range(correct_idx.size(0)),
args.subset_num)].cpu()
# use a very small batch size so that we can sample different layers multiple times
subset_loader = utils.get_testloader(args,
batch_size=10,
shuffle=True,
subset_idx=subset_idx)
eps_list = [0.07]
steps = 10
rob = {}
rob['steps'] = steps
criterion = nn.CrossEntropyLoss(reduction='none')
for eps in tqdm(eps_list, desc='eps', leave=False, position=0):
losses = torch.zeros((len(models), len(models), args.subset_num))
torch.manual_seed(0)
loader = utils.DistillationLoader(subset_loader, subset_loader)
test_iter = tqdm(loader, desc='Batch', leave=False, position=1)
random.seed(0)
total = 0
for batch_idx, (si, sl, ti, tl) in enumerate(test_iter):
si, sl = si.cuda(), sl.cuda()
ti, tl = ti.cuda(), tl.cuda()
layer = random.randint(1, args.depth)
adv_list = []
for i, m in enumerate(models):
adv = Linf_distillation(m,
si,
ti,
eps,
eps / steps,
steps,
layer,
before_relu=True,
mu=1,
momentum=True,
rand_start=False)
adv_list.append(adv)
with torch.no_grad():
for i, adv in enumerate(adv_list):
for j, m in enumerate(models):
if j == i:
outputs = m(si)
_, pred = outputs.max(1)
assert pred.eq(sl).all()
outputs = m(adv)
loss = criterion(outputs, tl)
losses[i, j, total:total + si.size(0)] = loss
total += si.size(0)
losses_np = torch.mean(losses, dim=-1).numpy()
tqdm.write("eps: {:.2f}".format(eps))
for i in range(len(models)):
message = ''
for j in range(len(models)):
message += '\t{}: {:.3f}'.format(j, losses_np[i, j])
tqdm.write(message)
rob[str(eps)] = losses_np
# save to file
if args.save_to_file:
output_root = os.path.join('results', 'diversity', train_alg,
train_seed)
if not os.path.exists(output_root):
os.makedirs(output_root)
output_filename = args.model_file.split('/')[-2]
output = os.path.join(output_root, '.'.join((output_filename, 'pkl')))
with open(output, 'wb') as f:
pickle.dump(rob, f, pickle.HIGHEST_PROTOCOL)
type=str,
help='Run Configuration',
default='run_config.yaml')
parser.add_argument('--runs',
type=int,
help='Number of Runs for each model',
default=1)
#parser.add_argument('--early_stop', type=bool, help='no improvement during 10 epoch -> stop', default=True)
args = parser.parse_args()
# Load configuration file for pipeline
with open(args.run_config) as f:
config = yaml.load(f, Loader=yaml.Loader)
# Load Available models
models = utils.get_models()
logger.info('Available models %s' % list(models.keys()))
# Load and normalize data
input_data = pd.read_csv(args.train_data)
input_data.iloc[:, 1:] = MinMaxScaler().fit_transform(input_data.iloc[:, 1:])
train, validate, test = np.split(
input_data.sample(frac=1),
[int(.6 * len(input_data)),
int(.8 * len(input_data))])
train_Y = train.iloc[:, 0].values
train_X = train.iloc[:, 1:].values
validation_Y = validate.iloc[:, 0].values
validation_X = validate.iloc[:, 1:].values
def main():
# get args
args = get_args()
# set up gpus
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
assert torch.cuda.is_available()
# load models
if 'gal' in args.model_file:
leaky_relu = True
else:
leaky_relu = False
ensemble = utils.get_models(args, train=False, as_ensemble=True, model_file=args.model_file, leaky_relu=leaky_relu)
# get data loaders
total_sample_num = 10000
if args.subset_num:
random.seed(0)
subset_idx = random.sample(range(total_sample_num), args.subset_num)
testloader = utils.get_testloader(args, batch_size=200, shuffle=False, subset_idx=subset_idx)
else:
testloader = utils.get_testloader(args, batch_size=200, shuffle=False)
loss_fn = nn.CrossEntropyLoss() if args.loss_fn == 'xent' else CarliniWagnerLoss(conf=args.cw_conf)
rob = {}
rob['sample_num'] = args.subset_num if args.subset_num else total_sample_num
rob['loss_fn'] = 'xent' if args.loss_fn == 'xent' else 'cw_{:.1f}'.format(args.cw_conf)
train_seed = args.model_file.split('/')[-3]
train_alg = args.model_file.split('/')[-4]
if args.convergence_check:
eps = 0.01
steps_list = [50, 500, 1000]
random_start = 1
rob['random_start'] = random_start
rob['eps'] = eps
# FGSM
test_iter = tqdm(testloader, desc='FGSM', leave=False, position=0)
adversary = GradientSignAttack(
ensemble, loss_fn=nn.CrossEntropyLoss(), eps=eps,
clip_min=0., clip_max=1., targeted=False)
_, label, pred, advpred = attack_whole_dataset(adversary, test_iter, device="cuda")
print("Accuracy: {:.2f}%, FGSM Accuracy: {:.2f}%".format(
100. * (label == pred).sum().item() / len(label),
100. * (label == advpred).sum().item() / len(label)))
rob['clean'] = 100. * (label == pred).sum().item() / len(label)
rob['fgsm'] = 100. * (label == advpred).sum().item() / len(label)
for steps in tqdm(steps_list, desc='PGD steps', leave=False, position=0):
correct_or_not = []
for i in tqdm(range(random_start), desc='Random Start', leave=False, position=1):
torch.manual_seed(i)
test_iter = tqdm(testloader, desc='Batch', leave=False, position=2)
adversary = LinfPGDAttack(
ensemble, loss_fn=loss_fn, eps=eps,
nb_iter=steps, eps_iter=eps/5, rand_init=True, clip_min=0., clip_max=1.,
targeted=False)
_, label, pred, advpred = attack_whole_dataset(adversary, test_iter, device="cuda")
correct_or_not.append(label == advpred)
correct_or_not = torch.stack(correct_or_not, dim=-1).all(dim=-1)
tqdm.write("Accuracy: {:.2f}%, steps: {:d}, PGD Accuracy: {:.2f}%".format(
100. * (label == pred).sum().item() / len(label),
steps,
100. * correct_or_not.sum().item() / len(label)))
rob[str(steps)] = 100. * correct_or_not.sum().item() / len(label)
# save to file
if args.save_to_csv:
output_root = os.path.join('results', 'wbox', train_alg, train_seed, 'convergence_check')
if not os.path.exists(output_root):
os.makedirs(output_root)
output_filename = args.model_file.split('/')[-2]
output = os.path.join(output_root, '.'.join((output_filename, 'csv')))
df = pd.DataFrame(rob, index=[0])
if args.append_out and os.path.isfile(output):
with open(output, 'a') as f:
f.write('\n')
df.to_csv(output, sep=',', mode='a', header=False, index=False, float_format='%.2f')
else:
df.to_csv(output, sep=',', index=False, float_format='%.2f')
else:
eps_list = [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07]
rob['random_start'] = args.random_start
rob['steps'] = args.steps
for eps in tqdm(eps_list, desc='PGD eps', leave=True, position=0):
correct_or_not = []
for i in tqdm(range(args.random_start), desc='Random Start', leave=False, position=1):
torch.manual_seed(i)
test_iter = tqdm(testloader, desc='Batch', leave=False, position=2)
adversary = LinfPGDAttack(
ensemble, loss_fn=loss_fn, eps=eps,
nb_iter=args.steps, eps_iter=eps/5, rand_init=True, clip_min=0., clip_max=1.,
targeted=False)
_, label, pred, advpred = attack_whole_dataset(adversary, test_iter, device="cuda")
correct_or_not.append(label == advpred)
correct_or_not = torch.stack(correct_or_not, dim=-1).all(dim=-1)
tqdm.write("Accuracy: {:.2f}%, eps: {:.2f}, PGD Accuracy: {:.2f}%".format(
100. * (label == pred).sum().item() / len(label),
eps,
100. * correct_or_not.sum().item() / len(label)))
rob['clean'] = 100. * (label == pred).sum().item() / len(label)
rob[str(eps)] = 100. * correct_or_not.sum().item() / len(label)
# save to file
if args.save_to_csv:
output_root = os.path.join('results', 'wbox', train_alg, train_seed)
if not os.path.exists(output_root):
os.makedirs(output_root)
output_filename = args.model_file.split('/')[-2]
output = os.path.join(output_root, '.'.join((output_filename, 'csv')))
df = pd.DataFrame(rob, index=[0])
if args.append_out and os.path.isfile(output):
with open(output, 'a') as f:
f.write('\n')
df.to_csv(output, sep=',', mode='a', header=False, index=False, float_format='%.2f')
else:
df.to_csv(output, sep=',', index=False, float_format='%.2f')
# Set device
cuda = torch.cuda.is_available()
device = torch.device(f"cuda:{args.gpu}" if cuda else "cpu")
print(f'Using device: {device}')
print('Loading inception model')
block_idx = InceptionV3.BLOCK_INDEX_BY_DIM[args.dims]
inception = InceptionV3([block_idx]).to(device)
print('Finished loading inception')
with torch.cuda.device(device) if cuda else nullcontext:
# Load params from text file
models = get_models(args.hparams, device, load_discr=False)
print('Entering Hyperparameter Loop')
for h, gen, _ in models:
fid = get_fid_score(gen,
h.name,
h.dataroot,
inception,
num_samples=args.samples,
cuda=cuda)
print(fid)
def get_args():
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--dexp', help='root experiment folder', default='exp')
parser.add_argument('--model', help='which model to use', default='glad', choices=get_models())
parser.add_argument('--epoch', help='max epoch to run for', default=150, type=int)
parser.add_argument('--demb', help='word embedding size', default=400, type=int)
parser.add_argument('--dhid', help='hidden state size', default=200, type=int)
parser.add_argument('--batch_size', help='batch size', default=50, type=int)
parser.add_argument('--lr', help='learning rate', default=1e-3, type=float)
parser.add_argument('--eps', help='adam eplison', default=1e-8, type=float)
parser.add_argument('--stop', help='slot to early stop on', default='joint_goal')
parser.add_argument('--resume', help='save directory to resume from', type=str)#action='store_true')
parser.add_argument('-n', '--nick', help='nickname for model', default='default')
parser.add_argument('--seed', default=42, help='random seed', type=int)
parser.add_argument('--test', action='store_true', help='run in evaluation only mode')
parser.add_argument('--gpu', type=int, help='which GPU to use')
parser.add_argument('--emb_dropout', type=float, default=0.2, help='embedding dropout')
parser.add_argument('--local_dropout', type=float, default=0.2, help='local dropout')
parser.add_argument('--global_dropout', type=float, default=0.2, help='global dropout')
parser.add_argument('--selfattn_dropout', type=float, default=0.0, help='self attention dropout')
#parser.add_argument('--dropout', nargs='*', help='dropout rates', default=['emb=0.2', 'local=0.2', 'global=0.2'])
parser.add_argument("--word_dropout", help='rate at which word embedding is set to 0', default=0.3, type=float)
parser.add_argument('--dataset', type=str, help='dataset to use: woz or dstc', default='woz')
parser.add_argument('--threshold', help='sigmoid threshold', default=0.5, type=float)
parser.add_argument('--infer_with_asr', action='store_true', help='use asr for inference')
parser.add_argument('--infer_with_confnet', action='store_true', help='use confnet for inference')
parser.add_argument('--asr_number', type=int, default=1, help='number of asr utterance used during inference')
parser.add_argument('--asr_average_method', type=str, default='sum', help='method to accumulate ASR utterances: sum, wrighted_sum, mean')
parser.add_argument('--train_using', type=str, default='transcript', help='train using [asr, transcript, confnet, aug_confnet, aug_asr]')
parser.add_argument('--max_par_arc', type=int, default=5, help='max number of parallel arcs in the confnet')
parser.add_argument('--infer_with_confnet_best_pass', action='store_true', help='use confnet for inference')
parser.add_argument('--visualize_attention', action='store_true', help='Visualize the attention weights during eval')
parser.add_argument('--joint_training', action='store_true', help='Train asr or confnet embeddings jointly with transcript')
parser.add_argument('--ver1', action='store_true', help="use pi instead of alpha in eq 4")
parser.add_argument('--ver2', action='store_true', help="summation of p")
parser.add_argument('--ver3', action='store_true', help='remove pi from eq 1')
parser.add_argument('--ver4', action='store_true', help="pi*tanh(w1*Emb)")
parser.add_argument('--forward_pass_time', action='store_true', help='forward pass time')
args = parser.parse_args()
args.dout = os.path.join(args.dexp, args.model, args.nick)
if args.resume:
args.resume = os.path.join(args.resume)
#args.dropout = {d.split('=')[0]: float(d.split('=')[1]) for d in args.dropout}
if not os.path.isdir(args.dout):
os.makedirs(args.dout)
return args
