def test(classifier, generator, data_loader, dataset="MNIST"):
"""Evaluate classifier on source or target domains."""
# set eval state for Dropout and BN layers
generator.eval()
classifier.eval()
# init loss and accuracy
loss = 0
acc = 0
# set loss function
criterion = nn.CrossEntropyLoss()
# evaluate network
for (images, labels) in data_loader:
images = Variable(images, volatile=True)
labels = Variable(labels.squeeze_())
if use_cuda:
images = images.cuda()
labels = labels.cuda()
preds = classifier(generator(images))
loss += criterion(preds, labels).data[0]
pred_cls = preds.data.max(1)[1]
acc += pred_cls.eq(labels.data).cpu().sum()
loss /= len(data_loader)
acc /= len(data_loader.dataset)
print("Avg Loss = {:.5f}, Avg Accuracy = {:2.5%}".format(loss, acc))
def train_classifier(data):
image_shape = [image_size, image_size, 3]
n_classes = data["n_classes"]
training_steps = 10000
steps_per_epoch = 100
epochs = training_steps // steps_per_epoch
model = m.classifier(image_shape, n_classes)
model.summary()
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=[tf.keras.metrics.CategoricalAccuracy()])
dataset = make_infinite_dataset(data["n_train"],
(data["x_train"], data["y_train"]))
val_dataset = make_infinite_dataset(data["n_val"],
(data["x_val"], data["y_val"]))
model.fit(dataset,
validation_data=val_dataset,
validation_steps=120,
epochs=epochs,
steps_per_epoch=steps_per_epoch)
return model
def predict(id, t,predicted=0):
start=time.time()
stockwin=dp.createWinData(logr,id,t)
tr, te = dp.splitData(stockwin, ratio = 0.9)
tr[:,-1]=dp.transUpDown1(tr[:,-1])
te[:,-1]=dp.transUpDown1(te[:,-1])
tr1=[]
for i in range(len(tr)):
if tr[i,-1] >= 0:
tr1.append(tr[i,:])
tr1=np.array(tr1)
te1=[]
for i in range(len(te)):
if te[i,-1] >= 0:
te1.append(te[i,:])
te1=np.array(te1)
np.random.shuffle(tr1)
xtr=tr1[:,:-1]
ytr=tr1[:,-1]
xte=te1[:,:-1]
yte=te1[:,-1]
scaler = StandardScaler()
scaler.fit(xtr)
xtr=scaler.transform(xtr)
xte=scaler.transform(xte)
'''
logit=sm.Logit(ytr,xtr)
result=logit.fit()
ypr0=result.predict(xte)
ypr0=np.reshape(ypr0,(ypr0.shape[0],1))
temp1, tmep2=dp.para(0.5,0.5,ypr0,yte,1)
'''
xtr=np.reshape(xtr,(xtr.shape[0],t,int(xtr.shape[1]/t)))
xte=np.reshape(xte,(xte.shape[0],t,int(xte.shape[1]/t)))
model=models.classifier(t,36)
#model.summary()
model.fit(xtr, ytr, epochs = 100, batch_size=32,verbose=0,
validation_split = 0.1)#, callbacks = [earlystop,history])
#history.loss_plot('epoch')
model.evaluate(xte, yte)
ypr=model.predict(xte)
if predicted==0:
print('time cost:',time.time()-start)
return yte, ypr
else:
yptr=model.predict(xtr)
a,b=dp.maxpara(yptr,ytr)
print('alpha:',a)
print('beta:',b)
true,pre=dp.para(a,b,ypr,yte,1)
print('time cost:',time.time()-start)
return true, pre
def __init__(self, args):
super(Model, self).__init__()
self.device = args.device
self.layers = args.num_layers
self.input_size_graph = args.input_size_graph
self.output_size_graph = args.output_size_graph
self.train_data = args.train_data
self.test_data = args.test_data
self.train_labels = args.train_labels
self.test_labels = args.test_labels
self.latent_size = args.latent_size
self.hidden_size = args.hidden_size
self.learning_rate = args.lr
self.batch_size = args.batch_size
self.input_dim = args.input_dim
self.warmup = args.warmup
self.num_labels = args.num_labels
self.graph = args.graph
self.sequence = args.sequence
self.recons = args.recons
self.use_attn = args.attn
self.use_fusion = args.fusion
self.graph_pretrain = graph_models.GraphSage(self.layers,
self.input_size_graph,
self.output_size_graph,
device=self.device,
gcn="True",
agg_func="MEAN")
self.VAE = seq_models.VAE(args)
self.AtomEmbedding = nn.Embedding(self.input_size_graph,
self.hidden_size).to(self.device)
self.AtomEmbedding.weight.requires_grad = True
self.output_layer = models.classifier(self.latent_size,
self.num_labels, self.device)
self.label_criterion = nn.BCEWithLogitsLoss(reduction="none")
if self.use_attn:
self.attention = models.SelfAttention(self.hidden_size)
self.optimizer = optim.Adam(self.parameters(),
lr=self.learning_rate,
weight_decay=1e-8,
amsgrad=True)
for name, para in self.named_parameters():
if para.requires_grad:
print(name, para.data.shape)
def InceptionScore(img, splits=1):
with torch.no_grad():
N = len(img)
batch_size = 32
data_loader = DataLoader(img, batch_size=batch_size)
if cfg['data_name'] in ['COIL100', 'Omniglot']:
model = models.classifier().to(cfg['device'])
model_tag = ['0', cfg['data_name'], cfg['subset'], 'classifier']
model_tag = '_'.join(filter(None, model_tag))
checkpoint = load(
'./metrics_tf/res/classifier/{}_best.pt'.format(model_tag))
model.load_state_dict(checkpoint['model_dict'])
model.train(False)
pred = torch.zeros((N, cfg['classes_size']))
for i, input in enumerate(data_loader):
input = {
'img': input,
'label': input.new_zeros(input.size(0)).long()
}
input = to_device(input, cfg['device'])
input_size_i = input['img'].size(0)
output = model(input)
pred[i * batch_size:i * batch_size + input_size_i] = F.softmax(
output['label'], dim=-1).cpu()
else:
model = inception_v3(pretrained=True,
transform_input=False).to(cfg['device'])
model.train(False)
up = nn.Upsample(size=(299, 299),
mode='bilinear',
align_corners=False)
pred = torch.zeros((N, 1000))
for i, input in enumerate(data_loader):
input = input.to(cfg['device'])
input_size_i = input.size(0)
input = up(input)
output = model(input)
pred[i * batch_size:i * batch_size + input_size_i] = F.softmax(
output, dim=-1).cpu()
split_scores = []
for k in range(splits):
part = pred[k * (N // splits):(k + 1) * (N // splits), :]
py = torch.mean(part, dim=0)
scores = F.kl_div(py.log().view(1, -1).expand_as(part),
part,
reduction='batchmean').exp()
split_scores.append(scores)
inception_score = np.mean(split_scores).item()
return inception_score
parser = argparse.ArgumentParser("classification.py")
text = parser.add_argument_group(
"The following arguments are mandatory for text option")
text.add_argument("text", metavar="TEXT", help="text to predict", nargs="?")
file = parser.add_argument_group(
"The following arguments are mandatory for file option")
file.add_argument("--fin", help="text file input")
args = parser.parse_args()
if not (args.text or args.fin):
parser.print_help()
if args.text:
text = args.text
try:
predict = classifier(text)[0]
label = predict.replace("_", " ").capitalize()
print(label)
except:
print("Don't predict label")
if args.fin:
if not args.fin:
parser.error("File option requires --fin")
fin = args.fin
with open(fin) as f:
text = f.read()
try:
predict = classifier(text)[0]
label = predict.replace("_", " ").capitalize()
print(label)
def main(args):
exp_name = "exp3_train"
# setting up logging
log = get_logger(args.logging_dir, exp_name)
# setting a save directory
save_dir = get_save_dir("./checkpoints",
exp_name,
training=True,
id_max=200)
# setting up tensor board
tbx = SummaryWriter(save_dir)
# setting up saver
saver = CheckpointSaver(save_dir=save_dir,
max_checkpoints=args.max_checkpoints,
metric_name="BCELoss",
log=log)
# setting the random seed
log.info(f"Using random seed {args.random_seed}...")
random.seed(args.random_seed)
np.random.seed(args.random_seed)
torch.manual_seed(args.random_seed)
torch.cuda.manual_seed_all(args.random_seed)
# grabbing a gpu if it is available
gpu_ids = []
gpu_ids += [gpu_id for gpu_id in range(torch.cuda.device_count())]
device = torch.device(f'cuda:{gpu_ids[0]}')
torch.cuda.set_device(device)
log.info(f"Using device type: {device}")
# getting word embeddings
with open(args.word_emb_file, 'r') as fh:
word_vectors = np.array(json.load(fh))
word_vectors = torch.from_numpy(word_vectors)
# setting up the datasets
train_dataset = qcd(data_path=args.train_feature_file,
num_categories=args.num_categories)
train_loader = data.DataLoader(train_dataset,
shuffle=True,
batch_size=args.batch_size,
collate_fn=collate_fn)
dev_dataset = qcd(data_path=args.dev_feature_file,
num_categories=args.num_categories)
dev_loader = data.DataLoader(dev_dataset,
shuffle=False,
batch_size=args.batch_size,
collate_fn=collate_fn)
# setting up the model
model = classifier(args=args, word_vectors=word_vectors)
model = nn.DataParallel(model, gpu_ids)
model.to(device)
model.train()
ema = EMA(model, args.ema_decay)
# optimizer = optim.Adadelta(model.parameters(), args.learning_rate,
# weight_decay=args.learning_rate_decay)
# scheduler = sched.LambdaLR(optimizer, lambda s: 1.) # Constant LR
optimizer = optim.SGD(model.parameters(), lr=args.learning_rate)
step = 0
steps_till_eval = args.eval_steps
log.info(f"Vars: {json.dumps(vars(args), indent=4, sort_keys=True)}")
for epoch in range(args.num_epochs):
log.info(f"Starting epoch {epoch+1}")
with torch.enable_grad(), tqdm(
total=len(train_loader.dataset)) as progress_bar:
for qw_idxs, ids, topic_ids, lengths in train_loader:
qw_idxs = qw_idxs.to(device)
batch_size = qw_idxs.size(0)
if batch_size != args.batch_size:
log.info(
'Did not process because did not meet batch_size threshold'
)
continue
topic_ids = topic_ids.to(device)
lengths = lengths.to(device)
optimizer.zero_grad()
# targets = [torch.zeros(args.num_categories) for _ in topic_ids]
# targets = torch.stack(targets).to(device)
# for tid, t in zip(topic_ids, targets):
# t[tid] = 1
res = model(qw_idxs, lengths)
# for loss, either nn.softmax_cross_entropy_with_logits or nn.BCELoss or nn.BCEWithLogitsLoss
# not really sure why this is working and the others aren't
# loss = nn.CrossEntropyLoss()
# loss = nn.BCELoss()
# loss = nn.BCEWithLogitsLoss()
loss_output = F.nll_loss(F.log_softmax(res, dim=1), topic_ids)
loss_output.backward()
loss_val = loss_output.item()
optimizer.step()
# scheduler.step(step//batch_size)
ema(model, step // batch_size)
step += batch_size
steps_till_eval -= batch_size
progress_bar.update(batch_size)
progress_bar.set_postfix(NLL=(loss_val), Epoch=(epoch + 1))
if steps_till_eval <= 0:
steps_till_eval = args.eval_steps
log.info(f"Evaluating at step: {step}")
ema.assign(model)
perc_correct, vis_examples, avg_loss = evaluate(
model, dev_loader, device, args.dev_eval_file)
log.info(
f"Out of Sample BCE loss: {avg_loss} at step {step} in epoch {epoch+1}, resulting in {perc_correct} percent correct"
)
tbx.add_scalar("NLL Loss", loss_val, step)
tbx.add_scalar("Percent Accuracy", perc_correct, step)
for i, example in enumerate(vis_examples):
tbl_fmt = (
f'- **Question:** {example["question"]}\n' +
f'- **Topic ID:** {example["answer"]}\n' +
f'- **Prediction:** {example["prediction"]}')
tbx.add_text(tag=f'{i}_of_{len(vis_examples)}',
text_string=tbl_fmt,
global_step=step)
saver.save(model=model,
step=step,
epoch=epoch,
metric_val=loss_val,
device=device)
ema.resume(model)
model.to(device)
model.train()
log.info(f"resuming training on device {device}")
def train_classifier(data_path,
num_epoch=3,
lr=5e-3,
save=None,
test_data_path=None,
eval_period=10,
model_params=None,
save_period=1):
datas = loaders.load_from_disk(data_path)
loaders.set_dataset_format_with_hidden(datas)
test_datas = None
if test_data_path:
test_datas = loaders.load_from_disk(test_data_path)
loaders.set_dataset_format_with_hidden(test_datas)
model = models.classifier(models.tokenizer())
if model_params:
model.load_state_dict(torch.load(model_params, map_location=device))
model.to(device)
criterion = torch.nn.BCEWithLogitsLoss(reduction='sum')
optimizer = torch.optim.SGD(model.parameters(), lr=lr)
dataloader = torch.utils.data.DataLoader(datas, batch_size=16)
losses = []
periods = []
period_losses = []
test_losses = []
test_accs = []
period_loss = 0
period_sample = 0
num_checkpoint = 0
save_i = 0
for n_epoch in trange(num_epoch, desc="epoch"):
epoch_loss = 0
epoch_sample = 0
for batch in tqdm(dataloader, desc="batches", leave=False):
pred = model(batch['token'].to(device),
batch['hidden_state'].to(torch.float).to(device))
loss = criterion(pred, batch['label'].to(torch.float).to(device))
epoch_loss += loss.item()
epoch_sample += len(batch['token'])
period_loss += loss.item()
period_sample += len(batch['token'])
optimizer.zero_grad()
loss.backward()
optimizer.step()
if period_sample > eval_period:
save_i += 1
if save and save_i % save_period == 0:
torch.save(
model.state_dict(),
'{}/checkpoint_{}.json'.format(save, num_checkpoint))
periods.append(period_sample)
period_losses.append(period_loss / period_sample)
if test_datas is not None:
test_loss, test_acc, _ = evaluate_classifier(
test_datas, model)
test_losses.append(test_loss)
test_accs.append(test_acc)
print(
"{} :: train loss: {} test loss: {} test acc: {}".
format(num_checkpoint, period_loss / period_sample,
test_loss, test_acc))
else:
print("train loss: {}".format(num_checkpoint,
period_loss / period_sample))
num_checkpoint += 1
period_sample = 0
period_loss = 0
losses.append(epoch_loss / epoch_sample)
return model, {
'epoch_loss': losses,
'periods': periods,
'period_loss': period_losses,
'test_loss': test_losses,
'test_acc': test_accs
}
def build_networks(self):
# feature extractor
self.src_features = src_features = encoder(
self.xs,
'features',
hidden_layer_dims=self.encoder_hidden_layer_dims,
dropout_keep_prob=self.dropout_keep_prob_ph)
self.tgt_features = tgt_features = encoder(
self.xt,
'features',
hidden_layer_dims=self.encoder_hidden_layer_dims,
dropout_keep_prob=self.dropout_keep_prob_ph,
reuse=True)
# task src classifier
src_classifier_feat = encoder(
src_features,
'src_classifier_feat',
hidden_layer_dims=self.classifier_hidden_layer_dims,
dropout_keep_prob=self.dropout_keep_prob_ph)
self.src_task_scores = classifier(
src_classifier_feat,
self.output_dim,
'src_task_scores',
)
self.src_task_probs = tf.nn.softmax(self.src_task_scores)
# task tgt classifier with optional reuse
if self.shared_classifier:
reuse = True
tgt_classifier_feat_name = 'src_classifier_feat'
tgt_classifier_name = 'src_task_scores'
else:
reuse = False
tgt_classifier_feat_name = 'tgt_classifier_feat'
tgt_classifier_name = 'tgt_task_scores'
tgt_classifier_feat = encoder(
tgt_features,
tgt_classifier_feat_name,
hidden_layer_dims=self.classifier_hidden_layer_dims,
dropout_keep_prob=self.dropout_keep_prob_ph,
reuse=reuse)
self.tgt_task_scores = classifier(tgt_classifier_feat,
self.output_dim,
tgt_classifier_name,
reuse=reuse)
self.tgt_task_probs = tf.nn.softmax(self.tgt_task_scores)
# domain classifier
self.lmbda = lmbda = tf.train.polynomial_decay(
self.lambda_initial,
self.global_step,
self.lambda_steps,
end_learning_rate=self.lambda_final,
power=2.0,
name='lambda')
if self.src_only_adversarial:
tgt_features = tf.stop_gradient(tgt_features)
self.features = tf.concat((src_features, tgt_features), axis=0)
flipped_features = flip_gradient(self.features, lmbda)
src_d = tf.zeros((tf.shape(src_features)[0], ), dtype=tf.int32)
tgt_d = tf.ones((tf.shape(tgt_features)[0], ), dtype=tf.int32)
self.d = tf.concat((src_d, tgt_d), axis=0)
domain_features = encoder(
flipped_features,
'domain_features',
hidden_layer_dims=self.domain_classifier_hidden_layer_dims,
dropout_keep_prob=self.dropout_keep_prob_ph)
self.domain_scores = classifier(domain_features, 2,
'domain_classifier')
self.domain_probs = tf.nn.softmax(self.domain_scores)
pred = tf.cast(tf.argmax(self.domain_scores, axis=-1), tf.int32)
self.domain_acc = tf.reduce_mean(
tf.cast(tf.equal(pred, self.d), tf.float32))
def main(args):
# setting up logging
logger = get_logger(log_dir=args.logging_dir, name="exp3_evaluation")
# grabbing GPU
gpu_ids = []
if torch.cuda.is_available():
gpu_ids += [gpu_id for gpu_id in range(torch.cuda.device_count())]
device = torch.device(f'cuda:{gpu_ids[0]}')
torch.cuda.set_device(device)
else:
device = torch.device('cpu')
logger.info(f"Using device type: {device}")
# getting word embeddings
with open(args.word_emb_file, 'r') as fh:
word_vectors = np.array(json.load(fh))
word_vectors = torch.from_numpy(word_vectors)
# loading in the model
model = classifier(args=args, word_vectors=word_vectors)
model = nn.DataParallel(model, gpu_ids)
ckpt_dict = torch.load("./checkpoints/train/exp3_train-34/best.pth.tar",
map_location=device)
model.load_state_dict(ckpt_dict['model_state'])
dataset = qcd(data_path=args.dev_feature_file,
num_categories=args.num_categories)
loader = data.DataLoader(dataset,
shuffle=True,
batch_size=args.batch_size,
collate_fn=collate_fn)
# loading eval_file
with open(args.dev_eval_file, 'r') as fh:
gold_dict = json.load(fh)
all_predicted_indexes = {}
predicted_indexes = {}
with torch.no_grad():
for qw_idxs, ids, topic_ids, lengths in loader:
qw_idxs.to(device)
ids.to(device)
topic_ids.to(device)
batch_size = qw_idxs.size(0)
if batch_size != args.batch_size:
logger.info(
'Did not process because did not meet batch_size threshold'
)
continue
targets = [torch.zeros(args.num_categories) for _ in topic_ids]
targets = torch.stack(targets)
for tid, t in zip(topic_ids, targets):
t[tid] = 1
res = model(qw_idxs, lengths)
predicted_indexes = {
int(idx): int(torch.argmax(i))
for i, idx in zip(res, ids)
}
all_predicted_indexes.update(predicted_indexes)
print(
f"Was able to predict {len(all_predicted_indexes)}/{len(gold_dict)} total examples."
)
correct = 0
total_eval = 0
for i in all_predicted_indexes:
if i in gold_dict:
if all_predicted_indexes[i] == gold_dict[i]:
correct += 1
total_eval += 1
logger.info(f"Got {correct}/{total_eval} correct")
def get_attack_stats(args,
encoder,
classifier,
discriminator,
loader,
log,
type="class"):
clean_errors = AverageMeter()
adv_errors = AverageMeter()
mi_meter = AverageMeter()
mi_adv_adv_meter = AverageMeter()
mi_adv_clean_meter = AverageMeter()
mi_clean_adv_meter = AverageMeter()
c_l2_norms = AverageMeter()
c_l2_frac = AverageMeter()
fc_l2_norms = AverageMeter()
fc_l2_frac = AverageMeter()
z_l2_norms = AverageMeter()
z_l2_frac = AverageMeter()
batch = tqdm(loader, total=len(loader) // loader.batch_size)
for i, (X, y) in enumerate(batch):
if args.gpu:
X, y = X.cuda(), y.cuda()
if type == "class" and classifier is not None:
# adv samples using classifier
if args.attack == "pgd":
X_adv, delta, out, out_adv = pgd(model=classifier,
X=X,
y=y,
epsilon=args.epsilon,
alpha=args.alpha,
num_steps=args.num_steps,
p='inf')
elif args.attack == "fgsm":
X_adv, delta, out, out_adv = fgsm(model=classifier,
X=X,
y=y,
epsilon=args.epsilon)
elif type == "encoder":
X_adv, E_adv, diff, max_diff = encoder_attack(X,
encoder,
args.num_steps,
args.epsilon,
args.alpha,
random_restart=True)
batch.set_description("Avg Diff {} Max Diff {}".format(
diff, max_diff))
elif type == "impostor":
batch_size = X.shape[0]
# using the given batch form X_s X_t pairs
X_s = X[0:batch_size // 2]
X_t = X[batch_size // 2:]
# set y to the labels for X_s and X to X_s for later computation and logging
y = y[0:batch_size // 2]
X = X_s
X_adv, E_adv, diff, min_diff = cw_infomax_encoder_attack(
X_s,
X_t,
encoder=encoder,
num_steps=2000,
alpha=0.001,
c=0.1,
p=2)
batch.set_description("Avg Diff {} Min Diff {}".format(
diff, min_diff))
elif type == "random":
delta = torch.rand_like(X).sign() * args.epsilon
X_adv = X + delta
_, _, E = encoder(X)
_, _, E_d = encoder(X_adv)
norm = torch.norm(E - E_d, p=2, dim=-1)
batch.set_description("Avg Diff {} Max Diff {} ".format(
norm.mean(), norm.max()))
if classifier is not None:
# UPDATE CLEAN and ADV ERRORS
logits_clean = classifier(X)
logits_adv = classifier(X_adv)
out = logits_clean.max(1)[1]
out_adv = logits_adv.max(1)[1]
err_clean = (out.data != y).float().sum() / X.size(0)
err_adv = (out_adv.data != y).float().sum() / X.size(0)
clean_errors.update(err_clean)
adv_errors.update(err_adv)
# UPDATE L2 NORM METERS
C_clean, FC_clean, Z_clean = encoder(X)
C_adv, FC_adv, Z_adv = encoder(X_adv)
l2 = torch.norm(Z_clean - Z_adv, p=2, dim=-1, keepdim=True)
fraction = (l2 / torch.norm(Z_clean, p=2, dim=-1, keepdim=True))
z_l2_norms.update(l2.mean())
z_l2_frac.update(fraction.mean())
l2 = torch.norm(C_clean - C_adv, p=2, dim=(-1, -2, -3), keepdim=True)
fraction = (l2 /
torch.norm(C_clean, p=2, dim=(-1, -2, -3), keepdim=True))
c_l2_norms.update(l2.mean())
c_l2_frac.update(fraction.mean())
l2 = torch.norm(FC_clean - FC_adv, p=2, dim=-1, keepdim=True)
fraction = (l2 / torch.norm(FC_clean, p=2, dim=-1, keepdim=True))
fc_l2_norms.update(l2.mean())
fc_l2_frac.update(fraction.mean())
with torch.no_grad():
# evaluate the critic scores for X and E
mi, E = discriminator(X=X)
# evaluate the critic scores for X_adv and E_adv
mi_adv_adv, E_adv = discriminator(X=X_adv)
# evaluate the critic scores for X_adv and E_clean
mi_adv_clean, _ = discriminator(X_adv, E=E)
# evaluate the critic scores for X, E_adv
mi_clean_adv, _ = discriminator(X, E=E_adv)
# UPDATE MI METERS
mi_meter.update(mi)
mi_adv_adv_meter.update(mi_adv_adv)
mi_adv_clean_meter.update(mi_adv_clean)
mi_clean_adv_meter.update(mi_clean_adv)
batch.set_description(
"MI(X, E) {} MI(X_adv, E_adv) {} MI(X_adv, E) {} MI(X, E_adv) {}".
format(mi, mi_adv_adv, mi_adv_clean, mi_clean_adv))
# print to logfile
print("Error Clean {clean_errors.avg:.3f}\t Error Adv{adv_errors.avg:.3f}\t "
"C L2 {c_l2_norms.avg:.3f}\t C L2 Frac{c_l2_frac.avg:.3f}\t"
"FC L2 {fc_l2_norms.avg:.3f}\t FC L2 Frac{fc_l2_frac.avg:.3f}\t"
"Z L2 {z_l2_norms.avg:.3f}\t Z L2 Frac{z_l2_frac.avg:.3f}\t"
"MI(X, E) {mi.avg:.3f}\t MI(X_adv, E_adv) {mi_adv_adv.avg:.3f}\t "
"MI(X_adv, E) {mi_adv_clean.avg:.3f}\t MI(X, E_adv) {mi_clean_adv.avg:.3f}\t".format(
clean_errors=clean_errors, adv_errors=adv_errors,
c_l2_norms=c_l2_norms, c_l2_frac=c_l2_frac,
fc_l2_norms=fc_l2_norms, fc_l2_frac=fc_l2_frac,
z_l2_norms=z_l2_norms, z_l2_frac=z_l2_frac,
mi=mi_meter, mi_adv_adv=mi_adv_adv_meter, mi_adv_clean=mi_adv_clean_meter, mi_clean_adv=mi_clean_adv_meter), \
file=log)
log.flush()
def FID(img):
with torch.no_grad():
batch_size = 32
cfg['batch_size']['train'] = batch_size
dataset = fetch_dataset(cfg['data_name'], cfg['subset'], verbose=False)
real_data_loader = make_data_loader(dataset)['train']
generated_data_loader = DataLoader(img, batch_size=batch_size)
if cfg['data_name'] in ['COIL100', 'Omniglot']:
model = models.classifier().to(cfg['device'])
model_tag = ['0', cfg['data_name'], cfg['subset'], 'classifier']
model_tag = '_'.join(filter(None, model_tag))
checkpoint = load(
'./metrics_tf/res/classifier/{}_best.pt'.format(model_tag))
model.load_state_dict(checkpoint['model_dict'])
model.train(False)
real_feature = []
for i, input in enumerate(real_data_loader):
input = collate(input)
input = to_device(input, cfg['device'])
real_feature_i = model.feature(input)
real_feature.append(real_feature_i.cpu().numpy())
real_feature = np.concatenate(real_feature, axis=0)
generated_feature = []
for i, input in enumerate(generated_data_loader):
input = {
'img': input,
'label': input.new_zeros(input.size(0)).long()
}
input = to_device(input, cfg['device'])
generated_feature_i = model.feature(input)
generated_feature.append(generated_feature_i.cpu().numpy())
generated_feature = np.concatenate(generated_feature, axis=0)
else:
model = inception_v3(pretrained=True,
transform_input=False).to(cfg['device'])
up = nn.Upsample(size=(299, 299),
mode='bilinear',
align_corners=False)
model.feature = nn.Sequential(*[
up, model.Conv2d_1a_3x3, model.Conv2d_2a_3x3,
model.Conv2d_2b_3x3,
nn.MaxPool2d(kernel_size=3, stride=2), model.Conv2d_3b_1x1,
model.Conv2d_4a_3x3,
nn.MaxPool2d(kernel_size=3, stride=2), model.Mixed_5b,
model.Mixed_5c, model.Mixed_5d, model.Mixed_6a, model.Mixed_6b,
model.Mixed_6c, model.Mixed_6d, model.Mixed_6e, model.Mixed_7a,
model.Mixed_7b, model.Mixed_7c,
nn.AdaptiveAvgPool2d(1),
nn.Flatten()
])
model.train(False)
real_feature = []
for i, input in enumerate(real_data_loader):
input = collate(input)
input = to_device(input, cfg['device'])
real_feature_i = model.feature(input['img'])
real_feature.append(real_feature_i.cpu().numpy())
real_feature = np.concatenate(real_feature, axis=0)
generated_feature = []
for i, input in enumerate(generated_data_loader):
input = to_device(input, cfg['device'])
generated_feature_i = model.feature(input)
generated_feature.append(generated_feature_i.cpu().numpy())
generated_feature = np.concatenate(generated_feature, axis=0)
mu1 = np.mean(real_feature, axis=0)
sigma1 = np.cov(real_feature, rowvar=False)
mu2 = np.mean(generated_feature, axis=0)
sigma2 = np.cov(generated_feature, rowvar=False)
mu1 = np.atleast_1d(mu1)
mu2 = np.atleast_1d(mu2)
sigma1 = np.atleast_2d(sigma1)
sigma2 = np.atleast_2d(sigma2)
assert mu1.shape == mu2.shape, "Training and test mean vectors have different lengths"
assert sigma1.shape == sigma2.shape, "Training and test covariances have different dimensions"
diff = mu1 - mu2
# product might be almost singular
covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
if not np.isfinite(covmean).all():
offset = np.eye(sigma1.shape[0]) * 1e-6
covmean = linalg.sqrtm((sigma1 + offset).dot(sigma2 + offset))
# numerical error might give slight imaginary component
if np.iscomplexobj(covmean):
if not np.allclose(np.diagonal(covmean).imag, 0, atol=1e-3):
m = np.max(np.abs(covmean.imag))
raise ValueError("Imaginary component {}".format(m))
covmean = covmean.real
tr_covmean = np.trace(covmean)
fid = diff.dot(diff) + np.trace(sigma1) + np.trace(
sigma2) - 2 * tr_covmean
fid = fid.item()
return fid
def get_attack_stats(args, encoder, classifier, loader, log, type="class"):
clean_errors = AverageMeter()
adv_errors = AverageMeter()
c_l2_norms = AverageMeter()
c_l2_frac = AverageMeter()
fc_l2_norms = AverageMeter()
fc_l2_frac = AverageMeter()
z_l2_norms = AverageMeter()
z_l2_frac = AverageMeter()
classifier.eval()
batch = tqdm(loader, total=len(loader) // loader.batch_size)
for i, (X, y) in enumerate(batch):
if args.gpu:
X, y = X.cuda(), y.cuda()
if type == "class":
# adv samples using classifier
if args.attack == "pgd":
X_adv, delta, out, out_adv = pgd(model=classifier, X=X, y=y, epsilon=args.epsilon,
alpha=args.alpha, num_steps=args.num_steps, p='inf')
elif args.attack == "fgsm":
X_adv, delta, out, out_adv = fgsm(model=classifier, X=X, y=y, epsilon=args.epsilon)
elif type == "encoder":
X_adv, E_adv, diff, max_diff = encoder_attack(X, encoder, args.num_steps, args.epsilon, args.alpha,
random_restart=True)
batch.set_description("Avg Diff {} Max Diff {}".format(diff, max_diff))
# run classifier on adversarial representations
logits_clean = classifier(X)
logits_adv = classifier(X_adv)
out = logits_clean.max(1)[1]
out_adv = logits_adv.max(1)[1]
elif type == "impostor":
batch_size = X.shape[0]
# using the given batch form X_s X_t pairs
X_s = X[0:batch_size // 2]
X_t = X[batch_size // 2:]
# set y to the labels for X_s and X to X_s for later computation and logging
y = y[0:batch_size // 2]
X = X_s
X_adv, E_adv, diff, min_diff = source2target(X_s, X_t, encoder=encoder, epsilon=2.0,
max_steps=70000, step_size=0.001)
# run classifier on adversarial representations
logits_clean = classifier(X_s)
logits_adv = classifier(X_adv)
out = logits_clean.max(1)[1]
out_adv = logits_adv.max(1)[1]
batch.set_description("Avg Diff {} Min Diff {}".format(diff, min_diff))
elif type == "random":
delta = torch.rand_like(X).sign() * args.epsilon
X_adv = X + delta
_, _, E = encoder(X)
_, _, E_d = encoder(X_adv)
norm = torch.norm(E - E_d, p=2, dim=-1)
# run classifier on adversarial representations
logits_clean = classifier(X)
logits_adv = classifier(X_adv)
out = logits_clean.max(1)[1]
out_adv = logits_adv.max(1)[1]
batch.set_description("Avg Diff {} Max Diff {} ".format(norm.mean(), norm.max()))
# UPDATE CLEAN and ADV ERRORS
err_clean = (out.data != y).float().sum() / X.size(0)
err_adv = (out_adv.data != y).float().sum() / X.size(0)
clean_errors.update(err_clean)
adv_errors.update(err_adv)
# UPDATE L2 NORM METERS
C_clean, FC_clean, Z_clean = encoder(X)
C_adv, FC_adv, Z_adv = encoder(X_adv)
l2 = torch.norm(Z_clean - Z_adv, p=2, dim=-1, keepdim=True)
fraction = (l2 / torch.norm(Z_clean, p=2, dim=-1, keepdim=True))
z_l2_norms.update(l2.mean())
z_l2_frac.update(fraction.mean())
l2 = torch.norm(C_clean - C_adv, p=2, dim=(-1, -2, -3), keepdim=True)
fraction = (l2 / torch.norm(C_clean, p=2, dim=(-1, -2, -3), keepdim=True))
c_l2_norms.update(l2.mean())
c_l2_frac.update(fraction.mean())
l2 = torch.norm(FC_clean - FC_adv, p=2, dim=-1, keepdim=True)
fraction = (l2 / torch.norm(FC_clean, p=2, dim=-1, keepdim=True))
fc_l2_norms.update(l2.mean())
fc_l2_frac.update(fraction.mean())
# print to logfile
print("Error Clean {clean_errors.avg:.3f}\t Error Adv{adv_errors.avg:.3f}\t "
"C L2 {c_l2_norms.avg:.3f}\t C L2 Frac{c_l2_frac.avg:.3f}\t"
"FC L2 {fc_l2_norms.avg:.3f}\t FC L2 Frac{fc_l2_frac.avg:.3f}\t"
"Z L2 {z_l2_norms.avg:.3f}\t Z L2 Frac{z_l2_frac.avg:.3f}\t".format(
clean_errors=clean_errors, adv_errors=adv_errors,
c_l2_norms=c_l2_norms, c_l2_frac=c_l2_frac,
fc_l2_norms=fc_l2_norms, fc_l2_frac=fc_l2_frac,
z_l2_norms=z_l2_norms, z_l2_frac=z_l2_frac),
file=log)
log.flush()
def __init__(self, opts, tag):
tf.reset_default_graph()
logging.error('Building the Tensorflow Graph')
gpu_options = tf.GPUOptions(allow_growth=True)
config = tf.ConfigProto(gpu_options=gpu_options)
self.sess = tf.Session(config=config)
self.opts = opts
assert opts['dataset'] in datashapes, 'Unknown dataset.'
# Add placeholders
shape = datashapes[opts['dataset']]
self.sample_points = tf.placeholder(tf.float32, [None] + shape,
name='real_points_ph')
self.labels = tf.placeholder(tf.int64, shape=[None], name='label_ph')
self.sample_noise = tf.placeholder(tf.float32, [None, opts['zdim']],
name='noise_ph')
self.lr_decay = tf.placeholder(tf.float32, name='rate_decay_ph')
self.is_training = tf.placeholder(tf.bool, name='is_training_ph')
self.mean_ph = tf.placeholder(tf.float32, shape=[None, opts['zdim']])
self.sigma_ph = tf.placeholder(tf.float32, shape=[None, opts['zdim']])
# Build training computation graph
sample_size = tf.shape(self.sample_points)[0]
enc_mean, enc_sigmas = encoder(opts,
inputs=self.sample_points,
is_training=self.is_training,
y=self.labels)
enc_sigmas = tf.clip_by_value(enc_sigmas, -50, 50)
self.enc_mean, self.enc_sigmas = enc_mean, enc_sigmas
self.encoded = self.get_encoded(opts, self.enc_mean, self.enc_sigmas)
self.encoded2 = self.get_encoded(opts, self.mean_ph, self.sigma_ph)
self.reconstructed = decoder(opts,
noise=self.encoded,
is_training=self.is_training)
self.probs1 = classifier(opts, self.encoded)
self.probs2 = classifier(opts, self.encoded2)
self.correct_sum = tf.reduce_sum(
tf.cast(tf.equal(tf.argmax(self.probs1, axis=1), self.labels),
tf.float32))
self.decoded = decoder(opts,
noise=self.sample_noise,
is_training=self.is_training)
self.loss_cls = self.cls_loss(self.labels, self.probs1)
self.loss_cls2 = self.cls_loss(self.labels, self.probs2)
self.loss_mmd = self.mmd_penalty(self.sample_noise, self.encoded)
self.loss_recon = self.reconstruction_loss(self.opts,
self.sample_points,
self.reconstructed)
self.objective = self.loss_recon + opts[
'lambda'] * self.loss_mmd + self.loss_cls
# Build evaluate computation graph
logpxy = []
dimY = opts['n_classes']
N = sample_size
S = opts['sampling_size']
x_rep = tf.tile(self.sample_points, [S, 1, 1, 1])
for i in range(dimY):
y = tf.fill((N, ), i)
mu, log_sig = encoder(opts,
inputs=self.sample_points,
is_training=False,
y=y)
mu = tf.tile(mu, [S, 1])
log_sig = tf.tile(log_sig, [S, 1])
y = tf.tile(y, [
S,
])
z = self.get_encoded(opts, mu, log_sig)
z_sample = tf.random_normal((tf.shape(z)[0], opts['zdim']),
0.,
1.,
dtype=tf.float32)
mu_x = decoder(opts, noise=z, is_training=False)
logit_y = classifier(opts, z)
logp = -tf.reduce_sum((x_rep - mu_x)**2, axis=[1, 2, 3])
log_pyz = -tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=y, logits=logit_y)
mmd_loss = self.mmd_penalty(z_sample, z)
bound = 0.5 * logp + log_pyz + opts['lambda'] * mmd_loss
bound = tf.reshape(bound, [S, N])
bound = self.logsumexp(bound) - tf.log(float(S))
logpxy.append(tf.expand_dims(bound, 1))
logpxy = tf.concat(logpxy, 1)
y_pred = tf.nn.softmax(logpxy)
self.eval_probs = y_pred
self.loss_pretrain = self.pretrain_loss(
) if opts['e_pretrain'] else None
self.add_optimizers()
self.add_savers()
self.tag = tag
opt.trainRoot = "check_" + opt.optimizer
if opt.isDropOut:
opt.trainRoot += '_dropout'
if opt.testRoot is None:
opt.testRoot = opt.trainRoot.replace('check', 'test')
os.system('mkdir {0}'.format(opt.testRoot))
# Hyper parameter
imSize = 28
# data batch
imBatch = Variable(torch.FloatTensor(opt.batchSize, 1, imSize, imSize))
labelBatch = Variable(torch.LongTensor(opt.batchSize, 1))
# Network
classifier = models.classifier(isDropOut=opt.isDropOut).eval()
classifier.load_state_dict(
torch.load('{0}/classifier_{1}.pth'.format(opt.trainRoot, opt.nepoch - 1)))
# DataLoader
mnistDataset = dataLoader.BatchLoader(dataRoot=opt.dataRoot, phase='TEST')
mnistLoader = DataLoader(mnistDataset,
batch_size=opt.batchSize,
num_workers=4,
shuffle=False)
# Move data and network to gpu
if opt.cuda:
imBatch = imBatch.cuda(opt.gpuId)
labelBatch = labelBatch.cuda(opt.gpuId)
classifier = classifier.cuda(opt.gpuId)
# compute gradient penalty
gradient_penalty = calc_gradient_penalty(critic, gen_src.data,
gen_tgt.data)
gradient_penalty.backward()
d_loss = d_loss + gradient_penalty
# optimize weights of discriminator
#d_loss = - d_loss_src + d_loss_tgt + gradient_penalty
optimizer_d.step()
optimizer_d.step()
#break
# 训练 分类器
optimizer_c.zero_grad()
pred_c = classifier(generator(image_src).detach())
c_loss = criterion(pred_c, label_src) # 公式6 求交叉熵
c_loss.backward()
optimizer_c.step()
# 训练 生成器
# 训练 生成器时 鉴别器的梯度不下降
for p in critic.parameters():
p.requires_grad = False
optimizer_g.zero_grad()
# 计算 src 分类器的损失
gen_src = generator(image_src)
pred_c = classifier(gen_src)
g_loss_cls = criterion(pred_c, label_src)
# load classifier from checkpoint
classifier.load_state_dict(
torch.load(args.classifier_ckpt)["classifier_state_dict"])
classifier.to(args.device)
Z = []
pred = []
Y = []
Z_adv = []
pred_adv = []
for X, y in test_loader:
if args.gpu:
X, y = X.cuda(), y.cuda()
with torch.no_grad():
z, logits = classifier(X, intermediate=True)
Z.append(z.cpu().detach().numpy())
pred.append(logits.max(-1)[1].cpu().detach().numpy())
Y.append(y.cpu().detach().numpy())
X_adv, delta, out, out_adv = pgd(model=classifier,
X=X,
y=y,
epsilon=args.epsilon,
alpha=args.alpha,
num_steps=args.num_steps,
p='inf')
z, logits = classifier(X_adv, intermediate=True)
Z_adv.append(z.cpu().detach().numpy())
pred_adv.append(out_adv.cpu().detach().numpy())
def load_classifier(checkpoint):
model = models.classifier(models.tokenizer())
model.load_state_dict(torch.load(checkpoint, map_location=device))
return model.to(device)
validation_list = np.load('validation_number.npy')
validation_length = len(validation_list)
test_list = np.load('test_number.npy')
test_length = len(test_list)
except:
print("Create train/validation/test sets before using this code")
sys.exit()
label_size = FLAGS.num_of_sens
X = tf.placeholder(tf.float32, [None, None, None, 3])
Y = tf.placeholder(tf.float32, [None, label_size])
H = tf.placeholder(tf.float32, [None, None, FLAGS.c_dim])
weights, biases = classifier_weights(label_size)
classification = classifier(X, weights, biases)
classification = tf.reduce_mean(classification, 0)
classification = tf.reduce_mean(classification, 0)
classification = tf.reduce_mean(classification, 0)
prediction = tf.nn.softmax(classification)
# Define loss and optimizer
loss_op = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=classification, labels=Y))
global_step = tf.Variable(0, trainable=False)
starter_learning_rate = np.copy(FLAGS.learning_rate)
learning_rate = tf.train.exponential_decay(starter_learning_rate,
global_step,
1000,
0.93,
def train_model(alpha=0.25, batch_size=32, epochs=200):
X_s, y_s, _, _ = sample_source_data()
X_t, y_t, X_test, y_test = sample_target_data()
X1, X2, y1, y2, yc = create_pairs(X_s, y_s, X_t, y_t)
#X1, X2, y1, y2, yc, X_test, y_test = load_pairs()
base_model = tiny_base_model()
input_s = Input(shape=(X_s.shape[1], X_s.shape[2], X_s.shape[3]))
input_t = Input(shape=(X_t.shape[1], X_t.shape[2], X_t.shape[3]))
processed_s = base_model(input_s)
processed_t = base_model(input_t)
output = classifier(processed_s)
dist = Lambda(eucl_dist, output_shape=eucl_dist_output_shape, name='CSA')([processed_s, processed_t])
model = Model(inputs=[input_s, input_t], outputs=[output, dist])
model.compile(loss={'classification': 'categorical_crossentropy', 'CSA': contrastive_loss},
optimizer='adadelta', loss_weights={'classification': 1 - alpha, 'CSA': alpha})
print('Model Compiled.')
model.summary()
if not os.path.exists('./images/'):
os.mkdir('./images/')
plot_model(model, to_file='./images/resnet.png')
# Deploy TensorBoard as callbacks
log_dir = './logs'
if os.path.exists(log_dir):
shutil.rmtree(log_dir)
cb = TensorBoard(log_dir='./logs',
histogram_freq=0,
write_graph=True,
write_grads=True,
write_images=True,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None)
print('Start training model.')
#history = model.fit(x=[X1, X2], y=[y1, yc], batch_size=batch_size, epochs=epochs, callbacks=[cb], validation_split=0.2)
#plot_loss(history)
best_acc = 0
acc_his = [0]
total_loss_his = []
classification_loss_his = []
CSA_loss_his = []
for e in range(epochs):
if e % 10 == 0:
printn(str(e) + '->')
for i in range(int(len(y2) / batch_size)):
loss = model.train_on_batch([X1[i * batch_size:(i + 1) * batch_size, :, :, :], X2[i * batch_size:(i + 1) * batch_size, :, :, :]],
[y1[i * batch_size:(i + 1) * batch_size, :], yc[i * batch_size:(i + 1) * batch_size, ]])
loss = model.train_on_batch([X2[i * batch_size:(i + 1) * batch_size, :, :, :], X1[i * batch_size:(i + 1) * batch_size, :, :, :]],
[y2[i * batch_size:(i + 1) * batch_size, :], yc[i * batch_size:(i + 1) * batch_size, ]])
out = model.predict([X_test, X_test])
acc_v = np.argmax(out[0], axis=1) - np.argmax(y_test, axis=1)
acc = (len(acc_v) - np.count_nonzero(acc_v) + .0000001) / len(acc_v)
if best_acc < acc:
best_acc = acc
acc_his.append(acc)
total_loss_his.append(loss[2])
classification_loss_his.append(loss[1])
CSA_loss_his.append(loss[0])
"""
if not os.path.exists('./model/'):
os.mkdir('./model/')
model.save('./model/CCSA.h5')
"""
plot_loss(total_loss_his, classification_loss_his, CSA_loss_his)
plot_acc(acc_his)
return best_acc
