def test(classifier, generator, data_loader, dataset="MNIST"): """Evaluate classifier on source or target domains.""" # set eval state for Dropout and BN layers generator.eval() classifier.eval() # init loss and accuracy loss = 0 acc = 0 # set loss function criterion = nn.CrossEntropyLoss() # evaluate network for (images, labels) in data_loader: images = Variable(images, volatile=True) labels = Variable(labels.squeeze_()) if use_cuda: images = images.cuda() labels = labels.cuda() preds = classifier(generator(images)) loss += criterion(preds, labels).data[0] pred_cls = preds.data.max(1)[1] acc += pred_cls.eq(labels.data).cpu().sum() loss /= len(data_loader) acc /= len(data_loader.dataset) print("Avg Loss = {:.5f}, Avg Accuracy = {:2.5%}".format(loss, acc))
def train_classifier(data): image_shape = [image_size, image_size, 3] n_classes = data["n_classes"] training_steps = 10000 steps_per_epoch = 100 epochs = training_steps // steps_per_epoch model = m.classifier(image_shape, n_classes) model.summary() model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[tf.keras.metrics.CategoricalAccuracy()]) dataset = make_infinite_dataset(data["n_train"], (data["x_train"], data["y_train"])) val_dataset = make_infinite_dataset(data["n_val"], (data["x_val"], data["y_val"])) model.fit(dataset, validation_data=val_dataset, validation_steps=120, epochs=epochs, steps_per_epoch=steps_per_epoch) return model
def predict(id, t,predicted=0): start=time.time() stockwin=dp.createWinData(logr,id,t) tr, te = dp.splitData(stockwin, ratio = 0.9) tr[:,-1]=dp.transUpDown1(tr[:,-1]) te[:,-1]=dp.transUpDown1(te[:,-1]) tr1=[] for i in range(len(tr)): if tr[i,-1] >= 0: tr1.append(tr[i,:]) tr1=np.array(tr1) te1=[] for i in range(len(te)): if te[i,-1] >= 0: te1.append(te[i,:]) te1=np.array(te1) np.random.shuffle(tr1) xtr=tr1[:,:-1] ytr=tr1[:,-1] xte=te1[:,:-1] yte=te1[:,-1] scaler = StandardScaler() scaler.fit(xtr) xtr=scaler.transform(xtr) xte=scaler.transform(xte) ''' logit=sm.Logit(ytr,xtr) result=logit.fit() ypr0=result.predict(xte) ypr0=np.reshape(ypr0,(ypr0.shape[0],1)) temp1, tmep2=dp.para(0.5,0.5,ypr0,yte,1) ''' xtr=np.reshape(xtr,(xtr.shape[0],t,int(xtr.shape[1]/t))) xte=np.reshape(xte,(xte.shape[0],t,int(xte.shape[1]/t))) model=models.classifier(t,36) #model.summary() model.fit(xtr, ytr, epochs = 100, batch_size=32,verbose=0, validation_split = 0.1)#, callbacks = [earlystop,history]) #history.loss_plot('epoch') model.evaluate(xte, yte) ypr=model.predict(xte) if predicted==0: print('time cost:',time.time()-start) return yte, ypr else: yptr=model.predict(xtr) a,b=dp.maxpara(yptr,ytr) print('alpha:',a) print('beta:',b) true,pre=dp.para(a,b,ypr,yte,1) print('time cost:',time.time()-start) return true, pre
def __init__(self, args): super(Model, self).__init__() self.device = args.device self.layers = args.num_layers self.input_size_graph = args.input_size_graph self.output_size_graph = args.output_size_graph self.train_data = args.train_data self.test_data = args.test_data self.train_labels = args.train_labels self.test_labels = args.test_labels self.latent_size = args.latent_size self.hidden_size = args.hidden_size self.learning_rate = args.lr self.batch_size = args.batch_size self.input_dim = args.input_dim self.warmup = args.warmup self.num_labels = args.num_labels self.graph = args.graph self.sequence = args.sequence self.recons = args.recons self.use_attn = args.attn self.use_fusion = args.fusion self.graph_pretrain = graph_models.GraphSage(self.layers, self.input_size_graph, self.output_size_graph, device=self.device, gcn="True", agg_func="MEAN") self.VAE = seq_models.VAE(args) self.AtomEmbedding = nn.Embedding(self.input_size_graph, self.hidden_size).to(self.device) self.AtomEmbedding.weight.requires_grad = True self.output_layer = models.classifier(self.latent_size, self.num_labels, self.device) self.label_criterion = nn.BCEWithLogitsLoss(reduction="none") if self.use_attn: self.attention = models.SelfAttention(self.hidden_size) self.optimizer = optim.Adam(self.parameters(), lr=self.learning_rate, weight_decay=1e-8, amsgrad=True) for name, para in self.named_parameters(): if para.requires_grad: print(name, para.data.shape)
def InceptionScore(img, splits=1): with torch.no_grad(): N = len(img) batch_size = 32 data_loader = DataLoader(img, batch_size=batch_size) if cfg['data_name'] in ['COIL100', 'Omniglot']: model = models.classifier().to(cfg['device']) model_tag = ['0', cfg['data_name'], cfg['subset'], 'classifier'] model_tag = '_'.join(filter(None, model_tag)) checkpoint = load( './metrics_tf/res/classifier/{}_best.pt'.format(model_tag)) model.load_state_dict(checkpoint['model_dict']) model.train(False) pred = torch.zeros((N, cfg['classes_size'])) for i, input in enumerate(data_loader): input = { 'img': input, 'label': input.new_zeros(input.size(0)).long() } input = to_device(input, cfg['device']) input_size_i = input['img'].size(0) output = model(input) pred[i * batch_size:i * batch_size + input_size_i] = F.softmax( output['label'], dim=-1).cpu() else: model = inception_v3(pretrained=True, transform_input=False).to(cfg['device']) model.train(False) up = nn.Upsample(size=(299, 299), mode='bilinear', align_corners=False) pred = torch.zeros((N, 1000)) for i, input in enumerate(data_loader): input = input.to(cfg['device']) input_size_i = input.size(0) input = up(input) output = model(input) pred[i * batch_size:i * batch_size + input_size_i] = F.softmax( output, dim=-1).cpu() split_scores = [] for k in range(splits): part = pred[k * (N // splits):(k + 1) * (N // splits), :] py = torch.mean(part, dim=0) scores = F.kl_div(py.log().view(1, -1).expand_as(part), part, reduction='batchmean').exp() split_scores.append(scores) inception_score = np.mean(split_scores).item() return inception_score
parser = argparse.ArgumentParser("classification.py") text = parser.add_argument_group( "The following arguments are mandatory for text option") text.add_argument("text", metavar="TEXT", help="text to predict", nargs="?") file = parser.add_argument_group( "The following arguments are mandatory for file option") file.add_argument("--fin", help="text file input") args = parser.parse_args() if not (args.text or args.fin): parser.print_help() if args.text: text = args.text try: predict = classifier(text)[0] label = predict.replace("_", " ").capitalize() print(label) except: print("Don't predict label") if args.fin: if not args.fin: parser.error("File option requires --fin") fin = args.fin with open(fin) as f: text = f.read() try: predict = classifier(text)[0] label = predict.replace("_", " ").capitalize() print(label)
def main(args): exp_name = "exp3_train" # setting up logging log = get_logger(args.logging_dir, exp_name) # setting a save directory save_dir = get_save_dir("./checkpoints", exp_name, training=True, id_max=200) # setting up tensor board tbx = SummaryWriter(save_dir) # setting up saver saver = CheckpointSaver(save_dir=save_dir, max_checkpoints=args.max_checkpoints, metric_name="BCELoss", log=log) # setting the random seed log.info(f"Using random seed {args.random_seed}...") random.seed(args.random_seed) np.random.seed(args.random_seed) torch.manual_seed(args.random_seed) torch.cuda.manual_seed_all(args.random_seed) # grabbing a gpu if it is available gpu_ids = [] gpu_ids += [gpu_id for gpu_id in range(torch.cuda.device_count())] device = torch.device(f'cuda:{gpu_ids[0]}') torch.cuda.set_device(device) log.info(f"Using device type: {device}") # getting word embeddings with open(args.word_emb_file, 'r') as fh: word_vectors = np.array(json.load(fh)) word_vectors = torch.from_numpy(word_vectors) # setting up the datasets train_dataset = qcd(data_path=args.train_feature_file, num_categories=args.num_categories) train_loader = data.DataLoader(train_dataset, shuffle=True, batch_size=args.batch_size, collate_fn=collate_fn) dev_dataset = qcd(data_path=args.dev_feature_file, num_categories=args.num_categories) dev_loader = data.DataLoader(dev_dataset, shuffle=False, batch_size=args.batch_size, collate_fn=collate_fn) # setting up the model model = classifier(args=args, word_vectors=word_vectors) model = nn.DataParallel(model, gpu_ids) model.to(device) model.train() ema = EMA(model, args.ema_decay) # optimizer = optim.Adadelta(model.parameters(), args.learning_rate, # weight_decay=args.learning_rate_decay) # scheduler = sched.LambdaLR(optimizer, lambda s: 1.) # Constant LR optimizer = optim.SGD(model.parameters(), lr=args.learning_rate) step = 0 steps_till_eval = args.eval_steps log.info(f"Vars: {json.dumps(vars(args), indent=4, sort_keys=True)}") for epoch in range(args.num_epochs): log.info(f"Starting epoch {epoch+1}") with torch.enable_grad(), tqdm( total=len(train_loader.dataset)) as progress_bar: for qw_idxs, ids, topic_ids, lengths in train_loader: qw_idxs = qw_idxs.to(device) batch_size = qw_idxs.size(0) if batch_size != args.batch_size: log.info( 'Did not process because did not meet batch_size threshold' ) continue topic_ids = topic_ids.to(device) lengths = lengths.to(device) optimizer.zero_grad() # targets = [torch.zeros(args.num_categories) for _ in topic_ids] # targets = torch.stack(targets).to(device) # for tid, t in zip(topic_ids, targets): # t[tid] = 1 res = model(qw_idxs, lengths) # for loss, either nn.softmax_cross_entropy_with_logits or nn.BCELoss or nn.BCEWithLogitsLoss # not really sure why this is working and the others aren't # loss = nn.CrossEntropyLoss() # loss = nn.BCELoss() # loss = nn.BCEWithLogitsLoss() loss_output = F.nll_loss(F.log_softmax(res, dim=1), topic_ids) loss_output.backward() loss_val = loss_output.item() optimizer.step() # scheduler.step(step//batch_size) ema(model, step // batch_size) step += batch_size steps_till_eval -= batch_size progress_bar.update(batch_size) progress_bar.set_postfix(NLL=(loss_val), Epoch=(epoch + 1)) if steps_till_eval <= 0: steps_till_eval = args.eval_steps log.info(f"Evaluating at step: {step}") ema.assign(model) perc_correct, vis_examples, avg_loss = evaluate( model, dev_loader, device, args.dev_eval_file) log.info( f"Out of Sample BCE loss: {avg_loss} at step {step} in epoch {epoch+1}, resulting in {perc_correct} percent correct" ) tbx.add_scalar("NLL Loss", loss_val, step) tbx.add_scalar("Percent Accuracy", perc_correct, step) for i, example in enumerate(vis_examples): tbl_fmt = ( f'- **Question:** {example["question"]}\n' + f'- **Topic ID:** {example["answer"]}\n' + f'- **Prediction:** {example["prediction"]}') tbx.add_text(tag=f'{i}_of_{len(vis_examples)}', text_string=tbl_fmt, global_step=step) saver.save(model=model, step=step, epoch=epoch, metric_val=loss_val, device=device) ema.resume(model) model.to(device) model.train() log.info(f"resuming training on device {device}")
def train_classifier(data_path, num_epoch=3, lr=5e-3, save=None, test_data_path=None, eval_period=10, model_params=None, save_period=1): datas = loaders.load_from_disk(data_path) loaders.set_dataset_format_with_hidden(datas) test_datas = None if test_data_path: test_datas = loaders.load_from_disk(test_data_path) loaders.set_dataset_format_with_hidden(test_datas) model = models.classifier(models.tokenizer()) if model_params: model.load_state_dict(torch.load(model_params, map_location=device)) model.to(device) criterion = torch.nn.BCEWithLogitsLoss(reduction='sum') optimizer = torch.optim.SGD(model.parameters(), lr=lr) dataloader = torch.utils.data.DataLoader(datas, batch_size=16) losses = [] periods = [] period_losses = [] test_losses = [] test_accs = [] period_loss = 0 period_sample = 0 num_checkpoint = 0 save_i = 0 for n_epoch in trange(num_epoch, desc="epoch"): epoch_loss = 0 epoch_sample = 0 for batch in tqdm(dataloader, desc="batches", leave=False): pred = model(batch['token'].to(device), batch['hidden_state'].to(torch.float).to(device)) loss = criterion(pred, batch['label'].to(torch.float).to(device)) epoch_loss += loss.item() epoch_sample += len(batch['token']) period_loss += loss.item() period_sample += len(batch['token']) optimizer.zero_grad() loss.backward() optimizer.step() if period_sample > eval_period: save_i += 1 if save and save_i % save_period == 0: torch.save( model.state_dict(), '{}/checkpoint_{}.json'.format(save, num_checkpoint)) periods.append(period_sample) period_losses.append(period_loss / period_sample) if test_datas is not None: test_loss, test_acc, _ = evaluate_classifier( test_datas, model) test_losses.append(test_loss) test_accs.append(test_acc) print( "{} :: train loss: {} test loss: {} test acc: {}". format(num_checkpoint, period_loss / period_sample, test_loss, test_acc)) else: print("train loss: {}".format(num_checkpoint, period_loss / period_sample)) num_checkpoint += 1 period_sample = 0 period_loss = 0 losses.append(epoch_loss / epoch_sample) return model, { 'epoch_loss': losses, 'periods': periods, 'period_loss': period_losses, 'test_loss': test_losses, 'test_acc': test_accs }
def build_networks(self): # feature extractor self.src_features = src_features = encoder( self.xs, 'features', hidden_layer_dims=self.encoder_hidden_layer_dims, dropout_keep_prob=self.dropout_keep_prob_ph) self.tgt_features = tgt_features = encoder( self.xt, 'features', hidden_layer_dims=self.encoder_hidden_layer_dims, dropout_keep_prob=self.dropout_keep_prob_ph, reuse=True) # task src classifier src_classifier_feat = encoder( src_features, 'src_classifier_feat', hidden_layer_dims=self.classifier_hidden_layer_dims, dropout_keep_prob=self.dropout_keep_prob_ph) self.src_task_scores = classifier( src_classifier_feat, self.output_dim, 'src_task_scores', ) self.src_task_probs = tf.nn.softmax(self.src_task_scores) # task tgt classifier with optional reuse if self.shared_classifier: reuse = True tgt_classifier_feat_name = 'src_classifier_feat' tgt_classifier_name = 'src_task_scores' else: reuse = False tgt_classifier_feat_name = 'tgt_classifier_feat' tgt_classifier_name = 'tgt_task_scores' tgt_classifier_feat = encoder( tgt_features, tgt_classifier_feat_name, hidden_layer_dims=self.classifier_hidden_layer_dims, dropout_keep_prob=self.dropout_keep_prob_ph, reuse=reuse) self.tgt_task_scores = classifier(tgt_classifier_feat, self.output_dim, tgt_classifier_name, reuse=reuse) self.tgt_task_probs = tf.nn.softmax(self.tgt_task_scores) # domain classifier self.lmbda = lmbda = tf.train.polynomial_decay( self.lambda_initial, self.global_step, self.lambda_steps, end_learning_rate=self.lambda_final, power=2.0, name='lambda') if self.src_only_adversarial: tgt_features = tf.stop_gradient(tgt_features) self.features = tf.concat((src_features, tgt_features), axis=0) flipped_features = flip_gradient(self.features, lmbda) src_d = tf.zeros((tf.shape(src_features)[0], ), dtype=tf.int32) tgt_d = tf.ones((tf.shape(tgt_features)[0], ), dtype=tf.int32) self.d = tf.concat((src_d, tgt_d), axis=0) domain_features = encoder( flipped_features, 'domain_features', hidden_layer_dims=self.domain_classifier_hidden_layer_dims, dropout_keep_prob=self.dropout_keep_prob_ph) self.domain_scores = classifier(domain_features, 2, 'domain_classifier') self.domain_probs = tf.nn.softmax(self.domain_scores) pred = tf.cast(tf.argmax(self.domain_scores, axis=-1), tf.int32) self.domain_acc = tf.reduce_mean( tf.cast(tf.equal(pred, self.d), tf.float32))
def main(args): # setting up logging logger = get_logger(log_dir=args.logging_dir, name="exp3_evaluation") # grabbing GPU gpu_ids = [] if torch.cuda.is_available(): gpu_ids += [gpu_id for gpu_id in range(torch.cuda.device_count())] device = torch.device(f'cuda:{gpu_ids[0]}') torch.cuda.set_device(device) else: device = torch.device('cpu') logger.info(f"Using device type: {device}") # getting word embeddings with open(args.word_emb_file, 'r') as fh: word_vectors = np.array(json.load(fh)) word_vectors = torch.from_numpy(word_vectors) # loading in the model model = classifier(args=args, word_vectors=word_vectors) model = nn.DataParallel(model, gpu_ids) ckpt_dict = torch.load("./checkpoints/train/exp3_train-34/best.pth.tar", map_location=device) model.load_state_dict(ckpt_dict['model_state']) dataset = qcd(data_path=args.dev_feature_file, num_categories=args.num_categories) loader = data.DataLoader(dataset, shuffle=True, batch_size=args.batch_size, collate_fn=collate_fn) # loading eval_file with open(args.dev_eval_file, 'r') as fh: gold_dict = json.load(fh) all_predicted_indexes = {} predicted_indexes = {} with torch.no_grad(): for qw_idxs, ids, topic_ids, lengths in loader: qw_idxs.to(device) ids.to(device) topic_ids.to(device) batch_size = qw_idxs.size(0) if batch_size != args.batch_size: logger.info( 'Did not process because did not meet batch_size threshold' ) continue targets = [torch.zeros(args.num_categories) for _ in topic_ids] targets = torch.stack(targets) for tid, t in zip(topic_ids, targets): t[tid] = 1 res = model(qw_idxs, lengths) predicted_indexes = { int(idx): int(torch.argmax(i)) for i, idx in zip(res, ids) } all_predicted_indexes.update(predicted_indexes) print( f"Was able to predict {len(all_predicted_indexes)}/{len(gold_dict)} total examples." ) correct = 0 total_eval = 0 for i in all_predicted_indexes: if i in gold_dict: if all_predicted_indexes[i] == gold_dict[i]: correct += 1 total_eval += 1 logger.info(f"Got {correct}/{total_eval} correct")
def get_attack_stats(args, encoder, classifier, discriminator, loader, log, type="class"): clean_errors = AverageMeter() adv_errors = AverageMeter() mi_meter = AverageMeter() mi_adv_adv_meter = AverageMeter() mi_adv_clean_meter = AverageMeter() mi_clean_adv_meter = AverageMeter() c_l2_norms = AverageMeter() c_l2_frac = AverageMeter() fc_l2_norms = AverageMeter() fc_l2_frac = AverageMeter() z_l2_norms = AverageMeter() z_l2_frac = AverageMeter() batch = tqdm(loader, total=len(loader) // loader.batch_size) for i, (X, y) in enumerate(batch): if args.gpu: X, y = X.cuda(), y.cuda() if type == "class" and classifier is not None: # adv samples using classifier if args.attack == "pgd": X_adv, delta, out, out_adv = pgd(model=classifier, X=X, y=y, epsilon=args.epsilon, alpha=args.alpha, num_steps=args.num_steps, p='inf') elif args.attack == "fgsm": X_adv, delta, out, out_adv = fgsm(model=classifier, X=X, y=y, epsilon=args.epsilon) elif type == "encoder": X_adv, E_adv, diff, max_diff = encoder_attack(X, encoder, args.num_steps, args.epsilon, args.alpha, random_restart=True) batch.set_description("Avg Diff {} Max Diff {}".format( diff, max_diff)) elif type == "impostor": batch_size = X.shape[0] # using the given batch form X_s X_t pairs X_s = X[0:batch_size // 2] X_t = X[batch_size // 2:] # set y to the labels for X_s and X to X_s for later computation and logging y = y[0:batch_size // 2] X = X_s X_adv, E_adv, diff, min_diff = cw_infomax_encoder_attack( X_s, X_t, encoder=encoder, num_steps=2000, alpha=0.001, c=0.1, p=2) batch.set_description("Avg Diff {} Min Diff {}".format( diff, min_diff)) elif type == "random": delta = torch.rand_like(X).sign() * args.epsilon X_adv = X + delta _, _, E = encoder(X) _, _, E_d = encoder(X_adv) norm = torch.norm(E - E_d, p=2, dim=-1) batch.set_description("Avg Diff {} Max Diff {} ".format( norm.mean(), norm.max())) if classifier is not None: # UPDATE CLEAN and ADV ERRORS logits_clean = classifier(X) logits_adv = classifier(X_adv) out = logits_clean.max(1)[1] out_adv = logits_adv.max(1)[1] err_clean = (out.data != y).float().sum() / X.size(0) err_adv = (out_adv.data != y).float().sum() / X.size(0) clean_errors.update(err_clean) adv_errors.update(err_adv) # UPDATE L2 NORM METERS C_clean, FC_clean, Z_clean = encoder(X) C_adv, FC_adv, Z_adv = encoder(X_adv) l2 = torch.norm(Z_clean - Z_adv, p=2, dim=-1, keepdim=True) fraction = (l2 / torch.norm(Z_clean, p=2, dim=-1, keepdim=True)) z_l2_norms.update(l2.mean()) z_l2_frac.update(fraction.mean()) l2 = torch.norm(C_clean - C_adv, p=2, dim=(-1, -2, -3), keepdim=True) fraction = (l2 / torch.norm(C_clean, p=2, dim=(-1, -2, -3), keepdim=True)) c_l2_norms.update(l2.mean()) c_l2_frac.update(fraction.mean()) l2 = torch.norm(FC_clean - FC_adv, p=2, dim=-1, keepdim=True) fraction = (l2 / torch.norm(FC_clean, p=2, dim=-1, keepdim=True)) fc_l2_norms.update(l2.mean()) fc_l2_frac.update(fraction.mean()) with torch.no_grad(): # evaluate the critic scores for X and E mi, E = discriminator(X=X) # evaluate the critic scores for X_adv and E_adv mi_adv_adv, E_adv = discriminator(X=X_adv) # evaluate the critic scores for X_adv and E_clean mi_adv_clean, _ = discriminator(X_adv, E=E) # evaluate the critic scores for X, E_adv mi_clean_adv, _ = discriminator(X, E=E_adv) # UPDATE MI METERS mi_meter.update(mi) mi_adv_adv_meter.update(mi_adv_adv) mi_adv_clean_meter.update(mi_adv_clean) mi_clean_adv_meter.update(mi_clean_adv) batch.set_description( "MI(X, E) {} MI(X_adv, E_adv) {} MI(X_adv, E) {} MI(X, E_adv) {}". format(mi, mi_adv_adv, mi_adv_clean, mi_clean_adv)) # print to logfile print("Error Clean {clean_errors.avg:.3f}\t Error Adv{adv_errors.avg:.3f}\t " "C L2 {c_l2_norms.avg:.3f}\t C L2 Frac{c_l2_frac.avg:.3f}\t" "FC L2 {fc_l2_norms.avg:.3f}\t FC L2 Frac{fc_l2_frac.avg:.3f}\t" "Z L2 {z_l2_norms.avg:.3f}\t Z L2 Frac{z_l2_frac.avg:.3f}\t" "MI(X, E) {mi.avg:.3f}\t MI(X_adv, E_adv) {mi_adv_adv.avg:.3f}\t " "MI(X_adv, E) {mi_adv_clean.avg:.3f}\t MI(X, E_adv) {mi_clean_adv.avg:.3f}\t".format( clean_errors=clean_errors, adv_errors=adv_errors, c_l2_norms=c_l2_norms, c_l2_frac=c_l2_frac, fc_l2_norms=fc_l2_norms, fc_l2_frac=fc_l2_frac, z_l2_norms=z_l2_norms, z_l2_frac=z_l2_frac, mi=mi_meter, mi_adv_adv=mi_adv_adv_meter, mi_adv_clean=mi_adv_clean_meter, mi_clean_adv=mi_clean_adv_meter), \ file=log) log.flush()
def FID(img): with torch.no_grad(): batch_size = 32 cfg['batch_size']['train'] = batch_size dataset = fetch_dataset(cfg['data_name'], cfg['subset'], verbose=False) real_data_loader = make_data_loader(dataset)['train'] generated_data_loader = DataLoader(img, batch_size=batch_size) if cfg['data_name'] in ['COIL100', 'Omniglot']: model = models.classifier().to(cfg['device']) model_tag = ['0', cfg['data_name'], cfg['subset'], 'classifier'] model_tag = '_'.join(filter(None, model_tag)) checkpoint = load( './metrics_tf/res/classifier/{}_best.pt'.format(model_tag)) model.load_state_dict(checkpoint['model_dict']) model.train(False) real_feature = [] for i, input in enumerate(real_data_loader): input = collate(input) input = to_device(input, cfg['device']) real_feature_i = model.feature(input) real_feature.append(real_feature_i.cpu().numpy()) real_feature = np.concatenate(real_feature, axis=0) generated_feature = [] for i, input in enumerate(generated_data_loader): input = { 'img': input, 'label': input.new_zeros(input.size(0)).long() } input = to_device(input, cfg['device']) generated_feature_i = model.feature(input) generated_feature.append(generated_feature_i.cpu().numpy()) generated_feature = np.concatenate(generated_feature, axis=0) else: model = inception_v3(pretrained=True, transform_input=False).to(cfg['device']) up = nn.Upsample(size=(299, 299), mode='bilinear', align_corners=False) model.feature = nn.Sequential(*[ up, model.Conv2d_1a_3x3, model.Conv2d_2a_3x3, model.Conv2d_2b_3x3, nn.MaxPool2d(kernel_size=3, stride=2), model.Conv2d_3b_1x1, model.Conv2d_4a_3x3, nn.MaxPool2d(kernel_size=3, stride=2), model.Mixed_5b, model.Mixed_5c, model.Mixed_5d, model.Mixed_6a, model.Mixed_6b, model.Mixed_6c, model.Mixed_6d, model.Mixed_6e, model.Mixed_7a, model.Mixed_7b, model.Mixed_7c, nn.AdaptiveAvgPool2d(1), nn.Flatten() ]) model.train(False) real_feature = [] for i, input in enumerate(real_data_loader): input = collate(input) input = to_device(input, cfg['device']) real_feature_i = model.feature(input['img']) real_feature.append(real_feature_i.cpu().numpy()) real_feature = np.concatenate(real_feature, axis=0) generated_feature = [] for i, input in enumerate(generated_data_loader): input = to_device(input, cfg['device']) generated_feature_i = model.feature(input) generated_feature.append(generated_feature_i.cpu().numpy()) generated_feature = np.concatenate(generated_feature, axis=0) mu1 = np.mean(real_feature, axis=0) sigma1 = np.cov(real_feature, rowvar=False) mu2 = np.mean(generated_feature, axis=0) sigma2 = np.cov(generated_feature, rowvar=False) mu1 = np.atleast_1d(mu1) mu2 = np.atleast_1d(mu2) sigma1 = np.atleast_2d(sigma1) sigma2 = np.atleast_2d(sigma2) assert mu1.shape == mu2.shape, "Training and test mean vectors have different lengths" assert sigma1.shape == sigma2.shape, "Training and test covariances have different dimensions" diff = mu1 - mu2 # product might be almost singular covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False) if not np.isfinite(covmean).all(): offset = np.eye(sigma1.shape[0]) * 1e-6 covmean = linalg.sqrtm((sigma1 + offset).dot(sigma2 + offset)) # numerical error might give slight imaginary component if np.iscomplexobj(covmean): if not np.allclose(np.diagonal(covmean).imag, 0, atol=1e-3): m = np.max(np.abs(covmean.imag)) raise ValueError("Imaginary component {}".format(m)) covmean = covmean.real tr_covmean = np.trace(covmean) fid = diff.dot(diff) + np.trace(sigma1) + np.trace( sigma2) - 2 * tr_covmean fid = fid.item() return fid
def get_attack_stats(args, encoder, classifier, loader, log, type="class"): clean_errors = AverageMeter() adv_errors = AverageMeter() c_l2_norms = AverageMeter() c_l2_frac = AverageMeter() fc_l2_norms = AverageMeter() fc_l2_frac = AverageMeter() z_l2_norms = AverageMeter() z_l2_frac = AverageMeter() classifier.eval() batch = tqdm(loader, total=len(loader) // loader.batch_size) for i, (X, y) in enumerate(batch): if args.gpu: X, y = X.cuda(), y.cuda() if type == "class": # adv samples using classifier if args.attack == "pgd": X_adv, delta, out, out_adv = pgd(model=classifier, X=X, y=y, epsilon=args.epsilon, alpha=args.alpha, num_steps=args.num_steps, p='inf') elif args.attack == "fgsm": X_adv, delta, out, out_adv = fgsm(model=classifier, X=X, y=y, epsilon=args.epsilon) elif type == "encoder": X_adv, E_adv, diff, max_diff = encoder_attack(X, encoder, args.num_steps, args.epsilon, args.alpha, random_restart=True) batch.set_description("Avg Diff {} Max Diff {}".format(diff, max_diff)) # run classifier on adversarial representations logits_clean = classifier(X) logits_adv = classifier(X_adv) out = logits_clean.max(1)[1] out_adv = logits_adv.max(1)[1] elif type == "impostor": batch_size = X.shape[0] # using the given batch form X_s X_t pairs X_s = X[0:batch_size // 2] X_t = X[batch_size // 2:] # set y to the labels for X_s and X to X_s for later computation and logging y = y[0:batch_size // 2] X = X_s X_adv, E_adv, diff, min_diff = source2target(X_s, X_t, encoder=encoder, epsilon=2.0, max_steps=70000, step_size=0.001) # run classifier on adversarial representations logits_clean = classifier(X_s) logits_adv = classifier(X_adv) out = logits_clean.max(1)[1] out_adv = logits_adv.max(1)[1] batch.set_description("Avg Diff {} Min Diff {}".format(diff, min_diff)) elif type == "random": delta = torch.rand_like(X).sign() * args.epsilon X_adv = X + delta _, _, E = encoder(X) _, _, E_d = encoder(X_adv) norm = torch.norm(E - E_d, p=2, dim=-1) # run classifier on adversarial representations logits_clean = classifier(X) logits_adv = classifier(X_adv) out = logits_clean.max(1)[1] out_adv = logits_adv.max(1)[1] batch.set_description("Avg Diff {} Max Diff {} ".format(norm.mean(), norm.max())) # UPDATE CLEAN and ADV ERRORS err_clean = (out.data != y).float().sum() / X.size(0) err_adv = (out_adv.data != y).float().sum() / X.size(0) clean_errors.update(err_clean) adv_errors.update(err_adv) # UPDATE L2 NORM METERS C_clean, FC_clean, Z_clean = encoder(X) C_adv, FC_adv, Z_adv = encoder(X_adv) l2 = torch.norm(Z_clean - Z_adv, p=2, dim=-1, keepdim=True) fraction = (l2 / torch.norm(Z_clean, p=2, dim=-1, keepdim=True)) z_l2_norms.update(l2.mean()) z_l2_frac.update(fraction.mean()) l2 = torch.norm(C_clean - C_adv, p=2, dim=(-1, -2, -3), keepdim=True) fraction = (l2 / torch.norm(C_clean, p=2, dim=(-1, -2, -3), keepdim=True)) c_l2_norms.update(l2.mean()) c_l2_frac.update(fraction.mean()) l2 = torch.norm(FC_clean - FC_adv, p=2, dim=-1, keepdim=True) fraction = (l2 / torch.norm(FC_clean, p=2, dim=-1, keepdim=True)) fc_l2_norms.update(l2.mean()) fc_l2_frac.update(fraction.mean()) # print to logfile print("Error Clean {clean_errors.avg:.3f}\t Error Adv{adv_errors.avg:.3f}\t " "C L2 {c_l2_norms.avg:.3f}\t C L2 Frac{c_l2_frac.avg:.3f}\t" "FC L2 {fc_l2_norms.avg:.3f}\t FC L2 Frac{fc_l2_frac.avg:.3f}\t" "Z L2 {z_l2_norms.avg:.3f}\t Z L2 Frac{z_l2_frac.avg:.3f}\t".format( clean_errors=clean_errors, adv_errors=adv_errors, c_l2_norms=c_l2_norms, c_l2_frac=c_l2_frac, fc_l2_norms=fc_l2_norms, fc_l2_frac=fc_l2_frac, z_l2_norms=z_l2_norms, z_l2_frac=z_l2_frac), file=log) log.flush()
def __init__(self, opts, tag): tf.reset_default_graph() logging.error('Building the Tensorflow Graph') gpu_options = tf.GPUOptions(allow_growth=True) config = tf.ConfigProto(gpu_options=gpu_options) self.sess = tf.Session(config=config) self.opts = opts assert opts['dataset'] in datashapes, 'Unknown dataset.' # Add placeholders shape = datashapes[opts['dataset']] self.sample_points = tf.placeholder(tf.float32, [None] + shape, name='real_points_ph') self.labels = tf.placeholder(tf.int64, shape=[None], name='label_ph') self.sample_noise = tf.placeholder(tf.float32, [None, opts['zdim']], name='noise_ph') self.lr_decay = tf.placeholder(tf.float32, name='rate_decay_ph') self.is_training = tf.placeholder(tf.bool, name='is_training_ph') self.mean_ph = tf.placeholder(tf.float32, shape=[None, opts['zdim']]) self.sigma_ph = tf.placeholder(tf.float32, shape=[None, opts['zdim']]) # Build training computation graph sample_size = tf.shape(self.sample_points)[0] enc_mean, enc_sigmas = encoder(opts, inputs=self.sample_points, is_training=self.is_training, y=self.labels) enc_sigmas = tf.clip_by_value(enc_sigmas, -50, 50) self.enc_mean, self.enc_sigmas = enc_mean, enc_sigmas self.encoded = self.get_encoded(opts, self.enc_mean, self.enc_sigmas) self.encoded2 = self.get_encoded(opts, self.mean_ph, self.sigma_ph) self.reconstructed = decoder(opts, noise=self.encoded, is_training=self.is_training) self.probs1 = classifier(opts, self.encoded) self.probs2 = classifier(opts, self.encoded2) self.correct_sum = tf.reduce_sum( tf.cast(tf.equal(tf.argmax(self.probs1, axis=1), self.labels), tf.float32)) self.decoded = decoder(opts, noise=self.sample_noise, is_training=self.is_training) self.loss_cls = self.cls_loss(self.labels, self.probs1) self.loss_cls2 = self.cls_loss(self.labels, self.probs2) self.loss_mmd = self.mmd_penalty(self.sample_noise, self.encoded) self.loss_recon = self.reconstruction_loss(self.opts, self.sample_points, self.reconstructed) self.objective = self.loss_recon + opts[ 'lambda'] * self.loss_mmd + self.loss_cls # Build evaluate computation graph logpxy = [] dimY = opts['n_classes'] N = sample_size S = opts['sampling_size'] x_rep = tf.tile(self.sample_points, [S, 1, 1, 1]) for i in range(dimY): y = tf.fill((N, ), i) mu, log_sig = encoder(opts, inputs=self.sample_points, is_training=False, y=y) mu = tf.tile(mu, [S, 1]) log_sig = tf.tile(log_sig, [S, 1]) y = tf.tile(y, [ S, ]) z = self.get_encoded(opts, mu, log_sig) z_sample = tf.random_normal((tf.shape(z)[0], opts['zdim']), 0., 1., dtype=tf.float32) mu_x = decoder(opts, noise=z, is_training=False) logit_y = classifier(opts, z) logp = -tf.reduce_sum((x_rep - mu_x)**2, axis=[1, 2, 3]) log_pyz = -tf.nn.sparse_softmax_cross_entropy_with_logits( labels=y, logits=logit_y) mmd_loss = self.mmd_penalty(z_sample, z) bound = 0.5 * logp + log_pyz + opts['lambda'] * mmd_loss bound = tf.reshape(bound, [S, N]) bound = self.logsumexp(bound) - tf.log(float(S)) logpxy.append(tf.expand_dims(bound, 1)) logpxy = tf.concat(logpxy, 1) y_pred = tf.nn.softmax(logpxy) self.eval_probs = y_pred self.loss_pretrain = self.pretrain_loss( ) if opts['e_pretrain'] else None self.add_optimizers() self.add_savers() self.tag = tag
opt.trainRoot = "check_" + opt.optimizer if opt.isDropOut: opt.trainRoot += '_dropout' if opt.testRoot is None: opt.testRoot = opt.trainRoot.replace('check', 'test') os.system('mkdir {0}'.format(opt.testRoot)) # Hyper parameter imSize = 28 # data batch imBatch = Variable(torch.FloatTensor(opt.batchSize, 1, imSize, imSize)) labelBatch = Variable(torch.LongTensor(opt.batchSize, 1)) # Network classifier = models.classifier(isDropOut=opt.isDropOut).eval() classifier.load_state_dict( torch.load('{0}/classifier_{1}.pth'.format(opt.trainRoot, opt.nepoch - 1))) # DataLoader mnistDataset = dataLoader.BatchLoader(dataRoot=opt.dataRoot, phase='TEST') mnistLoader = DataLoader(mnistDataset, batch_size=opt.batchSize, num_workers=4, shuffle=False) # Move data and network to gpu if opt.cuda: imBatch = imBatch.cuda(opt.gpuId) labelBatch = labelBatch.cuda(opt.gpuId) classifier = classifier.cuda(opt.gpuId)
# compute gradient penalty gradient_penalty = calc_gradient_penalty(critic, gen_src.data, gen_tgt.data) gradient_penalty.backward() d_loss = d_loss + gradient_penalty # optimize weights of discriminator #d_loss = - d_loss_src + d_loss_tgt + gradient_penalty optimizer_d.step() optimizer_d.step() #break # 训练 分类器 optimizer_c.zero_grad() pred_c = classifier(generator(image_src).detach()) c_loss = criterion(pred_c, label_src) # 公式6 求交叉熵 c_loss.backward() optimizer_c.step() # 训练 生成器 # 训练 生成器时 鉴别器的梯度不下降 for p in critic.parameters(): p.requires_grad = False optimizer_g.zero_grad() # 计算 src 分类器的损失 gen_src = generator(image_src) pred_c = classifier(gen_src) g_loss_cls = criterion(pred_c, label_src)
# load classifier from checkpoint classifier.load_state_dict( torch.load(args.classifier_ckpt)["classifier_state_dict"]) classifier.to(args.device) Z = [] pred = [] Y = [] Z_adv = [] pred_adv = [] for X, y in test_loader: if args.gpu: X, y = X.cuda(), y.cuda() with torch.no_grad(): z, logits = classifier(X, intermediate=True) Z.append(z.cpu().detach().numpy()) pred.append(logits.max(-1)[1].cpu().detach().numpy()) Y.append(y.cpu().detach().numpy()) X_adv, delta, out, out_adv = pgd(model=classifier, X=X, y=y, epsilon=args.epsilon, alpha=args.alpha, num_steps=args.num_steps, p='inf') z, logits = classifier(X_adv, intermediate=True) Z_adv.append(z.cpu().detach().numpy()) pred_adv.append(out_adv.cpu().detach().numpy())
def load_classifier(checkpoint): model = models.classifier(models.tokenizer()) model.load_state_dict(torch.load(checkpoint, map_location=device)) return model.to(device)
validation_list = np.load('validation_number.npy') validation_length = len(validation_list) test_list = np.load('test_number.npy') test_length = len(test_list) except: print("Create train/validation/test sets before using this code") sys.exit() label_size = FLAGS.num_of_sens X = tf.placeholder(tf.float32, [None, None, None, 3]) Y = tf.placeholder(tf.float32, [None, label_size]) H = tf.placeholder(tf.float32, [None, None, FLAGS.c_dim]) weights, biases = classifier_weights(label_size) classification = classifier(X, weights, biases) classification = tf.reduce_mean(classification, 0) classification = tf.reduce_mean(classification, 0) classification = tf.reduce_mean(classification, 0) prediction = tf.nn.softmax(classification) # Define loss and optimizer loss_op = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=classification, labels=Y)) global_step = tf.Variable(0, trainable=False) starter_learning_rate = np.copy(FLAGS.learning_rate) learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, 1000, 0.93,
def train_model(alpha=0.25, batch_size=32, epochs=200): X_s, y_s, _, _ = sample_source_data() X_t, y_t, X_test, y_test = sample_target_data() X1, X2, y1, y2, yc = create_pairs(X_s, y_s, X_t, y_t) #X1, X2, y1, y2, yc, X_test, y_test = load_pairs() base_model = tiny_base_model() input_s = Input(shape=(X_s.shape[1], X_s.shape[2], X_s.shape[3])) input_t = Input(shape=(X_t.shape[1], X_t.shape[2], X_t.shape[3])) processed_s = base_model(input_s) processed_t = base_model(input_t) output = classifier(processed_s) dist = Lambda(eucl_dist, output_shape=eucl_dist_output_shape, name='CSA')([processed_s, processed_t]) model = Model(inputs=[input_s, input_t], outputs=[output, dist]) model.compile(loss={'classification': 'categorical_crossentropy', 'CSA': contrastive_loss}, optimizer='adadelta', loss_weights={'classification': 1 - alpha, 'CSA': alpha}) print('Model Compiled.') model.summary() if not os.path.exists('./images/'): os.mkdir('./images/') plot_model(model, to_file='./images/resnet.png') # Deploy TensorBoard as callbacks log_dir = './logs' if os.path.exists(log_dir): shutil.rmtree(log_dir) cb = TensorBoard(log_dir='./logs', histogram_freq=0, write_graph=True, write_grads=True, write_images=True, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) print('Start training model.') #history = model.fit(x=[X1, X2], y=[y1, yc], batch_size=batch_size, epochs=epochs, callbacks=[cb], validation_split=0.2) #plot_loss(history) best_acc = 0 acc_his = [0] total_loss_his = [] classification_loss_his = [] CSA_loss_his = [] for e in range(epochs): if e % 10 == 0: printn(str(e) + '->') for i in range(int(len(y2) / batch_size)): loss = model.train_on_batch([X1[i * batch_size:(i + 1) * batch_size, :, :, :], X2[i * batch_size:(i + 1) * batch_size, :, :, :]], [y1[i * batch_size:(i + 1) * batch_size, :], yc[i * batch_size:(i + 1) * batch_size, ]]) loss = model.train_on_batch([X2[i * batch_size:(i + 1) * batch_size, :, :, :], X1[i * batch_size:(i + 1) * batch_size, :, :, :]], [y2[i * batch_size:(i + 1) * batch_size, :], yc[i * batch_size:(i + 1) * batch_size, ]]) out = model.predict([X_test, X_test]) acc_v = np.argmax(out[0], axis=1) - np.argmax(y_test, axis=1) acc = (len(acc_v) - np.count_nonzero(acc_v) + .0000001) / len(acc_v) if best_acc < acc: best_acc = acc acc_his.append(acc) total_loss_his.append(loss[2]) classification_loss_his.append(loss[1]) CSA_loss_his.append(loss[0]) """ if not os.path.exists('./model/'): os.mkdir('./model/') model.save('./model/CCSA.h5') """ plot_loss(total_loss_his, classification_loss_his, CSA_loss_his) plot_acc(acc_his) return best_acc