def get_embs(generator): ''' given a generator, runs all the data through one pass of the model to calculate embeddings used when BERT weights are frozen, calculates embeddings first to save compute ''' features = [] model.eval() with torch.no_grad(): for input_ids, input_mask, segment_ids, y, group, guid, other_vars in generator: input_ids = input_ids.to(device) segment_ids = segment_ids.to(device) input_mask = input_mask.to(device) hidden_states, _ = model(input_ids, token_type_ids=segment_ids, attention_mask=input_mask) bert_out = extract_embeddings(hidden_states, args.emb_method) for c, i in enumerate(guid): note_id, seq_id = i.split('-') emb = bert_out[c, :].detach().cpu().numpy() features.append( EmbFeature(emb=emb, y=y[c], guid=i, group=group, other_fields=[i[c] for i in other_vars])) return features
def main(): parser = argparse.ArgumentParser() parser.add_argument('--coco-path', type=str, help='', default='') parser.add_argument('--set-name', type=str, help='', default='') parser.add_argument('--target-size', type=int, help='Resize/padding input image to target-size.', default=224) parser.add_argument('--snapshot', type=str, help='', default=None) parser.add_argument('--emb-size', type=int, help='Embedding size', default=2048) parser.add_argument('--backbone', type=str, help='ResNet18/34/50/101/152', default='ResNet50') parser.add_argument('--num-workers', type=int, help='Number of workers for data loader', default=1) args = parser.parse_args() # Set up data loaders parameters kwargs = {'num_workers': args.num_workers, 'pin_memory': True} if cuda else {} # transforms_args = [transforms.ToTensor(), transforms.Normalize([127.5, 127.5, 127.5], [1.0, 1.0, 1.0])] dataset = CoCoDataset(args.coco_path, args.set_name, target_size=args.target_size, transform=transforms.Compose(transforms_args)) data_loader = DataLoader(dataset, batch_size=8, **kwargs) # init model model, _, _ = load_model(args.backbone, args.snapshot) if cuda: model.cuda() embeddings, labels = extract_embeddings(data_loader, model, embedding_size=args.emb_size, cuda=cuda) # using PCA to reduce a reasonable amount of dimensionality first # pca = PCA(n_components=50) # embeddings_pca = pca.fit_transform(embeddings) tsne = TSNE(n_components=2, metric="euclidean").fit_transform(embeddings) plot_embeddings(dataset, tsne, labels)
def log_topk_retrieval_acc(self, engine): """ For tracking the performance during training top K Precision """ loader_kwargs = { 'pin_memory': True, 'num_workers': os.cpu_count(), 'batch_size': 100 } train_loader = DataLoader(self.train_ds, **loader_kwargs) val_loader = DataLoader(self.val_ds, **loader_kwargs) # ---------------------------------- train_embs, train_labels = extract_embeddings(self.model, train_loader) val_embs, val_labels = extract_embeddings(self.model, val_loader) emb_dim = train_embs.shape[1] # ---------------------------------- t = AnnoyIndex(emb_dim, metric='euclidean') n_trees = 100 for i, emb_vec in enumerate(train_embs): t.add_item(i, emb_vec) # build a forest of trees tqdm.write("Building ANN forest...") t.build(n_trees) # ---------------------------------- top_k_corrects = dict() # Meassure Prec@[5, 10, 20, 30] for i, emb_vec in enumerate(val_embs): correct_cls = val_labels[i] for k in [5, 10, 20, 30]: idx = t.get_nns_by_vector(emb_vec, k) top_k_classes = train_labels[idx] correct = np.sum(top_k_classes == correct_cls) accum_corr = top_k_corrects.get(k, 0) top_k_corrects[k] = accum_corr + correct # ------------------------------------------------- # calculate back the acc top_k_acc = dict() for k in [5, 10, 20, 30]: top_k_acc[k] = top_k_corrects[k] / k / val_embs.shape[0] tqdm.write( "Top K Retrieval Results - Epoch: {} Avg top-k accuracy:".format( engine.state.epoch)) for k in [5, 10, 20, 30]: tqdm.write(" Prec@{} = {:.2f}".format(k, top_k_acc[k]))
def main(args): token_to_index, _ = Vocabulary.load(args.vocab_path) if os.path.exists(args.embed_array_path) and os.path.exists( args.embed_dict_path): with open(args.embed_dict_path, 'rb') as f: pretrained_token_to_index = pickle.load(f) embeddings = extract_embeddings(token_to_index, pretrained_token_to_index, np.load(args.embed_array_path)) else: if os.path.exists(args.embed_path): pretrained_token_to_index, embeddings = save_word_embedding_as_npy( args.embed_path, args.dim) else: raise FileNotFoundError( 'Please download pre-trained embedding file') root, _ = os.path.splitext(args.vocab_path) basepath, basename = os.path.split(root) filename = f'{basepath}/embedding_{basename}.npy' np.save(filename, embeddings)
def get_embs(generator): model.eval() embs = { str(idx): np.zeros(shape=(row['num_seqs'], EMB_SIZE), dtype=np.float32) for idx, row in df.iterrows() } with torch.no_grad(): for input_ids, input_mask, segment_ids, _, _, guid, _ in tqdm( generator): input_ids = input_ids.to(device) segment_ids = segment_ids.to(device) input_mask = input_mask.to(device) hidden_states, _ = model(input_ids, token_type_ids=segment_ids, attention_mask=input_mask) bert_out = extract_embeddings(hidden_states, args.emb_method) for c, i in enumerate(guid): note_id, seq_id = i.split('-') emb = bert_out[c, :].detach().cpu().numpy() embs[note_id][int(seq_id), :] = emb return embs
def run_validation(engine): # loader_kwargs = { # 'pin_memory': True, # 'num_workers': 4, # 'batch_size': 100, # } siamese_val_loader = DataLoader(siamese_val_ds, **loader_kwargs) # ---------------------------------- siamese_evaluator.run(siamese_val_loader) avg_acc = siamese_evaluator.state.metrics['accuracy'] print("run_validation: SimilarityAccuracy accuracy: {}".format( avg_acc)) val_embs, val_labels = extract_embeddings(emb_net, val_loader) val_emb_ds = TensorDataset(val_embs, val_labels) clsf_evaluator.run(DataLoader(val_emb_ds, **loader_kwargs)) metrics = clsf_evaluator.state.metrics avg_accuracy = metrics['accuracy'] avg_loss = metrics['loss'] print("run_validation: clsf accuracy: {}, loss: {}".format( avg_accuracy, avg_loss)) return
def evaluate_on_set(generator, predictor, emb_gen=False, c_val=2): ''' Input: a pytorch data loader, whether the generator is an embedding or text generator Outputs: prediction_dict: a dictionary mapping note_id (str) to list of predicted probabilities merged_preds: a dictionary mapping note_id (str) to a single merged probability embs: a dictionary mapping note_id (str) to a numpy 2d array (shape num_seq * 768) ''' model.eval() predictor.eval() if generator.dataset.gen_type == 'val': prediction_dict = {str(idx): [0]*row['num_seqs'] for idx, row in val_df.iterrows()} embs = {str(idx): np.zeros( shape=(row['num_seqs'], EMB_SIZE)) for idx, row in val_df.iterrows()} elif generator.dataset.gen_type == 'test': prediction_dict = {str(idx): [0]*row['num_seqs'] for idx, row in test_df.iterrows()} embs = {str(idx): np.zeros( shape=(row['num_seqs'], EMB_SIZE)) for idx, row in test_df.iterrows()} elif generator.dataset.gen_type == 'train': prediction_dict = {str(idx): [0]*row['num_seqs'] for idx, row in train_df.iterrows()} embs = {str(idx): np.zeros( shape=(row['num_seqs'], EMB_SIZE)) for idx, row in train_df.iterrows()} if emb_gen: with torch.no_grad(): for embs, y, guid, other_vars in generator: embs = embs.to(device) y = y.to(device) for i in other_vars: embs = torch.cat( [embs, i.float().unsqueeze(dim=1).to(device)], 1) preds = predictor(embs).detach().cpu() for c, i in enumerate(guid): note_id, seq_id = i.split('-') if args.task_type in ['binary', 'regression']: prediction_dict[note_id][int(seq_id)] = preds[c].item() else: prediction_dict[note_id][int( seq_id)] = preds[c, :].numpy() else: with torch.no_grad(): for input_ids, input_mask, segment_ids, y, group, guid, other_vars in generator: input_ids = input_ids.to(device) segment_ids = segment_ids.to(device) input_mask = input_mask.to(device) y = y.to(device) group = group.to(device) hidden_states, _ = model( input_ids, token_type_ids=segment_ids, attention_mask=input_mask) bert_out = extract_embeddings(hidden_states, args.emb_method) for i in other_vars: bert_out = torch.cat( [bert_out, i.float().unsqueeze(dim=1).to(device)], 1) preds = predictor(bert_out).detach().cpu() for c, i in enumerate(guid): note_id, seq_id = i.split('-') if args.task_type in ['binary', 'regression']: prediction_dict[note_id][int(seq_id)] = preds[c].item() else: prediction_dict[note_id][int( seq_id)] = preds[c, :].numpy() embs[note_id][int(seq_id), :] = bert_out[c, :EMB_SIZE].detach().cpu().numpy() merged_preds = merge_preds(prediction_dict, c_val) return (prediction_dict, merged_preds, embs)
if args.use_adversary: discriminator.train() running_loss = 0.0 num_steps = 0 with tqdm(total=len(training_generator), desc="Epoch %s" % epoch) as pbar: if not args.freeze_bert: for input_ids, input_mask, segment_ids, y, group, _, other_vars in training_generator: input_ids = input_ids.to(device) segment_ids = segment_ids.to(device) input_mask = input_mask.to(device) y = y.to(device) group = group.to(device) hidden_states, _ = model( input_ids, token_type_ids=segment_ids, attention_mask=input_mask) bert_out = extract_embeddings( hidden_states, args.emb_method) for i in other_vars: bert_out = torch.cat( [bert_out, i.float().unsqueeze(dim=1).to(device)], 1) preds = predictor(bert_out) loss = criterion(preds, y) if args.use_adversary: adv_input = bert_out[:, :-len(other_vars)] if args.fairness_def == 'odds': adv_input = torch.cat( [adv_input, y.unsqueeze(dim=1)], 1) adv_pred = discriminator(adv_input) adv_loss = criterion_adv(adv_pred, group)
print("Experiment result folder:", exp_folder) # Mdoels emb_net = ResidualEmbNetwork() emb_net.load_state_dict(torch.load(join(exp_folder, "_emb_net_20.pth"))) # Dataset trans = Compose([Resize(cfg.sizes), ToTensor(), Normalize(cfg.mean, cfg.std)]) # train_ds = DeepFashionDataset(cfg.root_dir, 'train', transform=trans) val_ds = DeepFashionDataset(cfg.root_dir, 'val', transform=trans) test_ds = DeepFashionDataset(cfg.root_dir, 'test', transform=trans) # Extract embedding vectors load_kwargs = {'batch_size': 128, 'num_workers': os.cpu_count()} test_embs, _ = extract_embeddings(emb_net, DataLoader(test_ds, **load_kwargs)) val_embs, _ = extract_embeddings(emb_net, DataLoader(val_ds, **load_kwargs)) # search tree building search_tree = AnnoyIndex(emb_net.emb_dim, metric='euclidean') for i, vec in enumerate(val_embs): search_tree.add_item(i, vec.cpu().numpy()) search_tree.build(100) def plot_search_results(query_img_dix): n_search = 5 # prepate the plot result_idx = search_tree.get_nns_by_vector( test_embs[query_img_dix].cpu().numpy(), n_search)
def run_validation(engine): # loader_kwargs = { # 'pin_memory': True, # 'num_workers': 4, # 'batch_size': 100, # } siamese_val_loader = DataLoader(siamese_val_ds, **loader_kwargs) # ---------------------------------- siamese_evaluator.run(siamese_val_loader) avg_acc = siamese_evaluator.state.metrics['accuracy'] print( "run_validation: SimilarityAccuracy accuracy: {}".format(avg_acc)) val_embs, val_labels = extract_embeddings(emb_net, val_loader) val_emb_ds = TensorDataset(val_embs, val_labels) clsf_evaluator.run(DataLoader(val_emb_ds, **loader_kwargs)) metrics = clsf_evaluator.state.metrics avg_accuracy = metrics['accuracy'] avg_loss = metrics['loss'] print("run_validation: clsf accuracy: {}, loss: {}".format( avg_accuracy, avg_loss)) if engine.state.epoch % 5 != 0: return # ---------------------------------------------------------------------- train_loader = DataLoader(train_ds, **loader_kwargs) train_embs, train_labels = extract_embeddings(emb_net, train_loader) emb_dim = train_embs.shape[1] # ---------------------------------- from annoy import AnnoyIndex from tqdm import tqdm t = AnnoyIndex(emb_dim, metric='euclidean') n_trees = 100 for i, emb_vec in enumerate(train_embs): t.add_item(i, emb_vec.cpu().numpy()) # build a forest of trees tqdm.write("Building ANN forest...") t.build(n_trees) # ---------------------------------- top_k_corrects = dict() # Meassure Prec@[5, 10, 20, 30] k_vals = [10, 30, 50, 100, 500, 1000] for i, emb_vec in enumerate(val_embs): correct_cls = val_labels[i] for k in k_vals: idx = t.get_nns_by_vector(emb_vec.cpu().numpy(), k) top_k_classes = train_labels[idx] correct = torch.sum(top_k_classes == correct_cls) accum_corr = top_k_corrects.get(k, 0) top_k_corrects[k] = accum_corr + correct.item() # ------------------------------------------------- # calculate back the acc top_k_acc = dict() for k in k_vals: top_k_acc[k] = top_k_corrects[k] / k / val_embs.shape[0] tqdm.write( "Top K Retrieval Results - Epoch: {} Avg top-k accuracy:".format( engine.state.epoch)) for k in k_vals: tqdm.write(" Prec@{} = {:.2f}, Corrects@{} = {}".format( k, top_k_acc[k], k, top_k_corrects[k]))
# Mdoels emb_net = ResidualEmbNetwork() emb_net.load_state_dict(torch.load(join(exp_folder, "_emb_net_20.pth"))) # Dataset trans = Compose([Resize(cfg.sizes), ToTensor(), Normalize(cfg.mean, cfg.std)]) train_ds = DeepFashionDataset(cfg.root_dir, 'val', transform=trans) rnd_state = np.random.RandomState(200) samples = rnd_state.choice(len(train_ds), 5000, replace=False) train_ds = Subset(train_ds, samples) # Extract embedding vectors load_kwargs = {'batch_size': 128, 'num_workers': os.cpu_count()} # test_embs, _ = extract_embeddings(emb_net, DataLoader(test_ds, **load_kwargs)) embs, labels = extract_embeddings(emb_net, DataLoader(train_ds, **load_kwargs)) # translate them to cpu + numpy embs = embs.cpu().numpy() labels = labels.cpu().numpy() # ----------------------------------------------------------------------------- print("Plotting T-sne....") from cuml.manifold import TSNE tsne = TSNE(n_iter=1000, metric="euclidean") projected_emb = tsne.fit_transform(embs) fig = plot_embeddings(projected_emb, labels) png_fname = join(exp_folder, 't-sne.png') fig.savefig(png_fname, bbox_inches='tight') pdf_fname = join(exp_folder, 't-sne.pdf') fig.savefig(pdf_fname, bbox_inches='tight') # -----------------------------------------------------------------------------
# --------------------------------------------------------------------- model = SimpleCNN() loss_fn = torch.nn.CrossEntropyLoss() lr = 1e-2 if has_cuda: model.cuda() optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1) n_epochs = 1 log_interval = 50 fit(train_loader, test_loader, model, loss_fn, optimizer, scheduler, n_epochs, has_cuda, log_interval, metrics=[AccumulatedAccuracyMetric()]) # --------------------------------------------------------------------------- # Obtain the embeddings embeddings, labels = extract_embeddings(model.emb_net, test_loader) print(embeddings.shape) # # Project it with pca pca = PCA(n_components=2) projected_emb = pca.fit_transform(embeddings) # The default of 1,000 iterations gives fine results, but I'm training for longer just to eke # out some marginal improvements. NB: This takes almost an hour! # tsne = TSNE(random_state=1, n_iter=1000, metric="cosine") # projected_emb = tsne.fit_transform(embeddings) fig = plot_embeddings(projected_emb, labels) fig.savefig('baseline.png', bbox_inches='tight')
def main(): parser = argparse.ArgumentParser() parser.add_argument('--coco-path', type=str, help='', default='') parser.add_argument('--train-set-name', type=str, help='', default='training') parser.add_argument('--test-set-name', type=str, help='', default='validation_wo_occlusion') parser.add_argument('--target-size', type=int, help='Resize/padding input image to target-size.', default=224) parser.add_argument('--snapshot', type=str, help='', default=None) parser.add_argument('--emb-size', type=int, help='Embedding size', default=2048) parser.add_argument('--backbone', type=str, help='ResNet18/34/50/101/152', default='ResNet50') parser.add_argument('--snapshot-path', type=str, help='Path to save snapshot', default='.') parser.add_argument('--num-workers', type=int, help='Number of workers for data loader', default=1) parser.add_argument('--n-neighbors', type=int, help='Number of neighbors for KNN classifier', default=1) args = parser.parse_args() # Set up data loaders parameters kwargs = { 'num_workers': args.num_workers, 'pin_memory': True } if cuda else {} # transforms_args = [ transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ] train_dataset = CoCoDataset(args.coco_path, args.train_set_name, target_size=args.target_size, transform=transforms.Compose(transforms_args)) test_dataset = CoCoDataset(args.coco_path, args.test_set_name, target_size=args.target_size, transform=transforms.Compose(transforms_args)) train_loader = DataLoader(train_dataset, batch_size=8, shuffle=False, **kwargs) test_loader = DataLoader(test_dataset, batch_size=8, shuffle=False, **kwargs) # init model model, _, _ = load_model(args.backbone, args.snapshot) if cuda: model.cuda() train_embeddings, train_labels = extract_embeddings( train_loader, model, embedding_size=args.emb_size, cuda=cuda) test_embeddings, test_labels = extract_embeddings( test_loader, model, embedding_size=args.emb_size, cuda=cuda) dist_mtx = pdist(test_embeddings, train_embeddings) indices = np.argmin(dist_mtx, axis=1) y_pred = train_labels[indices] clf = KNeighborsClassifier(n_neighbors=args.n_neighbors, metric='l2', n_jobs=-1, weights="distance") clf.fit(train_embeddings, train_labels) pickle.dump( clf, open( os.path.join( args.snapshot_path, '%s.pkl' % (os.path.basename(args.snapshot).split(".")[0])), 'wb')) print(classification_report(test_labels, y_pred)) plt.figure(figsize=(20, 20)) columns = 4 rows = 4 for i in range(rows * 2): # plot test image index = np.random.randint(len(test_dataset)) image, label = test_dataset._load_image(index) ax = plt.subplot(rows, columns, i * 2 + 1) ax.imshow(image) ax.title.set_text( test_dataset.coco_label_to_name( test_dataset.label_to_coco_label(label))) # plot most similar image from train_dataset gt_index = indices[index] gt_image, gt_label = train_dataset._load_image(gt_index) ax = plt.subplot(rows, columns, i * 2 + 2) ax.imshow(gt_image) ax.title.set_text( train_dataset.coco_label_to_name( train_dataset.label_to_coco_label(gt_label))) plt.tight_layout() plt.savefig("evaluation.png")
def hard_triplet_baseline_exp(device='3', lr=1e-3, n_epochs=300, n_classes=10, n_samples=12, margin=0.3, log_interval=50): """ :param device: :param lr: :param n_epochs: :param n_classes: :param n_samples: :return: """ os.environ['CUDA_VISIBLE_DEVICES'] = str(device) # get the mean and std of dataset train/a standarizer = TaskbStandarizer(data_manager=Dcase18TaskbData()) mu, sigma = standarizer.load_mu_sigma(mode='train', device='a') # get the normalized train dataset train_dataset = DevSet(mode='train', device='a', transform=Compose( [Normalize(mean=mu, std=sigma), ToTensor()])) test_dataset = DevSet(mode='test', device='a', transform=Compose( [Normalize(mean=mu, std=sigma), ToTensor()])) train_batch_sampler = BalanceBatchSampler(dataset=train_dataset, n_classes=n_classes, n_samples=n_samples) train_batch_loader = DataLoader(dataset=train_dataset, batch_sampler=train_batch_sampler, num_workers=1) test_batch_sampler = BalanceBatchSampler(dataset=test_dataset, n_classes=n_classes, n_samples=n_samples) test_batch_loader = DataLoader(dataset=test_dataset, batch_sampler=test_batch_sampler, num_workers=1) model = networks.embedding_net_shallow() model = model.cuda() loss_fn = OnlineTripletLoss( margin=margin, triplet_selector=utils.RandomNegativeTripletSelector(margin=margin)) optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = lr_scheduler.StepLR(optimizer=optimizer, step_size=30, gamma=0.5) fit(train_loader=train_batch_loader, val_loader=test_batch_loader, model=model, loss_fn=loss_fn, optimizer=optimizer, scheduler=scheduler, n_epochs=n_epochs, log_interval=log_interval, metrics=[AverageNoneZeroTripletsMetric()]) train_embedding_tl, train_labels_tl = utils.extract_embeddings( train_batch_loader, model) utils.plot_embeddings(train_embedding_tl, train_labels_tl) test_embedding_tl, test_labels_tl = utils.extract_embeddings( test_batch_loader, model) utils.plot_embeddings(test_embedding_tl, test_labels_tl)
# Step 3 model = SiameseNet(embedding_net) margin = 1. loss_fn = ContrastiveLoss(margin) lr = 1e-3 if has_cuda: model.cuda() optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1, last_epoch=-1) n_epochs = 20 log_interval = 50 fit(siamese_train_loader, siamese_test_loader, model, loss_fn, optimizer, scheduler, n_epochs, has_cuda, log_interval) # --------------------------------------------------------------------------- # Obtain the embeddings embeddings, labels = extract_embeddings(embedding_net, test_loader) tsne = TSNE(random_state=1, n_iter=1000, metric="euclidean") projected_emb = tsne.fit_transform(embeddings) fig = plot_embeddings(projected_emb, labels) fig.savefig('siamese.png', bbox_inches='tight')
model.load_state_dict(torch.load('siamese_resnet18.pth')) deep_fashion_root_dir = "./deepfashion_data" train_ds = DeepFashionDataset(deep_fashion_root_dir, 'train', transform=trans) emb_net = model.emb_net emb_net.cuda() # subset n_samples = 25000 sel_idx = np.random.choice(list(range(len(train_ds))), n_samples, replace=False) assert len(set(sel_idx)) == n_samples ds = Subset(train_ds, sel_idx) loader = DataLoader(ds, batch_size=100, pin_memory=True, num_workers=os.cpu_count()) print("extracting...") embeddings, labels = extract_embeddings(emb_net, loader) tsne = TSNE(n_iter=400, metric="euclidean") projected_emb = tsne.fit_transform(embeddings) with open('projected_emb.pkl', 'wb') as handle: pickle.dump(projected_emb, handle, protocol=pickle.HIGHEST_PROTOCOL) with open('labels.pkl', 'wb') as handle: pickle.dump(labels, handle, protocol=pickle.HIGHEST_PROTOCOL)