def get_embs(generator):
'''
given a generator, runs all the data through one pass of the model to calculate embeddings
used when BERT weights are frozen, calculates embeddings first to save compute
'''
features = []
model.eval()
with torch.no_grad():
for input_ids, input_mask, segment_ids, y, group, guid, other_vars in generator:
input_ids = input_ids.to(device)
segment_ids = segment_ids.to(device)
input_mask = input_mask.to(device)
hidden_states, _ = model(input_ids,
token_type_ids=segment_ids,
attention_mask=input_mask)
bert_out = extract_embeddings(hidden_states, args.emb_method)
for c, i in enumerate(guid):
note_id, seq_id = i.split('-')
emb = bert_out[c, :].detach().cpu().numpy()
features.append(
EmbFeature(emb=emb,
y=y[c],
guid=i,
group=group,
other_fields=[i[c] for i in other_vars]))
return features
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--coco-path', type=str, help='', default='')
parser.add_argument('--set-name', type=str, help='', default='')
parser.add_argument('--target-size', type=int, help='Resize/padding input image to target-size.', default=224)
parser.add_argument('--snapshot', type=str, help='', default=None)
parser.add_argument('--emb-size', type=int, help='Embedding size', default=2048)
parser.add_argument('--backbone', type=str, help='ResNet18/34/50/101/152', default='ResNet50')
parser.add_argument('--num-workers', type=int, help='Number of workers for data loader', default=1)
args = parser.parse_args()
# Set up data loaders parameters
kwargs = {'num_workers': args.num_workers, 'pin_memory': True} if cuda else {} #
transforms_args = [transforms.ToTensor(), transforms.Normalize([127.5, 127.5, 127.5], [1.0, 1.0, 1.0])]
dataset = CoCoDataset(args.coco_path, args.set_name, target_size=args.target_size, transform=transforms.Compose(transforms_args))
data_loader = DataLoader(dataset, batch_size=8, **kwargs)
# init model
model, _, _ = load_model(args.backbone, args.snapshot)
if cuda:
model.cuda()
embeddings, labels = extract_embeddings(data_loader, model, embedding_size=args.emb_size, cuda=cuda)
# using PCA to reduce a reasonable amount of dimensionality first
# pca = PCA(n_components=50)
# embeddings_pca = pca.fit_transform(embeddings)
tsne = TSNE(n_components=2, metric="euclidean").fit_transform(embeddings)
plot_embeddings(dataset, tsne, labels)
def log_topk_retrieval_acc(self, engine):
"""
For tracking the performance during training top K Precision
"""
loader_kwargs = {
'pin_memory': True,
'num_workers': os.cpu_count(),
'batch_size': 100
}
train_loader = DataLoader(self.train_ds, **loader_kwargs)
val_loader = DataLoader(self.val_ds, **loader_kwargs)
# ----------------------------------
train_embs, train_labels = extract_embeddings(self.model, train_loader)
val_embs, val_labels = extract_embeddings(self.model, val_loader)
emb_dim = train_embs.shape[1]
# ----------------------------------
t = AnnoyIndex(emb_dim, metric='euclidean')
n_trees = 100
for i, emb_vec in enumerate(train_embs):
t.add_item(i, emb_vec)
# build a forest of trees
tqdm.write("Building ANN forest...")
t.build(n_trees)
# ----------------------------------
top_k_corrects = dict()
# Meassure Prec@[5, 10, 20, 30]
for i, emb_vec in enumerate(val_embs):
correct_cls = val_labels[i]
for k in [5, 10, 20, 30]:
idx = t.get_nns_by_vector(emb_vec, k)
top_k_classes = train_labels[idx]
correct = np.sum(top_k_classes == correct_cls)
accum_corr = top_k_corrects.get(k, 0)
top_k_corrects[k] = accum_corr + correct
# -------------------------------------------------
# calculate back the acc
top_k_acc = dict()
for k in [5, 10, 20, 30]:
top_k_acc[k] = top_k_corrects[k] / k / val_embs.shape[0]
tqdm.write(
"Top K Retrieval Results - Epoch: {} Avg top-k accuracy:".format(
engine.state.epoch))
for k in [5, 10, 20, 30]:
tqdm.write(" Prec@{} = {:.2f}".format(k, top_k_acc[k]))
def main(args):
token_to_index, _ = Vocabulary.load(args.vocab_path)
if os.path.exists(args.embed_array_path) and os.path.exists(
args.embed_dict_path):
with open(args.embed_dict_path, 'rb') as f:
pretrained_token_to_index = pickle.load(f)
embeddings = extract_embeddings(token_to_index,
pretrained_token_to_index,
np.load(args.embed_array_path))
else:
if os.path.exists(args.embed_path):
pretrained_token_to_index, embeddings = save_word_embedding_as_npy(
args.embed_path, args.dim)
else:
raise FileNotFoundError(
'Please download pre-trained embedding file')
root, _ = os.path.splitext(args.vocab_path)
basepath, basename = os.path.split(root)
filename = f'{basepath}/embedding_{basename}.npy'
np.save(filename, embeddings)
def get_embs(generator):
model.eval()
embs = {
str(idx): np.zeros(shape=(row['num_seqs'], EMB_SIZE), dtype=np.float32)
for idx, row in df.iterrows()
}
with torch.no_grad():
for input_ids, input_mask, segment_ids, _, _, guid, _ in tqdm(
generator):
input_ids = input_ids.to(device)
segment_ids = segment_ids.to(device)
input_mask = input_mask.to(device)
hidden_states, _ = model(input_ids,
token_type_ids=segment_ids,
attention_mask=input_mask)
bert_out = extract_embeddings(hidden_states, args.emb_method)
for c, i in enumerate(guid):
note_id, seq_id = i.split('-')
emb = bert_out[c, :].detach().cpu().numpy()
embs[note_id][int(seq_id), :] = emb
return embs
def run_validation(engine):
# loader_kwargs = {
# 'pin_memory': True,
# 'num_workers': 4,
# 'batch_size': 100,
# }
siamese_val_loader = DataLoader(siamese_val_ds, **loader_kwargs)
# ----------------------------------
siamese_evaluator.run(siamese_val_loader)
avg_acc = siamese_evaluator.state.metrics['accuracy']
print("run_validation: SimilarityAccuracy accuracy: {}".format(
avg_acc))
val_embs, val_labels = extract_embeddings(emb_net, val_loader)
val_emb_ds = TensorDataset(val_embs, val_labels)
clsf_evaluator.run(DataLoader(val_emb_ds, **loader_kwargs))
metrics = clsf_evaluator.state.metrics
avg_accuracy = metrics['accuracy']
avg_loss = metrics['loss']
print("run_validation: clsf accuracy: {}, loss: {}".format(
avg_accuracy, avg_loss))
return
def evaluate_on_set(generator, predictor, emb_gen=False, c_val=2):
'''
Input: a pytorch data loader, whether the generator is an embedding or text generator
Outputs:
prediction_dict: a dictionary mapping note_id (str) to list of predicted probabilities
merged_preds: a dictionary mapping note_id (str) to a single merged probability
embs: a dictionary mapping note_id (str) to a numpy 2d array (shape num_seq * 768)
'''
model.eval()
predictor.eval()
if generator.dataset.gen_type == 'val':
prediction_dict = {str(idx): [0]*row['num_seqs']
for idx, row in val_df.iterrows()}
embs = {str(idx): np.zeros(
shape=(row['num_seqs'], EMB_SIZE)) for idx, row in val_df.iterrows()}
elif generator.dataset.gen_type == 'test':
prediction_dict = {str(idx): [0]*row['num_seqs']
for idx, row in test_df.iterrows()}
embs = {str(idx): np.zeros(
shape=(row['num_seqs'], EMB_SIZE)) for idx, row in test_df.iterrows()}
elif generator.dataset.gen_type == 'train':
prediction_dict = {str(idx): [0]*row['num_seqs']
for idx, row in train_df.iterrows()}
embs = {str(idx): np.zeros(
shape=(row['num_seqs'], EMB_SIZE)) for idx, row in train_df.iterrows()}
if emb_gen:
with torch.no_grad():
for embs, y, guid, other_vars in generator:
embs = embs.to(device)
y = y.to(device)
for i in other_vars:
embs = torch.cat(
[embs, i.float().unsqueeze(dim=1).to(device)], 1)
preds = predictor(embs).detach().cpu()
for c, i in enumerate(guid):
note_id, seq_id = i.split('-')
if args.task_type in ['binary', 'regression']:
prediction_dict[note_id][int(seq_id)] = preds[c].item()
else:
prediction_dict[note_id][int(
seq_id)] = preds[c, :].numpy()
else:
with torch.no_grad():
for input_ids, input_mask, segment_ids, y, group, guid, other_vars in generator:
input_ids = input_ids.to(device)
segment_ids = segment_ids.to(device)
input_mask = input_mask.to(device)
y = y.to(device)
group = group.to(device)
hidden_states, _ = model(
input_ids, token_type_ids=segment_ids, attention_mask=input_mask)
bert_out = extract_embeddings(hidden_states, args.emb_method)
for i in other_vars:
bert_out = torch.cat(
[bert_out, i.float().unsqueeze(dim=1).to(device)], 1)
preds = predictor(bert_out).detach().cpu()
for c, i in enumerate(guid):
note_id, seq_id = i.split('-')
if args.task_type in ['binary', 'regression']:
prediction_dict[note_id][int(seq_id)] = preds[c].item()
else:
prediction_dict[note_id][int(
seq_id)] = preds[c, :].numpy()
embs[note_id][int(seq_id), :] = bert_out[c,
:EMB_SIZE].detach().cpu().numpy()
merged_preds = merge_preds(prediction_dict, c_val)
return (prediction_dict, merged_preds, embs)
if args.use_adversary:
discriminator.train()
running_loss = 0.0
num_steps = 0
with tqdm(total=len(training_generator), desc="Epoch %s" % epoch) as pbar:
if not args.freeze_bert:
for input_ids, input_mask, segment_ids, y, group, _, other_vars in training_generator:
input_ids = input_ids.to(device)
segment_ids = segment_ids.to(device)
input_mask = input_mask.to(device)
y = y.to(device)
group = group.to(device)
hidden_states, _ = model(
input_ids, token_type_ids=segment_ids, attention_mask=input_mask)
bert_out = extract_embeddings(
hidden_states, args.emb_method)
for i in other_vars:
bert_out = torch.cat(
[bert_out, i.float().unsqueeze(dim=1).to(device)], 1)
preds = predictor(bert_out)
loss = criterion(preds, y)
if args.use_adversary:
adv_input = bert_out[:, :-len(other_vars)]
if args.fairness_def == 'odds':
adv_input = torch.cat(
[adv_input, y.unsqueeze(dim=1)], 1)
adv_pred = discriminator(adv_input)
adv_loss = criterion_adv(adv_pred, group)
print("Experiment result folder:", exp_folder)
# Mdoels
emb_net = ResidualEmbNetwork()
emb_net.load_state_dict(torch.load(join(exp_folder, "_emb_net_20.pth")))
# Dataset
trans = Compose([Resize(cfg.sizes), ToTensor(), Normalize(cfg.mean, cfg.std)])
# train_ds = DeepFashionDataset(cfg.root_dir, 'train', transform=trans)
val_ds = DeepFashionDataset(cfg.root_dir, 'val', transform=trans)
test_ds = DeepFashionDataset(cfg.root_dir, 'test', transform=trans)
# Extract embedding vectors
load_kwargs = {'batch_size': 128, 'num_workers': os.cpu_count()}
test_embs, _ = extract_embeddings(emb_net, DataLoader(test_ds, **load_kwargs))
val_embs, _ = extract_embeddings(emb_net, DataLoader(val_ds, **load_kwargs))
# search tree building
search_tree = AnnoyIndex(emb_net.emb_dim, metric='euclidean')
for i, vec in enumerate(val_embs):
search_tree.add_item(i, vec.cpu().numpy())
search_tree.build(100)
def plot_search_results(query_img_dix):
n_search = 5
# prepate the plot
result_idx = search_tree.get_nns_by_vector(
test_embs[query_img_dix].cpu().numpy(), n_search)
def run_validation(engine):
# loader_kwargs = {
# 'pin_memory': True,
# 'num_workers': 4,
# 'batch_size': 100,
# }
siamese_val_loader = DataLoader(siamese_val_ds, **loader_kwargs)
# ----------------------------------
siamese_evaluator.run(siamese_val_loader)
avg_acc = siamese_evaluator.state.metrics['accuracy']
print(
"run_validation: SimilarityAccuracy accuracy: {}".format(avg_acc))
val_embs, val_labels = extract_embeddings(emb_net, val_loader)
val_emb_ds = TensorDataset(val_embs, val_labels)
clsf_evaluator.run(DataLoader(val_emb_ds, **loader_kwargs))
metrics = clsf_evaluator.state.metrics
avg_accuracy = metrics['accuracy']
avg_loss = metrics['loss']
print("run_validation: clsf accuracy: {}, loss: {}".format(
avg_accuracy, avg_loss))
if engine.state.epoch % 5 != 0:
return
# ----------------------------------------------------------------------
train_loader = DataLoader(train_ds, **loader_kwargs)
train_embs, train_labels = extract_embeddings(emb_net, train_loader)
emb_dim = train_embs.shape[1]
# ----------------------------------
from annoy import AnnoyIndex
from tqdm import tqdm
t = AnnoyIndex(emb_dim, metric='euclidean')
n_trees = 100
for i, emb_vec in enumerate(train_embs):
t.add_item(i, emb_vec.cpu().numpy())
# build a forest of trees
tqdm.write("Building ANN forest...")
t.build(n_trees)
# ----------------------------------
top_k_corrects = dict()
# Meassure Prec@[5, 10, 20, 30]
k_vals = [10, 30, 50, 100, 500, 1000]
for i, emb_vec in enumerate(val_embs):
correct_cls = val_labels[i]
for k in k_vals:
idx = t.get_nns_by_vector(emb_vec.cpu().numpy(), k)
top_k_classes = train_labels[idx]
correct = torch.sum(top_k_classes == correct_cls)
accum_corr = top_k_corrects.get(k, 0)
top_k_corrects[k] = accum_corr + correct.item()
# -------------------------------------------------
# calculate back the acc
top_k_acc = dict()
for k in k_vals:
top_k_acc[k] = top_k_corrects[k] / k / val_embs.shape[0]
tqdm.write(
"Top K Retrieval Results - Epoch: {} Avg top-k accuracy:".format(
engine.state.epoch))
for k in k_vals:
tqdm.write(" Prec@{} = {:.2f}, Corrects@{} = {}".format(
k, top_k_acc[k], k, top_k_corrects[k]))
# Mdoels
emb_net = ResidualEmbNetwork()
emb_net.load_state_dict(torch.load(join(exp_folder, "_emb_net_20.pth")))
# Dataset
trans = Compose([Resize(cfg.sizes), ToTensor(), Normalize(cfg.mean, cfg.std)])
train_ds = DeepFashionDataset(cfg.root_dir, 'val', transform=trans)
rnd_state = np.random.RandomState(200)
samples = rnd_state.choice(len(train_ds), 5000, replace=False)
train_ds = Subset(train_ds, samples)
# Extract embedding vectors
load_kwargs = {'batch_size': 128, 'num_workers': os.cpu_count()}
# test_embs, _ = extract_embeddings(emb_net, DataLoader(test_ds, **load_kwargs))
embs, labels = extract_embeddings(emb_net, DataLoader(train_ds, **load_kwargs))
# translate them to cpu + numpy
embs = embs.cpu().numpy()
labels = labels.cpu().numpy()
# -----------------------------------------------------------------------------
print("Plotting T-sne....")
from cuml.manifold import TSNE
tsne = TSNE(n_iter=1000, metric="euclidean")
projected_emb = tsne.fit_transform(embs)
fig = plot_embeddings(projected_emb, labels)
png_fname = join(exp_folder, 't-sne.png')
fig.savefig(png_fname, bbox_inches='tight')
pdf_fname = join(exp_folder, 't-sne.pdf')
fig.savefig(pdf_fname, bbox_inches='tight')
# -----------------------------------------------------------------------------
# ---------------------------------------------------------------------
model = SimpleCNN()
loss_fn = torch.nn.CrossEntropyLoss()
lr = 1e-2
if has_cuda:
model.cuda()
optimizer = optim.Adam(model.parameters(), lr=lr)
scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1)
n_epochs = 1
log_interval = 50
fit(train_loader, test_loader, model, loss_fn, optimizer, scheduler,
n_epochs, has_cuda, log_interval, metrics=[AccumulatedAccuracyMetric()])
# ---------------------------------------------------------------------------
# Obtain the embeddings
embeddings, labels = extract_embeddings(model.emb_net, test_loader)
print(embeddings.shape)
# # Project it with pca
pca = PCA(n_components=2)
projected_emb = pca.fit_transform(embeddings)
# The default of 1,000 iterations gives fine results, but I'm training for longer just to eke
# out some marginal improvements. NB: This takes almost an hour!
# tsne = TSNE(random_state=1, n_iter=1000, metric="cosine")
# projected_emb = tsne.fit_transform(embeddings)
fig = plot_embeddings(projected_emb, labels)
fig.savefig('baseline.png', bbox_inches='tight')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--coco-path', type=str, help='', default='')
parser.add_argument('--train-set-name',
type=str,
help='',
default='training')
parser.add_argument('--test-set-name',
type=str,
help='',
default='validation_wo_occlusion')
parser.add_argument('--target-size',
type=int,
help='Resize/padding input image to target-size.',
default=224)
parser.add_argument('--snapshot', type=str, help='', default=None)
parser.add_argument('--emb-size',
type=int,
help='Embedding size',
default=2048)
parser.add_argument('--backbone',
type=str,
help='ResNet18/34/50/101/152',
default='ResNet50')
parser.add_argument('--snapshot-path',
type=str,
help='Path to save snapshot',
default='.')
parser.add_argument('--num-workers',
type=int,
help='Number of workers for data loader',
default=1)
parser.add_argument('--n-neighbors',
type=int,
help='Number of neighbors for KNN classifier',
default=1)
args = parser.parse_args()
# Set up data loaders parameters
kwargs = {
'num_workers': args.num_workers,
'pin_memory': True
} if cuda else {} #
transforms_args = [
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]
train_dataset = CoCoDataset(args.coco_path,
args.train_set_name,
target_size=args.target_size,
transform=transforms.Compose(transforms_args))
test_dataset = CoCoDataset(args.coco_path,
args.test_set_name,
target_size=args.target_size,
transform=transforms.Compose(transforms_args))
train_loader = DataLoader(train_dataset,
batch_size=8,
shuffle=False,
**kwargs)
test_loader = DataLoader(test_dataset,
batch_size=8,
shuffle=False,
**kwargs)
# init model
model, _, _ = load_model(args.backbone, args.snapshot)
if cuda:
model.cuda()
train_embeddings, train_labels = extract_embeddings(
train_loader, model, embedding_size=args.emb_size, cuda=cuda)
test_embeddings, test_labels = extract_embeddings(
test_loader, model, embedding_size=args.emb_size, cuda=cuda)
dist_mtx = pdist(test_embeddings, train_embeddings)
indices = np.argmin(dist_mtx, axis=1)
y_pred = train_labels[indices]
clf = KNeighborsClassifier(n_neighbors=args.n_neighbors,
metric='l2',
n_jobs=-1,
weights="distance")
clf.fit(train_embeddings, train_labels)
pickle.dump(
clf,
open(
os.path.join(
args.snapshot_path,
'%s.pkl' % (os.path.basename(args.snapshot).split(".")[0])),
'wb'))
print(classification_report(test_labels, y_pred))
plt.figure(figsize=(20, 20))
columns = 4
rows = 4
for i in range(rows * 2):
# plot test image
index = np.random.randint(len(test_dataset))
image, label = test_dataset._load_image(index)
ax = plt.subplot(rows, columns, i * 2 + 1)
ax.imshow(image)
ax.title.set_text(
test_dataset.coco_label_to_name(
test_dataset.label_to_coco_label(label)))
# plot most similar image from train_dataset
gt_index = indices[index]
gt_image, gt_label = train_dataset._load_image(gt_index)
ax = plt.subplot(rows, columns, i * 2 + 2)
ax.imshow(gt_image)
ax.title.set_text(
train_dataset.coco_label_to_name(
train_dataset.label_to_coco_label(gt_label)))
plt.tight_layout()
plt.savefig("evaluation.png")
def hard_triplet_baseline_exp(device='3',
lr=1e-3,
n_epochs=300,
n_classes=10,
n_samples=12,
margin=0.3,
log_interval=50):
"""
:param device:
:param lr:
:param n_epochs:
:param n_classes:
:param n_samples:
:return:
"""
os.environ['CUDA_VISIBLE_DEVICES'] = str(device)
# get the mean and std of dataset train/a
standarizer = TaskbStandarizer(data_manager=Dcase18TaskbData())
mu, sigma = standarizer.load_mu_sigma(mode='train', device='a')
# get the normalized train dataset
train_dataset = DevSet(mode='train',
device='a',
transform=Compose(
[Normalize(mean=mu, std=sigma),
ToTensor()]))
test_dataset = DevSet(mode='test',
device='a',
transform=Compose(
[Normalize(mean=mu, std=sigma),
ToTensor()]))
train_batch_sampler = BalanceBatchSampler(dataset=train_dataset,
n_classes=n_classes,
n_samples=n_samples)
train_batch_loader = DataLoader(dataset=train_dataset,
batch_sampler=train_batch_sampler,
num_workers=1)
test_batch_sampler = BalanceBatchSampler(dataset=test_dataset,
n_classes=n_classes,
n_samples=n_samples)
test_batch_loader = DataLoader(dataset=test_dataset,
batch_sampler=test_batch_sampler,
num_workers=1)
model = networks.embedding_net_shallow()
model = model.cuda()
loss_fn = OnlineTripletLoss(
margin=margin,
triplet_selector=utils.RandomNegativeTripletSelector(margin=margin))
optimizer = optim.Adam(model.parameters(), lr=lr)
scheduler = lr_scheduler.StepLR(optimizer=optimizer,
step_size=30,
gamma=0.5)
fit(train_loader=train_batch_loader,
val_loader=test_batch_loader,
model=model,
loss_fn=loss_fn,
optimizer=optimizer,
scheduler=scheduler,
n_epochs=n_epochs,
log_interval=log_interval,
metrics=[AverageNoneZeroTripletsMetric()])
train_embedding_tl, train_labels_tl = utils.extract_embeddings(
train_batch_loader, model)
utils.plot_embeddings(train_embedding_tl, train_labels_tl)
test_embedding_tl, test_labels_tl = utils.extract_embeddings(
test_batch_loader, model)
utils.plot_embeddings(test_embedding_tl, test_labels_tl)
# Step 3
model = SiameseNet(embedding_net)
margin = 1.
loss_fn = ContrastiveLoss(margin)
lr = 1e-3
if has_cuda:
model.cuda()
optimizer = optim.Adam(model.parameters(), lr=lr)
scheduler = lr_scheduler.StepLR(optimizer,
step_size=5,
gamma=0.1,
last_epoch=-1)
n_epochs = 20
log_interval = 50
fit(siamese_train_loader, siamese_test_loader, model, loss_fn, optimizer,
scheduler, n_epochs, has_cuda, log_interval)
# ---------------------------------------------------------------------------
# Obtain the embeddings
embeddings, labels = extract_embeddings(embedding_net, test_loader)
tsne = TSNE(random_state=1, n_iter=1000, metric="euclidean")
projected_emb = tsne.fit_transform(embeddings)
fig = plot_embeddings(projected_emb, labels)
fig.savefig('siamese.png', bbox_inches='tight')
model.load_state_dict(torch.load('siamese_resnet18.pth'))
deep_fashion_root_dir = "./deepfashion_data"
train_ds = DeepFashionDataset(deep_fashion_root_dir, 'train', transform=trans)
emb_net = model.emb_net
emb_net.cuda()
# subset
n_samples = 25000
sel_idx = np.random.choice(list(range(len(train_ds))),
n_samples,
replace=False)
assert len(set(sel_idx)) == n_samples
ds = Subset(train_ds, sel_idx)
loader = DataLoader(ds,
batch_size=100,
pin_memory=True,
num_workers=os.cpu_count())
print("extracting...")
embeddings, labels = extract_embeddings(emb_net, loader)
tsne = TSNE(n_iter=400, metric="euclidean")
projected_emb = tsne.fit_transform(embeddings)
with open('projected_emb.pkl', 'wb') as handle:
pickle.dump(projected_emb, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open('labels.pkl', 'wb') as handle:
pickle.dump(labels, handle, protocol=pickle.HIGHEST_PROTOCOL)
