def data_loader():
datasets, word_to_i_map = load_swbd_labelled(rng, data_dir, min_count)
ntokens = len(word_to_i_map)
# Loading train set
train_x, train_y = datasets[0]
train_x = torch.cuda.FloatTensor(
train_x) if torch.cuda.is_available() else torch.FloatTensor(train_x)
train_y = torch.cuda.LongTensor(
train_y) if torch.cuda.is_available() else torch.FloatTensor(train_y)
train = TensorDataset(train_x, train_y) # comment out if using get_batch2
train_ldr = make_loader(train, batch_size)
# Loading dev and test data
datasets, _ = load_swbd_labelled(rng, data_dir, 1)
dev_x, dev_y = datasets[1]
test_x, test_y = datasets[2]
dev = TensorDataset(
torch.cuda.FloatTensor(dev_x), torch.cuda.LongTensor(
dev_y)) if torch.cuda.is_available() else TensorDataset(
torch.FloatTensor(dev_x), torch.LongTensor(dev_y))
test = TensorDataset(
torch.cuda.FloatTensor(test_x), torch.cuda.LongTensor(
test_y)) if torch.cuda.is_available() else TensorDataset(
torch.FloatTensor(test_x), torch.LongTensor(test_y))
dev_ldr = make_loader(dev, batch_size)
test_ldr = make_loader(test, batch_size)
return ntokens, train_ldr, dev_ldr, test_ldr
def data_loader():
datasets, word_to_i_map = load_swbd_labelled(rng, data_dir, min_count)
ntokens = len(word_to_i_map)
# Loading train set
train_x, train_y = datasets[0]
train_x = torch.cuda.FloatTensor(train_x) if torch.cuda.is_available() else torch.FloatTensor(train_x)
train_y = torch.cuda.LongTensor(train_y) if torch.cuda.is_available() else torch.FloatTensor(train_y)
train = TensorDataset(train_x, train_y) # comment out if using get_batch2
train_ldr = make_loader(train, batch_size)
# Loading dev and test data
datasets, _ = load_swbd_labelled(rng, data_dir, 1)
dev_x, dev_y = datasets[1]
test_x, test_y = datasets[2]
dev = TensorDataset(torch.cuda.FloatTensor(dev_x), torch.cuda.LongTensor(dev_y)) if torch.cuda.is_available() else TensorDataset(torch.FloatTensor(dev_x), torch.LongTensor(dev_y))
test = TensorDataset(torch.cuda.FloatTensor(test_x), torch.cuda.LongTensor(test_y)) if torch.cuda.is_available() else TensorDataset(torch.FloatTensor(test_x), torch.LongTensor(test_y))
dev_ldr = make_loader(dev, batch_size)
test_ldr = make_loader(test, batch_size)
dictionary = list(word_to_i_map)
word_to_onehot_map = {}
for w in dictionary:
if w == "okay_1":
word_to_onehot_map[w] = word2onehot("okay")
elif w == "7-eleven":
word_to_onehot_map[w] = word2onehot("7-eleven")
elif w == "401k’s":
word_to_onehot_map[w] = word2onehot("fourohoneks")
else:
word_to_onehot_map[w] = word2onehot(w)
i_to_word_map = {value: key for key, value in word_to_i_map.items()}
return ntokens, train_x, train_y, train_ldr, dev_ldr, test_ldr, word_to_onehot_map, i_to_word_map
def run_experiment(cfg_dict):
device = utils.get_device()
expand_cfg(cfg_dict)
wandb.login()
with wandb.init(project=cfg_dict['project_name'],
config=cfg_dict,
notes=cfg_dict['run_description']) as wandb_run:
cfg = wandb_run.config
model = models.make_model(**cfg.model).to(device)
model = model.apply(models.init_weights)
trainloader = data.make_loader(**cfg.train_dataset)
testloader = data.make_loader(**cfg.test_dataset)
samples = [testloader.dataset[i][0] for i in range(8)]
if wandb_run.name:
filename = wandb_run.name
else:
filename = "checkpoint_" + datetime.date.today().strftime("%d%m%Y")
save_path = os.path.join(cfg.save_dir, filename + "_best.pt")
train = trainer.Trainer(save_path, device, model, trainloader,
testloader, samples, **cfg.trainer)
train.train()
def query(cfg, query_index):
# device
num_gpus = 0
if cfg.DEVICE == 'cuda':
os.environ['CUDA_VISIBLE_DEVICES'] = cfg.DEVICE_ID
num_gpus = len(cfg.DEVICE_ID.split(','))
print("Using {} GPUs.\n".format(num_gpus))
device = torch.device(cfg.DEVICE)
# data
train_loader, query_loader, gallery_loader, num_classes = make_loader(cfg)
# model
model = make_model(cfg, num_classes=num_classes)
model.load_state_dict(
torch.load(
os.path.join(cfg.OUTPUT_DIR,
model.__class__.__name__ + '_best.pth')))
if num_gpus > 1:
model = nn.DataParallel(model)
queryer = Queryer(model=model,
query_loader=query_loader,
gallery_loader=gallery_loader,
device=device)
queryer.query(idx=query_index)
print('Done.')
def train(cfg):
# output
output_dir = cfg.OUTPUT_DIR
if os.path.exists(output_dir):
raise KeyError("Existing path: ", output_dir)
else:
os.makedirs(output_dir)
with open(os.path.join(output_dir, 'config.yaml'), 'w') as f_out:
print(cfg, file=f_out)
# logger
logger = make_logger("project", output_dir, 'log')
# device
num_gpus = 0
if cfg.DEVICE == 'cuda':
os.environ['CUDA_VISIBLE_DEVICES'] = cfg.DEVICE_ID
num_gpus = len(cfg.DEVICE_ID.split(','))
logger.info("Using {} GPUs.\n".format(num_gpus))
cudnn.benchmark = True
device = torch.device(cfg.DEVICE)
# data
train_loader, query_loader, gallery_loader, num_classes = make_loader(cfg)
# model
model = make_model(cfg, num_classes=num_classes)
if num_gpus > 1:
model = nn.DataParallel(model)
# solver
criterion = make_loss(cfg, num_classes)
optimizer = make_optimizer(cfg, model)
scheduler = make_scheduler(cfg, optimizer)
# do_train
trainer = Trainer(model=model,
optimizer=optimizer,
criterion=criterion,
logger=logger,
scheduler=scheduler,
device=device)
trainer.run(start_epoch=0,
total_epoch=cfg.SOLVER.MAX_EPOCHS,
train_loader=train_loader,
query_loader=query_loader,
gallery_loader=gallery_loader,
print_freq=cfg.SOLVER.PRINT_FREQ,
eval_period=cfg.SOLVER.EVAL_PERIOD,
out_dir=output_dir)
print('Done.')
def test(cfg, flip):
# device
num_gpus = 0
if cfg.DEVICE == 'cuda':
os.environ['CUDA_VISIBLE_DEVICES'] = cfg.DEVICE_ID
num_gpus = len(cfg.DEVICE_ID.split(','))
print("Using {} GPUs.\n".format(num_gpus))
cudnn.benchmark = True
device = torch.device(cfg.DEVICE)
# data
train_loader, query_loader, gallery_loader, num_classes = make_loader(cfg)
# model
model = make_model(cfg, num_classes=num_classes)
model.load_state_dict(
torch.load(
os.path.join(cfg.OUTPUT_DIR,
model.__class__.__name__ + '_best.pth')))
if num_gpus > 1:
model = nn.DataParallel(model)
model = model.to(device)
evaluator = Evaluator(model)
# Results
cmc, mAP = evaluator.evaluate(query_loader, gallery_loader)
ranks = [1, 5, 10]
print("Results ----------")
print("mAP: {:.1%}".format(mAP))
print("CMC curve")
for r in ranks:
print("Rank-{:<3}: {:.1%}".format(r, cmc[r - 1]))
print("------------------\n")
# Results with flip
if flip:
print("Results with flip --------------")
query_flip_loader, gallery_flip_loader = make_loader_flip(cfg)
cmc, mAP = evaluator.evaluate_flip(query_loader, gallery_loader,
query_flip_loader,
gallery_flip_loader)
print("Results ----------")
print("mAP: {:.1%}".format(mAP))
print("CMC curve")
for r in ranks:
print("Rank-{:<3}: {:.1%}".format(r, cmc[r - 1]))
print("------------------\n")
print('Done')