def compare_semeval():
names = {
"NTN",
"TransE",
"GD",
"GD + NLM (0.5)",
"GD + NLM (0.25)",
"GD + NLM (0.75)",
"NLM",
"NTN + NLM (0.5)",
"NTN + NLM (0.25)",
"NTN + NLM (0.75)",
"TransE + NLM (0.5)",
"TransE + NLM (0.25)",
"TransE + NLM (0.75)",
}
paths = {name: DEFAULT_MODELS[name] for name in names}
models = map_dict(get_model, paths)
vocabs = map_dict(get_vocab, paths)
def semeval_score(name):
model = models[name]
vocab = vocabs[name]
rho, accuracy, scores = evaluation.semeval.run(model.embeddings, vocab)
return scores.groupby("category")[["rho", "accuracy"]].mean()
return {name: semeval_score(name) for name in names}
def component_updates(component, name=''):
update_fn = lambda lr, w: lr.get_value() / w.get_value()
d = map_dict(partial(update_fn, component.learning_rate), component.ada_weights)
if name:
return prefix_keys(d, name)
else:
return d
def component_updates(component, name=''):
update_fn = lambda lr, w: lr.get_value() / w.get_value()
d = map_dict(partial(update_fn, component.learning_rate),
component.ada_weights)
if name:
return prefix_keys(d, name)
else:
return d
def _map_image(self, obj):
keys = [
('caption', 'photo_title'),
('url', 'photo_file_url'),
('provider', lambda obj: self.provider_name),
('date',
lambda obj: datetime.strptime(obj['upload_date'], '%d %B %Y')),
]
return map_dict(keys, obj)
def __validate(self):
N = len(self.loaders["val"])
val_metrics = {}
train_metrics = {}
utils.iterate_loader(self.DEVICE,
self.loaders["val"],
self.__val_step_fn,
args=(val_metrics, ))
utils.iterate_loader(self.DEVICE,
self.loaders["train"],
self.__val_step_fn,
args=(train_metrics, ),
end=N)
val_metrics = utils.map_dict(val_metrics, lambda v: v / N)
train_metrics = utils.map_dict(train_metrics, lambda v: v / N)
return val_metrics, train_metrics
def compare_kb():
names = {
"NTN",
"TransE",
"NTN + NLM (0.5)",
"NTN + NLM (0.25)",
"NTN + NLM (0.75)",
"TransE + NLM (0.5)",
"TransE + NLM (0.25)",
"TransE + NLM (0.75)",
"NTN (NLM embeddings)",
"TransE (NLM embeddings)",
}
paths = {name: DEFAULT_MODELS[name] for name in names}
models = map_dict(get_model, paths)
vocabs = map_dict(get_vocab, paths)
rels = get_rels(iter(paths.values()).next())
def kb_accuracy(model):
return evaluation.kb_ranking.test_socher(model, rels)
return map_dict(kb_accuracy, models)
def make_comparison_function(model_directories, model_number=None):
model_paths = map_dict(models_in_folder, model_directories)
if model_number is None:
model_number = max(reduce(lambda p, q: p.intersection(q), map(lambda d: set(d.keys()), model_paths.values())))
paths = map_dict(lambda d: (d[model_number] if d else None), model_paths)
models = map_dict(load_model, paths)
vocab_containers = map_dict(get_vocab, models)
# since we turned off compile on load...
for v in vocab_containers.values():
try:
v._initialize()
except:
pass
def display_cluster(word, n=10):
for name, model in models.iteritems():
print "-------"
print name
pprint(query(model, vocab_containers[name], word, n = n))
print
return display_cluster, model_number
def proc_kb_scores(scores, rels):
x = map_dict(lambda (p, q): {rels.relations[ix]: score for (ix, score) in p.items()}, scores)
return pandas.DataFrame(x)
def clip_norm(params, thr=1., p=2.):
pnorm = norm(params, p=p)
if pnorm <= thr:
return params
return map_dict(lambda x: x / pnorm * thr, params)
def main_worker(gpu, ngpus_per_node, args):
args.gpu = gpu
cudnn.benchmark = True
if args.gpu is not None:
print("Using GPU:{} for training.".format(args.gpu))
if args.distributed:
args.rank = args.rank * ngpus_per_node + gpu
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size,
rank=args.rank)
is_main = not args.distributed or args.rank % ngpus_per_node == 0
if is_main: print("[!] Building model ... ", end=' ', flush=True)
networks = dict(D=Discriminator(img_size=64, sn=True),
G=Generator(latent_size=args.latent_size))
if is_main:
networks_on_main = dict(G_running=Generator(
latent_size=args.latent_size).train(False))
if args.distributed:
args.batch_size = args.gpu_batch_size
args.workers = args.workers // ngpus_per_node
torch.cuda.set_device(args.gpu)
networks = map_dict(
lambda x: torch.nn.parallel.DistributedDataParallel(
x.cuda(args.gpu), device_ids=[args.gpu]), networks)
if is_main:
networks_on_main = map_dict(lambda x: x.cuda(args.gpu),
networks_on_main)
networks.update(networks_on_main)
elif args.gpu is not None:
torch.cuda.set_device(args.gpu)
networks = map_dict(lambda x: x.cuda(args.gpu), networks)
if is_main:
networks_on_main = map_dict(lambda x: x.cuda(args.gpu),
networks_on_main)
networks.update(networks_on_main)
else:
networks = map_dict(lambda x: x.cuda(), networks)
if is_main:
networks.update(networks_on_main)
if is_main: accumulate(networks['G_running'], networks['G'].module, 0)
if is_main: print("Done !")
if is_main: print("[!] Building optimizer ... ", end=' ', flush=True)
d_opt = torch.optim.Adam(networks['D'].parameters(),
args.lr,
betas=(0.5, 0.999),
weight_decay=args.weight_decay)
g_opt = torch.optim.Adam(networks['G'].parameters(),
args.lr,
betas=(0.5, 0.999),
weight_decay=args.weight_decay)
optimizers = dict(D=d_opt, G=g_opt)
if is_main: print("Done !")
if args.load_model is not None:
if is_main: print("[!] Restoring model ... ")
with open(os.path.join(args.load_model, "ckpt", "checkpoint.txt"),
"r") as f:
to_restore = f.readlines()[-1].strip()
load_file = os.path.join(args.load_model, "ckpt", to_restore)
if os.path.isfile(load_file):
if is_main:
print(" => loading checkpoint '{}'".format(load_file))
checkpoint = torch.load(load_file, map_location='cpu')
args.start_epoch = checkpoint['epoch']
networks['D'].load_state_dict(
checkpoint['discriminator_state_dict'])
networks['G'].load_state_dict(
checkpoint['generator_state_dict'])
networks['G_running'].load_state_dict(
checkpoint['generator_running_state_dict'])
optimizers['G'].load_state_dict(checkpoint['d_opt'])
optimizers['D'].load_state_dict(checkpoint['g_opt'])
if is_main:
print(" => loaded checkpoint '{}' (epoch {})".format(
load_file, checkpoint['epoch']))
else:
if is_main:
print(" => no checkpoint found at '{}'".format(
args.load_model))
if is_main: print("[!] Loading dataset ... ")
transform = transforms.Compose([
transforms.CenterCrop(128),
transforms.Resize(64),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
train_dataset = CelebAANNO(transform, 'train', args.dataset_dir)
val_dataset = CelebAANNO(transform, 'val', args.dataset_dir)
if args.distributed:
train_sampler = torch.utils.data.distributed.DistributedSampler(
train_dataset)
else:
train_sampler = None
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=(train_sampler is None),
num_workers=args.workers,
pin_memory=True,
sampler=train_sampler,
drop_last=True)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
num_workers=args.workers,
pin_memory=True,
drop_last=True)
if is_main: print("Done !")
if args.validation:
validate(val_loader, networks, 0, args)
for epoch in range(args.start_epoch, args.epochs):
if args.distributed:
train_sampler.set_epoch(epoch)
if epoch % args.val_epoch == 0 and is_main:
validate(val_loader, networks, epoch, args)
# train for one epoch
train(train_loader, networks, optimizers, epoch, args, is_main)
# Save and validate model only with main gpu
if is_main:
with open(os.path.join(args.ckpt_dir, "checkpoint.txt"),
"a+") as check_list:
save_checkpoint(
dict(
epoch=epoch + 1,
generator_state_dict=networks['G'].state_dict,
generator_running_state_dict=networks['G_running'].
state_dict,
discriminator_state_dict=networks['D'].state_dict,
g_opt=optimizers['G'],
d_opt=optimizers['D'],
), check_list, args.ckpt_dir, epoch + 1)
dump = False
indices=range(0, 1050, 20)
# names = ['NTN + NLM (0.5)', 'GD + NLM (0.5)', 'TransE + NLM (0.5)']
# names = ['GD + NLM (\\rho=0.1)', 'GD + NLM (\\rho=0.05)', 'GD + NLM (\\rho=0.005)']
names = ['GD + NLM ($\\rho=0.1$)', 'GD + NLM ($\\rho=0.05$)', 'GD + NLM ($\\rho=0.005$)']
stats_frames = {}
for pretty_name in names:
name = pretty_name.replace('$', '')
fname = name.replace(' ', '_') + '.pkl'
if dump:
models = load_models(DEFAULT_MODELS[name], indices=indices)
stats = pandas.DataFrame(map_dict(agg, models))
stats.to_pickle(fname)
else:
stats = pandas.read_pickle(fname)
stats_frames[pretty_name] = stats
def filter_(df):
cols = df.columns.tolist()
return df[df.index <= 1000][list(reversed(cols))]
if not dump:
w_df = filter_(pandas.DataFrame(map_dict(lambda d: d.transpose().w, stats_frames)))
v_df = filter_(pandas.DataFrame(map_dict(lambda d: d.transpose().v, stats_frames)))
sr_df = filter_(pandas.DataFrame(map_dict(lambda d: d.transpose().scaled_residuals, stats_frames)))
for df, title in [(w_df, "mean $||\mathbf{w}||_2$"), (v_df, "mean $||\mathbf{v}||_2$"), (sr_df, "mean $||\mathbf{w} - \mathbf{v}||_2 / (\sqrt{||\mathbf{w}||_2 ||\mathbf{v}||_2})$")]:
ax = df.plot(legend=False)
