def reconstruction_eval(adj, opt, epoch, elapsed, loss, pth, best):
chkpnt = th.load(pth, map_location='cpu')
model = build_model(opt, chkpnt['embeddings'].size(0))
model.load_state_dict(chkpnt['model'])
meanrank, maprank = eval_reconstruction(adj, model)
sqnorms = manifold_file_norm(model.lt)
filename = 'eval_log_nouns.csv'
if os.path.exists(filename):
append_write = 'a' # append if already exists
else:
# write header
with open(filename, 'w') as file:
writer = csv.writer(file)
writer.writerow(["epoch", "elapsed", "loss", "sqnorm_min", "sqnorm_avg", "sqnorm_max", "mean_rank", "map_rank", "best"])
append_write = 'a' # make a new file if not
with open(filename, append_write) as file:
writer = csv.writer(file)
writer.writerow([epoch, elapsed, loss, sqnorms.min().item(), sqnorms.mean().item(), sqnorms.max().item(), meanrank, maprank, bool(best is None or loss < best['loss'])])
return {
'epoch': epoch,
'elapsed': elapsed,
'loss': loss,
'sqnorm_min': sqnorms.min().item(),
'sqnorm_avg': sqnorms.mean().item(),
'sqnorm_max': sqnorms.max().item(),
'mean_rank': meanrank,
'map_rank': maprank,
'best': bool(best is None or loss < best['loss']),
}
def async_eval(adj, q, logQ, opt):
manifold = MANIFOLDS[opt.manifold]()
while True:
temp = q.get()
if temp is None:
return
if not q.empty():
continue
epoch, elapsed, loss, pth = temp
chkpnt = th.load(pth, map_location='cpu')
lt = chkpnt['embeddings']
meanrank, maprank = eval_reconstruction(adj, lt, manifold.distance)
sqnorms = manifold.pnorm(lt)
lmsg = {
'epoch': epoch,
'elapsed': elapsed,
'loss': loss,
'sqnorm_min': sqnorms.min().item(),
'sqnorm_avg': sqnorms.mean().item(),
'sqnorm_max': sqnorms.max().item(),
'mean_rank': meanrank,
'map_rank': maprank
}
logQ.put((lmsg, pth))
def reconstruction_eval(adj, opt, epoch, elapsed, loss, pth, best):
chkpnt = th.load(pth, map_location='cpu')
model = build_model(opt, chkpnt['embeddings'].size(0))
model.load_state_dict(chkpnt['model'])
meanrank, maprank = eval_reconstruction(adj, model)
sqnorms = model.manifold.norm(model.lt)
return {
'epoch': epoch,
'elapsed': elapsed,
'loss': loss,
'sqnorm_min': sqnorms.min().item(),
'sqnorm_avg': sqnorms.mean().item(),
'sqnorm_max': sqnorms.max().item(),
'mean_rank': meanrank,
'map_rank': maprank,
'best': bool(best is None or loss < best['loss']),
}
def calc_rank_map(fpath='', adj=None):
chkpnt = torch.load(fpath)
manifold = MANIFOLDS[chkpnt['conf']['manifold']]()
lt = chkpnt['embeddings']
if not isinstance(lt, torch.Tensor):
lt = torch.from_numpy(lt).cuda()
tstart = timeit.default_timer()
meanrank, maprank = eval_reconstruction(adj,
lt,
manifold.distance,
workers=2,
progress=False)
etime = timeit.default_timer() - tstart
print(f'Mean rank: {meanrank}, mAP rank: {maprank}, time: {etime}')
return (chkpnt, manifold, lt, meanrank, maprank, etime)
def main():
parser = argparse.ArgumentParser(description='Train Hyperbolic Embeddings')
parser.add_argument('-dset', type=str, required=True,
help='Dataset identifier')
parser.add_argument('-dim', type=int, default=20,
help='Embedding dimension')
parser.add_argument('-com_n', type=int, default=2,
help='Embedding components number')
parser.add_argument('-manifold', type=str, default='lorentz',
choices=MANIFOLDS.keys(), help='Embedding manifold')
parser.add_argument('-lr', type=float, default=1000,
help='Learning rate')
parser.add_argument('-epochs', type=int, default=100,
help='Number of epochs')
parser.add_argument('-batchsize', type=int, default=12800,
help='Batchsize')
parser.add_argument('-negs', type=int, default=50,
help='Number of negatives')
parser.add_argument('-burnin', type=int, default=20,
help='Epochs of burn in')
parser.add_argument('-dampening', type=float, default=0.75,
help='Sample dampening during burnin')
parser.add_argument('-ndproc', type=int, default=8,
help='Number of data loading processes')
parser.add_argument('-eval_each', type=int, default=1,
help='Run evaluation every n-th epoch')
parser.add_argument('-debug', action='store_true', default=False,
help='Print debuggin output')
parser.add_argument('-gpu', default=-1, type=int,
help='Which GPU to run on (-1 for no gpu)')
parser.add_argument('-sym', action='store_true', default=False,
help='Symmetrize dataset')
parser.add_argument('-maxnorm', '-no-maxnorm', default='500000',
action=Unsettable, type=int)
parser.add_argument('-sparse', default=False, action='store_true',
help='Use sparse gradients for embedding table')
parser.add_argument('-burnin_multiplier', default=0.01, type=float)
parser.add_argument('-neg_multiplier', default=1.0, type=float)
parser.add_argument('-quiet', action='store_true', default=True)
parser.add_argument('-lr_type', choices=['scale', 'constant'], default='constant')
parser.add_argument('-train_threads', type=int, default=1,
help='Number of threads to use in training')
parser.add_argument('-eval_embedding', default=False, help='path for the embedding to be evaluated')
opt = parser.parse_args()
if 'LTiling' in opt.manifold:
opt.nor = 'LTiling'
opt.norevery = 20
opt.stre = 50
elif 'HTiling' in opt.manifold:
opt.nor = 'HTiling'
opt.norevery = 1
opt.stre = 0
else:
opt.nor = 'none'
# setup debugging and logigng
log_level = logging.DEBUG if opt.debug else logging.INFO
log = logging.getLogger('tiling model')
logging.basicConfig(level=log_level, format='%(message)s', stream=sys.stdout)
# set default tensor type
th.set_default_tensor_type('torch.DoubleTensor')####FloatTensor DoubleTensor
# set device
# device = th.device(f'cuda:{opt.gpu}' if opt.gpu >= 0 else 'cpu')
device = th.device('cpu')
# select manifold to optimize on
manifold = MANIFOLDS[opt.manifold](debug=opt.debug, max_norm=opt.maxnorm, com_n=opt.com_n)
if 'Halfspace' not in opt.manifold:
opt.dim = manifold.dim(opt.dim)
if 'csv' in opt.dset:
log.info('Using edge list dataloader')
idx, objects, weights = load_edge_list(opt.dset, opt.sym)
model, data, model_name, conf = initialize(
manifold, opt, idx, objects, weights, sparse=opt.sparse
)
else:
log.info('Using adjacency matrix dataloader')
dset = load_adjacency_matrix(opt.dset, 'hdf5')
log.info('Setting up dataset...')
data = AdjacencyDataset(dset, opt.negs, opt.batchsize, opt.ndproc,
opt.burnin > 0, sample_dampening=opt.dampening)
model = Embedding(data.N, opt.dim, manifold, sparse=opt.sparse, com_n=opt.com_n)
objects = dset['objects']
print('the total dimension', model.lt.weight.data.size(-1), 'com_n', opt.com_n)
# set burnin parameters
data.neg_multiplier = opt.neg_multiplier
train._lr_multiplier = opt.burnin_multiplier
# Build config string for log
log.info(f'json_conf: {json.dumps(vars(opt))}')
if opt.lr_type == 'scale':
opt.lr = opt.lr * opt.batchsize
# setup optimizer
optimizer = RiemannianSGD(model.optim_params(manifold), lr=opt.lr)
opt.epoch_start = 0
adj = {}
for inputs, _ in data:
for row in inputs:
x = row[0].item()
y = row[1].item()
if x in adj:
adj[x].add(y)
else:
adj[x] = {y}
if not opt.eval_embedding:
opt.adj = adj
model = model.to(device)
if hasattr(model, 'w_avg'):
model.w_avg = model.w_avg.to(device)
if opt.train_threads > 1:
threads = []
model = model.share_memory()
if 'LTiling' in opt.manifold:
model.int_matrix.share_memory_()
kwargs = {'progress' : not opt.quiet}
for i in range(opt.train_threads):
args = (i, device, model, data, optimizer, opt, log)
threads.append(mp.Process(target=train.train, args=args, kwargs=kwargs))
threads[-1].start()
[t.join() for t in threads]
else:
train.train(device, model, data, optimizer, opt, log, progress=not opt.quiet)
else:
model = th.load(opt.eval_embedding, map_location='cpu')['embeddings']
if 'LTiling' in opt.manifold:
meanrank, maprank = eval_reconstruction(adj, model.lt.weight.data.clone(), manifold.distance, lt_int_matrix = model.int_matrix.data.clone(), workers = opt.ndproc)
sqnorms = manifold.pnorm(model.lt.weight.data.clone(), model.int_matrix.data.clone())
else:
meanrank, maprank = eval_reconstruction(adj, model.lt.weight.data.clone(), manifold.distance, workers = opt.ndproc)
sqnorms = manifold.pnorm(model.lt.weight.data.clone())
log.info(
'json_stats final test: {'
f'"sqnorm_min": {sqnorms.min().item()}, '
f'"sqnorm_avg": {sqnorms.mean().item()}, '
f'"sqnorm_max": {sqnorms.max().item()}, '
f'"mean_rank": {meanrank}, '
f'"map": {maprank}, '
'}'
)
print(model.lt.weight.data[0])
raise ValueError("Can't find dset!")
format = 'hdf5' if dset.endswith('.h5') else 'csv'
dset = load_adjacency_matrix(dset, format)
sample_size = args.sample or len(dset['ids'])
sample = np.random.choice(len(dset['ids']), size=sample_size, replace=False)
adj = {}
for i in sample:
end = dset['offsets'][i + 1] if i + 1 < len(dset['offsets']) \
else len(dset['neighbors'])
adj[i] = set(dset['neighbors'][dset['offsets'][i]:end])
manifold = MANIFOLDS[chkpnt['conf']['manifold']]()
lt = chkpnt['embeddings']
if not isinstance(lt, torch.Tensor):
lt = torch.from_numpy(lt).cuda()
tstart = timeit.default_timer()
meanrank, maprank = eval_reconstruction(adj,
lt,
manifold.distance,
workers=args.workers,
progress=not args.quiet)
etime = timeit.default_timer() - tstart
print(f'Mean rank: {meanrank}, mAP rank: {maprank}, time: {etime}')
def train(device, model, data, optimizer, opt, log, progress=False):
if isinstance(data, torch_data.Dataset):
loader = torch_data.DataLoader(data,
batch_size=opt.batchsize,
shuffle=False,
num_workers=opt.ndproc)
else:
loader = data
epoch_loss = th.Tensor(len(loader))
counts = th.zeros(model.nobjects, 1).to(device)
LOSS = np.zeros(opt.epochs)
for epoch in range(opt.epoch_start, opt.epochs):
print(th.abs(model.lt.weight.data).max().item())
#if 'bugaenko6' in opt.manifold or 'group' in opt.manifold or 'vinberg17' in opt.manifold:
# print(model.int_matrix.max().item(), model.int_matrix.min().item())
epoch_loss.fill_(0)
data.burnin = False
t_start = timeit.default_timer()
lr = opt.lr
if epoch < opt.burnin:
data.burnin = True
lr = opt.lr * _lr_multiplier
loader_iter = tqdm(loader) if progress else loader
for i_batch, (inputs, targets) in enumerate(loader_iter):
elapsed = timeit.default_timer() - t_start
inputs = inputs.to(device)
targets = targets.to(device)
# count occurrences of objects in batch
if hasattr(opt, 'asgd') and opt.asgd:
counts = th.bincount(inputs.view(-1), minlength=model.nobjects)
counts.clamp_(min=1).div_(inputs.size(0))
counts = counts.double().unsqueeze(-1)
optimizer.zero_grad()
preds = model(inputs)
#print(preds)
loss = model.loss(preds, targets, size_average=True)
loss.backward()
optimizer.step(lr=lr, counts=counts)
epoch_loss[i_batch] = loss.cpu().item()
LOSS[epoch] = th.mean(epoch_loss).item()
log.info('json_stats: {'
f'"epoch": {epoch}, '
f'"elapsed": {elapsed}, '
f'"loss": {LOSS[epoch]}, '
'}')
if opt.nor != 'none' and epoch > opt.stre and (
epoch - opt.stre) % opt.norevery == 0:
if opt.nor == 'group':
NMD, NMD_int_matrix = normalize_gmatrix(
model.lt.weight.data.cpu().clone(),
model.int_matrix.data.cpu().clone())
model.int_matrix.data.copy_(NMD_int_matrix)
model.lt.weight.data.copy_(NMD)
elif opt.nor == 'bugaenko6':
NMD, NMD_int_matrix = normalize_bugaenko6_gmatrix(
model.lt.weight.data.cpu().clone(),
model.int_matrix.data.cpu().clone())
model.int_matrix.data.copy_(NMD_int_matrix)
model.lt.weight.data.copy_(NMD)
elif opt.nor == 'vinberg17':
print('normalizuje!')
NMD, NMD_int_matrix = normalize_vinberg17_gmatrix(
model.lt.weight.data.cpu().clone(),
model.int_matrix.data.cpu().clone())
model.int_matrix.data.copy_(NMD_int_matrix)
model.lt.weight.data.copy_(NMD)
elif opt.nor == 'halfspace':
NMD = normalize_halfspace_matrix(model.lt.weight.data.clone())
model.lt.weight.data.copy_(NMD)
if (epoch + 1) % opt.eval_each == 0:
manifold = MANIFOLDS[opt.manifold](debug=opt.debug,
max_norm=opt.maxnorm)
if 'group' in opt.manifold:
meanrank, maprank = eval_reconstruction(
opt.adj,
model.lt.weight.data.clone(),
manifold.distance,
lt_int_matrix=model.int_matrix.data.clone())
sqnorms = manifold.pnorm(model.lt.weight.data.clone(),
model.int_matrix.data.clone())
elif 'bugaenko6' in opt.manifold or 'vinberg17' in opt.manifold:
meanrank, maprank = eval_reconstruction(
opt.adj,
model.lt.weight.data.clone(),
manifold.distance,
g=model.g,
lt_int_matrix=model.int_matrix.data.clone())
sqnorms = manifold.pnorm(model.lt.weight.data.clone(),
model.int_matrix.data.clone())
imax = th.argmax(sqnorms).item()
imin = th.argmin(sqnorms).item()
else:
meanrank, maprank = eval_reconstruction(
opt.adj, model.lt.weight.data.clone(), manifold.distance)
sqnorms = manifold.pnorm(model.lt.weight.data.clone())
print('max matrix:\n{}\n,min matrix:\n{}'.format(
model.int_matrix[imax], model.int_matrix[imin]))
print('max: {}, min: {}'.format(model.int_matrix.max().item(),
model.int_matrix.min().item()))
log.info('json_stats: {'
f'"sqnorm_min": {sqnorms.min().item()}, '
f'"sqnorm_avg": {sqnorms.mean().item()}, '
f'"sqnorm_max": {sqnorms.max().item()}, '
f'"mean_rank": {meanrank}, '
f'"map_rank": {maprank}, '
'}')
print(LOSS)
raise ValueError("Can't find dset!")
adj = {}
obj_embedding = chkpnt['objects']
translate = {el:i for i,el in enumerate(obj_embedding)}
idx, obj, weights = load_edge_list(dset, False)
for row in idx:
if(obj[row[0]] in translate and obj[row[1]] in translate):
x = translate[obj[row[0]]]
y = translate[obj[row[1]]]
if x in adj:
adj[x].add(y)
else:
adj[x] = {y}
manifold = MANIFOLDS[chkpnt['conf']['manifold']]()
lt = chkpnt['embeddings']
if not isinstance(lt, torch.Tensor):
lt = torch.from_numpy(lt).cuda()
tstart = timeit.default_timer()
meanrank, maprank = eval_reconstruction(adj, lt, manifold.distance)
etime = timeit.default_timer() - tstart
print(f'Mean rank: {meanrank}, mAP rank: {maprank}, time: {etime}')
sample_size = args.sample or len(dset['ids'])
sample = np.random.choice(len(dset['ids']), size=sample_size, replace=False)
adj = {}
for i in sample:
end = dset['offsets'][i + 1] if i + 1 < len(dset['offsets']) \
else len(dset['neighbors'])
adj[dset['ids'][i]] = set(dset['neighbors'][dset['offsets'][i]:end])
manifold = MANIFOLDS[chkpnt['conf']['manifold']]()
manifold = MANIFOLDS[chkpnt['conf']['manifold']]()
model = MODELS[chkpnt['conf']['model']](manifold,
dim=chkpnt['conf']['dim'],
size=chkpnt['embeddings'].size(0),
sparse=chkpnt['conf']['sparse'])
model.load_state_dict(chkpnt['model'])
lt = chkpnt['embeddings']
if not isinstance(lt, torch.Tensor):
lt = torch.from_numpy(lt).cuda()
tstart = timeit.default_timer()
meanrank, maprank = eval_reconstruction(adj,
model,
workers=args.workers,
progress=not args.quiet)
etime = timeit.default_timer() - tstart
print(f'Mean rank: {meanrank}, mAP rank: {maprank}, time: {etime}')