def reconstruction_eval(adj, opt, epoch, elapsed, loss, pth, best): chkpnt = th.load(pth, map_location='cpu') model = build_model(opt, chkpnt['embeddings'].size(0)) model.load_state_dict(chkpnt['model']) meanrank, maprank = eval_reconstruction(adj, model) sqnorms = manifold_file_norm(model.lt) filename = 'eval_log_nouns.csv' if os.path.exists(filename): append_write = 'a' # append if already exists else: # write header with open(filename, 'w') as file: writer = csv.writer(file) writer.writerow(["epoch", "elapsed", "loss", "sqnorm_min", "sqnorm_avg", "sqnorm_max", "mean_rank", "map_rank", "best"]) append_write = 'a' # make a new file if not with open(filename, append_write) as file: writer = csv.writer(file) writer.writerow([epoch, elapsed, loss, sqnorms.min().item(), sqnorms.mean().item(), sqnorms.max().item(), meanrank, maprank, bool(best is None or loss < best['loss'])]) return { 'epoch': epoch, 'elapsed': elapsed, 'loss': loss, 'sqnorm_min': sqnorms.min().item(), 'sqnorm_avg': sqnorms.mean().item(), 'sqnorm_max': sqnorms.max().item(), 'mean_rank': meanrank, 'map_rank': maprank, 'best': bool(best is None or loss < best['loss']), }
def async_eval(adj, q, logQ, opt): manifold = MANIFOLDS[opt.manifold]() while True: temp = q.get() if temp is None: return if not q.empty(): continue epoch, elapsed, loss, pth = temp chkpnt = th.load(pth, map_location='cpu') lt = chkpnt['embeddings'] meanrank, maprank = eval_reconstruction(adj, lt, manifold.distance) sqnorms = manifold.pnorm(lt) lmsg = { 'epoch': epoch, 'elapsed': elapsed, 'loss': loss, 'sqnorm_min': sqnorms.min().item(), 'sqnorm_avg': sqnorms.mean().item(), 'sqnorm_max': sqnorms.max().item(), 'mean_rank': meanrank, 'map_rank': maprank } logQ.put((lmsg, pth))
def reconstruction_eval(adj, opt, epoch, elapsed, loss, pth, best): chkpnt = th.load(pth, map_location='cpu') model = build_model(opt, chkpnt['embeddings'].size(0)) model.load_state_dict(chkpnt['model']) meanrank, maprank = eval_reconstruction(adj, model) sqnorms = model.manifold.norm(model.lt) return { 'epoch': epoch, 'elapsed': elapsed, 'loss': loss, 'sqnorm_min': sqnorms.min().item(), 'sqnorm_avg': sqnorms.mean().item(), 'sqnorm_max': sqnorms.max().item(), 'mean_rank': meanrank, 'map_rank': maprank, 'best': bool(best is None or loss < best['loss']), }
def calc_rank_map(fpath='', adj=None): chkpnt = torch.load(fpath) manifold = MANIFOLDS[chkpnt['conf']['manifold']]() lt = chkpnt['embeddings'] if not isinstance(lt, torch.Tensor): lt = torch.from_numpy(lt).cuda() tstart = timeit.default_timer() meanrank, maprank = eval_reconstruction(adj, lt, manifold.distance, workers=2, progress=False) etime = timeit.default_timer() - tstart print(f'Mean rank: {meanrank}, mAP rank: {maprank}, time: {etime}') return (chkpnt, manifold, lt, meanrank, maprank, etime)
def main(): parser = argparse.ArgumentParser(description='Train Hyperbolic Embeddings') parser.add_argument('-dset', type=str, required=True, help='Dataset identifier') parser.add_argument('-dim', type=int, default=20, help='Embedding dimension') parser.add_argument('-com_n', type=int, default=2, help='Embedding components number') parser.add_argument('-manifold', type=str, default='lorentz', choices=MANIFOLDS.keys(), help='Embedding manifold') parser.add_argument('-lr', type=float, default=1000, help='Learning rate') parser.add_argument('-epochs', type=int, default=100, help='Number of epochs') parser.add_argument('-batchsize', type=int, default=12800, help='Batchsize') parser.add_argument('-negs', type=int, default=50, help='Number of negatives') parser.add_argument('-burnin', type=int, default=20, help='Epochs of burn in') parser.add_argument('-dampening', type=float, default=0.75, help='Sample dampening during burnin') parser.add_argument('-ndproc', type=int, default=8, help='Number of data loading processes') parser.add_argument('-eval_each', type=int, default=1, help='Run evaluation every n-th epoch') parser.add_argument('-debug', action='store_true', default=False, help='Print debuggin output') parser.add_argument('-gpu', default=-1, type=int, help='Which GPU to run on (-1 for no gpu)') parser.add_argument('-sym', action='store_true', default=False, help='Symmetrize dataset') parser.add_argument('-maxnorm', '-no-maxnorm', default='500000', action=Unsettable, type=int) parser.add_argument('-sparse', default=False, action='store_true', help='Use sparse gradients for embedding table') parser.add_argument('-burnin_multiplier', default=0.01, type=float) parser.add_argument('-neg_multiplier', default=1.0, type=float) parser.add_argument('-quiet', action='store_true', default=True) parser.add_argument('-lr_type', choices=['scale', 'constant'], default='constant') parser.add_argument('-train_threads', type=int, default=1, help='Number of threads to use in training') parser.add_argument('-eval_embedding', default=False, help='path for the embedding to be evaluated') opt = parser.parse_args() if 'LTiling' in opt.manifold: opt.nor = 'LTiling' opt.norevery = 20 opt.stre = 50 elif 'HTiling' in opt.manifold: opt.nor = 'HTiling' opt.norevery = 1 opt.stre = 0 else: opt.nor = 'none' # setup debugging and logigng log_level = logging.DEBUG if opt.debug else logging.INFO log = logging.getLogger('tiling model') logging.basicConfig(level=log_level, format='%(message)s', stream=sys.stdout) # set default tensor type th.set_default_tensor_type('torch.DoubleTensor')####FloatTensor DoubleTensor # set device # device = th.device(f'cuda:{opt.gpu}' if opt.gpu >= 0 else 'cpu') device = th.device('cpu') # select manifold to optimize on manifold = MANIFOLDS[opt.manifold](debug=opt.debug, max_norm=opt.maxnorm, com_n=opt.com_n) if 'Halfspace' not in opt.manifold: opt.dim = manifold.dim(opt.dim) if 'csv' in opt.dset: log.info('Using edge list dataloader') idx, objects, weights = load_edge_list(opt.dset, opt.sym) model, data, model_name, conf = initialize( manifold, opt, idx, objects, weights, sparse=opt.sparse ) else: log.info('Using adjacency matrix dataloader') dset = load_adjacency_matrix(opt.dset, 'hdf5') log.info('Setting up dataset...') data = AdjacencyDataset(dset, opt.negs, opt.batchsize, opt.ndproc, opt.burnin > 0, sample_dampening=opt.dampening) model = Embedding(data.N, opt.dim, manifold, sparse=opt.sparse, com_n=opt.com_n) objects = dset['objects'] print('the total dimension', model.lt.weight.data.size(-1), 'com_n', opt.com_n) # set burnin parameters data.neg_multiplier = opt.neg_multiplier train._lr_multiplier = opt.burnin_multiplier # Build config string for log log.info(f'json_conf: {json.dumps(vars(opt))}') if opt.lr_type == 'scale': opt.lr = opt.lr * opt.batchsize # setup optimizer optimizer = RiemannianSGD(model.optim_params(manifold), lr=opt.lr) opt.epoch_start = 0 adj = {} for inputs, _ in data: for row in inputs: x = row[0].item() y = row[1].item() if x in adj: adj[x].add(y) else: adj[x] = {y} if not opt.eval_embedding: opt.adj = adj model = model.to(device) if hasattr(model, 'w_avg'): model.w_avg = model.w_avg.to(device) if opt.train_threads > 1: threads = [] model = model.share_memory() if 'LTiling' in opt.manifold: model.int_matrix.share_memory_() kwargs = {'progress' : not opt.quiet} for i in range(opt.train_threads): args = (i, device, model, data, optimizer, opt, log) threads.append(mp.Process(target=train.train, args=args, kwargs=kwargs)) threads[-1].start() [t.join() for t in threads] else: train.train(device, model, data, optimizer, opt, log, progress=not opt.quiet) else: model = th.load(opt.eval_embedding, map_location='cpu')['embeddings'] if 'LTiling' in opt.manifold: meanrank, maprank = eval_reconstruction(adj, model.lt.weight.data.clone(), manifold.distance, lt_int_matrix = model.int_matrix.data.clone(), workers = opt.ndproc) sqnorms = manifold.pnorm(model.lt.weight.data.clone(), model.int_matrix.data.clone()) else: meanrank, maprank = eval_reconstruction(adj, model.lt.weight.data.clone(), manifold.distance, workers = opt.ndproc) sqnorms = manifold.pnorm(model.lt.weight.data.clone()) log.info( 'json_stats final test: {' f'"sqnorm_min": {sqnorms.min().item()}, ' f'"sqnorm_avg": {sqnorms.mean().item()}, ' f'"sqnorm_max": {sqnorms.max().item()}, ' f'"mean_rank": {meanrank}, ' f'"map": {maprank}, ' '}' ) print(model.lt.weight.data[0])
raise ValueError("Can't find dset!") format = 'hdf5' if dset.endswith('.h5') else 'csv' dset = load_adjacency_matrix(dset, format) sample_size = args.sample or len(dset['ids']) sample = np.random.choice(len(dset['ids']), size=sample_size, replace=False) adj = {} for i in sample: end = dset['offsets'][i + 1] if i + 1 < len(dset['offsets']) \ else len(dset['neighbors']) adj[i] = set(dset['neighbors'][dset['offsets'][i]:end]) manifold = MANIFOLDS[chkpnt['conf']['manifold']]() lt = chkpnt['embeddings'] if not isinstance(lt, torch.Tensor): lt = torch.from_numpy(lt).cuda() tstart = timeit.default_timer() meanrank, maprank = eval_reconstruction(adj, lt, manifold.distance, workers=args.workers, progress=not args.quiet) etime = timeit.default_timer() - tstart print(f'Mean rank: {meanrank}, mAP rank: {maprank}, time: {etime}')
def train(device, model, data, optimizer, opt, log, progress=False): if isinstance(data, torch_data.Dataset): loader = torch_data.DataLoader(data, batch_size=opt.batchsize, shuffle=False, num_workers=opt.ndproc) else: loader = data epoch_loss = th.Tensor(len(loader)) counts = th.zeros(model.nobjects, 1).to(device) LOSS = np.zeros(opt.epochs) for epoch in range(opt.epoch_start, opt.epochs): print(th.abs(model.lt.weight.data).max().item()) #if 'bugaenko6' in opt.manifold or 'group' in opt.manifold or 'vinberg17' in opt.manifold: # print(model.int_matrix.max().item(), model.int_matrix.min().item()) epoch_loss.fill_(0) data.burnin = False t_start = timeit.default_timer() lr = opt.lr if epoch < opt.burnin: data.burnin = True lr = opt.lr * _lr_multiplier loader_iter = tqdm(loader) if progress else loader for i_batch, (inputs, targets) in enumerate(loader_iter): elapsed = timeit.default_timer() - t_start inputs = inputs.to(device) targets = targets.to(device) # count occurrences of objects in batch if hasattr(opt, 'asgd') and opt.asgd: counts = th.bincount(inputs.view(-1), minlength=model.nobjects) counts.clamp_(min=1).div_(inputs.size(0)) counts = counts.double().unsqueeze(-1) optimizer.zero_grad() preds = model(inputs) #print(preds) loss = model.loss(preds, targets, size_average=True) loss.backward() optimizer.step(lr=lr, counts=counts) epoch_loss[i_batch] = loss.cpu().item() LOSS[epoch] = th.mean(epoch_loss).item() log.info('json_stats: {' f'"epoch": {epoch}, ' f'"elapsed": {elapsed}, ' f'"loss": {LOSS[epoch]}, ' '}') if opt.nor != 'none' and epoch > opt.stre and ( epoch - opt.stre) % opt.norevery == 0: if opt.nor == 'group': NMD, NMD_int_matrix = normalize_gmatrix( model.lt.weight.data.cpu().clone(), model.int_matrix.data.cpu().clone()) model.int_matrix.data.copy_(NMD_int_matrix) model.lt.weight.data.copy_(NMD) elif opt.nor == 'bugaenko6': NMD, NMD_int_matrix = normalize_bugaenko6_gmatrix( model.lt.weight.data.cpu().clone(), model.int_matrix.data.cpu().clone()) model.int_matrix.data.copy_(NMD_int_matrix) model.lt.weight.data.copy_(NMD) elif opt.nor == 'vinberg17': print('normalizuje!') NMD, NMD_int_matrix = normalize_vinberg17_gmatrix( model.lt.weight.data.cpu().clone(), model.int_matrix.data.cpu().clone()) model.int_matrix.data.copy_(NMD_int_matrix) model.lt.weight.data.copy_(NMD) elif opt.nor == 'halfspace': NMD = normalize_halfspace_matrix(model.lt.weight.data.clone()) model.lt.weight.data.copy_(NMD) if (epoch + 1) % opt.eval_each == 0: manifold = MANIFOLDS[opt.manifold](debug=opt.debug, max_norm=opt.maxnorm) if 'group' in opt.manifold: meanrank, maprank = eval_reconstruction( opt.adj, model.lt.weight.data.clone(), manifold.distance, lt_int_matrix=model.int_matrix.data.clone()) sqnorms = manifold.pnorm(model.lt.weight.data.clone(), model.int_matrix.data.clone()) elif 'bugaenko6' in opt.manifold or 'vinberg17' in opt.manifold: meanrank, maprank = eval_reconstruction( opt.adj, model.lt.weight.data.clone(), manifold.distance, g=model.g, lt_int_matrix=model.int_matrix.data.clone()) sqnorms = manifold.pnorm(model.lt.weight.data.clone(), model.int_matrix.data.clone()) imax = th.argmax(sqnorms).item() imin = th.argmin(sqnorms).item() else: meanrank, maprank = eval_reconstruction( opt.adj, model.lt.weight.data.clone(), manifold.distance) sqnorms = manifold.pnorm(model.lt.weight.data.clone()) print('max matrix:\n{}\n,min matrix:\n{}'.format( model.int_matrix[imax], model.int_matrix[imin])) print('max: {}, min: {}'.format(model.int_matrix.max().item(), model.int_matrix.min().item())) log.info('json_stats: {' f'"sqnorm_min": {sqnorms.min().item()}, ' f'"sqnorm_avg": {sqnorms.mean().item()}, ' f'"sqnorm_max": {sqnorms.max().item()}, ' f'"mean_rank": {meanrank}, ' f'"map_rank": {maprank}, ' '}') print(LOSS)
raise ValueError("Can't find dset!") adj = {} obj_embedding = chkpnt['objects'] translate = {el:i for i,el in enumerate(obj_embedding)} idx, obj, weights = load_edge_list(dset, False) for row in idx: if(obj[row[0]] in translate and obj[row[1]] in translate): x = translate[obj[row[0]]] y = translate[obj[row[1]]] if x in adj: adj[x].add(y) else: adj[x] = {y} manifold = MANIFOLDS[chkpnt['conf']['manifold']]() lt = chkpnt['embeddings'] if not isinstance(lt, torch.Tensor): lt = torch.from_numpy(lt).cuda() tstart = timeit.default_timer() meanrank, maprank = eval_reconstruction(adj, lt, manifold.distance) etime = timeit.default_timer() - tstart print(f'Mean rank: {meanrank}, mAP rank: {maprank}, time: {etime}')
sample_size = args.sample or len(dset['ids']) sample = np.random.choice(len(dset['ids']), size=sample_size, replace=False) adj = {} for i in sample: end = dset['offsets'][i + 1] if i + 1 < len(dset['offsets']) \ else len(dset['neighbors']) adj[dset['ids'][i]] = set(dset['neighbors'][dset['offsets'][i]:end]) manifold = MANIFOLDS[chkpnt['conf']['manifold']]() manifold = MANIFOLDS[chkpnt['conf']['manifold']]() model = MODELS[chkpnt['conf']['model']](manifold, dim=chkpnt['conf']['dim'], size=chkpnt['embeddings'].size(0), sparse=chkpnt['conf']['sparse']) model.load_state_dict(chkpnt['model']) lt = chkpnt['embeddings'] if not isinstance(lt, torch.Tensor): lt = torch.from_numpy(lt).cuda() tstart = timeit.default_timer() meanrank, maprank = eval_reconstruction(adj, model, workers=args.workers, progress=not args.quiet) etime = timeit.default_timer() - tstart print(f'Mean rank: {meanrank}, mAP rank: {maprank}, time: {etime}')