def gen_results_for_hw(X, y): for m in [1, 20, 200, 1000, 2000]: results = [] plt.figure() plt.xlabel('Iteration (every 10th saved)') plt.ylabel('Loss') plt.title('SVRG ojective function progression for m = {}'.format(m)) for i in range(5): svrg = SVRG(M=m, calc_loss=True) svrg.fit(X, y) results.append(svrg.fit_time) plt.plot(svrg.training_loss, label='Run {}'.format(i)) print('Run {} time: {}'.format(i, round(results[-1], 3))) plt.legend(loc='upper right') plt.savefig('writeup/plots/svrg/m{}.pdf'.format(m), format='pdf') plt.close('all') print('Avg: {}s, Std: {}s'.format(round(np.mean(results), 3), round(np.std(results), 3)))
def gen_results(X, y, m, numruns, loss=False): results = [] if loss: plt.figure() plt.xlabel('Iteration (every 10th saved)') plt.ylabel('Loss') plt.title( 'Objective function progression across {} runs'.format(numruns)) for i in range(numruns): svrg = SVRG(M=m, calc_loss=loss) svrg.fit(X, y) results.append(svrg.fit_time) if loss: plt.plot(svrg.training_loss, label='Run {}'.format(i)) print('Run {} time: {}'.format(i, round(results[-1], 3))) if loss: plt.legend(loc='upper right') plt.savefig('plots/svrg/loss_vs_ite_{}.pdf'.format( strftime("%Y.%m.%d_%H.%M.%S", localtime()) + '__m_{}'.format(m), format='pdf')) plt.close('all') return results
def learn(dataset, dim=2, hyp=1, edim=1, euc=0, sdim=1, sph=0, scale=1., riemann=False, learning_rate=1e-1, decay_length=1000, decay_step=1.0, momentum=0.0, tol=1e-8, epochs=100, burn_in=0, use_yellowfin=False, use_adagrad=False, resample_freq=1000, print_freq=1, model_save_file=None, model_load_file=None, batch_size=16, num_workers=None, lazy_generation=False, log_name=None, log=False, warm_start=None, learn_scale=False, checkpoint_freq=100, sample=1., subsample=None, logloss=False, distloss=False, squareloss=False, symloss=False, exponential_rescale=None, extra_steps=1, use_svrg=False, T=10, use_hmds=False, visualize=False): # Log configuration formatter = logging.Formatter('%(asctime)s %(message)s') logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(message)s', datefmt='%FT%T', ) if log_name is None and log: log_name = f"{os.path.splitext(dataset)[0]}.H{dim}-{hyp}.E{edim}-{euc}.S{sdim}-{sph}.lr{learning_rate}.log" if log_name is not None: logging.info(f"Logging to {log_name}") log = logging.getLogger() fh = logging.FileHandler(log_name) fh.setFormatter(formatter) log.addHandler(fh) ############# loss_list = [] ############# logging.info(f"Commandline {sys.argv}") if model_save_file is None: logging.warning("No Model Save selected!") G = load_graph.load_graph(dataset) GM = nx.to_scipy_sparse_matrix(G, nodelist=list(range(G.order()))) # grab scale if warm starting: if warm_start: scale = pandas.read_csv(warm_start, index_col=0).as_matrix()[0, -1] n = G.order() logging.info(f"Loaded Graph {dataset} with {n} nodes scale={scale}") if exponential_rescale is not None: # torch.exp(exponential_rescale * -d) def weight_fn(d): if d <= 2.0: return 5.0 elif d > 4.0: return 0.01 else: return 1.0 else: def weight_fn(d): return 1.0 Z, z = build_dataset(G, lazy_generation, sample, subsample, scale, batch_size, weight_fn, num_workers) if model_load_file is not None: logging.info(f"Loading {model_load_file}...") m = torch.load(model_load_file).to(device) logging.info( f"Loaded scale {unwrap(m.scale())} {torch.sum(m.embedding().data)} {m.epoch}" ) else: logging.info(f"Creating a fresh model warm_start?={warm_start}") m_init = None if warm_start: # load from DataFrame; assume that the julia combinatorial embedding has been saved ws_data = pandas.read_csv(warm_start, index_col=0).as_matrix() scale = ws_data[0, ws_data.shape[1] - 1] m_init = torch.DoubleTensor(ws_data[:, range(ws_data.shape[1] - 1)]) elif use_hmds: # m_init = torch.DoubleTensor(mds_warmstart.get_normalized_hyperbolic(mds_warmstart.get_model(dataset,dim,scale)[1])) m_init = torch.DoubleTensor( mds_warmstart.get_model(dataset, dim, scale)[1]) logging.info( f"\t Warmstarting? {warm_start} {m_init.size() if warm_start else None} {G.order()}" ) # initial_scale = z.dataset.max_dist / 3.0 # print("MAX DISTANCE", z.dataset.max_dist) # print("AVG DISTANCE", torch.mean(z.dataset.val_cache)) initial_scale = 0.0 m = ProductEmbedding(G.order(), dim, hyp, edim, euc, sdim, sph, initialize=m_init, learn_scale=learn_scale, initial_scale=initial_scale, logrel_loss=logloss, dist_loss=distloss, square_loss=squareloss, sym_loss=symloss, exponential_rescale=exponential_rescale, riemann=riemann).to(device) m.normalize() m.epoch = 0 logging.info( f"Constructed model with dim={dim} and epochs={m.epoch} isnan={np.any(np.isnan(m.embedding().cpu().data.numpy()))}" ) if visualize: name = 'animations/' + f"{os.path.split(os.path.splitext(dataset)[0])[1]}.H{dim}-{hyp}.E{edim}-{euc}.S{sdim}-{sph}.lr{learning_rate}.ep{epochs}.seed{seed}" fig, ax, writer = vis.setup_plot(m=m, name=name, draw_circle=True) else: fig = None ax = None writer = None # # Build the Optimizer # # TODO: Redo this in a sensible way!! # per-parameter learning rates exp_params = [p for p in m.embed_params if p.use_exp] learn_params = [p for p in m.embed_params if not p.use_exp] hyp_params = [p for p in m.hyp_params if not p.use_exp] euc_params = [p for p in m.euc_params if not p.use_exp] sph_params = [p for p in m.sph_params if not p.use_exp] scale_params = m.scale_params # model_params = [{'params': m.embed_params}, {'params': m.scale_params, 'lr': 1e-4*learning_rate}] # model_params = [{'params': learn_params}, {'params': m.scale_params, 'lr': 1e-4*learning_rate}] model_params = [{ 'params': hyp_params }, { 'params': euc_params }, { 'params': sph_params, 'lr': 0.1 * learning_rate }, { 'params': m.scale_params, 'lr': 1e-4 * learning_rate }] # opt = None if len(model_params) > 0: opt = torch.optim.SGD(model_params, lr=learning_rate / 10, momentum=momentum) # opt = torch.optim.SGD(learn_params, lr=learning_rate/10, momentum=momentum) # opt = torch.optim.SGD(model_params, lr=learning_rate/10, momentum=momentum) # exp = None # if len(exp_params) > 0: # exp = torch.optim.SGD(exp_params, lr=1.0) # dummy for zeroing if len(scale_params) > 0: scale_opt = torch.optim.SGD(scale_params, lr=1e-3 * learning_rate) scale_decay = torch.optim.lr_scheduler.StepLR(scale_opt, step_size=1, gamma=.99) else: scale_opt = None scale_decay = None lr_burn_in = torch.optim.lr_scheduler.MultiStepLR(opt, milestones=[burn_in], gamma=10) # lr_decay = torch.optim.lr_scheduler.StepLR(opt, decay_length, decay_step) #TODO reconcile multiple LR schedulers if use_yellowfin: from yellowfin import YFOptimizer opt = YFOptimizer(model_params) if use_adagrad: opt = torch.optim.Adagrad(model_params) if use_svrg: from svrg import SVRG base_opt = torch.optim.Adagrad if use_adagrad else torch.optim.SGD opt = SVRG(m.parameters(), lr=learning_rate, T=T, data_loader=z, opt=base_opt) # TODO add ability for SVRG to take parameter groups logging.info(opt) # Log stats from import: when warmstarting, check that it matches Julia's stats logging.info(f"*** Initial Checkpoint. Computing Stats") major_stats(GM, n, m, lazy_generation, Z, z, fig, ax, writer, visualize, subsample) logging.info("*** End Initial Checkpoint\n") # track best stats best_loss = 1.0e10 best_dist = 1.0e10 best_wcdist = 1.0e10 best_map = 0.0 for i in range(m.epoch + 1, m.epoch + epochs + 1): lr_burn_in.step() # lr_decay.step() # scale_decay.step() # print(scale_opt.param_groups[0]['lr']) # for param_group in opt.param_groups: # print(param_group['lr']) # print(type(opt.param_groups), opt.param_groups) l, n_edges = 0.0, 0.0 # track average loss per edge m.train(True) if use_svrg: for data in z: def closure(data=data, target=None): _data = data if target is None else (data, target) c = m.loss(_data.to(device)) c.backward() return c.data[0] l += opt.step(closure) # Projection m.normalize() else: # scale_opt.zero_grad() for the_step in range(extra_steps): # Accumulate the gradient for u in z: # Zero out the gradients # if opt is not None: opt.zero_grad() # This is handled by the SVRG. # if exp is not None: exp.zero_grad() opt.zero_grad() for p in exp_params: if p.grad is not None: p.grad.detach_() p.grad.zero_() # Compute loss _loss = m.loss(cu_var(u)) _loss.backward() l += _loss.item() * u[0].size(0) # print(weight) n_edges += u[0].size(0) # modify gradients if necessary RParameter.correct_metric(m.parameters()) # step opt.step() for p in exp_params: lr = opt.param_groups[0]['lr'] p.exp(lr) # Projection m.normalize() # scale_opt.step() l /= n_edges # m.epoch refers to num of training epochs finished m.epoch += 1 # Logging code # if l < tol: # logging.info("Found a {l} solution. Done at iteration {i}!") # break if i % print_freq == 0: logging.info(f"{i} loss={l}") ############ if log_name is not None: loss_list.append(l) ############# if (i <= burn_in and i % (checkpoint_freq / 5) == 0) or i % checkpoint_freq == 0: logging.info(f"\n*** Major Checkpoint. Computing Stats and Saving") avg_dist, wc_dist, me, mc, mapscore = major_stats( GM, n, m, True, Z, z, fig, ax, writer, visualize, subsample) best_loss = min(best_loss, l) best_dist = min(best_dist, avg_dist) best_wcdist = min(best_wcdist, wc_dist) best_map = max(best_map, mapscore) if model_save_file is not None: fname = f"{model_save_file}.{m.epoch}" logging.info( f"Saving model into {fname} {torch.sum(m.embedding().data)} " ) torch.save(m, fname) logging.info("*** End Major Checkpoint\n") if i % resample_freq == 0: if sample < 1. or subsample is not None: Z, z = build_dataset(G, lazy_generation, sample, subsample, scale, batch_size, weight_fn, num_workers) logging.info(f"final loss={l}") logging.info( f"best loss={best_loss}, distortion={best_dist}, map={best_map}, wc_dist={best_wcdist}" ) final_dist, final_wc, final_me, final_mc, final_map = major_stats( GM, n, m, lazy_generation, Z, z, fig, ax, writer, False, subsample) if log_name is not None: ### with open(log_name + "_loss.stat", "w") as f: for loss in loss_list: f.write("%f\n" % loss) ### with open(log_name + '_final.stat', "w") as f: f.write("Best-loss MAP dist wc Final-loss MAP dist wc me mc\n") f.write( f"{best_loss:10.6f} {best_map:8.4f} {best_dist:8.4f} {best_wcdist:8.4f} {l:10.6f} {final_map:8.4f} {final_dist:8.4f} {final_wc:8.4f} {final_me:8.4f} {final_mc:8.4f}" ) if visualize: writer.finish() if model_save_file is not None: fname = f"{model_save_file}.final" logging.info( f"Saving model into {fname}-final {torch.sum(m.embedding().data)} {unwrap(m.scale())}" ) torch.save(m, fname)
model_AVRG_0 = copy.deepcopy(model_SGD) model_AVRG_1 = copy.deepcopy(model_SGD) model_SAGA = copy.deepcopy(model_SGD) lr = args.lr # learning rate n_epoch = args.n_epoch # the number of epochs loss_fn = nn.CrossEntropyLoss() N = len(train_loader) # The optimizer optimizer_SGD = torch.optim.SGD(model_SGD.parameters(), lr=lr, weight_decay=args.weight_decay) optimizer_SVRG_1 = SVRG(model_SVRG_1.parameters(), lr=lr, N=N) optimizer_SVRG_0 = SVRG_0(model_SVRG_0.parameters()) optimizer_AVRG_1 = AVRG(model_AVRG_1.parameters(), lr=lr, N=N) optimizer_AVRG_0 = AVRG_0(model_AVRG_0.parameters()) optimizer_SAGA = SAGA(model_SAGA.parameters(), lr=lr, N=N) best_train_SGD = 0 best_test_SGD = 0 best_train_SVRG = 0 best_test_SVRG = 0 best_train_AVRG = 0 best_test_AVRG = 0 best_train_SAGA = 0
torch.nn.Linear(H1, H2), torch.nn.ReLU(), torch.nn.Linear(H2, D_out)) ## model2 = copy.deepcopy(model) # Use same initial weights for both networks loss_fn = torch.nn.CrossEntropyLoss() alpha = 1e-2 freq = 100 # how often to recompute large gradient # The optimizer will not update model parameters on iterations # where the large batches are calculated lg_batch = 3000 # size of large gradient batch min_batch = 300 # size of mini batch optimizer = SVRG(model.parameters(), lr=alpha, freq=freq) optimizer2 = torch.optim.SGD(model2.parameters(), lr=alpha) epochs = 50 iterations = int(epochs * (60000 / min_batch)) # SVRG Training counter = 0 total = time.time() while (counter < iterations): # compute large batch gradient temp = np.random.choice(x.size()[0], lg_batch, replace=False) # calculate batch indices indices = torch.from_numpy(temp) temp2 = torch.index_select(x, 0, indices) y_pred = model(temp2)
def learn(dataset, rank=2, scale=1., learning_rate=1e-1, tol=1e-8, epochs=100, use_yellowfin=False, use_adagrad=False, print_freq=1, model_save_file=None, model_load_file=None, batch_size=16, num_workers=None, lazy_generation=False, log_name=None, warm_start=None, learn_scale=False, checkpoint_freq=1000, sample=1., subsample=None, exponential_rescale=None, extra_steps=1, use_svrg=False, T=10, use_hmds=False): # Log configuration formatter = logging.Formatter('%(asctime)s %(message)s') logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(message)s', datefmt='%FT%T', ) if log_name is not None: logging.info(f"Logging to {log_name}") log = logging.getLogger() fh = logging.FileHandler(log_name) fh.setFormatter(formatter) log.addHandler(fh) logging.info(f"Commandline {sys.argv}") if model_save_file is None: logging.warn("No Model Save selected!") G = load_graph.load_graph(dataset) GM = nx.to_scipy_sparse_matrix(G) # grab scale if warm starting: if warm_start: scale = pandas.read_csv(warm_start, index_col=0).as_matrix()[0, -1] n = G.order() logging.info(f"Loaded Graph {dataset} with {n} nodes scale={scale}") Z = None def collate(ls): x, y = zip(*ls) return torch.cat(x), torch.cat(y) if lazy_generation: if subsample is not None: z = DataLoader(GraphRowSubSampler(G, scale, subsample), batch_size, shuffle=True, collate_fn=collate) else: z = DataLoader(GraphRowSampler(G, scale), batch_size, shuffle=True, collate_fn=collate) logging.info("Built Data Sampler") else: Z = gh.build_distance(G, scale, num_workers=int(num_workers) if num_workers is not None else 16) # load the whole matrix logging.info(f"Built distance matrix with {scale} factor") if subsample is not None: z = DataLoader(GraphRowSubSampler(G, scale, subsample, Z=Z), batch_size, shuffle=True, collate_fn=collate) else: idx = torch.LongTensor([(i, j) for i in range(n) for j in range(i + 1, n)]) Z_sampled = gh.dist_sample_rebuild_pos_neg( Z, sample) if sample < 1 else Z vals = torch.DoubleTensor( [Z_sampled[i, j] for i in range(n) for j in range(i + 1, n)]) z = DataLoader(TensorDataset(idx, vals), batch_size=batch_size, shuffle=True, pin_memory=torch.cuda.is_available()) logging.info("Built data loader") if model_load_file is not None: logging.info(f"Loading {model_load_file}...") m = cudaify(torch.load(model_load_file)) logging.info( f"Loaded scale {m.scale.data[0]} {torch.sum(m.w.data)} {m.epoch}") else: logging.info(f"Creating a fresh model warm_start?={warm_start}") m_init = None if warm_start: # load from DataFrame; assume that the julia combinatorial embedding has been saved ws_data = pandas.read_csv(warm_start, index_col=0).as_matrix() scale = ws_data[0, ws_data.shape[1] - 1] m_init = torch.DoubleTensor(ws_data[:, range(ws_data.shape[1] - 1)]) elif use_hmds: # m_init = torch.DoubleTensor(mds_warmstart.get_normalized_hyperbolic(mds_warmstart.get_model(dataset,rank,scale)[1])) m_init = torch.DoubleTensor( mds_warmstart.get_model(dataset, rank, scale)[1]) logging.info( f"\t Warmstarting? {warm_start} {m_init.size() if warm_start else None} {G.order()}" ) m = cudaify( Hyperbolic_Emb(G.order(), rank, initialize=m_init, learn_scale=learn_scale, exponential_rescale=exponential_rescale)) m.normalize() m.epoch = 0 logging.info( f"Constructed model with rank={rank} and epochs={m.epoch} isnan={np.any(np.isnan(m.w.cpu().data.numpy()))}" ) # # Build the Optimizer # # TODO: Redo this in a sensible way!! # opt = torch.optim.SGD(m.parameters(), lr=learning_rate) if use_yellowfin: from yellowfin import YFOptimizer opt = YFOptimizer(m.parameters()) if use_adagrad: opt = torch.optim.Adagrad(m.parameters()) if use_svrg: from svrg import SVRG base_opt = torch.optim.Adagrad if use_adagrad else torch.optim.SGD opt = SVRG(m.parameters(), lr=learning_rate, T=T, data_loader=z, opt=base_opt) logging.info(opt) # Log stats from import: when warmstarting, check that it matches Julia's stats logging.info(f"*** Initial Checkpoint. Computing Stats") major_stats(GM, 1 + m.scale.data[0], n, m, lazy_generation, Z, z) logging.info("*** End Initial Checkpoint\n") for i in range(m.epoch, m.epoch + epochs): l = 0.0 m.train(True) if use_svrg: for data in z: def closure(data=data, target=None): _data = data if target is None else (data, target) c = m.loss(cu_var(_data)) c.backward() return c.data[0] l += opt.step(closure) # Projection m.normalize() else: opt.zero_grad() # This is handled by the SVRG. for the_step in range(extra_steps): # Accumulate the gradient for u in z: _loss = m.loss(cu_var(u, requires_grad=False)) _loss.backward() l += _loss.data[0] Hyperbolic_Parameter.correct_metric( m.parameters()) # NB: THIS IS THE NEW CALL # print("Scale before step: ", m.scale.data) opt.step() # print("Scale after step: ", m.scale.data) # Projection m.normalize() #l += step(m, opt, u).data[0] # Logging code if l < tol: logging.info("Found a {l} solution. Done at iteration {i}!") break if i % print_freq == 0: logging.info(f"{i} loss={l}") if i % checkpoint_freq == 0: logging.info(f"\n*** Major Checkpoint. Computing Stats and Saving") major_stats(GM, 1 + m.scale.data[0], n, m, True, Z, z) if model_save_file is not None: fname = f"{model_save_file}.{m.epoch}" logging.info( f"Saving model into {fname} {torch.sum(m.w.data)} ") torch.save(m, fname) logging.info("*** End Major Checkpoint\n") m.epoch += 1 logging.info(f"final loss={l}") if model_save_file is not None: fname = f"{model_save_file}.final" logging.info( f"Saving model into {fname}-final {torch.sum(m.w.data)} {m.scale.data[0]}" ) torch.save(m, fname) major_stats(GM, 1 + m.scale.data[0], n, m, lazy_generation, Z, z)