def test_measurement(zero_debias=True): dtype = torch.FloatTensor w = Variable(torch.ones(n_dim, 1).type(dtype), requires_grad=True) b = Variable(torch.ones(1).type(dtype), requires_grad=True) x = Variable(torch.ones(1, n_dim).type(dtype), requires_grad=False) opt = YFOptimizer([w, b], lr=1.0, mu=0.0, zero_debias=zero_debias) target_h_max = 0.0 target_h_min = 0.0 g_norm_squared_avg = 0.0 g_norm_avg = 0.0 g_avg = 0.0 target_dist = 0.0 for i in range(n_iter): opt.zero_grad() loss = (x.mm(w) + b).sum() loss.backward() w.grad.data = (i + 1) * torch.ones([ n_dim, ]).type(dtype) b.grad.data = (i + 1) * torch.ones([ 1, ]).type(dtype) opt.step() res = [opt._h_max, opt._h_min, opt._grad_var, opt._dist_to_opt] g_norm_squared_avg = 0.999 * g_norm_squared_avg \ + 0.001 * np.sum(((i + 1) * np.ones([n_dim + 1, ]))**2) g_norm_avg = 0.999 * g_norm_avg \ + 0.001 * np.linalg.norm((i + 1) * np.ones([n_dim + 1, ])) g_avg = 0.999 * g_avg + 0.001 * (i + 1) target_h_max = 0.999 * target_h_max + 0.001 * (i + 1)**2 * (n_dim + 1) target_h_min = 0.999 * target_h_min + 0.001 * \ max(1, i + 2 - 20)**2 * (n_dim + 1) if zero_debias: target_var = g_norm_squared_avg / (1 - 0.999**(i + 1)) \ - g_avg**2 * (n_dim + 1) / (1 - 0.999**(i + 1))**2 else: target_var = g_norm_squared_avg - g_avg**2 * (n_dim + 1) target_dist = 0.999 * target_dist + 0.001 * g_norm_avg / g_norm_squared_avg if i == 0: continue if zero_debias: # print "iter ", i, " h max ", res[0], target_h_max/(1-0.999**(i + 1) ), \ # " h min ", res[1], target_h_min/(1-0.999**(i + 1) ), \ # " var ", res[2], target_var, \ # " dist ", res[3], target_dist/(1-0.999**(i + 1) ) assert np.abs(target_h_max / (1 - 0.999**(i + 1)) - res[0]) < np.abs(res[0]) * 1e-3 assert np.abs(target_h_min / (1 - 0.999**(i + 1)) - res[1]) < np.abs(res[1]) * 1e-3 assert np.abs(target_var - res[2]) < np.abs(target_var) * 1e-3 assert np.abs(target_dist / (1 - 0.999**(i + 1)) - res[3]) < np.abs(res[3]) * 1e-3 else: # print "iter ", i, " h max ", res[0], target_h_max, " h min ", res[1], target_h_min, \ # " var ", res[2], target_var, " dist ", res[3], target_dist assert np.abs(target_h_max - res[0]) < np.abs(target_h_max) * 1e-3 assert np.abs(target_h_min - res[1]) < np.abs(target_h_min) * 1e-3 assert np.abs(target_var - res[2]) < np.abs(res[2]) * 1e-3 assert np.abs(target_dist - res[3]) < np.abs(res[3]) * 1e-3 print "sync measurement test passed!"
def learn(dataset, dim=2, hyp=1, edim=1, euc=0, sdim=1, sph=0, scale=1., riemann=False, learning_rate=1e-1, decay_length=1000, decay_step=1.0, momentum=0.0, tol=1e-8, epochs=100, burn_in=0, use_yellowfin=False, use_adagrad=False, resample_freq=1000, print_freq=1, model_save_file=None, model_load_file=None, batch_size=16, num_workers=None, lazy_generation=False, log_name=None, log=False, warm_start=None, learn_scale=False, checkpoint_freq=100, sample=1., subsample=None, logloss=False, distloss=False, squareloss=False, symloss=False, exponential_rescale=None, extra_steps=1, use_svrg=False, T=10, use_hmds=False, visualize=False): # Log configuration formatter = logging.Formatter('%(asctime)s %(message)s') logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(message)s', datefmt='%FT%T', ) if log_name is None and log: log_name = f"{os.path.splitext(dataset)[0]}.H{dim}-{hyp}.E{edim}-{euc}.S{sdim}-{sph}.lr{learning_rate}.log" if log_name is not None: logging.info(f"Logging to {log_name}") log = logging.getLogger() fh = logging.FileHandler(log_name) fh.setFormatter(formatter) log.addHandler(fh) ############# loss_list = [] ############# logging.info(f"Commandline {sys.argv}") if model_save_file is None: logging.warning("No Model Save selected!") G = load_graph.load_graph(dataset) GM = nx.to_scipy_sparse_matrix(G, nodelist=list(range(G.order()))) # grab scale if warm starting: if warm_start: scale = pandas.read_csv(warm_start, index_col=0).as_matrix()[0, -1] n = G.order() logging.info(f"Loaded Graph {dataset} with {n} nodes scale={scale}") if exponential_rescale is not None: # torch.exp(exponential_rescale * -d) def weight_fn(d): if d <= 2.0: return 5.0 elif d > 4.0: return 0.01 else: return 1.0 else: def weight_fn(d): return 1.0 Z, z = build_dataset(G, lazy_generation, sample, subsample, scale, batch_size, weight_fn, num_workers) if model_load_file is not None: logging.info(f"Loading {model_load_file}...") m = torch.load(model_load_file).to(device) logging.info( f"Loaded scale {unwrap(m.scale())} {torch.sum(m.embedding().data)} {m.epoch}" ) else: logging.info(f"Creating a fresh model warm_start?={warm_start}") m_init = None if warm_start: # load from DataFrame; assume that the julia combinatorial embedding has been saved ws_data = pandas.read_csv(warm_start, index_col=0).as_matrix() scale = ws_data[0, ws_data.shape[1] - 1] m_init = torch.DoubleTensor(ws_data[:, range(ws_data.shape[1] - 1)]) elif use_hmds: # m_init = torch.DoubleTensor(mds_warmstart.get_normalized_hyperbolic(mds_warmstart.get_model(dataset,dim,scale)[1])) m_init = torch.DoubleTensor( mds_warmstart.get_model(dataset, dim, scale)[1]) logging.info( f"\t Warmstarting? {warm_start} {m_init.size() if warm_start else None} {G.order()}" ) # initial_scale = z.dataset.max_dist / 3.0 # print("MAX DISTANCE", z.dataset.max_dist) # print("AVG DISTANCE", torch.mean(z.dataset.val_cache)) initial_scale = 0.0 m = ProductEmbedding(G.order(), dim, hyp, edim, euc, sdim, sph, initialize=m_init, learn_scale=learn_scale, initial_scale=initial_scale, logrel_loss=logloss, dist_loss=distloss, square_loss=squareloss, sym_loss=symloss, exponential_rescale=exponential_rescale, riemann=riemann).to(device) m.normalize() m.epoch = 0 logging.info( f"Constructed model with dim={dim} and epochs={m.epoch} isnan={np.any(np.isnan(m.embedding().cpu().data.numpy()))}" ) if visualize: name = 'animations/' + f"{os.path.split(os.path.splitext(dataset)[0])[1]}.H{dim}-{hyp}.E{edim}-{euc}.S{sdim}-{sph}.lr{learning_rate}.ep{epochs}.seed{seed}" fig, ax, writer = vis.setup_plot(m=m, name=name, draw_circle=True) else: fig = None ax = None writer = None # # Build the Optimizer # # TODO: Redo this in a sensible way!! # per-parameter learning rates exp_params = [p for p in m.embed_params if p.use_exp] learn_params = [p for p in m.embed_params if not p.use_exp] hyp_params = [p for p in m.hyp_params if not p.use_exp] euc_params = [p for p in m.euc_params if not p.use_exp] sph_params = [p for p in m.sph_params if not p.use_exp] scale_params = m.scale_params # model_params = [{'params': m.embed_params}, {'params': m.scale_params, 'lr': 1e-4*learning_rate}] # model_params = [{'params': learn_params}, {'params': m.scale_params, 'lr': 1e-4*learning_rate}] model_params = [{ 'params': hyp_params }, { 'params': euc_params }, { 'params': sph_params, 'lr': 0.1 * learning_rate }, { 'params': m.scale_params, 'lr': 1e-4 * learning_rate }] # opt = None if len(model_params) > 0: opt = torch.optim.SGD(model_params, lr=learning_rate / 10, momentum=momentum) # opt = torch.optim.SGD(learn_params, lr=learning_rate/10, momentum=momentum) # opt = torch.optim.SGD(model_params, lr=learning_rate/10, momentum=momentum) # exp = None # if len(exp_params) > 0: # exp = torch.optim.SGD(exp_params, lr=1.0) # dummy for zeroing if len(scale_params) > 0: scale_opt = torch.optim.SGD(scale_params, lr=1e-3 * learning_rate) scale_decay = torch.optim.lr_scheduler.StepLR(scale_opt, step_size=1, gamma=.99) else: scale_opt = None scale_decay = None lr_burn_in = torch.optim.lr_scheduler.MultiStepLR(opt, milestones=[burn_in], gamma=10) # lr_decay = torch.optim.lr_scheduler.StepLR(opt, decay_length, decay_step) #TODO reconcile multiple LR schedulers if use_yellowfin: from yellowfin import YFOptimizer opt = YFOptimizer(model_params) if use_adagrad: opt = torch.optim.Adagrad(model_params) if use_svrg: from svrg import SVRG base_opt = torch.optim.Adagrad if use_adagrad else torch.optim.SGD opt = SVRG(m.parameters(), lr=learning_rate, T=T, data_loader=z, opt=base_opt) # TODO add ability for SVRG to take parameter groups logging.info(opt) # Log stats from import: when warmstarting, check that it matches Julia's stats logging.info(f"*** Initial Checkpoint. Computing Stats") major_stats(GM, n, m, lazy_generation, Z, z, fig, ax, writer, visualize, subsample) logging.info("*** End Initial Checkpoint\n") # track best stats best_loss = 1.0e10 best_dist = 1.0e10 best_wcdist = 1.0e10 best_map = 0.0 for i in range(m.epoch + 1, m.epoch + epochs + 1): lr_burn_in.step() # lr_decay.step() # scale_decay.step() # print(scale_opt.param_groups[0]['lr']) # for param_group in opt.param_groups: # print(param_group['lr']) # print(type(opt.param_groups), opt.param_groups) l, n_edges = 0.0, 0.0 # track average loss per edge m.train(True) if use_svrg: for data in z: def closure(data=data, target=None): _data = data if target is None else (data, target) c = m.loss(_data.to(device)) c.backward() return c.data[0] l += opt.step(closure) # Projection m.normalize() else: # scale_opt.zero_grad() for the_step in range(extra_steps): # Accumulate the gradient for u in z: # Zero out the gradients # if opt is not None: opt.zero_grad() # This is handled by the SVRG. # if exp is not None: exp.zero_grad() opt.zero_grad() for p in exp_params: if p.grad is not None: p.grad.detach_() p.grad.zero_() # Compute loss _loss = m.loss(cu_var(u)) _loss.backward() l += _loss.item() * u[0].size(0) # print(weight) n_edges += u[0].size(0) # modify gradients if necessary RParameter.correct_metric(m.parameters()) # step opt.step() for p in exp_params: lr = opt.param_groups[0]['lr'] p.exp(lr) # Projection m.normalize() # scale_opt.step() l /= n_edges # m.epoch refers to num of training epochs finished m.epoch += 1 # Logging code # if l < tol: # logging.info("Found a {l} solution. Done at iteration {i}!") # break if i % print_freq == 0: logging.info(f"{i} loss={l}") ############ if log_name is not None: loss_list.append(l) ############# if (i <= burn_in and i % (checkpoint_freq / 5) == 0) or i % checkpoint_freq == 0: logging.info(f"\n*** Major Checkpoint. Computing Stats and Saving") avg_dist, wc_dist, me, mc, mapscore = major_stats( GM, n, m, True, Z, z, fig, ax, writer, visualize, subsample) best_loss = min(best_loss, l) best_dist = min(best_dist, avg_dist) best_wcdist = min(best_wcdist, wc_dist) best_map = max(best_map, mapscore) if model_save_file is not None: fname = f"{model_save_file}.{m.epoch}" logging.info( f"Saving model into {fname} {torch.sum(m.embedding().data)} " ) torch.save(m, fname) logging.info("*** End Major Checkpoint\n") if i % resample_freq == 0: if sample < 1. or subsample is not None: Z, z = build_dataset(G, lazy_generation, sample, subsample, scale, batch_size, weight_fn, num_workers) logging.info(f"final loss={l}") logging.info( f"best loss={best_loss}, distortion={best_dist}, map={best_map}, wc_dist={best_wcdist}" ) final_dist, final_wc, final_me, final_mc, final_map = major_stats( GM, n, m, lazy_generation, Z, z, fig, ax, writer, False, subsample) if log_name is not None: ### with open(log_name + "_loss.stat", "w") as f: for loss in loss_list: f.write("%f\n" % loss) ### with open(log_name + '_final.stat', "w") as f: f.write("Best-loss MAP dist wc Final-loss MAP dist wc me mc\n") f.write( f"{best_loss:10.6f} {best_map:8.4f} {best_dist:8.4f} {best_wcdist:8.4f} {l:10.6f} {final_map:8.4f} {final_dist:8.4f} {final_wc:8.4f} {final_me:8.4f} {final_mc:8.4f}" ) if visualize: writer.finish() if model_save_file is not None: fname = f"{model_save_file}.final" logging.info( f"Saving model into {fname}-final {torch.sum(m.embedding().data)} {unwrap(m.scale())}" ) torch.save(m, fname)
def test_lr_mu(zero_debias=False): dtype = torch.FloatTensor w = Variable(torch.ones(n_dim, 1).type(dtype), requires_grad=True) b = Variable(torch.ones(1).type(dtype), requires_grad=True) x = Variable(torch.ones(1, n_dim).type(dtype), requires_grad=False) opt = YFOptimizer([w, b], lr=1.0, mu=0.0, zero_debias=zero_debias) target_h_max = 0.0 target_h_min = 0.0 g_norm_squared_avg = 0.0 g_norm_avg = 0.0 g_avg = 0.0 target_dist = 0.0 target_lr = 1.0 target_mu = 0.0 for i in range(n_iter): opt.zero_grad() loss = (x.mm(w) + b).sum() loss.backward() w.grad.data = (i + 1) * torch.ones([ n_dim, ]).type(dtype) b.grad.data = (i + 1) * torch.ones([ 1, ]).type(dtype) opt.step() res = [ opt._h_max, opt._h_min, opt._grad_var, opt._dist_to_opt, opt._lr, opt._mu ] g_norm_squared_avg = 0.999 * g_norm_squared_avg \ + 0.001 * np.sum(((i + 1) * np.ones([n_dim + 1, ]))**2) g_norm_avg = 0.999 * g_norm_avg \ + 0.001 * np.linalg.norm((i + 1) * np.ones([n_dim + 1, ])) g_avg = 0.999 * g_avg + 0.001 * (i + 1) target_h_max = 0.999 * target_h_max + 0.001 * (i + 1)**2 * (n_dim + 1) target_h_min = 0.999 * target_h_min + 0.001 * \ max(1, i + 2 - 20)**2 * (n_dim + 1) if zero_debias: target_var = g_norm_squared_avg / (1 - 0.999**(i + 1)) \ - g_avg**2 * (n_dim + 1) / (1 - 0.999**(i + 1))**2 else: target_var = g_norm_squared_avg - g_avg**2 * (n_dim + 1) target_dist = 0.999 * target_dist + 0.001 * g_norm_avg / g_norm_squared_avg if i == 0: continue if zero_debias: # print "iter ", i, " h max ", res[0], target_h_max/(1-0.999**(i + 1) ), \ # " h min ", res[1], target_h_min/(1-0.999**(i + 1) ), \ # " var ", res[2], target_var, \ # " dist ", res[3], target_dist/(1-0.999**(i + 1) ) assert np.abs(target_h_max / (1 - 0.999**(i + 1)) - res[0]) < np.abs(res[0]) * 1e-3 assert np.abs(target_h_min / (1 - 0.999**(i + 1)) - res[1]) < np.abs(res[1]) * 1e-3 assert np.abs(target_var - res[2]) < np.abs(target_var) * 1e-3 assert np.abs(target_dist / (1 - 0.999**(i + 1)) - res[3]) < np.abs(res[3]) * 1e-3 else: # print "iter ", i, " h max ", res[0], target_h_max, " h min ", res[1], target_h_min, \ # " var ", res[2], target_var, " dist ", res[3], target_dist assert np.abs(target_h_max - res[0]) < np.abs(target_h_max) * 1e-3 assert np.abs(target_h_min - res[1]) < np.abs(target_h_min) * 1e-3 assert np.abs(target_var - res[2]) < np.abs(res[2]) * 1e-3 assert np.abs(target_dist - res[3]) < np.abs(res[3]) * 1e-3 if i > 0: if zero_debias: lr, mu = tune_everything( (target_dist / (1 - 0.999**(i + 1)))**2, target_var, 1, target_h_min / (1 - 0.999**(i + 1)), target_h_max / (1 - 0.999**(i + 1))) else: lr, mu = tune_everything(target_dist**2, target_var, 1, target_h_min, target_h_max) lr = np.real(lr) mu = np.real(mu) target_lr = 0.999 * target_lr + 0.001 * lr target_mu = 0.999 * target_mu + 0.001 * mu # print "lr ", target_lr, res[4], " mu ", target_mu, res[5] assert target_lr == 0.0 or np.abs(target_lr - res[4]) < np.abs(res[4]) * 1e-3 assert target_mu == 0.0 or np.abs(target_mu - res[5]) < np.abs(res[5]) * 5e-3 print "lr and mu computing test passed!"
# optimizer which Variables it should update. min_loss_so_far = np.inf optimizer = YFOptimizer(model.parameters(), lr=0.0001, mu=0.0) for t in range(6600): # Forward pass: compute predicted y by passing x to the model. y_pred = model(x) # Compute and print loss. loss = loss_fn(y_pred, y) # Before the backward pass, use the optimizer object to zero all of the # gradients for the variables it will update (which are the learnable weights # of the model) optimizer.zero_grad() # Backward pass: compute gradient of the loss with respect to model parameters loss.backward() ####### MEDDLING AND MONITORING if loss.data.item() < min_loss_so_far: min_loss_so_far = loss.data.item() if loss.data.item() > 5 * min_loss_so_far: # JIAN: THIS IS THE CHECK FOR BOUNCING BACK # We might want to throw an exception here (if that's how TravisCI works) raise Exception("loss dramatically bounces back after gradient spike") if t > 10 and t < 6500: if t == 11:
def learn(dataset, rank=2, scale=1., learning_rate=1e-1, tol=1e-8, epochs=100, use_yellowfin=False, use_adagrad=False, print_freq=1, model_save_file=None, model_load_file=None, batch_size=16, num_workers=None, lazy_generation=False, log_name=None, warm_start=None, learn_scale=False, checkpoint_freq=1000, sample=1., subsample=None, exponential_rescale=None, extra_steps=1, use_svrg=False, T=10, use_hmds=False): # Log configuration formatter = logging.Formatter('%(asctime)s %(message)s') logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(message)s', datefmt='%FT%T', ) if log_name is not None: logging.info(f"Logging to {log_name}") log = logging.getLogger() fh = logging.FileHandler(log_name) fh.setFormatter(formatter) log.addHandler(fh) logging.info(f"Commandline {sys.argv}") if model_save_file is None: logging.warn("No Model Save selected!") G = load_graph.load_graph(dataset) GM = nx.to_scipy_sparse_matrix(G) # grab scale if warm starting: if warm_start: scale = pandas.read_csv(warm_start, index_col=0).as_matrix()[0, -1] n = G.order() logging.info(f"Loaded Graph {dataset} with {n} nodes scale={scale}") Z = None def collate(ls): x, y = zip(*ls) return torch.cat(x), torch.cat(y) if lazy_generation: if subsample is not None: z = DataLoader(GraphRowSubSampler(G, scale, subsample), batch_size, shuffle=True, collate_fn=collate) else: z = DataLoader(GraphRowSampler(G, scale), batch_size, shuffle=True, collate_fn=collate) logging.info("Built Data Sampler") else: Z = gh.build_distance(G, scale, num_workers=int(num_workers) if num_workers is not None else 16) # load the whole matrix logging.info(f"Built distance matrix with {scale} factor") if subsample is not None: z = DataLoader(GraphRowSubSampler(G, scale, subsample, Z=Z), batch_size, shuffle=True, collate_fn=collate) else: idx = torch.LongTensor([(i, j) for i in range(n) for j in range(i + 1, n)]) Z_sampled = gh.dist_sample_rebuild_pos_neg( Z, sample) if sample < 1 else Z vals = torch.DoubleTensor( [Z_sampled[i, j] for i in range(n) for j in range(i + 1, n)]) z = DataLoader(TensorDataset(idx, vals), batch_size=batch_size, shuffle=True, pin_memory=torch.cuda.is_available()) logging.info("Built data loader") if model_load_file is not None: logging.info(f"Loading {model_load_file}...") m = cudaify(torch.load(model_load_file)) logging.info( f"Loaded scale {m.scale.data[0]} {torch.sum(m.w.data)} {m.epoch}") else: logging.info(f"Creating a fresh model warm_start?={warm_start}") m_init = None if warm_start: # load from DataFrame; assume that the julia combinatorial embedding has been saved ws_data = pandas.read_csv(warm_start, index_col=0).as_matrix() scale = ws_data[0, ws_data.shape[1] - 1] m_init = torch.DoubleTensor(ws_data[:, range(ws_data.shape[1] - 1)]) elif use_hmds: # m_init = torch.DoubleTensor(mds_warmstart.get_normalized_hyperbolic(mds_warmstart.get_model(dataset,rank,scale)[1])) m_init = torch.DoubleTensor( mds_warmstart.get_model(dataset, rank, scale)[1]) logging.info( f"\t Warmstarting? {warm_start} {m_init.size() if warm_start else None} {G.order()}" ) m = cudaify( Hyperbolic_Emb(G.order(), rank, initialize=m_init, learn_scale=learn_scale, exponential_rescale=exponential_rescale)) m.normalize() m.epoch = 0 logging.info( f"Constructed model with rank={rank} and epochs={m.epoch} isnan={np.any(np.isnan(m.w.cpu().data.numpy()))}" ) # # Build the Optimizer # # TODO: Redo this in a sensible way!! # opt = torch.optim.SGD(m.parameters(), lr=learning_rate) if use_yellowfin: from yellowfin import YFOptimizer opt = YFOptimizer(m.parameters()) if use_adagrad: opt = torch.optim.Adagrad(m.parameters()) if use_svrg: from svrg import SVRG base_opt = torch.optim.Adagrad if use_adagrad else torch.optim.SGD opt = SVRG(m.parameters(), lr=learning_rate, T=T, data_loader=z, opt=base_opt) logging.info(opt) # Log stats from import: when warmstarting, check that it matches Julia's stats logging.info(f"*** Initial Checkpoint. Computing Stats") major_stats(GM, 1 + m.scale.data[0], n, m, lazy_generation, Z, z) logging.info("*** End Initial Checkpoint\n") for i in range(m.epoch, m.epoch + epochs): l = 0.0 m.train(True) if use_svrg: for data in z: def closure(data=data, target=None): _data = data if target is None else (data, target) c = m.loss(cu_var(_data)) c.backward() return c.data[0] l += opt.step(closure) # Projection m.normalize() else: opt.zero_grad() # This is handled by the SVRG. for the_step in range(extra_steps): # Accumulate the gradient for u in z: _loss = m.loss(cu_var(u, requires_grad=False)) _loss.backward() l += _loss.data[0] Hyperbolic_Parameter.correct_metric( m.parameters()) # NB: THIS IS THE NEW CALL # print("Scale before step: ", m.scale.data) opt.step() # print("Scale after step: ", m.scale.data) # Projection m.normalize() #l += step(m, opt, u).data[0] # Logging code if l < tol: logging.info("Found a {l} solution. Done at iteration {i}!") break if i % print_freq == 0: logging.info(f"{i} loss={l}") if i % checkpoint_freq == 0: logging.info(f"\n*** Major Checkpoint. Computing Stats and Saving") major_stats(GM, 1 + m.scale.data[0], n, m, True, Z, z) if model_save_file is not None: fname = f"{model_save_file}.{m.epoch}" logging.info( f"Saving model into {fname} {torch.sum(m.w.data)} ") torch.save(m, fname) logging.info("*** End Major Checkpoint\n") m.epoch += 1 logging.info(f"final loss={l}") if model_save_file is not None: fname = f"{model_save_file}.final" logging.info( f"Saving model into {fname}-final {torch.sum(m.w.data)} {m.scale.data[0]}" ) torch.save(m, fname) major_stats(GM, 1 + m.scale.data[0], n, m, lazy_generation, Z, z)
optimizer._sparsity_debias = True ######################### # TRAINING WITH NEWMODEL# ######################### iter = 0 for epoch in range(num_epoches): for i, (images, labels) in enumerate(train_loader): if torch.cuda.is_available(): images = Variable(images.cuda()) labels = Variable(labels.cuda()) else: images = Variable(images) labels = Variable(labels) optimizer.zero_grad() # Clear gradients first outputs = newmodel(images) # Forward to get outputs if type(outputs) is tuple: outputs = outputs[0] # print(outputs) loss = criterion(outputs, labels) # Cross-entropy loss function loss.backward() optimizer.step() iter += 1 if iter % 100 == 0: # Calculate Accuracy correct = 0 total = 0 for images, labels in test_loader: if torch.cuda.is_available():
for x in train: t = -time.time() #import ipdb; ipdb.set_trace() #if not (i == 0 and epoch == 0): #fast_view_act_list.append(opt._fast_bound_const/( (math.sqrt(opt._global_state["grad_norm_squared"] ) + math.sqrt(opt._h_min) )**2 + 1e-6)) #lr_grad_norm_clamp_act_list.append(opt._lr_grad_norm_thresh / (math.sqrt(opt._global_state["grad_norm_squared"] ) + 1e-6) ) x = numpy.asarray(x, dtype=numpy.float32) x = torch.from_numpy(x) x = x.view(x.size()[0], x.size()[1], input_size) y = torch.cat((x[:, 1:, :], torch.zeros([x.size()[0], 1, input_size])), 1) images = Variable(x).cuda() labels = Variable(y).long().cuda() opt.zero_grad() outputs = rnn(images) shp = outputs.size() outputs_reshp = outputs.view([shp[0] * shp[1], num_classes]) labels_reshp = labels.view(shp[0] * shp[1]) loss = criterion(outputs_reshp, labels_reshp) loss.backward() opt.step() # For plotting #unclip_g_norm_list.append(opt._unclip_grad_norm) t += time.time() loss_list.append(loss.data[0])