def gen_results_for_hw(X, y):
for m in [1, 20, 200, 1000, 2000]:
results = []
plt.figure()
plt.xlabel('Iteration (every 10th saved)')
plt.ylabel('Loss')
plt.title('SVRG ojective function progression for m = {}'.format(m))
for i in range(5):
svrg = SVRG(M=m, calc_loss=True)
svrg.fit(X, y)
results.append(svrg.fit_time)
plt.plot(svrg.training_loss, label='Run {}'.format(i))
print('Run {} time: {}'.format(i, round(results[-1], 3)))
plt.legend(loc='upper right')
plt.savefig('writeup/plots/svrg/m{}.pdf'.format(m), format='pdf')
plt.close('all')
print('Avg: {}s, Std: {}s'.format(round(np.mean(results), 3),
round(np.std(results), 3)))
def gen_results(X, y, m, numruns, loss=False):
results = []
if loss:
plt.figure()
plt.xlabel('Iteration (every 10th saved)')
plt.ylabel('Loss')
plt.title(
'Objective function progression across {} runs'.format(numruns))
for i in range(numruns):
svrg = SVRG(M=m, calc_loss=loss)
svrg.fit(X, y)
results.append(svrg.fit_time)
if loss:
plt.plot(svrg.training_loss, label='Run {}'.format(i))
print('Run {} time: {}'.format(i, round(results[-1], 3)))
if loss:
plt.legend(loc='upper right')
plt.savefig('plots/svrg/loss_vs_ite_{}.pdf'.format(
strftime("%Y.%m.%d_%H.%M.%S", localtime()) + '__m_{}'.format(m),
format='pdf'))
plt.close('all')
return results
def learn(dataset,
dim=2,
hyp=1,
edim=1,
euc=0,
sdim=1,
sph=0,
scale=1.,
riemann=False,
learning_rate=1e-1,
decay_length=1000,
decay_step=1.0,
momentum=0.0,
tol=1e-8,
epochs=100,
burn_in=0,
use_yellowfin=False,
use_adagrad=False,
resample_freq=1000,
print_freq=1,
model_save_file=None,
model_load_file=None,
batch_size=16,
num_workers=None,
lazy_generation=False,
log_name=None,
log=False,
warm_start=None,
learn_scale=False,
checkpoint_freq=100,
sample=1.,
subsample=None,
logloss=False,
distloss=False,
squareloss=False,
symloss=False,
exponential_rescale=None,
extra_steps=1,
use_svrg=False,
T=10,
use_hmds=False,
visualize=False):
# Log configuration
formatter = logging.Formatter('%(asctime)s %(message)s')
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(message)s',
datefmt='%FT%T',
)
if log_name is None and log:
log_name = f"{os.path.splitext(dataset)[0]}.H{dim}-{hyp}.E{edim}-{euc}.S{sdim}-{sph}.lr{learning_rate}.log"
if log_name is not None:
logging.info(f"Logging to {log_name}")
log = logging.getLogger()
fh = logging.FileHandler(log_name)
fh.setFormatter(formatter)
log.addHandler(fh)
#############
loss_list = []
#############
logging.info(f"Commandline {sys.argv}")
if model_save_file is None: logging.warning("No Model Save selected!")
G = load_graph.load_graph(dataset)
GM = nx.to_scipy_sparse_matrix(G, nodelist=list(range(G.order())))
# grab scale if warm starting:
if warm_start:
scale = pandas.read_csv(warm_start, index_col=0).as_matrix()[0, -1]
n = G.order()
logging.info(f"Loaded Graph {dataset} with {n} nodes scale={scale}")
if exponential_rescale is not None:
# torch.exp(exponential_rescale * -d)
def weight_fn(d):
if d <= 2.0: return 5.0
elif d > 4.0: return 0.01
else: return 1.0
else:
def weight_fn(d):
return 1.0
Z, z = build_dataset(G, lazy_generation, sample, subsample, scale,
batch_size, weight_fn, num_workers)
if model_load_file is not None:
logging.info(f"Loading {model_load_file}...")
m = torch.load(model_load_file).to(device)
logging.info(
f"Loaded scale {unwrap(m.scale())} {torch.sum(m.embedding().data)} {m.epoch}"
)
else:
logging.info(f"Creating a fresh model warm_start?={warm_start}")
m_init = None
if warm_start:
# load from DataFrame; assume that the julia combinatorial embedding has been saved
ws_data = pandas.read_csv(warm_start, index_col=0).as_matrix()
scale = ws_data[0, ws_data.shape[1] - 1]
m_init = torch.DoubleTensor(ws_data[:,
range(ws_data.shape[1] - 1)])
elif use_hmds:
# m_init = torch.DoubleTensor(mds_warmstart.get_normalized_hyperbolic(mds_warmstart.get_model(dataset,dim,scale)[1]))
m_init = torch.DoubleTensor(
mds_warmstart.get_model(dataset, dim, scale)[1])
logging.info(
f"\t Warmstarting? {warm_start} {m_init.size() if warm_start else None} {G.order()}"
)
# initial_scale = z.dataset.max_dist / 3.0
# print("MAX DISTANCE", z.dataset.max_dist)
# print("AVG DISTANCE", torch.mean(z.dataset.val_cache))
initial_scale = 0.0
m = ProductEmbedding(G.order(),
dim,
hyp,
edim,
euc,
sdim,
sph,
initialize=m_init,
learn_scale=learn_scale,
initial_scale=initial_scale,
logrel_loss=logloss,
dist_loss=distloss,
square_loss=squareloss,
sym_loss=symloss,
exponential_rescale=exponential_rescale,
riemann=riemann).to(device)
m.normalize()
m.epoch = 0
logging.info(
f"Constructed model with dim={dim} and epochs={m.epoch} isnan={np.any(np.isnan(m.embedding().cpu().data.numpy()))}"
)
if visualize:
name = 'animations/' + f"{os.path.split(os.path.splitext(dataset)[0])[1]}.H{dim}-{hyp}.E{edim}-{euc}.S{sdim}-{sph}.lr{learning_rate}.ep{epochs}.seed{seed}"
fig, ax, writer = vis.setup_plot(m=m, name=name, draw_circle=True)
else:
fig = None
ax = None
writer = None
#
# Build the Optimizer
#
# TODO: Redo this in a sensible way!!
# per-parameter learning rates
exp_params = [p for p in m.embed_params if p.use_exp]
learn_params = [p for p in m.embed_params if not p.use_exp]
hyp_params = [p for p in m.hyp_params if not p.use_exp]
euc_params = [p for p in m.euc_params if not p.use_exp]
sph_params = [p for p in m.sph_params if not p.use_exp]
scale_params = m.scale_params
# model_params = [{'params': m.embed_params}, {'params': m.scale_params, 'lr': 1e-4*learning_rate}]
# model_params = [{'params': learn_params}, {'params': m.scale_params, 'lr': 1e-4*learning_rate}]
model_params = [{
'params': hyp_params
}, {
'params': euc_params
}, {
'params': sph_params,
'lr': 0.1 * learning_rate
}, {
'params': m.scale_params,
'lr': 1e-4 * learning_rate
}]
# opt = None
if len(model_params) > 0:
opt = torch.optim.SGD(model_params,
lr=learning_rate / 10,
momentum=momentum)
# opt = torch.optim.SGD(learn_params, lr=learning_rate/10, momentum=momentum)
# opt = torch.optim.SGD(model_params, lr=learning_rate/10, momentum=momentum)
# exp = None
# if len(exp_params) > 0:
# exp = torch.optim.SGD(exp_params, lr=1.0) # dummy for zeroing
if len(scale_params) > 0:
scale_opt = torch.optim.SGD(scale_params, lr=1e-3 * learning_rate)
scale_decay = torch.optim.lr_scheduler.StepLR(scale_opt,
step_size=1,
gamma=.99)
else:
scale_opt = None
scale_decay = None
lr_burn_in = torch.optim.lr_scheduler.MultiStepLR(opt,
milestones=[burn_in],
gamma=10)
# lr_decay = torch.optim.lr_scheduler.StepLR(opt, decay_length, decay_step) #TODO reconcile multiple LR schedulers
if use_yellowfin:
from yellowfin import YFOptimizer
opt = YFOptimizer(model_params)
if use_adagrad:
opt = torch.optim.Adagrad(model_params)
if use_svrg:
from svrg import SVRG
base_opt = torch.optim.Adagrad if use_adagrad else torch.optim.SGD
opt = SVRG(m.parameters(),
lr=learning_rate,
T=T,
data_loader=z,
opt=base_opt)
# TODO add ability for SVRG to take parameter groups
logging.info(opt)
# Log stats from import: when warmstarting, check that it matches Julia's stats
logging.info(f"*** Initial Checkpoint. Computing Stats")
major_stats(GM, n, m, lazy_generation, Z, z, fig, ax, writer, visualize,
subsample)
logging.info("*** End Initial Checkpoint\n")
# track best stats
best_loss = 1.0e10
best_dist = 1.0e10
best_wcdist = 1.0e10
best_map = 0.0
for i in range(m.epoch + 1, m.epoch + epochs + 1):
lr_burn_in.step()
# lr_decay.step()
# scale_decay.step()
# print(scale_opt.param_groups[0]['lr'])
# for param_group in opt.param_groups:
# print(param_group['lr'])
# print(type(opt.param_groups), opt.param_groups)
l, n_edges = 0.0, 0.0 # track average loss per edge
m.train(True)
if use_svrg:
for data in z:
def closure(data=data, target=None):
_data = data if target is None else (data, target)
c = m.loss(_data.to(device))
c.backward()
return c.data[0]
l += opt.step(closure)
# Projection
m.normalize()
else:
# scale_opt.zero_grad()
for the_step in range(extra_steps):
# Accumulate the gradient
for u in z:
# Zero out the gradients
# if opt is not None: opt.zero_grad() # This is handled by the SVRG.
# if exp is not None: exp.zero_grad()
opt.zero_grad()
for p in exp_params:
if p.grad is not None:
p.grad.detach_()
p.grad.zero_()
# Compute loss
_loss = m.loss(cu_var(u))
_loss.backward()
l += _loss.item() * u[0].size(0)
# print(weight)
n_edges += u[0].size(0)
# modify gradients if necessary
RParameter.correct_metric(m.parameters())
# step
opt.step()
for p in exp_params:
lr = opt.param_groups[0]['lr']
p.exp(lr)
# Projection
m.normalize()
# scale_opt.step()
l /= n_edges
# m.epoch refers to num of training epochs finished
m.epoch += 1
# Logging code
# if l < tol:
# logging.info("Found a {l} solution. Done at iteration {i}!")
# break
if i % print_freq == 0:
logging.info(f"{i} loss={l}")
############
if log_name is not None:
loss_list.append(l)
#############
if (i <= burn_in and i %
(checkpoint_freq / 5) == 0) or i % checkpoint_freq == 0:
logging.info(f"\n*** Major Checkpoint. Computing Stats and Saving")
avg_dist, wc_dist, me, mc, mapscore = major_stats(
GM, n, m, True, Z, z, fig, ax, writer, visualize, subsample)
best_loss = min(best_loss, l)
best_dist = min(best_dist, avg_dist)
best_wcdist = min(best_wcdist, wc_dist)
best_map = max(best_map, mapscore)
if model_save_file is not None:
fname = f"{model_save_file}.{m.epoch}"
logging.info(
f"Saving model into {fname} {torch.sum(m.embedding().data)} "
)
torch.save(m, fname)
logging.info("*** End Major Checkpoint\n")
if i % resample_freq == 0:
if sample < 1. or subsample is not None:
Z, z = build_dataset(G, lazy_generation, sample, subsample,
scale, batch_size, weight_fn, num_workers)
logging.info(f"final loss={l}")
logging.info(
f"best loss={best_loss}, distortion={best_dist}, map={best_map}, wc_dist={best_wcdist}"
)
final_dist, final_wc, final_me, final_mc, final_map = major_stats(
GM, n, m, lazy_generation, Z, z, fig, ax, writer, False, subsample)
if log_name is not None:
###
with open(log_name + "_loss.stat", "w") as f:
for loss in loss_list:
f.write("%f\n" % loss)
###
with open(log_name + '_final.stat', "w") as f:
f.write("Best-loss MAP dist wc Final-loss MAP dist wc me mc\n")
f.write(
f"{best_loss:10.6f} {best_map:8.4f} {best_dist:8.4f} {best_wcdist:8.4f} {l:10.6f} {final_map:8.4f} {final_dist:8.4f} {final_wc:8.4f} {final_me:8.4f} {final_mc:8.4f}"
)
if visualize:
writer.finish()
if model_save_file is not None:
fname = f"{model_save_file}.final"
logging.info(
f"Saving model into {fname}-final {torch.sum(m.embedding().data)} {unwrap(m.scale())}"
)
torch.save(m, fname)
model_AVRG_0 = copy.deepcopy(model_SGD)
model_AVRG_1 = copy.deepcopy(model_SGD)
model_SAGA = copy.deepcopy(model_SGD)
lr = args.lr # learning rate
n_epoch = args.n_epoch # the number of epochs
loss_fn = nn.CrossEntropyLoss()
N = len(train_loader)
# The optimizer
optimizer_SGD = torch.optim.SGD(model_SGD.parameters(),
lr=lr,
weight_decay=args.weight_decay)
optimizer_SVRG_1 = SVRG(model_SVRG_1.parameters(), lr=lr, N=N)
optimizer_SVRG_0 = SVRG_0(model_SVRG_0.parameters())
optimizer_AVRG_1 = AVRG(model_AVRG_1.parameters(), lr=lr, N=N)
optimizer_AVRG_0 = AVRG_0(model_AVRG_0.parameters())
optimizer_SAGA = SAGA(model_SAGA.parameters(), lr=lr, N=N)
best_train_SGD = 0
best_test_SGD = 0
best_train_SVRG = 0
best_test_SVRG = 0
best_train_AVRG = 0
best_test_AVRG = 0
best_train_SAGA = 0
torch.nn.Linear(H1, H2), torch.nn.ReLU(),
torch.nn.Linear(H2, D_out))
## model2 = copy.deepcopy(model) # Use same initial weights for both networks
loss_fn = torch.nn.CrossEntropyLoss()
alpha = 1e-2
freq = 100 # how often to recompute large gradient
# The optimizer will not update model parameters on iterations
# where the large batches are calculated
lg_batch = 3000 # size of large gradient batch
min_batch = 300 # size of mini batch
optimizer = SVRG(model.parameters(), lr=alpha, freq=freq)
optimizer2 = torch.optim.SGD(model2.parameters(), lr=alpha)
epochs = 50
iterations = int(epochs * (60000 / min_batch))
# SVRG Training
counter = 0
total = time.time()
while (counter < iterations):
# compute large batch gradient
temp = np.random.choice(x.size()[0], lg_batch,
replace=False) # calculate batch indices
indices = torch.from_numpy(temp)
temp2 = torch.index_select(x, 0, indices)
y_pred = model(temp2)
def learn(dataset,
rank=2,
scale=1.,
learning_rate=1e-1,
tol=1e-8,
epochs=100,
use_yellowfin=False,
use_adagrad=False,
print_freq=1,
model_save_file=None,
model_load_file=None,
batch_size=16,
num_workers=None,
lazy_generation=False,
log_name=None,
warm_start=None,
learn_scale=False,
checkpoint_freq=1000,
sample=1.,
subsample=None,
exponential_rescale=None,
extra_steps=1,
use_svrg=False,
T=10,
use_hmds=False):
# Log configuration
formatter = logging.Formatter('%(asctime)s %(message)s')
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(message)s',
datefmt='%FT%T',
)
if log_name is not None:
logging.info(f"Logging to {log_name}")
log = logging.getLogger()
fh = logging.FileHandler(log_name)
fh.setFormatter(formatter)
log.addHandler(fh)
logging.info(f"Commandline {sys.argv}")
if model_save_file is None: logging.warn("No Model Save selected!")
G = load_graph.load_graph(dataset)
GM = nx.to_scipy_sparse_matrix(G)
# grab scale if warm starting:
if warm_start:
scale = pandas.read_csv(warm_start, index_col=0).as_matrix()[0, -1]
n = G.order()
logging.info(f"Loaded Graph {dataset} with {n} nodes scale={scale}")
Z = None
def collate(ls):
x, y = zip(*ls)
return torch.cat(x), torch.cat(y)
if lazy_generation:
if subsample is not None:
z = DataLoader(GraphRowSubSampler(G, scale, subsample),
batch_size,
shuffle=True,
collate_fn=collate)
else:
z = DataLoader(GraphRowSampler(G, scale),
batch_size,
shuffle=True,
collate_fn=collate)
logging.info("Built Data Sampler")
else:
Z = gh.build_distance(G,
scale,
num_workers=int(num_workers) if num_workers
is not None else 16) # load the whole matrix
logging.info(f"Built distance matrix with {scale} factor")
if subsample is not None:
z = DataLoader(GraphRowSubSampler(G, scale, subsample, Z=Z),
batch_size,
shuffle=True,
collate_fn=collate)
else:
idx = torch.LongTensor([(i, j) for i in range(n)
for j in range(i + 1, n)])
Z_sampled = gh.dist_sample_rebuild_pos_neg(
Z, sample) if sample < 1 else Z
vals = torch.DoubleTensor(
[Z_sampled[i, j] for i in range(n) for j in range(i + 1, n)])
z = DataLoader(TensorDataset(idx, vals),
batch_size=batch_size,
shuffle=True,
pin_memory=torch.cuda.is_available())
logging.info("Built data loader")
if model_load_file is not None:
logging.info(f"Loading {model_load_file}...")
m = cudaify(torch.load(model_load_file))
logging.info(
f"Loaded scale {m.scale.data[0]} {torch.sum(m.w.data)} {m.epoch}")
else:
logging.info(f"Creating a fresh model warm_start?={warm_start}")
m_init = None
if warm_start:
# load from DataFrame; assume that the julia combinatorial embedding has been saved
ws_data = pandas.read_csv(warm_start, index_col=0).as_matrix()
scale = ws_data[0, ws_data.shape[1] - 1]
m_init = torch.DoubleTensor(ws_data[:,
range(ws_data.shape[1] - 1)])
elif use_hmds:
# m_init = torch.DoubleTensor(mds_warmstart.get_normalized_hyperbolic(mds_warmstart.get_model(dataset,rank,scale)[1]))
m_init = torch.DoubleTensor(
mds_warmstart.get_model(dataset, rank, scale)[1])
logging.info(
f"\t Warmstarting? {warm_start} {m_init.size() if warm_start else None} {G.order()}"
)
m = cudaify(
Hyperbolic_Emb(G.order(),
rank,
initialize=m_init,
learn_scale=learn_scale,
exponential_rescale=exponential_rescale))
m.normalize()
m.epoch = 0
logging.info(
f"Constructed model with rank={rank} and epochs={m.epoch} isnan={np.any(np.isnan(m.w.cpu().data.numpy()))}"
)
#
# Build the Optimizer
#
# TODO: Redo this in a sensible way!!
#
opt = torch.optim.SGD(m.parameters(), lr=learning_rate)
if use_yellowfin:
from yellowfin import YFOptimizer
opt = YFOptimizer(m.parameters())
if use_adagrad:
opt = torch.optim.Adagrad(m.parameters())
if use_svrg:
from svrg import SVRG
base_opt = torch.optim.Adagrad if use_adagrad else torch.optim.SGD
opt = SVRG(m.parameters(),
lr=learning_rate,
T=T,
data_loader=z,
opt=base_opt)
logging.info(opt)
# Log stats from import: when warmstarting, check that it matches Julia's stats
logging.info(f"*** Initial Checkpoint. Computing Stats")
major_stats(GM, 1 + m.scale.data[0], n, m, lazy_generation, Z, z)
logging.info("*** End Initial Checkpoint\n")
for i in range(m.epoch, m.epoch + epochs):
l = 0.0
m.train(True)
if use_svrg:
for data in z:
def closure(data=data, target=None):
_data = data if target is None else (data, target)
c = m.loss(cu_var(_data))
c.backward()
return c.data[0]
l += opt.step(closure)
# Projection
m.normalize()
else:
opt.zero_grad() # This is handled by the SVRG.
for the_step in range(extra_steps):
# Accumulate the gradient
for u in z:
_loss = m.loss(cu_var(u, requires_grad=False))
_loss.backward()
l += _loss.data[0]
Hyperbolic_Parameter.correct_metric(
m.parameters()) # NB: THIS IS THE NEW CALL
# print("Scale before step: ", m.scale.data)
opt.step()
# print("Scale after step: ", m.scale.data)
# Projection
m.normalize()
#l += step(m, opt, u).data[0]
# Logging code
if l < tol:
logging.info("Found a {l} solution. Done at iteration {i}!")
break
if i % print_freq == 0:
logging.info(f"{i} loss={l}")
if i % checkpoint_freq == 0:
logging.info(f"\n*** Major Checkpoint. Computing Stats and Saving")
major_stats(GM, 1 + m.scale.data[0], n, m, True, Z, z)
if model_save_file is not None:
fname = f"{model_save_file}.{m.epoch}"
logging.info(
f"Saving model into {fname} {torch.sum(m.w.data)} ")
torch.save(m, fname)
logging.info("*** End Major Checkpoint\n")
m.epoch += 1
logging.info(f"final loss={l}")
if model_save_file is not None:
fname = f"{model_save_file}.final"
logging.info(
f"Saving model into {fname}-final {torch.sum(m.w.data)} {m.scale.data[0]}"
)
torch.save(m, fname)
major_stats(GM, 1 + m.scale.data[0], n, m, lazy_generation, Z, z)