def run_isomap(emb_name, dataset, r):
#emb_name = 'isomap_test/smalltree.E10-1.lr10.emb.final'
#dataset = 'data/edges/smalltree.edges'
dataset = 'data/edges/' + dataset + '.edges'
m = torch.load(emb_name)
emb_orig = unwrap(m.E[0].w)
# perform the isomap dim reduction
embedding = Isomap(n_components=r)
emb_transformed = embedding.fit_transform(emb_orig)
#print(emb_transformed.shape)
num_workers = 1
scale = 1
# compute d_avg
G = load_graph.load_graph(dataset)
n = G.order()
H = gh.build_distance(G, scale, num_workers=int(num_workers) if num_workers is not None else 16)
#Hrec = unwrap(m.dist_matrix())
Hrec = dist_matrix(emb_transformed)
mc, me, avg_dist, nan_elements = dis.distortion(H, Hrec, n, num_workers)
wc_dist = me*mc
print("d_avg = ", avg_dist)
return avg_dist
def stats(dataset, d_file, procs=1, verbose=False):
start = timer()
# Load graph
G = lg.load_graph(dataset, directed=True)
n = G.order()
weighted = gu.is_weighted(G)
print("G: ", G.edges)
adj_mat_original = nx.to_scipy_sparse_matrix(G, range(0, n))
print(f"Finished loading graph. Elapsed time {timer()-start}")
# Load distance matrix chunks
hyp_dist_df = pandas.read_csv(d_file, index_col=0)
loaded = timer()
print(f"Finished loading distance matrix. Elapsed time {loaded-start}")
rows = hyp_dist_df.index.values
hyp_dist_mat = hyp_dist_df.as_matrix()
n_ = rows.size
_map = np.zeros(n_)
_d_avg = np.zeros(n_)
_dc = np.zeros(n_)
_de = np.zeros(n_)
for (i, row) in enumerate(rows):
# if row == 0: continue
(_map[i], _d_avg[i], _dc[i],
_de[i]) = compute_row_stats(row,
n,
adj_mat_original,
hyp_dist_mat[i, :],
weighted=weighted,
verbose=verbose)
map_ = np.sum(_map)
d_avg_ = np.sum(_d_avg)
dc_ = np.max(_dc)
de_ = np.max(_de)
if weighted:
print("Note: MAP is not well defined for weighted graphs")
# Final stats:
# n_ -= 1
print(
f"MAP = {map_/n_}, d_avg = {d_avg_/n_}, d_wc = {dc_*de_}, d_c = {dc_}, d_e = {de_}"
)
end = timer()
print(f"Finished computing stats. Total elapsed time {end-start}")
with open(f"{d_file}.stats", "w") as stats_log:
stats_log.write(f"{n_},{map_},{d_avg_},{dc_},{de_}\n")
print(f"Stats saved to {d_file}.stats")
print()
def estimate(dataset='data/edges/smalltree.edges', n_samples=100000):
G = load_graph.load_graph(dataset)
n = G.order()
GM = nx.to_scipy_sparse_matrix(G, nodelist=list(range(G.order())))
num_workers = 16
D = gh.build_distance(G, 1.0, num_workers) # load the whole matrix
# n_samples = 100000
n1 = 5
n2 = 5
samples = sample_G(G, D, n, n_samples)
# coefs = sample_K(n1, n2, n_samples)
print("stats", np.mean(samples), np.std(samples)**2, np.mean(samples**2))
m1 = np.mean(samples)
m2 = np.mean(samples**2)
solns = sample_K(m1, m2, n1, n2, n_samples)
print(solns)
def get_model(dataset, max_k, scale=1.0):
#G = dp.load_graph(dataset)
G = load_graph.load_graph(dataset)
H = gh.build_distance(G, 1.0)
(n, n) = H.shape
Z = (np.cosh(scale * H) - 1) / 2
# Find Perron vector
(d, U) = get_eig(Z, 1)
idx = np.argmax(d)
l0 = d[idx]
u = U[:, idx]
u = u if u[0] > 0 else -u
(d1, dv, v) = compute_d(u, l0, n)
inv_d = 1. / d1
#Q = (np.eye(n)-np.ones( (n,n)) /n)*np.diag(inv_d)
#G = -Q@[email protected]/2
G = Z # This does make a copy.
center_numpy_inplace(G, inv_d, v)
G /= -2.0
# Recover our points
(emb_d, points_d) = get_eig(G, max_k)
# good_idx = emb_d > 0
# our_points = np.real(points_d[:,good_idx]@np.diag(np.sqrt(emb_d[good_idx])))
bad_idx = emb_d <= 0
emb_d[bad_idx] = 0
our_points = points_d @ np.diag(np.sqrt(emb_d))
# Just for evaluation
(Z, Hrec) = data_rec(our_points, scale)
# np.set_printoptions(threshold=np.nan)
# print(f"Distortion Score {dis.distortion(H, Hrec, n, 2)}")
# this will get done in the preliminary stats pass:
#print(f"Map Score {dis.map_score(H/scale, Hrec, n, 2)}")
return (H, our_points)
def learn(dataset,
dim=2,
hyp=1,
edim=1,
euc=0,
sdim=1,
sph=0,
scale=1.,
riemann=False,
learning_rate=1e-1,
decay_length=1000,
decay_step=1.0,
momentum=0.0,
tol=1e-8,
epochs=100,
burn_in=0,
use_yellowfin=False,
use_adagrad=False,
resample_freq=1000,
print_freq=1,
model_save_file=None,
model_load_file=None,
batch_size=16,
num_workers=None,
lazy_generation=False,
log_name=None,
log=False,
warm_start=None,
learn_scale=False,
checkpoint_freq=100,
sample=1.,
subsample=None,
logloss=False,
distloss=False,
squareloss=False,
symloss=False,
exponential_rescale=None,
extra_steps=1,
use_svrg=False,
T=10,
use_hmds=False,
visualize=False):
# Log configuration
formatter = logging.Formatter('%(asctime)s %(message)s')
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(message)s',
datefmt='%FT%T',
)
if log_name is None and log:
log_name = f"{os.path.splitext(dataset)[0]}.H{dim}-{hyp}.E{edim}-{euc}.S{sdim}-{sph}.lr{learning_rate}.log"
if log_name is not None:
logging.info(f"Logging to {log_name}")
log = logging.getLogger()
fh = logging.FileHandler(log_name)
fh.setFormatter(formatter)
log.addHandler(fh)
#############
loss_list = []
#############
logging.info(f"Commandline {sys.argv}")
if model_save_file is None: logging.warning("No Model Save selected!")
G = load_graph.load_graph(dataset)
GM = nx.to_scipy_sparse_matrix(G, nodelist=list(range(G.order())))
# grab scale if warm starting:
if warm_start:
scale = pandas.read_csv(warm_start, index_col=0).as_matrix()[0, -1]
n = G.order()
logging.info(f"Loaded Graph {dataset} with {n} nodes scale={scale}")
if exponential_rescale is not None:
# torch.exp(exponential_rescale * -d)
def weight_fn(d):
if d <= 2.0: return 5.0
elif d > 4.0: return 0.01
else: return 1.0
else:
def weight_fn(d):
return 1.0
Z, z = build_dataset(G, lazy_generation, sample, subsample, scale,
batch_size, weight_fn, num_workers)
if model_load_file is not None:
logging.info(f"Loading {model_load_file}...")
m = torch.load(model_load_file).to(device)
logging.info(
f"Loaded scale {unwrap(m.scale())} {torch.sum(m.embedding().data)} {m.epoch}"
)
else:
logging.info(f"Creating a fresh model warm_start?={warm_start}")
m_init = None
if warm_start:
# load from DataFrame; assume that the julia combinatorial embedding has been saved
ws_data = pandas.read_csv(warm_start, index_col=0).as_matrix()
scale = ws_data[0, ws_data.shape[1] - 1]
m_init = torch.DoubleTensor(ws_data[:,
range(ws_data.shape[1] - 1)])
elif use_hmds:
# m_init = torch.DoubleTensor(mds_warmstart.get_normalized_hyperbolic(mds_warmstart.get_model(dataset,dim,scale)[1]))
m_init = torch.DoubleTensor(
mds_warmstart.get_model(dataset, dim, scale)[1])
logging.info(
f"\t Warmstarting? {warm_start} {m_init.size() if warm_start else None} {G.order()}"
)
# initial_scale = z.dataset.max_dist / 3.0
# print("MAX DISTANCE", z.dataset.max_dist)
# print("AVG DISTANCE", torch.mean(z.dataset.val_cache))
initial_scale = 0.0
m = ProductEmbedding(G.order(),
dim,
hyp,
edim,
euc,
sdim,
sph,
initialize=m_init,
learn_scale=learn_scale,
initial_scale=initial_scale,
logrel_loss=logloss,
dist_loss=distloss,
square_loss=squareloss,
sym_loss=symloss,
exponential_rescale=exponential_rescale,
riemann=riemann).to(device)
m.normalize()
m.epoch = 0
logging.info(
f"Constructed model with dim={dim} and epochs={m.epoch} isnan={np.any(np.isnan(m.embedding().cpu().data.numpy()))}"
)
if visualize:
name = 'animations/' + f"{os.path.split(os.path.splitext(dataset)[0])[1]}.H{dim}-{hyp}.E{edim}-{euc}.S{sdim}-{sph}.lr{learning_rate}.ep{epochs}.seed{seed}"
fig, ax, writer = vis.setup_plot(m=m, name=name, draw_circle=True)
else:
fig = None
ax = None
writer = None
#
# Build the Optimizer
#
# TODO: Redo this in a sensible way!!
# per-parameter learning rates
exp_params = [p for p in m.embed_params if p.use_exp]
learn_params = [p for p in m.embed_params if not p.use_exp]
hyp_params = [p for p in m.hyp_params if not p.use_exp]
euc_params = [p for p in m.euc_params if not p.use_exp]
sph_params = [p for p in m.sph_params if not p.use_exp]
scale_params = m.scale_params
# model_params = [{'params': m.embed_params}, {'params': m.scale_params, 'lr': 1e-4*learning_rate}]
# model_params = [{'params': learn_params}, {'params': m.scale_params, 'lr': 1e-4*learning_rate}]
model_params = [{
'params': hyp_params
}, {
'params': euc_params
}, {
'params': sph_params,
'lr': 0.1 * learning_rate
}, {
'params': m.scale_params,
'lr': 1e-4 * learning_rate
}]
# opt = None
if len(model_params) > 0:
opt = torch.optim.SGD(model_params,
lr=learning_rate / 10,
momentum=momentum)
# opt = torch.optim.SGD(learn_params, lr=learning_rate/10, momentum=momentum)
# opt = torch.optim.SGD(model_params, lr=learning_rate/10, momentum=momentum)
# exp = None
# if len(exp_params) > 0:
# exp = torch.optim.SGD(exp_params, lr=1.0) # dummy for zeroing
if len(scale_params) > 0:
scale_opt = torch.optim.SGD(scale_params, lr=1e-3 * learning_rate)
scale_decay = torch.optim.lr_scheduler.StepLR(scale_opt,
step_size=1,
gamma=.99)
else:
scale_opt = None
scale_decay = None
lr_burn_in = torch.optim.lr_scheduler.MultiStepLR(opt,
milestones=[burn_in],
gamma=10)
# lr_decay = torch.optim.lr_scheduler.StepLR(opt, decay_length, decay_step) #TODO reconcile multiple LR schedulers
if use_yellowfin:
from yellowfin import YFOptimizer
opt = YFOptimizer(model_params)
if use_adagrad:
opt = torch.optim.Adagrad(model_params)
if use_svrg:
from svrg import SVRG
base_opt = torch.optim.Adagrad if use_adagrad else torch.optim.SGD
opt = SVRG(m.parameters(),
lr=learning_rate,
T=T,
data_loader=z,
opt=base_opt)
# TODO add ability for SVRG to take parameter groups
logging.info(opt)
# Log stats from import: when warmstarting, check that it matches Julia's stats
logging.info(f"*** Initial Checkpoint. Computing Stats")
major_stats(GM, n, m, lazy_generation, Z, z, fig, ax, writer, visualize,
subsample)
logging.info("*** End Initial Checkpoint\n")
# track best stats
best_loss = 1.0e10
best_dist = 1.0e10
best_wcdist = 1.0e10
best_map = 0.0
for i in range(m.epoch + 1, m.epoch + epochs + 1):
lr_burn_in.step()
# lr_decay.step()
# scale_decay.step()
# print(scale_opt.param_groups[0]['lr'])
# for param_group in opt.param_groups:
# print(param_group['lr'])
# print(type(opt.param_groups), opt.param_groups)
l, n_edges = 0.0, 0.0 # track average loss per edge
m.train(True)
if use_svrg:
for data in z:
def closure(data=data, target=None):
_data = data if target is None else (data, target)
c = m.loss(_data.to(device))
c.backward()
return c.data[0]
l += opt.step(closure)
# Projection
m.normalize()
else:
# scale_opt.zero_grad()
for the_step in range(extra_steps):
# Accumulate the gradient
for u in z:
# Zero out the gradients
# if opt is not None: opt.zero_grad() # This is handled by the SVRG.
# if exp is not None: exp.zero_grad()
opt.zero_grad()
for p in exp_params:
if p.grad is not None:
p.grad.detach_()
p.grad.zero_()
# Compute loss
_loss = m.loss(cu_var(u))
_loss.backward()
l += _loss.item() * u[0].size(0)
# print(weight)
n_edges += u[0].size(0)
# modify gradients if necessary
RParameter.correct_metric(m.parameters())
# step
opt.step()
for p in exp_params:
lr = opt.param_groups[0]['lr']
p.exp(lr)
# Projection
m.normalize()
# scale_opt.step()
l /= n_edges
# m.epoch refers to num of training epochs finished
m.epoch += 1
# Logging code
# if l < tol:
# logging.info("Found a {l} solution. Done at iteration {i}!")
# break
if i % print_freq == 0:
logging.info(f"{i} loss={l}")
############
if log_name is not None:
loss_list.append(l)
#############
if (i <= burn_in and i %
(checkpoint_freq / 5) == 0) or i % checkpoint_freq == 0:
logging.info(f"\n*** Major Checkpoint. Computing Stats and Saving")
avg_dist, wc_dist, me, mc, mapscore = major_stats(
GM, n, m, True, Z, z, fig, ax, writer, visualize, subsample)
best_loss = min(best_loss, l)
best_dist = min(best_dist, avg_dist)
best_wcdist = min(best_wcdist, wc_dist)
best_map = max(best_map, mapscore)
if model_save_file is not None:
fname = f"{model_save_file}.{m.epoch}"
logging.info(
f"Saving model into {fname} {torch.sum(m.embedding().data)} "
)
torch.save(m, fname)
logging.info("*** End Major Checkpoint\n")
if i % resample_freq == 0:
if sample < 1. or subsample is not None:
Z, z = build_dataset(G, lazy_generation, sample, subsample,
scale, batch_size, weight_fn, num_workers)
logging.info(f"final loss={l}")
logging.info(
f"best loss={best_loss}, distortion={best_dist}, map={best_map}, wc_dist={best_wcdist}"
)
final_dist, final_wc, final_me, final_mc, final_map = major_stats(
GM, n, m, lazy_generation, Z, z, fig, ax, writer, False, subsample)
if log_name is not None:
###
with open(log_name + "_loss.stat", "w") as f:
for loss in loss_list:
f.write("%f\n" % loss)
###
with open(log_name + '_final.stat', "w") as f:
f.write("Best-loss MAP dist wc Final-loss MAP dist wc me mc\n")
f.write(
f"{best_loss:10.6f} {best_map:8.4f} {best_dist:8.4f} {best_wcdist:8.4f} {l:10.6f} {final_map:8.4f} {final_dist:8.4f} {final_wc:8.4f} {final_me:8.4f} {final_mc:8.4f}"
)
if visualize:
writer.finish()
if model_save_file is not None:
fname = f"{model_save_file}.final"
logging.info(
f"Saving model into {fname}-final {torch.sum(m.embedding().data)} {unwrap(m.scale())}"
)
torch.save(m, fname)
def learn(dataset,
rank=2,
scale=1.,
learning_rate=1e-1,
tol=1e-8,
epochs=100,
use_yellowfin=False,
use_adagrad=False,
print_freq=1,
model_save_file=None,
model_load_file=None,
batch_size=16,
num_workers=None,
lazy_generation=False,
log_name=None,
warm_start=None,
learn_scale=False,
checkpoint_freq=1000,
sample=1.,
subsample=None,
exponential_rescale=None,
extra_steps=1,
use_svrg=False,
T=10,
use_hmds=False):
# Log configuration
formatter = logging.Formatter('%(asctime)s %(message)s')
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(message)s',
datefmt='%FT%T',
)
if log_name is not None:
logging.info(f"Logging to {log_name}")
log = logging.getLogger()
fh = logging.FileHandler(log_name)
fh.setFormatter(formatter)
log.addHandler(fh)
logging.info(f"Commandline {sys.argv}")
if model_save_file is None: logging.warn("No Model Save selected!")
G = load_graph.load_graph(dataset)
GM = nx.to_scipy_sparse_matrix(G)
# grab scale if warm starting:
if warm_start:
scale = pandas.read_csv(warm_start, index_col=0).as_matrix()[0, -1]
n = G.order()
logging.info(f"Loaded Graph {dataset} with {n} nodes scale={scale}")
Z = None
def collate(ls):
x, y = zip(*ls)
return torch.cat(x), torch.cat(y)
if lazy_generation:
if subsample is not None:
z = DataLoader(GraphRowSubSampler(G, scale, subsample),
batch_size,
shuffle=True,
collate_fn=collate)
else:
z = DataLoader(GraphRowSampler(G, scale),
batch_size,
shuffle=True,
collate_fn=collate)
logging.info("Built Data Sampler")
else:
Z = gh.build_distance(G,
scale,
num_workers=int(num_workers) if num_workers
is not None else 16) # load the whole matrix
logging.info(f"Built distance matrix with {scale} factor")
if subsample is not None:
z = DataLoader(GraphRowSubSampler(G, scale, subsample, Z=Z),
batch_size,
shuffle=True,
collate_fn=collate)
else:
idx = torch.LongTensor([(i, j) for i in range(n)
for j in range(i + 1, n)])
Z_sampled = gh.dist_sample_rebuild_pos_neg(
Z, sample) if sample < 1 else Z
vals = torch.DoubleTensor(
[Z_sampled[i, j] for i in range(n) for j in range(i + 1, n)])
z = DataLoader(TensorDataset(idx, vals),
batch_size=batch_size,
shuffle=True,
pin_memory=torch.cuda.is_available())
logging.info("Built data loader")
if model_load_file is not None:
logging.info(f"Loading {model_load_file}...")
m = cudaify(torch.load(model_load_file))
logging.info(
f"Loaded scale {m.scale.data[0]} {torch.sum(m.w.data)} {m.epoch}")
else:
logging.info(f"Creating a fresh model warm_start?={warm_start}")
m_init = None
if warm_start:
# load from DataFrame; assume that the julia combinatorial embedding has been saved
ws_data = pandas.read_csv(warm_start, index_col=0).as_matrix()
scale = ws_data[0, ws_data.shape[1] - 1]
m_init = torch.DoubleTensor(ws_data[:,
range(ws_data.shape[1] - 1)])
elif use_hmds:
# m_init = torch.DoubleTensor(mds_warmstart.get_normalized_hyperbolic(mds_warmstart.get_model(dataset,rank,scale)[1]))
m_init = torch.DoubleTensor(
mds_warmstart.get_model(dataset, rank, scale)[1])
logging.info(
f"\t Warmstarting? {warm_start} {m_init.size() if warm_start else None} {G.order()}"
)
m = cudaify(
Hyperbolic_Emb(G.order(),
rank,
initialize=m_init,
learn_scale=learn_scale,
exponential_rescale=exponential_rescale))
m.normalize()
m.epoch = 0
logging.info(
f"Constructed model with rank={rank} and epochs={m.epoch} isnan={np.any(np.isnan(m.w.cpu().data.numpy()))}"
)
#
# Build the Optimizer
#
# TODO: Redo this in a sensible way!!
#
opt = torch.optim.SGD(m.parameters(), lr=learning_rate)
if use_yellowfin:
from yellowfin import YFOptimizer
opt = YFOptimizer(m.parameters())
if use_adagrad:
opt = torch.optim.Adagrad(m.parameters())
if use_svrg:
from svrg import SVRG
base_opt = torch.optim.Adagrad if use_adagrad else torch.optim.SGD
opt = SVRG(m.parameters(),
lr=learning_rate,
T=T,
data_loader=z,
opt=base_opt)
logging.info(opt)
# Log stats from import: when warmstarting, check that it matches Julia's stats
logging.info(f"*** Initial Checkpoint. Computing Stats")
major_stats(GM, 1 + m.scale.data[0], n, m, lazy_generation, Z, z)
logging.info("*** End Initial Checkpoint\n")
for i in range(m.epoch, m.epoch + epochs):
l = 0.0
m.train(True)
if use_svrg:
for data in z:
def closure(data=data, target=None):
_data = data if target is None else (data, target)
c = m.loss(cu_var(_data))
c.backward()
return c.data[0]
l += opt.step(closure)
# Projection
m.normalize()
else:
opt.zero_grad() # This is handled by the SVRG.
for the_step in range(extra_steps):
# Accumulate the gradient
for u in z:
_loss = m.loss(cu_var(u, requires_grad=False))
_loss.backward()
l += _loss.data[0]
Hyperbolic_Parameter.correct_metric(
m.parameters()) # NB: THIS IS THE NEW CALL
# print("Scale before step: ", m.scale.data)
opt.step()
# print("Scale after step: ", m.scale.data)
# Projection
m.normalize()
#l += step(m, opt, u).data[0]
# Logging code
if l < tol:
logging.info("Found a {l} solution. Done at iteration {i}!")
break
if i % print_freq == 0:
logging.info(f"{i} loss={l}")
if i % checkpoint_freq == 0:
logging.info(f"\n*** Major Checkpoint. Computing Stats and Saving")
major_stats(GM, 1 + m.scale.data[0], n, m, True, Z, z)
if model_save_file is not None:
fname = f"{model_save_file}.{m.epoch}"
logging.info(
f"Saving model into {fname} {torch.sum(m.w.data)} ")
torch.save(m, fname)
logging.info("*** End Major Checkpoint\n")
m.epoch += 1
logging.info(f"final loss={l}")
if model_save_file is not None:
fname = f"{model_save_file}.final"
logging.info(
f"Saving model into {fname}-final {torch.sum(m.w.data)} {m.scale.data[0]}"
)
torch.save(m, fname)
major_stats(GM, 1 + m.scale.data[0], n, m, lazy_generation, Z, z)