# Build config string for log
conf = [
('distfn', '"{:s}"'),
('dim', '{:d}'),
('lr', '{:g}'),
('batchsize', '{:d}'),
('negs', '{:d}'),
] + conf
conf = ', '.join(['"{}": {}'.format(k, f).format(getattr(opt, k)) for k, f in conf])
log.info(f'json_conf: {{{conf}}}')
# initialize optimizer
optimizer = RiemannianSGD(
model.parameters(),
rgrad=opt.rgrad,
retraction=opt.retraction,
lr=opt.lr,
)
# if nproc == 0, run single threaded, otherwise run Hogwild
if opt.nproc == 0:
train.train(model, data, optimizer, opt, log, 0)
else:
queue = mp.Manager().Queue()
model.share_memory()
processes = []
for rank in range(opt.nproc):
p = mp.Process(
target=train.train_mp,
args=(model, data, optimizer, opt, log, rank + 1, queue)
)
def train_model(coordinates_from,
coordinates_to,
file_name,
method_type,
n_epochs=1000,
space='poincare',
lr=1e-4,
wd=1e-3,
batch_size=8,
n_warmup=3000,
cuda=0,
tb=0,
bn='before',
lrm=1.0):
if method_type == 'decoder':
space = 'euclidean'
if cuda:
device = th.device("cuda:0" if torch.cuda.is_available() else "cpu")
else:
device = th.device("cpu")
print(f"Computing on {device}")
encoder = Encoder(n_inputs=coordinates_from.shape[1],
n_outputs=coordinates_to.shape[1],
bn=bn)
encoder = encoder.to(device)
optimizer = torch.optim.Adam(encoder.parameters(), lr=lr, weight_decay=wd)
loss = torch.nn.MSELoss()
if tb:
writer = SummaryWriter()
X_train, X_test, y_train, y_test = train_test_split(coordinates_from,
coordinates_to,
test_size=0.3,
random_state=42)
if cuda:
t_X_train = torch.Tensor(X_train).cuda()
t_y_train = torch.Tensor(y_train).cuda()
t_X_test = torch.Tensor(X_test).cuda()
t_y_test = torch.Tensor(y_test).cuda()
loader = DataLoader(
TensorDataset(
torch.Tensor(X_train).cuda(),
torch.Tensor(y_train).cuda()),
batch_size=batch_size,
# pin_memory=True,
shuffle=True)
else:
t_X_train = torch.Tensor(X_train)
t_y_train = torch.Tensor(y_train)
t_X_test = torch.Tensor(X_test)
t_y_test = torch.Tensor(y_test)
loader = DataLoader(
TensorDataset(torch.Tensor(X_train), torch.Tensor(y_train)),
batch_size=batch_size,
# pin_memory=True,
shuffle=True)
poincare_distances = PoincareDistance()
train_error = []
test_error = []
pbar = tqdm(range(n_epochs), ncols=80)
t_start = timeit.default_timer()
n_iter = 0
for epoch in pbar:
epoch_error = 0
# if epoch == 100:
# optimizer = torch.optim.Adam(encoder.parameters(), lr=lr, weight_decay=wd)
if epoch == n_warmup:
optimizerSGD = RiemannianSGD(encoder.parameters(), lr=1e-4)
for inputs, targets in loader:
preds = encoder(inputs)
if epoch >= n_warmup and space == 'poincare':
z = PoincareDistance()(preds, targets)
error_encoder = th.mean(z)
optimizerSGD.zero_grad()
error_encoder.backward()
optimizerSGD.step()
else:
error_encoder = loss(preds, targets)
optimizer.zero_grad()
error_encoder.backward()
optimizer.step()
epoch_error += error_encoder.item()
if tb:
writer.add_scalar("data/train/error", error_encoder.item(),
n_iter)
writer.add_histogram("data/train/predictions", preds.data,
n_iter)
writer.add_histogram("data/train/targets", targets.data,
n_iter)
n_iter += 1
pbar.set_description("loss: {:.5e}".format(epoch_error))
encoder.eval()
if space == 'poincare':
test_error.append(
th.mean(poincare_distances(encoder(t_X_test),
t_y_test)).detach().cpu())
train_error.append(
th.mean(poincare_distances(encoder(t_X_train),
t_y_train)).detach().cpu())
else:
test_error.append(loss(encoder(t_X_test), t_y_test).detach().cpu())
train_error.append(
loss(encoder(t_X_train), t_y_train).detach().cpu())
if epoch % 100 == 0:
fig = plt.figure(figsize=(5, 5))
plt.plot(np.log10(train_error), label='train', color='red')
plt.plot(np.log10(test_error), label='test', color='green')
plt.legend(['train', 'test'])
plt.show()
plt.savefig(file_name + '_training_error.png',
format='png',
dpi=150)
if tb:
writer.add_scalar("data/test/epoch_error", test_error[-1], epoch)
writer.add_scalar("data/train/epoch_error", train_error[-1], epoch)
writer.add_histogram("data/test/predictions",
encoder(t_X_test).detach().cpu(), epoch)
writer.add_histogram("data/test/targets", t_y_test, epoch)
encoder.train()
encoder.eval()
print(f"epoch_error = {epoch_error:.5e}")
elapsed = timeit.default_timer() - t_start
print(f"Time: {elapsed:.2f}")
print(
f"Max norm = {torch.max(torch.sum(encoder(t_X_test)**2, dim=1)):.2f}")
encoder = encoder.to("cpu")
th.save(encoder.state_dict(), f"{file_name}_{method_type}.pth.tar")
if tb:
writer.close()
return encoder
def compute_poincare_maps(features,
labels,
fout,
mode='features',
k_neighbours=15,
distlocal='minkowski',
sigma=1.0,
gamma=2.0,
epochs=300,
color_dict=None,
debugplot=False,
batchsize=-1,
lr=0.1,
burnin=500,
lrm=1.0,
earlystop=0.0001,
cuda=0):
RFA = compute_rfa(features,
mode=mode,
k_neighbours=k_neighbours,
distlocal=distlocal,
distfn='MFIsym',
connected=True,
sigma=sigma)
if batchsize < 0:
batchsize = min(512, int(len(RFA) / 10))
print('batchsize = ', batchsize)
lr = batchsize / 16 * lr
indices = torch.arange(len(RFA))
if cuda:
indices = indices.cuda()
RFA = RFA.cuda()
dataset = TensorDataset(indices, RFA)
# instantiate our Embedding predictor
predictor = PoincareEmbedding(len(dataset),
2,
dist=PoincareDistance,
max_norm=1,
Qdist='laplace',
lossfn='klSym',
gamma=gamma,
cuda=cuda)
t_start = timeit.default_timer()
optimizer = RiemannianSGD(predictor.parameters(), lr=lr)
opt = PoincareOptions(debugplot=debugplot,
batchsize=batchsize,
lr=lr,
burnin=burnin,
lrm=lrm,
earlystop=earlystop,
cuda=cuda,
epochs=epochs)
# train predictor
print('Starting training...')
embeddings, loss, epoch = train(predictor,
dataset,
optimizer,
opt,
fout=fout,
labels=labels,
earlystop=earlystop,
color_dict=color_dict)
np.savetxt(fout + '.csv', embeddings, delimiter=",")
t = timeit.default_timer() - t_start
titlename = f"loss = {loss:.3e}\ntime = {t/60:.3f} min"
print(titlename)
return embeddings, titlename
sig_diff = -th.log(F.sigmoid(differences))
res=(sig_diff).mean()
return res
def bprl_loss_norm(embeddings):
res=th.mean(embeddings * embeddings).cuda(0)
return res
cf = CF_SNEmbedding(len(user_item_id_to_idx), n_dimensions, distfn).cuda(0)
# In[16]:
# define our optimizer. This is taken directly from Nickel and Kiela 2017
optimizer = RiemannianSGD(
cf.parameters(),
rgrad=rgrad,
retraction=retraction,
lr=learning_rate,
)
# In[17]:
bprl = bpr_loss(regularisation)
# In[18]:
# train the model. Each epoch compute new random negative examples. Split into batches and print loss after each epoch
reg_lam = th.FloatTensor([regularisation]).cuda(0)
# Build config string for log
conf = [
('distfn', '"{:s}"'),
('dim', '{:d}'),
('lr', '{:g}'),
('batchsize', '{:d}'),
('negs', '{:d}'),
] + conf
conf = ', '.join(
['"{}": {}'.format(k, f).format(getattr(opt, k)) for k, f in conf])
log.info(f'json_conf: {{{conf}}}')
# initialize optimizer
optimizer = RiemannianSGD(
model.parameters(),
rgrad=opt.rgrad,
retraction=opt.retraction,
lr=opt.lr * hvd.size(),
)
print(f'Size of hvd process : {hvd.size()}')
# optimizer = Adam(
# model.parameters(),
# lr=opt.lr * hvd.size(),
# )
lr = opt.lr
# Add Horovod Distributed Optimizer
optimizer = hvd.DistributedOptimizer(
optimizer, named_parameters=model.named_parameters())
dataset = TensorDataset(indices, RFA)
# instantiate our Embedding predictor
predictor = PoincareEmbedding(len(dataset),
opt.dim,
dist=PoincareDistance,
max_norm=1,
Qdist=opt.distr,
lossfn=opt.lossfn,
gamma=opt.gamma,
cuda=opt.cuda)
# instantiate the Riemannian optimizer
t_start = timeit.default_timer()
optimizer = RiemannianSGD(predictor.parameters(), lr=opt.lr)
# train predictor
print('Starting training...')
embeddings, loss, epoch = train(predictor,
dataset,
optimizer,
opt,
fout=fout,
labels=labels,
tb=opt.tb,
earlystop=opt.earlystop,
color_dict=color_dict)
np.savetxt(fout + '.csv', embeddings, delimiter=",")
