def multinomial_train(self,
X,
y,
C,
w0=None,
b0=None,
eta=0.5,
max_iterations=1000):
""" Inputs:
- X: training features, a N-by-D numpy array, where N is the
number of training points and D is the dimensionality of features
- y: multiclass training labels, a N dimensional numpy array where
N is the number of training points, indicating the labels of
training data
- C: number of classes in the data
- eta: learning rate
- max_iterations: maximum number for iterations to perform
Returns:
- w: C-by-D weight matrix of multinomial logistic regression, where
C is the number of classes and D is the dimensionality of features.
- b: bias vector of length C, where C is the number of classes
"""
N, D = X.shape
w = np.zeros((C, D))
if w0 is not None:
w = w0
assert w.shape == (
C, D), f"check your w0, its dimension should be: {(C, D)}"
b = np.zeros(C)
if b0 is not None:
b = b0
W = np.hstack(
(b.reshape(-1, 1),
w)).T # shape (D+1, C), you could ignore this, I use this
# shape(N, D + 1), to visulize vectorization when implement it
X = np.insert(X, 0, 1, axis=1)
Y = one_hot(y, nb_class=C)
P = softmax(X @ W) - Y
tol = 1e-5
for it in range(max_iterations):
idx = minibatch(X)
W_prev = W
W = W - eta / N * X[idx, :].T @ P[idx, :]
P[idx, :] = softmax(X[idx, :] @ W) - Y[idx, :]
if np.max(np.abs(W_prev - W)) < tol:
print(f"Converged in {it} iters.")
break
w = W.T[:, 1:]
b = W.T[:, 0]
assert w.shape == (C, D)
assert b.shape == (C, )
self.w = w
self.b = b
def run_one_epoch(self, ep, train_ds, valid_ds, batch_size):
losses = []
i = 0
score = 0
for xbatch, ybatch in minibatch(train_ds, batch_size):
i += 1
word_seq, sequence_len = pad_sequences(xbatch)
target_seq, _ = pad_sequences(ybatch)
# build feed dictionary
feed = {
self.word_ids: word_seq,
self.labels: target_seq,
self.sequence_lengths: sequence_len,
self.learning_rate: self.lr,
self.keep_dropout_rate: self.kdr
}
_, train_loss = self.sess.run([self.train_op, self.loss],
feed_dict=feed)
losses += [train_loss]
if i % 10 == 0:
print('ep:', ep, 'iter:', i, 'loss:', np.mean(losses))
if i % 50 == 0:
acc_score, _ = self.run_validation(valid_ds, batch_size)
print('accuracy', acc_score)
if acc_score == 0:
acc_score, _ = self.run_validation(valid_ds, batch_size)
metrics = {}
metrics['acc'] = acc_score
return metrics
def BPR_train_original(dataset, recommend_model, loss_class, epoch, neg_k=1, w=None):
Recmodel = recommend_model
Recmodel.train()
bpr: utils.BPRLoss = loss_class
allusers = list(range(dataset.n_users))
S, sam_time = utils.UniformSample_original(allusers, dataset)
print(f"BPR[sample time][{sam_time[0]:.1f}={sam_time[1]:.2f}+{sam_time[2]:.2f}]")
users = torch.Tensor(S[:, 0]).long()
posItems = torch.Tensor(S[:, 1]).long()
negItems = torch.Tensor(S[:, 2]).long()
users = users.to(world.device)
posItems = posItems.to(world.device)
negItems = negItems.to(world.device)
users, posItems, negItems = utils.shuffle(users, posItems, negItems)
total_batch = len(users) // world.config['bpr_batch_size'] + 1
aver_loss = 0.
for (batch_i,
(batch_users,
batch_pos,
batch_neg)) in enumerate(utils.minibatch(users,
posItems,
negItems,
batch_size=world.config['bpr_batch_size'])):
cri = bpr.stageOne(batch_users, batch_pos, batch_neg)
aver_loss += cri
if world.tensorboard:
w.add_scalar(f'BPRLoss/BPR', cri, epoch * int(len(users) / world.config['bpr_batch_size']) + batch_i)
aver_loss = aver_loss / total_batch
return f"[BPR[aver loss{aver_loss:.3e}]"
def BPR_train_original(cuda_loader,
recommend_model,
loss_class,
epoch,
neg_k=1,
w=None):
Recmodel = recommend_model
Recmodel.train()
bpr: utils.BPRLoss = loss_class
users, posItems, negItems = cuda_loader.get_train_data_at(epoch)
users, posItems, negItems = utils.shuffle(users, posItems, negItems)
total_batch = len(users) // world.config['bpr_batch_size'] + 1
aver_loss = 0.
for (batch_i, (batch_users, batch_pos, batch_neg)) in enumerate(
utils.minibatch(users,
posItems,
negItems,
batch_size=world.config['bpr_batch_size'])):
cri = bpr.stageOne(batch_users, batch_pos, batch_neg)
aver_loss += cri
if world.tensorboard:
w.add_scalar(
f'BPRLoss/BPR', cri,
epoch * int(len(users) / world.config['bpr_batch_size']) +
batch_i)
aver_loss = aver_loss / total_batch
return f"[BPR[aver loss{aver_loss:.3e}]"
def _run_epoch_dev(self, sess, inputs, labels, shuffle):
minibatches = utils.minibatch(self.config.batch_size,
inputs, labels=labels, shuffle=shuffle)
mean_loss = 0
for i, (inputs_batch, labels_batch) in enumerate(minibatches):
loss = self._loss_on_batch(sess, inputs_batch, labels_batch)
mean_loss += loss
mean_loss /= (i + 1)
return mean_loss
def _run_epoch_pred(self, sess, inputs):
minibatches = utils.minibatch(self.config.batch_size,
inputs, labels=None, shuffle=False)
list_y_prob = []
for i, inputs_batch in enumerate(minibatches):
prob = self._predict_on_batch(sess, inputs_batch)
list_y_prob.append(prob)
y_prob = np.vstack(list_y_prob)
return y_prob
def Test(dataset, Recmodel, epoch, cold=False, w=None):
u_batch_size = world.config['test_u_batch_size']
if cold:
testDict: dict = dataset.coldTestDict
else:
testDict: dict = dataset.testDict
Recmodel = Recmodel.eval()
max_K = max(world.topks)
results = {
'precision': np.zeros(len(world.topks)),
'recall': np.zeros(len(world.topks)),
'ndcg': np.zeros(len(world.topks))
}
with torch.no_grad():
users = list(testDict.keys())
try:
assert u_batch_size <= len(users) / 10
except AssertionError:
print(
f"test_u_batch_size is too big for this dataset, try a small one {len(users) // 10}"
)
users_list = []
rating_list = []
groundTrue_list = []
total_batch = len(users) // u_batch_size + 1
for batch_users in utils.minibatch(users, batch_size=u_batch_size):
allPos = dataset.getUserPosItems(batch_users)
groundTrue = [testDict[u] for u in batch_users]
batch_users_gpu = torch.Tensor(batch_users).long()
batch_users_gpu = batch_users_gpu.to(world.device)
rating = Recmodel.getUsersRating(batch_users_gpu)
exclude_index = []
exclude_items = []
for range_i, items in enumerate(allPos):
exclude_index.extend([range_i] * len(items))
exclude_items.extend(items)
rating[exclude_index, exclude_items] = -(1 << 10)
_, rating_K = torch.topk(rating, k=max_K)
del rating
users_list.append(batch_users)
rating_list.append(rating_K.cpu())
groundTrue_list.append(groundTrue)
assert total_batch == len(users_list)
X = zip(rating_list, groundTrue_list)
pre_results = []
for x in X:
pre_results.append(test_one_batch(x))
for result in pre_results:
results['recall'] += result['recall']
results['precision'] += result['precision']
results['ndcg'] += result['ndcg']
results['recall'] /= float(len(users))
results['precision'] /= float(len(users))
results['ndcg'] /= float(len(users))
print(results)
return results
def binary_train(self,
X,
y,
w0=None,
b0=None,
eta=0.5,
max_iterations=1000):
""" Inputs:
- X: training features, a N-by-D numpy array, where N is the
number of training points and D is the dimensionality of features
- y: binary training labels, a N dimensional numpy array where
N is the number of training points, indicating the labels of
training data
- eta: learning rate
Returns:
- w: D-dimensional vector, a numpy array which is the weight
vector of logistic regression
- b: scalar, which is the bias of logistic regression
Find the optimal parameters w and b for inputs X and y.
Use the Mini-batch gradient descent.
"""
N, D = X.shape
assert len(np.unique(y)) == 2
y = np.array(y)
w = np.zeros(D)
if w0 is not None:
w = w0
assert len(w) == D, f"w0 length does not match number of features: {D}"
b = 0
if b0 is not None:
b = b0
w = np.append(b, w)
X = np.insert(X, 0, 1, axis=1)
tol = 1e-7
for i in range(max_iterations):
idx = minibatch(X)
w_next = (
w + eta / N * np.dot(X[idx, :].T,
(y[idx] - sigmoid(np.dot(X[idx, :], w)))))
if sp.spatial.distance.euclidean(w, w_next) < tol:
print(f"Converged in {i} iterations.")
break
w = w_next
b = w[0]
w = w[1:]
assert w.shape == (D, )
self.w = w
self.b = b
return w, b
def BPR_train_original(dataset, recommend_model, loss_class, epoch, w=None):
"""Traininf procedure for uniform BPR
Args:
dataset (BasicDatset): defined in dataloader.BasicDataset, loaded in register.py
recommend_model (PairWiseModel): recommend model with small dim
loss_class (utils.BPRLoss): class to get BPR training loss, and BackPropagation
epoch (int):
w (SummaryWriter, optional): Tensorboard writer
Returns:
str: summary of aver loss and running time for one epoch
"""
global item_count
if item_count is None:
item_count = torch.zeros(dataset.m_items)
Recmodel = recommend_model
Recmodel.train()
bpr: utils.BPRLoss = loss_class
S = UniformSample_original(dataset)
S = torch.from_numpy(S).long()
# print(f"BPR[sample time][{sam_time[0]:.1f}={sam_time[1]:.2f}+{sam_time[2]:.2f}]")
users = S[:, 0]
posItems = S[:, 1]
negItems = S[:, 2]
users = users.to(world.DEVICE)
posItems = posItems.to(world.DEVICE)
negItems = negItems.to(world.DEVICE)
users, posItems, negItems = utils.shuffle(users, posItems, negItems)
total_batch = len(users) // world.config['bpr_batch_size'] + 1
aver_loss = 0.
for (batch_i, (batch_users, batch_pos, batch_neg)) in enumerate(
utils.minibatch(users,
posItems,
negItems,
batch_size=world.config['bpr_batch_size'])):
if world.ALLDATA:
print(world.ALLDATA)
weights = utils.getTestweight(batch_users, batch_pos, dataset)
else:
weights = None
item_count[batch_neg] += 1
cri = bpr.stageOne(batch_users, batch_pos, batch_neg, weights=weights)
aver_loss += cri
if world.tensorboard:
w.add_scalar(
f'BPRLoss/BPR', cri,
epoch * int(len(users) / world.config['bpr_batch_size']) +
batch_i)
aver_loss = aver_loss / total_batch
return f"[BPR[aver loss{aver_loss:.3e}]"
def fit(self, X_train, n_epochs=20, batchsize=128, verbose=True):
"""
Fit the VAE to a training dataset.
Parameters
----------
X_train : numpy array of shape (n_ex, in_rows, in_cols, in_ch)
The input volume
n_epochs : int (default: 20)
The maximum number of training epochs to run
batchsize : int (default: 128)
The desired number of examples in each training batch
verbose : bool (default: True)
Print batch information during training
"""
self.verbose = verbose
self.n_epochs = n_epochs
self.batchsize = batchsize
_, self.in_rows, self.in_cols, self.in_ch = X_train.shape
self.N = self.in_rows * self.in_cols * self.in_ch
prev_loss = np.inf
for i in range(n_epochs):
loss, estart = 0.0, time()
batch_generator, nb = minibatch(X_train, batchsize, shuffle=True)
# TODO: parallelize inner loop
for j, b_ix in enumerate(batch_generator):
bsize, bstart = len(b_ix), time()
X_batch = X_train[b_ix]
X_batch_col = X_train[b_ix].reshape(bsize, -1)
X_recon = self.forward(X_batch)
t_mean = self.derived_variables["t_mean"]
t_log_var = self.derived_variables["t_log_var"]
self.backward(X_batch, X_recon)
batch_loss = self.loss(X_batch_col, X_recon, t_mean, t_log_var)
loss += batch_loss
self.update(batch_loss)
if self.verbose:
fstr = "\t[Batch {}/{}] Train loss: {:.3f} ({:.1f}s/batch)"
print(fstr.format(j + 1, nb, batch_loss, time() - bstart))
loss /= nb
fstr = "[Epoch {}] Avg. loss: {:.3f} Delta: {:.3f} ({:.2f}m/epoch)"
print(fstr.format(i + 1, loss, prev_loss - loss, (time() - estart) / 60.0))
prev_loss = loss
def _run_epoch_train(self, sess, inputs, labels, shuffle):
n_minibatches = len(np.arange(0, len(inputs), self.config.batch_size))
prog = tf.keras.utils.Progbar(target=n_minibatches)
minibatches = utils.minibatch(self.config.batch_size,
inputs, labels=labels, shuffle=shuffle)
mean_loss = 0
for i, (inputs_batch, labels_batch) in enumerate(minibatches):
loss = self._train_on_batch(sess, inputs_batch, labels_batch)
mean_loss += loss
force = (i + 1) == n_minibatches
prog.update(i + 1, [('train_loss', loss)], force=force)
mean_loss /= (i + 1)
return mean_loss
def Distill_DNS(dataset, student, sampler, loss_class, epoch, w=None):
"""Training procedure for distillation methods
Args:
dataset (BasicDatset): defined in dataloader.BasicDataset, loaded in register.py
student (PairWiseModel): recommend model with small dim
sampler (DS|DL|RD|CD): tons of distill methods defined in sample.py
loss_class (utils.BPRLoss): class to get BPR training loss, and BackPropagation
epoch (int):
w (SummaryWriter, optional): Tensorboard writer
Returns:
str: summary of aver loss and running time for one epoch
"""
sampler: DistillLogits
bpr: utils.BPRLoss = loss_class
student.train()
aver_loss = 0
with timer(name='sampling'):
S = sampler.PerSample()
S = torch.from_numpy(S).long().to(world.DEVICE)
users, posItems, negItems = S[:, 0], S[:, 1], S[:, 2:]
users, posItems, negItems = utils.shuffle(users, posItems, negItems)
total_batch = len(users) // world.config['bpr_batch_size'] + 1
for (batch_i, (batch_users, batch_pos, batch_neg)) in enumerate(
utils.minibatch(users,
posItems,
negItems,
batch_size=world.config['bpr_batch_size'])):
with timer(name="KD"):
batch_neg, weights, KD_loss = sampler.Sample(
batch_users, batch_pos, batch_neg, epoch)
with timer(name="BP"):
cri = bpr.stageOne(batch_users,
batch_pos,
batch_neg,
add_loss=KD_loss,
weights=weights)
aver_loss += cri
# Additional section------------------------
#
# ------------------------------------------
if world.tensorboard:
w.add_scalar(
f'BPRLoss/BPR', cri,
epoch * int(len(users) / world.config['bpr_batch_size']) +
batch_i)
aver_loss = aver_loss / total_batch
info = f"{timer.dict()}[BPR loss{aver_loss:.3e}]"
timer.zero()
return info
def BPR_train_DNS_neg(dataset, recommend_model, loss_class, epoch, w=None):
"""Traininf procedure for DNS algorithms
Args:
dataset (BasicDatset): defined in dataloader.BasicDataset, loaded in register.py
recommend_model (PairWiseModel): recommend model with small dim
loss_class (utils.BPRLoss): class to get BPR training loss, and BackPropagation
epoch (int):
w (SummaryWriter, optional): Tensorboard writer
Returns:
str: summary of aver loss and running time for one epoch
"""
Recmodel: PairWiseModel = recommend_model
Recmodel.train()
bpr: utils.BPRLoss = loss_class
S = UniformSample_DNS(dataset, world.DNS_K)
users = torch.Tensor(S[:, 0]).long()
posItems = torch.Tensor(S[:, 1]).long()
negItems = torch.Tensor(S[:, 2:]).long()
users = users.to(world.DEVICE)
posItems = posItems.to(world.DEVICE)
negItems = negItems.to(world.DEVICE)
users, posItems, negItems = utils.shuffle(users, posItems, negItems)
total_batch = len(users) // world.config['bpr_batch_size'] + 1
aver_loss = 0.
for (batch_i, (batch_users, batch_pos, batch_neg)) in enumerate(
utils.minibatch(users,
posItems,
negItems,
batch_size=world.config['bpr_batch_size'])):
if world.ALLDATA:
weights = utils.getTestweight(batch_users, batch_pos, dataset)
else:
weights = None
batch_neg = DNS_sampling_neg(batch_users, batch_neg, dataset, Recmodel)
cri = bpr.stageOne(batch_users, batch_pos, batch_neg, weights=weights)
aver_loss += cri
if world.tensorboard:
w.add_scalar(
f'BPRLoss/BPR', cri,
epoch * int(len(users) / world.config['bpr_batch_size']) +
batch_i)
# print(f"DNS[sampling][{time()-DNS_time:.1f}={DNS_time1:.2f}+{DNS_time2:.2f}]")
aver_loss = aver_loss / total_batch
return f"[BPR[aver loss{aver_loss:.3e}]"
def Popularity_Bias(dataset, Recmodel, valid=True):
u_batch_size = world.config['test_u_batch_size']
dataset: utils.BasicDataset
testDict: dict
Popularity = np.zeros(dataset.m_items, ).astype('int')
if valid:
testDict = dataset.validDict
else:
testDict = dataset.testDict
Recmodel: model.LightGCN
perUser = int(dataset.trainDataSize / dataset.n_users)
# eval mode with no dropout
Recmodel.eval()
max_K = perUser
user_topk = np.zeros((dataset.n_users, max_K), dtype=int)
with torch.no_grad():
users = list(testDict.keys())
rating_list = []
# ratings = []
total_batch = len(users) // u_batch_size + 1
for batch_users in utils.minibatch(users, batch_size=u_batch_size):
allPos = dataset.getUserPosItems(batch_users)
batch_users_gpu = torch.Tensor(batch_users).long()
batch_users_gpu = batch_users_gpu.to(world.DEVICE)
rating = Recmodel.getUsersRating(batch_users_gpu)
rating = rating.cpu()
exclude_index = []
exclude_items = []
if not world.TESTDATA:
for range_i, items in enumerate(allPos):
exclude_index.extend([range_i] * len(items))
exclude_items.extend(items)
rating[exclude_index, exclude_items] = -1e5
_, rating_K = torch.topk(rating, k=max_K)
del rating
rating_K = rating_K.numpy().astype('int')
user_topk[batch_users] = rating_K
for i in range(len(batch_users)):
Popularity[rating_K[i]] += 1
return Popularity.astype('int'), user_topk.astype("int")
def BPR_train_original(dataset,
Recmodel,
bpr: utils.BPRLoss,
epoch,
w=None,
val=-1):
if not args.simutaneously:
# train alternatively
Recmodel.train()
for m in Recmodel.modules():
if isinstance(m, nn.Embedding):
m.requires_grad_(val <= 0)
elif hasattr(m, 'weight'):
m.requires_grad_(not (val <= 0))
print('mapping fixed') if val <= 0 else print('embedding fixed')
# for m in Recmodel.modules(): # test separate training
# if hasattr(m, 'weight'):
# if hasattr(m, 'bias') and m.bias is not None:
# print(m, 'weight:', m.weight.requires_grad, ' bias:', m.bias.requires_grad)
# else:
# print(m, 'weight', m.weight.requires_grad)
S, sam_time = utils.UniformSample_original(dataset, val) # bpr sample
print(f"sample time:{sam_time[0]:.1f}={sam_time[1]:.2f}+{sam_time[2]:.2f}")
users = torch.Tensor(S[:, 0]).long().to(args.device)
posItems = torch.Tensor(S[:, 1]).long().to(args.device)
negItems = torch.Tensor(S[:, 2]).long().to(args.device)
users, posItems, negItems = utils.shuffle(users, posItems, negItems)
total_batch = len(users) // args.bpr_batch_size + 1
aver_loss = 0.
for (batch_i, (batch_users, batch_pos, batch_neg)) \
in enumerate(utils.minibatch(users, posItems, negItems, batch_size=args.bpr_batch_size)):
# train on different graph
cri = bpr.stageOne(dataset.graph[val + 1], batch_users, batch_pos,
batch_neg)
aver_loss += cri
if args.tensorboard:
w.add_scalar(
f'BPRLoss/BPR', cri,
epoch * int(len(users) / args.bpr_batch_size) + batch_i)
aver_loss = aver_loss / total_batch
return aver_loss
def _run_epoch_train(self, sess, inputs, masks, labels, shuffle):
n_minibatches = len(np.arange(0, len(inputs), self.config.batch_size))
prog = tf.keras.utils.Progbar(target=n_minibatches)
minibatches = utils.minibatch(self.config.batch_size,
inputs,
masks=masks,
labels=labels,
shuffle=shuffle)
mean_loss = 0
mean_grad_norm = 0
for i, curr_batch in enumerate(minibatches):
loss, grad_norm = self._train_on_batch(sess, *curr_batch)
mean_loss += loss
mean_grad_norm += grad_norm
force = (i + 1) == n_minibatches
prog.update(i + 1, [('train_loss', loss),
('grad_norm', grad_norm)],
force=force)
mean_loss /= (i + 1)
mean_grad_norm /= (i + 1)
return mean_loss, mean_grad_norm
def _get_predictions(model):
model._net.eval()
sequences = model.test_sequence.sequences
users = np.arange(model._num_users)
n_train = sequences.shape[0]
indices = np.arange(n_train)
# convert from numpy to PyTorch tensor
all_item_ids = np.arange(model._num_items).reshape(1, -1)
all_item_ids = torch.from_numpy(all_item_ids.astype(np.int64)).clone()
all_item_var = Variable(gpu(all_item_ids, model._use_cuda))
sequences_tensor = gpu(torch.from_numpy(sequences), model._use_cuda)
user_tensor = gpu(torch.from_numpy(users), model._use_cuda)
top = 1000
all_predictions = np.zeros((model._num_users, top), dtype=np.int64)
for (batch_indices, batch_sequence, batch_user) in minibatch(indices,
sequences_tensor,
user_tensor,
batch_size=256):
batch_size = batch_user.shape[0]
sequence_var = Variable(batch_sequence)
user_var = Variable(batch_user)
batch_all_item_var = all_item_var.repeat(batch_size, 1)
prediction = model._net(sequence_var,
user_var,
batch_all_item_var)
_, tops = prediction.topk(top, dim=1)
all_predictions[batch_indices, :] = cpu(tops.data).numpy()
return all_predictions
def BPR_train_original(dataset,
recommend_model,
loss_class,
epoch,
neg_k=1,
w=None):
Recmodel = recommend_model
Recmodel.train()
bpr: utils.BPRLoss = loss_class
with timer(name="Sample"):
S = utils.UniformSample_original(dataset)
users = torch.Tensor(S[:, 0]).long()
posItems = torch.Tensor(S[:, 1]).long()
negItems = torch.Tensor(S[:, 2]).long()
users = users.to(world.device)
posItems = posItems.to(world.device)
negItems = negItems.to(world.device)
users, posItems, negItems = utils.shuffle(users, posItems, negItems)
total_batch = len(users) // world.config['bpr_batch_size'] + 1
aver_loss = 0.
for (batch_i, (batch_users, batch_pos, batch_neg)) in enumerate(
utils.minibatch(users,
posItems,
negItems,
batch_size=world.config['bpr_batch_size'])):
cri = bpr.stageOne(batch_users, batch_pos, batch_neg)
aver_loss += cri
if False: #world.tensorboard:
w.add_scalar(
f'BPRLoss/BPR', cri,
epoch * int(len(users) / world.config['bpr_batch_size']) +
batch_i)
aver_loss = aver_loss / total_batch
time_info = timer.dict()
timer.zero()
return f"loss{aver_loss:.3f}-{time_info}"
def run_validation(self, valid_dataset, batch_size):
accs = []
ret = []
for xbatch, labels in minibatch(valid_dataset, batch_size):
word_seq, sequence_len = pad_sequences(xbatch)
feed = {
self.word_ids: word_seq,
self.sequence_lengths: sequence_len,
self.keep_dropout_rate: 1.0
}
if self.use_crf:
viterbi_sequences = []
logits_v, trans_params_v = self.sess.run(
[self.logits, self.trans_params], feed_dict=feed)
for logit, seq_length in zip(logits_v, sequence_len):
logit = logit[:seq_length]
viterbi_seq, viterbi_score = tf.contrib.crf.viterbi_decode(
logit, trans_params_v)
viterbi_sequences += [viterbi_seq]
labels_pred_v = viterbi_sequences
else:
labels_pred_v = self.sess.run(label_pred, feed_dict=feed)
for words, lab, lab_pred, seq_length in zip(
xbatch, labels, labels_pred_v, sequence_len):
lab = lab[:seq_length]
lab_pred = lab_pred[:seq_length]
acc = [a == b for (a, b) in zip(lab, lab_pred)]
ret.append((words, lab, lab_pred, acc))
accs += acc
overall_acc = np.mean(accs)
return overall_acc, ret
def BPR_train_original(dataset, Recmodel, opt):
Recmodel.train()
S = utils.bpr_neg_samp(dataset.trainUniqueUsers, dataset.trainDataSize,
support_dict=dataset.allPos, item_array=dataset.trainUniqueItems)
users = torch.Tensor(S[:, 0]).long().to(args.device)
posItems = torch.Tensor(S[:, 1]).long().to(args.device)
negItems = torch.Tensor(S[:, 2]).long().to(args.device)
users, posItems, negItems = utils.shuffle(users, posItems, negItems)
# loss and update
total_batch = len(users) // args.batch_size + 1
aver_loss = 0.
for (batch_i, (batch_users, batch_pos, batch_neg)) \
in enumerate(utils.minibatch(users, posItems, negItems, batch_size=args.batch_size)):
loss, reg_loss = Recmodel.bpr_loss(dataset.graph, batch_users, batch_pos, batch_neg)
reg_loss = reg_loss * args.reg_lambda
loss = loss + reg_loss
opt.zero_grad()
loss.backward()
opt.step()
aver_loss += loss.cpu().item()
aver_loss = aver_loss / total_batch
return aver_loss
def Test(dataset, Recmodel, epoch, w=None, multicore=0, valid=True):
"""evaluate models
Args:
dataset (BasicDatset): defined in dataloader.BasicDataset, loaded in register.py
Recmodel (PairWiseModel):
epoch (int):
w (SummaryWriter, optional): Tensorboard writer
multicore (int, optional): The num of cpu cores for testing. Defaults to 0.
Returns:
dict: summary of metrics
"""
u_batch_size = world.config['test_u_batch_size']
dataset: utils.BasicDataset
testDict: dict
if valid:
testDict = dataset.validDict
else:
testDict = dataset.testDict
Recmodel: model.LightGCN
# eval mode with no dropout
Recmodel.eval()
max_K = max(world.topks)
if multicore == 1:
pool = multiprocessing.Pool(CORES)
with torch.no_grad():
users = list(testDict.keys())
users_list = []
rating_list = []
groundTrue_list = []
# ratings = []
total_batch = len(users) // u_batch_size + 1
for batch_users in utils.minibatch(users, batch_size=u_batch_size):
allPos = dataset.getUserPosItems(batch_users)
groundTrue = [testDict[u] for u in batch_users]
batch_users_gpu = torch.Tensor(batch_users).long()
batch_users_gpu = batch_users_gpu.to(world.DEVICE)
rating = Recmodel.getUsersRating(batch_users_gpu)
#rating = rating.cpu()
exclude_index = []
exclude_items = []
if not world.TESTDATA:
for range_i, items in enumerate(allPos):
exclude_index.extend([range_i] * len(items))
exclude_items.extend(items)
rating[exclude_index, exclude_items] = -1e5
_, rating_K = torch.topk(rating, k=max_K)
del rating
users_list.append(batch_users)
rating_list.append(rating_K.cpu())
groundTrue_list.append(groundTrue)
assert total_batch == len(users_list)
X = zip(rating_list, groundTrue_list)
if world.ONE:
results = {
'hr': np.zeros(len(world.topks)),
'ndcg': np.zeros(len(world.topks))
}
if multicore == 1:
pre_results = pool.map(test_one_batch, X)
else:
pre_results = []
for x in X:
pre_results.append(test_one_batch_ONE(x))
scale = float(u_batch_size / len(users))
for result in pre_results:
results['hr'] += result['hr']
results['ndcg'] += result['ndcg']
results['hr'] /= float(len(users))
results['ndcg'] /= float(len(users))
if w:
w.add_scalars(
f'Valid/HR@{world.topks}', {
str(world.topks[i]): results['hr'][i]
for i in range(len(world.topks))
}, epoch)
w.add_scalars(
f'Valid/NDCG@{world.topks}', {
str(world.topks[i]): results['ndcg'][i]
for i in range(len(world.topks))
}, epoch)
else:
results = {
'precision': np.zeros(len(world.topks)),
'recall': np.zeros(len(world.topks)),
'ndcg': np.zeros(len(world.topks))
}
if multicore == 1:
pre_results = pool.map(test_one_batch, X)
else:
pre_results = []
for x in X:
pre_results.append(test_one_batch(x))
scale = float(u_batch_size / len(users))
for result in pre_results:
results['recall'] += result['recall']
results['precision'] += result['precision']
results['ndcg'] += result['ndcg']
results['recall'] /= float(len(users))
results['precision'] /= float(len(users))
results['ndcg'] /= float(len(users))
if w:
w.add_scalars(
f'Valid/Recall@{world.topks}', {
str(world.topks[i]): results['recall'][i]
for i in range(len(world.topks))
}, epoch)
w.add_scalars(
f'Valid/Precision@{world.topks}', {
str(world.topks[i]): results['precision'][i]
for i in range(len(world.topks))
}, epoch)
w.add_scalars(
f'Valid/NDCG@{world.topks}', {
str(world.topks[i]): results['ndcg'][i]
for i in range(len(world.topks))
}, epoch)
if multicore == 1:
pool.close()
return results
# saver.restore(sess, "loc_model_mse.ckpt")
# print("Model restored.")
for epoch in xrange(20):
rand_batches = range(110)
shuffle(rand_batches)
for i in rand_batches:
data = np.load(
'/home/data2/vision6/ethpete/gaze_data/batch_{0}.npz'.format(
i))
imgs = data['imgs'][:, :, :, 0]
imgs = imgs.reshape(
(imgs.shape[0], imgs.shape[1], imgs.shape[2], 1))
gazes = data['gazes']
img_batches = minibatch(imgs, 32, 1000)
label_batches = minibatch(gazes, 32, 1000)
# print("Processing training batch {0}".format(i))
for images, labels in zip(img_batches, label_batches):
sess.run(optimizer,
feed_dict={
x: images,
y: labels,
keep_prob: dropout
})
# print("Processed training batch {0}".format(i))
avg_loss = 0
avg_acc = 0
nums = 0
for j in range(110, 126):
data = np.load(
def fit(
self,
X_real,
lambda_,
n_steps=1000,
batchsize=128,
c_updates_per_epoch=5,
verbose=True,
):
"""
Fit WGAN_GP on a training dataset.
Parameters
----------
X_real : numpy array of shape (n_ex, n_feats)
Training dataset
lambda_ : float
Gradient penalty coefficient for the critic loss
n_steps : int (default: 1000)
The maximum number of generator updates to perform
batchsize : int (default: 128)
Number of examples to use in each training minibatch
c_updates_per_epoch : int
The number of critic updates to perform at each generator update
verbose : bool (default : True)
Print loss values after each update. If false, only print loss
every 100 steps.
"""
self.lambda_ = lambda_
self.verbose = verbose
self.n_steps = n_steps
self.batchsize = batchsize
self.c_updates_per_epoch = c_updates_per_epoch
# adjust output of the generator to match the dimensionality of X
if not self.is_initialized:
self.n_feats = X_real.shape[1]
self._init_params()
# (re-)initialize loss
prev_C, prev_G = np.inf, np.inf
self.loss = WGAN_GPLoss(lambda_=self.lambda_)
# training loop
NC, NG = self.c_updates_per_epoch, self.n_steps
for i in range(NG):
estart = time()
batch_generator, _ = minibatch(X_real, batchsize, shuffle=False)
for j, b_ix in zip(range(NC), batch_generator):
bstart = time()
X_batch = X_real[b_ix]
C_loss = self.update_critic(X_batch)
# for testing, don't perform gradient update so we can inspect each grad
if not self.debug:
self.update("C", C_loss)
if self.verbose:
fstr = "\t[Critic batch {}] Critic loss: {:.3f} {:.3f}∆ ({:.1f}s/batch)"
print(fstr.format(j + 1, C_loss, prev_C - C_loss, time() - bstart))
prev_C = C_loss
# generator update
G_loss = self.update_generator(X_batch.shape)
# for testing, don't perform gradient update so we can inspect each grad
if not self.debug:
self.update("G", G_loss)
if i % 99 == 0:
fstr = "[Epoch {}] Gen. loss: {:.3f} Critic loss: {:.3f}"
print(fstr.format(i + 1, G_loss, C_loss))
elif self.verbose:
fstr = "[Epoch {}] Gen. loss: {:.3f} {:.3f}∆ ({:.1f}s/epoch)"
print(fstr.format(i + 1, G_loss, prev_G - G_loss, time() - estart))
prev_G = G_loss
def Test(dataset, Recmodel, epoch, w=None, multicore=0):
u_batch_size = world.config['test_u_batch_size']
dataset: utils.BasicDataset
testDict: dict = dataset.testDict
Recmodel: model.LightGCN
# eval mode with no dropout
Recmodel = Recmodel.eval()
max_K = max(world.topks)
if multicore == 1:
pool = multiprocessing.Pool(CORES)
results = {
'precision': np.zeros(len(world.topks)),
'recall': np.zeros(len(world.topks)),
'ndcg': np.zeros(len(world.topks))
}
with torch.no_grad():
users = list(testDict.keys())
try:
assert u_batch_size <= len(users) / 10
except AssertionError:
print(
f"test_u_batch_size is too big for this dataset, try a small one {len(users) // 10}"
)
users_list = []
rating_list = []
groundTrue_list = []
auc_record = []
# ratings = []
total_batch = len(users) // u_batch_size + 1
for batch_users in utils.minibatch(users, batch_size=u_batch_size):
allPos = dataset.getUserPosItems(batch_users)
groundTrue = [testDict[u] for u in batch_users]
batch_users_gpu = torch.Tensor(batch_users).long()
batch_users_gpu = batch_users_gpu.to(world.device)
rating = Recmodel.getUsersRating(batch_users_gpu)
#rating = rating.cpu()
exclude_index = []
exclude_items = []
for range_i, items in enumerate(allPos):
exclude_index.extend([range_i] * len(items))
exclude_items.extend(items)
rating[exclude_index, exclude_items] = -(1 << 10)
_, rating_K = torch.topk(rating, k=max_K)
#rating = rating.cpu().numpy()
'''aucs = [
utils.AUC(rating[i],
dataset,
test_data) for i, test_data in enumerate(groundTrue)
]
auc_record.extend(aucs)'''
del rating
users_list.append(batch_users)
rating_list.append(rating_K.cpu())
groundTrue_list.append(groundTrue)
assert total_batch == len(users_list)
X = zip(rating_list, groundTrue_list)
if multicore == 1:
pre_results = pool.map(test_one_batch, X)
else:
pre_results = []
for x in X:
pre_results.append(test_one_batch(x))
scale = float(u_batch_size / len(users))
for result in pre_results:
results['recall'] += result['recall']
results['precision'] += result['precision']
results['ndcg'] += result['ndcg']
results['recall'] /= float(len(users))
results['precision'] /= float(len(users))
results['ndcg'] /= float(len(users))
#results['auc'] = np.mean(auc_record)
if world.tensorboard:
w.add_scalars(
f'Test/Recall@{world.topks}', {
str(world.topks[i]): results['recall'][i]
for i in range(len(world.topks))
}, epoch)
w.add_scalars(
f'Test/Precision@{world.topks}', {
str(world.topks[i]): results['precision'][i]
for i in range(len(world.topks))
}, epoch)
w.add_scalars(
f'Test/NDCG@{world.topks}', {
str(world.topks[i]): results['ndcg'][i]
for i in range(len(world.topks))
}, epoch)
if multicore == 1:
pool.close()
print(results)
return results
def test(dataset, Recmodel, epoch, w=None, multicore=0, best_results=None):
u_batch_size = args.test_batch
testDict = dataset.testDict
Recmodel = Recmodel.eval() # eval mode with no dropout
max_K = max(args.topks)
if multicore == 1:
pool = multiprocessing.Pool(args.n_cores)
results = {
'precision': np.zeros(len(args.topks)),
'recall': np.zeros(len(args.topks)),
'ndcg': np.zeros(len(args.topks))
}
with torch.no_grad():
users = list(testDict.keys())
try:
assert u_batch_size <= len(users) / 10
except AssertionError:
print(
f"test_u_batch_size is too big for this dataset, try a small one {len(users) // 10}"
)
users_list, rating_list, groundTrue_list, auc_record = [], [], [], []
t0 = time.time()
total_batch = len(users) // u_batch_size + 1
for batch_users in utils.minibatch(users, batch_size=u_batch_size):
allPos = dataset.getUserPosItems(batch_users,
dataset.UserItemNet) # **********
groundTrue = [testDict[u] for u in batch_users]
batch_users_gpu = torch.Tensor(batch_users).long()
batch_users_gpu = batch_users_gpu.to(args.device)
rating = Recmodel.getUsersRating(dataset.graph[args.test_graph],
batch_users_gpu) # **********
exclude_index = []
exclude_items = []
for range_i, items in enumerate(allPos):
exclude_index.extend([range_i] * len(items))
exclude_items.extend(items)
rating[exclude_index, exclude_items] = -(1 << 10)
_, rating_K = torch.topk(rating, k=max_K)
rating = rating.cpu().numpy()
aucs = [
utils.AUC(rating[i], dataset, test_data)
for i, test_data in enumerate(groundTrue)
]
auc_record.extend(aucs)
del rating
users_list.append(batch_users)
rating_list.append(rating_K.cpu())
groundTrue_list.append(groundTrue)
assert total_batch == len(users_list)
X = zip(rating_list, groundTrue_list)
if multicore == 1:
pre_results = pool.map(test_one_batch, X)
else:
pre_results = []
for x in X:
pre_results.append(test_one_batch(x))
scale = float(u_batch_size / len(users))
for result in pre_results:
results['recall'] += result['recall']
results['precision'] += result['precision']
results['ndcg'] += result['ndcg']
results['recall'] /= float(len(users))
results['precision'] /= float(len(users))
results['ndcg'] /= float(len(users))
results['auc'] = np.mean(auc_record)
if args.tensorboard:
w.add_scalars(
f'Test/Recall@{args.topks}', {
str(args.topks[i]): results['recall'][i]
for i in range(len(args.topks))
}, epoch)
w.add_scalars(
f'Test/Precision@{args.topks}', {
str(args.topks[i]): results['precision'][i]
for i in range(len(args.topks))
}, epoch)
w.add_scalars(
f'Test/NDCG@{args.topks}', {
str(args.topks[i]): results['ndcg'][i]
for i in range(len(args.topks))
}, epoch)
if multicore == 1:
pool.close()
print('time consumption: %.2f' % (time.time() - t0))
print('recall:', results['recall'], 'precision:', results['precision'],
'ndcg:', results['ndcg'], 'auc:', results['auc'])
if results['recall'][0] > best_results['recall'][0]:
return results
else:
return best_results
# with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
with tf.Session() as sess:
sess.run(init)
saver.restore(sess, "loc_model.ckpt")
print("Model restored.")
for epoch in xrange(20):
rand_batches = range(110)
shuffle(rand_batches)
for i in rand_batches:
data = np.load('/home/data2/vision6/ethpete/gaze_data/batch_{0}.npz'.format(i))
imgs = data['imgs'][:, :, :, 0]
imgs = imgs.reshape((imgs.shape[0], imgs.shape[1], imgs.shape[2], 1))
buckets = data['buckets']
img_batches = minibatch(imgs, 32, 1000)
label_batches = minibatch(buckets, 32, 1000)
# print("Processing training batch {0}".format(i))
for images, labels in zip(img_batches, label_batches):
for label in labels:
if label.sum() != 1:
print(label, i)
sess.run(optimizer, feed_dict={x: images, y: labels, keep_prob: dropout})
# print("Processed training batch {0}".format(i))
avg_loss = 0
avg_acc = 0
nums = 0
for j in range(110, 126):
data = np.load('/home/data2/vision6/ethpete/gaze_data/batch_{0}.npz'.format(j))
imgs = data['imgs']
def test(dataset, testDict, Recmodel):
u_batch_size = args.test_batch
Recmodel = Recmodel.eval() # eval mode with no dropout
max_K = max(args.topks)
results = {'precision': np.zeros(len(args.topks)),
'recall': np.zeros(len(args.topks)),
'ndcg': np.zeros(len(args.topks))}
with torch.no_grad():
users = list(testDict.keys())
users_list, rating_list, groundTrue_list, auc_record = [], [], [], []
total_batch = len(users) // u_batch_size + 1
for batch_users in utils.minibatch(users, batch_size=u_batch_size):
allPos = dataset.getUserPosItems(batch_users) # batch user pos in train
groundTrue = [testDict[u] for u in batch_users] # batch user pos in test
batch_users_gpu = torch.Tensor(batch_users).long()
batch_users_gpu = batch_users_gpu.to(args.device)
rating = Recmodel.getUsersRating(dataset.graph, batch_users_gpu) # batch user rating / have used sigmoid
# mask pos in train
exclude_index = []
exclude_items = []
for range_i, items in enumerate(allPos):
exclude_index.extend([range_i] * len(items))
exclude_items.extend(items)
rating[exclude_index, exclude_items] = -(1 << 10)
# get top k rating item index
_, rating_K = torch.topk(rating, k=max_K)
rating_K = rating_K.cpu().numpy()
# compute auc
rating = rating.cpu().numpy()
aucs = [utils.AUC(rating[i], test_data) for i, test_data in enumerate(groundTrue)]
auc_record.extend(aucs)
# store batch
del rating
users_list.append(batch_users)
rating_list.append(rating_K)
groundTrue_list.append(groundTrue)
# compute metric
assert total_batch == len(users_list)
pre_results = []
for rl, gt in zip(rating_list, groundTrue_list):
batch_ks_result = np.zeros((3, len(args.topks)), dtype=np.float32)
for i, k in enumerate(args.topks):
batch_ks_result[:, i] = np.array([utils.test_batch(rl[:, :k], gt)], dtype=np.float32)
pre_results.append(batch_ks_result)
for result in pre_results:
results['precision'] += result[0]
results['recall'] += result[1]
results['ndcg'] += result[2]
results['precision'] /= float(len(users))
results['recall'] /= float(len(users))
results['ndcg'] /= float(len(users))
results['auc'] = np.mean(auc_record)
return results
