def train(self, eval_scheme=None, use_async_eval=True):
model_train = self.model_train
model_all_loss = self.model_all_loss
conf = self.conf
data_helper = self.data_helper
train = data_helper.data['train']
C = data_helper.data['C']
train_time = []
for epoch in range(conf.max_epoch + 1):
bb, b = 0, conf.batch_size_p
cost, it = 0, 0
monitor_losses = model_all_loss.keys()
costs = dict(zip(monitor_losses, np.zeros(len(monitor_losses))))
monitor_rate, monitor_it = self.monitor_rate, 0
np.random.shuffle(train)
t_start = time.time()
while epoch > 0 and bb < len(train):
it += 1
b = bb + conf.batch_size_p
if b > len(train):
# get rid of uneven tail so no need to dynamically adjust batch_size_p
break
train_batch = train[bb:b]
user_batch = train_batch[:, 0]
item_batch = train_batch[:, 1]
response_batch = train_batch[:, 2]
cost += model_train.train_on_batch([user_batch, item_batch],
[response_batch])
if random.random() < monitor_rate:
monitor_it += 1
for lname, m in model_all_loss.iteritems():
costs[lname] += m.train_on_batch(
[user_batch, item_batch], [response_batch])
bb = b
if epoch > 0:
train_time.append(time.time() - t_start)
print get_cur_time(), 'epoch %d (%d it)' % (epoch, it), \
'cost %.5f' % (cost / it if it > 0 else -1),
nan_detection('cost', cost)
if monitor_rate > 0:
print '(',
for loss in monitor_losses:
print '{}={}'.format(
loss,
costs[loss] / monitor_it if monitor_it > 0 else -1),
print ')',
if eval_scheme is None:
print ''
else:
async_eval = True \
if use_async_eval and epoch != conf.max_epoch else False
try:
ps[-1].join()
except:
pass
ps = self.test(eval_scheme, use_async_eval=async_eval)
print 'Training time (sec) per epoch:', np.mean(train_time)
def train(self, eval_scheme=None, use_async_eval=True):
model_train = self.model_train
neg_sign = self.neg_sign
conf = self.conf
data_helper = self.data_helper
num_negatives = conf.num_negatives
batch_size_p = conf.batch_size_p
sample_batch_u = self.sample_batch_u
train = data_helper.data['train']
C = data_helper.data['C']
train_time = []
for epoch in range(conf.max_epoch + 1):
bb, b = 0, batch_size_p
cost, it = 0, 0
np.random.shuffle(train)
t_start = time.time()
while epoch > 0 and bb < len(train):
it += 1
b = bb + batch_size_p
if b > len(train):
# get rid of uneven tail so no need to dynamically adjust batch_size_p
break
train_batch_p = train[bb:b]
train_batch_n = train_batch_p.repeat(num_negatives, axis=0)
train_batch_n[:,
0] = sample_batch_u(num_negatives * batch_size_p)
train_batch_n[:, 2] = neg_sign
train_batch = np.vstack((train_batch_p, train_batch_n))
user_batch = train_batch[:, 0]
item_batch = train_batch[:, 1]
response_batch = train_batch[:, 2]
cost += model_train.train_on_batch([user_batch, item_batch],
[response_batch])
bb = b
if epoch > 0:
train_time.append(time.time() - t_start)
print get_cur_time(), 'epoch %d (%d it)' % (epoch, it), \
'cost %.5f' % (cost / it if it > 0 else -1),
nan_detection('cost', cost)
if eval_scheme is None:
print ''
else:
async_eval = True \
if use_async_eval and epoch != conf.max_epoch else False
try:
ps[-1].join()
except:
pass
ps = self.test(eval_scheme, use_async_eval=async_eval)
print 'Training time (sec) per epoch:', np.mean(train_time)
def train(self, eval_scheme=None, use_async_eval=True):
model_train = self.model_train
conf = self.conf
data_helper = self.data_helper
train = data_helper.data['train']
C = data_helper.data['C']
num_negatives = conf.num_negatives
sample_batch = self.sample_batch
train_batch_n = np.zeros((num_negatives, 3))
response_batch = np.ones((conf.batch_size_p + num_negatives, #dummy
1 + num_negatives))
response_batch[:, 1:] = self.neg_sign
train_time = []
for epoch in range(conf.max_epoch + 1):
bb, b = 0, conf.batch_size_p
train = group_shuffle_train(train, by='item', \
chop=conf.chop_size, iidx=self._iidx['item'])
cost, it = 0, 0
t_start = time.time()
while epoch > 0 and bb < len(train):
it += 1
b = bb + conf.batch_size_p
if b > len(train):
# get rid of uneven tail so no need to dynamically adjust batch_size_p
break
train_batch_p = train[bb: b]
train_batch_n[:, 1] = sample_batch(train_batch_n.shape[0])
train_batch = np.vstack((train_batch_p, train_batch_n))
user_batch = train_batch[:, 0]
item_batch = train_batch[:, 1]
cost += model_train.train_on_batch([user_batch, item_batch],
[response_batch])
bb = b
if epoch > 0:
train_time.append(time.time() - t_start)
print get_cur_time(), 'epoch %d (%d it)' % (epoch, it), \
'cost %.5f' % (cost / it if it > 0 else -1),
nan_detection('cost', cost)
if eval_scheme is None:
print ''
else:
async_eval = True \
if use_async_eval and epoch != conf.max_epoch else False
try: ps[-1].join()
except: pass
ps = self.test(eval_scheme, use_async_eval=async_eval)
print 'Training time (sec) per epoch:', np.mean(train_time)
def train(self, eval_scheme=None, use_async_eval=True):
model_train = self.model_train
conf = self.conf
data_helper = self.data_helper
batch_size_p = conf.batch_size_p
train = data_helper.data['train']
C = data_helper.data['C']
group_shuffling_trick = self.group_shuffling_trick
train_time = []
for epoch in range(conf.max_epoch + 1):
bb, b = 0, batch_size_p
cost, it = 0, 0
if group_shuffling_trick:
train = group_shuffle_train(train, by='item', \
chop=conf.chop_size, iidx=self._iidx['item'])
t_start = time.time()
while epoch > 0 and bb < len(train):
it += 1
b = bb + batch_size_p
if b > len(train):
# get rid of uneven tail so no need to dynamically adjust batch_size_p
break
if group_shuffling_trick:
train_batch = train[bb: b]
else:
train_batch = self.group_sample(batch_size_p)
user_batch = train_batch[:, 0]
item_batch = train_batch[:, 1]
response_batch = train_batch[:, 2]
cost += model_train.train_on_batch([user_batch, item_batch],
[response_batch])
bb = b
if epoch > 0:
train_time.append(time.time() - t_start)
print get_cur_time(), 'epoch %d (%d it)' % (epoch, it), \
'cost %.5f' % (cost / it if it > 0 else -1),
nan_detection('cost', cost)
if eval_scheme is None:
print ''
else:
async_eval = True \
if use_async_eval and epoch != conf.max_epoch else False
try: ps[-1].join()
except: pass
ps = self.test(eval_scheme, use_async_eval=async_eval)
print 'Training time (sec) per epoch:', np.mean(train_time)
def train(self, eval_scheme=None, use_async_eval=True):
model_train = self.model_train
conf = self.conf
data_helper = self.data_helper
num_negatives = conf.num_negatives
batch_size_p = conf.batch_size_p
train = data_helper.data['train']
C = data_helper.data['C']
group_shuffling_trick = self.group_shuffling_trick
if group_shuffling_trick:
# Support both user and item based sampling
# Only when group_shuffling_trick is True
shuffle_st = conf.shuffle_st
if shuffle_st.startswith('by_user'):
by = 'user'
elif shuffle_st.startswith('by_item'):
by = 'item'
else:
assert False
print '[INFO] sampling group based on {}'.format(by)
else:
print '[INFO] sampling group based on item'
train_time = []
for epoch in range(conf.max_epoch + 1):
bb, b = 0, batch_size_p
cost, it = 0, 0
if group_shuffling_trick:
train = group_shuffle_train(train, by=by, \
chop=conf.chop_size, iidx=self._iidx[by])
t_start = time.time()
while epoch > 0 and bb < len(train):
it += 1
b = bb + batch_size_p
if b > len(train):
# get rid of uneven tail so no need to dynamically adjust batch_size_p
break
if group_shuffling_trick:
train_batch_p = train[bb: b]
train_batch_n = train_batch_p.repeat(num_negatives, axis=0)
if by == 'item':
train_batch_n[:, 0] = self.sample_batch_u( \
num_negatives * batch_size_p)
else:
train_batch[:, 1] = self.sample_batch( \
num_negatives * batch_size_p)
train_batch_n[:, 2] = self.neg_sign
train_batch = np.vstack((train_batch_p, train_batch_n))
else:
train_batch = self.group_sample_with_negs(batch_size_p,
num_negatives)
user_batch = train_batch[:, 0]
item_batch = train_batch[:, 1]
response_batch = train_batch[:, 2]
cost += model_train.train_on_batch([user_batch, item_batch],
[response_batch])
bb = b
if epoch > 0:
train_time.append(time.time() - t_start)
print get_cur_time(), 'epoch %d (%d it)' % (epoch, it), \
'cost %.5f' % (cost / it if it > 0 else -1),
nan_detection('cost', cost)
if eval_scheme is None:
print ''
else:
async_eval = True \
if use_async_eval and epoch != conf.max_epoch else False
try: ps[-1].join()
except: pass
ps = self.test(eval_scheme, use_async_eval=async_eval)
print 'Training time (sec) per epoch:', np.mean(train_time)
def train(self, eval_scheme=None, use_async_eval=True):
model_train = self.model_train
neg_sign = self.neg_sign
conf = self.conf
num_negatives = conf.num_negatives
batch_size = conf.batch_size_p * (1 + num_negatives)
data_helper = self.data_helper
sample_batch = self.sample_batch
sample_batch_u = self.sample_batch_u
train_p = data_helper.data['train']
C = data_helper.data['C']
chop = conf.chop_size
train_time = []
for epoch in range(conf.max_epoch + 1):
bb, b = 0, batch_size
cost, it = 0, 0
# pre-sample
if conf.shuffle_st == 'original':
# sampling as in original training scheme (given link, sample neg users)
np.random.shuffle(train_p)
train = train_p.repeat(1 + num_negatives, axis=0)
train[:, 1] = sample_batch(train.shape[0])
train[:, 2] = neg_sign
train[::1 + num_negatives] = train_p
elif conf.shuffle_st == 'reverse':
np.random.shuffle(train_p)
# sampling as in reverse training scheme (given link, sample neg users)
train = train_p.repeat(1 + num_negatives, axis=0)
train[:, 0] = sample_batch_u(train.shape[0])
train[:, 2] = neg_sign
train[::1 + num_negatives] = train_p
else: # random sampling and/or groupping
train_n = train_p.repeat(num_negatives, axis=0)
train_n[:, 1] = sample_batch(train_n.shape[0])
train_n[:, 2] = neg_sign
train = np.vstack((train_p, train_n))
if conf.shuffle_st == 'random':
np.random.shuffle(train)
elif conf.shuffle_st == 'by_user':
train = group_shuffle_train(train,
by='user',
iidx=self._iidx['user'])
elif conf.shuffle_st == 'by_item':
train = group_shuffle_train(train,
by='item',
iidx=self._iidx['item'])
elif conf.shuffle_st.startswith('by_user_chop'):
train = group_shuffle_train(train,
by='user',
chop=chop,
iidx=self._iidx['user'])
elif conf.shuffle_st.startswith('by_item_chop'):
train = group_shuffle_train(train,
by='item',
chop=chop,
iidx=self._iidx['item'])
elif conf.shuffle_st.startswith('by_useritem_chop'):
if epoch % 2 == 0:
train = group_shuffle_train(train,
by='user',
chop=chop,
iidx=self._iidx['user'])
else:
train = group_shuffle_train(train,
by='item',
chop=chop,
iidx=self._iidx['item'])
else:
assert False, 'ERROR: unknown shuffle strategy {}'.format(
conf.shuffle_st)
t_start = time.time()
while epoch > 0 and bb < len(train):
it += 1
b = bb + batch_size
if b > len(train):
# get rid of uneven tail, otherwise need to dynamically adjust batch_size
break
train_batch = train[bb:b]
user_batch = train_batch[:, 0]
item_batch = train_batch[:, 1]
response_batch = train_batch[:, 2]
cost += model_train.train_on_batch([user_batch, item_batch],
[response_batch])
bb = b
if epoch > 0:
train_time.append(time.time() - t_start)
print get_cur_time(), 'epoch %d (%d it)' % (epoch, it), \
'cost %.5f' % (cost / it if it > 0 else -1),
nan_detection('cost', cost)
if eval_scheme is None:
print ''
else:
async_eval = True \
if use_async_eval and epoch != conf.max_epoch else False
try:
ps[-1].join()
except:
pass
ps = self.test(eval_scheme, use_async_eval=async_eval)
print 'Training time (sec) per epoch:', np.mean(train_time)
def run(self, model, predict_only=False, verbose=True,
eval_scheme=None, batch_size=1024, use_async_eval=False):
''' predict_only: bool
if True, return prediction; otherwise, evaluate normally
eval_scheme: given or whole
if given, evaluate with given test list
if whole, predict for all items
batch_size: int
> 0 if want to use batch to predict
<= 0 if want to use whole to predict at once
use_async_eval: bool
if True, will create new thread for running part of eval func,
so that GPU will not be left empty
'''
if eval_scheme is None:
eval_scheme = self.eval_scheme
self._check_eval_sheme(eval_scheme)
C = self.C
test_seen = self.test_seen
test = self.test
eval_topk = self.eval_topk
user_count = self.user_count
item_count = self.item_count
train_true_mat = self.train_true_mat
test_true_mat = self.test_true_mat
train_items = self.train_items
test_items = self.test_items
if eval_scheme == 'given':
model_dict = model
test_eval_gen = test_eval([test_seen, test], eval_topk, \
model_dict, user_count, item_count, C, \
predict_only=predict_only)
train_results = test_eval_gen.next()
test_results = test_eval_gen.next()
if verbose and not predict_only:
print get_cur_time(), 'train map/auc', \
train_results['map@%s' % eval_topk], train_results['auc'], \
'test map/auc', test_results['map@%s' % eval_topk], test_results['auc']
return train_results, test_results
elif eval_scheme == 'whole':
assert eval_topk > 0, \
'[ERROR] eval_top {} must > 0'.format(eval_topk) + \
'when eval_scheme=whole'
model_predict = model
predict_only_local = True \
if predict_only or use_async_eval else False
train_results = test_eval_mat(model_predict, eval_topk, \
train_true_mat, range(user_count), train_items, C, \
predict_only=predict_only_local, batch_size=batch_size)
test_results = test_eval_mat(model_predict, eval_topk, \
test_true_mat, range(user_count), test_items, C, \
predict_only=predict_only_local, batch_size=batch_size)
if verbose and not predict_only:
if use_async_eval:
import threading
t = threading.Thread(target=evaluate_mat_thread,
args=(train_results, test_results,
eval_topk))
print get_cur_time(),
t.start()
train_results = test_results = t
else:
print get_cur_time(), \
'train recall/map', train_results['recall@%s' % eval_topk], \
train_results['map@%s' % eval_topk], \
'test recall/map', test_results['recall@%s' % eval_topk], \
test_results['map@%s' % eval_topk]
return train_results, test_results
def train(self, eval_scheme=None, use_async_eval=True):
model_train = self.model_train
model_all_loss = self.model_all_loss
neg_sign = self.neg_sign
conf = self.conf
num_negatives = conf.num_negatives
batch_size_p = conf.batch_size_p
data_helper = self.data_helper
sample_batch = self.sample_batch
train = data_helper.data['train']
C = data_helper.data['C']
train_time = []
for epoch in range(conf.max_epoch + 1):
bb, b = 0, batch_size_p
np.random.shuffle(train)
cost, it = 0, 0
monitor_losses = model_all_loss.keys()
costs = dict(zip(monitor_losses, np.zeros(len(monitor_losses))))
monitor_rate, monitor_it = self.monitor_rate, 0
t_start = time.time()
while epoch > 0 and bb < len(train):
it += 1
b = bb + batch_size_p
if b > len(train):
# get rid of uneven tail so no need to dynamically adjust batch_size_p
break
train_batch_p = train[bb:b]
train_batch_n = train_batch_p.repeat(num_negatives, axis=0)
train_batch_n[:,
1] = sample_batch(num_negatives * batch_size_p)
train_batch_n[:, 2] = neg_sign
train_batch = np.vstack((train_batch_p, train_batch_n))
user_batch = train_batch[:, 0]
item_batch = train_batch[:, 1]
response_batch = train_batch[:, 2]
cost += model_train.train_on_batch([user_batch, item_batch],
[response_batch])
if random.random() < monitor_rate:
monitor_it += 1
for lname, m in model_all_loss.iteritems():
neg_sign_tmp = np.array([-1], dtype='int32') \
if lname == 'skip-gram' else 0
response_batch[batch_size_p:] = neg_sign_tmp
costs[lname] += m.train_on_batch(
[user_batch, item_batch], [response_batch])
bb = b
if epoch > 0:
train_time.append(time.time() - t_start)
print get_cur_time(), 'epoch %d (%d it)' % (epoch, it), \
'cost %.5f' % (cost / it if it > 0 else -1),
nan_detection('cost', cost)
if monitor_rate > 0:
print '(',
for loss in monitor_losses:
print '{}={}'.format(
loss,
costs[loss] / monitor_it if monitor_it > 0 else -1),
print ')',
if eval_scheme is None:
print ''
else:
async_eval = True \
if use_async_eval and epoch != conf.max_epoch else False
try:
ps[-1].join()
except:
pass
ps = self.test(eval_scheme, use_async_eval=async_eval)
print 'Training time (sec) per epoch:', np.mean(train_time)