def callback(self, epoch):
metric_vars = [i for i in tf.local_variables() if 'auc_roc' in i.name.split('/')[1]]
keras.backend.get_session().run(tf.initializers.variables(metric_vars))
keras.backend.set_value(self.model.optimizer.lr,
keras.backend.get_value(self.model.optimizer.lr) * self.config.learning_rate_decay)
if epoch or True:
def __gen__():
for i, (y_pred, y_true) in zip(range(self.config.testing_impression), self.test):
auc = roc_auc_score(y_true, y_pred)
ndcgx = utils.ndcg_score(y_true, y_pred, 10)
ndcgv = utils.ndcg_score(y_true, y_pred, 5)
mrr = utils.mrr_score(y_true, y_pred)
pos = np.sum(y_true)
size = len(y_true)
yield auc, ndcgx, ndcgv, mrr, pos, size, i
values = [np.mean(x) for x in zip(*__gen__())]
utils.logging_evaluation(dict(auc=values[0], ndcgx=values[1], ndcgv=values[2], mrr=values[3]))
utils.logging_evaluation(dict(pos=values[4], size=values[5], num=values[6] * 2 + 1))
if epoch == self.config.epochs - 1 and tf.gfile.Exists(self.config.testing_data_input):
self.is_training = False
values = [np.mean(x) for x in zip(*__gen__())]
utils.logging_evaluation(dict(auc=values[0], ndcgx=values[1], ndcgv=values[2], mrr=values[3]))
utils.logging_evaluation(dict(pos=values[4], size=values[5], num=values[6] * 2 + 1))
def train(config):
# "1. log config"
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO,
handlers=[
logging.FileHandler(config.log_output),
logging.StreamHandler()
])
# "2. training loop: data, model and process"
UM = UserModeling(config)
training_data = UM.train
for i in range(config.rounds):
logging.info("launching the {} round of {}".format(i, m))
for epoch in range(config.epochs):
logging.info('[+] start epoch {}'.format(epoch))
model = UM.build_model(epoch)
history = model.fit_generator(
training_data,
config.training_step,
epochs=epoch + 1,
initial_epoch=epoch,
callbacks=None,
verbose=1 if config.debug and not config.background else 2)
utils.logging_history(history)
try:
evaluations = model.evaluate_generator(
UM.valid,
config.validation_step,
verbose=1 if config.debug and not config.background else 2)
assert len(evaluations) == len(model.metrics_names)
utils.logging_evaluation(
dict(zip(model.metrics_names, evaluations)))
except:
pass
if hasattr(UM, 'callback_valid'):
UM.callback_valid(epoch)
logging.info('[-] finish epoch {}'.format(epoch))
if hasattr(UM, 'callback'):
UM.callback(epoch)
# "3. save model to .json and .pkl"
print("saving model to file ...")
UM.save_model()
# K.clear_session()
return 0
def train(config: settings.Config):
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO,
handlers=[
logging.FileHandler(config.log_output),
logging.StreamHandler()
])
task_handler = task.get(config)
training_data = task_handler.train
for epoch in range(config.epochs):
logging.info('[+] start epoch {}'.format(epoch))
model = task_handler.build_model(epoch)
history = model.fit_generator(
training_data,
task_handler.training_step,
epochs=epoch + 1,
initial_epoch=epoch,
verbose=1 if config.debug and not config.background else 2)
utils.logging_history(history)
if hasattr(task_handler, 'callback'):
task_handler.callback(epoch)
try:
evaluations = model.evaluate_generator(
task_handler.valid,
task_handler.validation_step,
verbose=1 if config.debug and not config.background else 2)
utils.logging_evaluation(
dict(zip(model.metrics_names, evaluations)))
except:
pass
if hasattr(task_handler, 'callback_valid'):
task_handler.callback_valid(epoch)
logging.info('[-] finish epoch {}'.format(epoch))
task_handler.save_model()
return 0
def cook(config: settings.Config):
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO,
handlers=[
logging.FileHandler(config.log_output),
logging.StreamHandler()
])
task_handler = task.get(config)
for epoch in range(config.epochs):
logging.info('[+] start epoch {}'.format(epoch))
model = task_handler.build_model(epoch)
if config.use_generator:
history = model.fit_generator(
task_handler.train(),
task_handler.training_step,
epochs=epoch + 1,
initial_epoch=epoch,
verbose=1 if config.debug and not config.background else 2)
else:
history = model.fit(
*task_handler.train(),
config.batch_size,
epochs=epoch + 1,
initial_epoch=epoch,
shuffle=True,
verbose=1 if config.debug and not config.background else 2)
utils.logging_history(history)
if hasattr(task_handler, 'callback'):
task_handler.callback(epoch)
try:
if config.use_generator:
evaluations = model.evaluate_generator(
task_handler.valid(),
steps=task_handler.validation_step,
verbose=1 if config.debug and not config.background else 2)
else:
evaluations = task_handler.test_model.evaluate(
*task_handler.valid(),
config.batch_size,
verbose=1 if config.debug and not config.background else 0)
utils.logging_evaluation(
dict(zip(task_handler.test_model.metrics_names, evaluations)))
except Exception as e:
print(e)
if hasattr(task_handler, 'callback_valid'):
task_handler.callback_valid(epoch)
logging.info('[-] finish epoch {}'.format(epoch))
if config.use_generator:
users_ = []
imprs_ = []
mask_ = []
y_true_ = []
y_pred_ = []
for feature, [users, imprs, mask, y_true] in task_handler.test():
users_.append(users)
imprs_.append(imprs)
mask_.append(mask)
y_true_.append(y_true)
y_pred_.append(
task_handler.test_model.predict_on_batch(feature).reshape(
(-1, )))
users = np.hstack(users_)
imprs = np.hstack(imprs_)
mask = np.hstack(mask_)
y_true = np.hstack(y_true_)
y_pred = np.hstack(y_pred_)
else:
feature, [users, imprs, mask, y_true] = task_handler.test()
y_pred = task_handler.test_model.predict(
feature,
batch_size=config.batch_size,
verbose=1
if config.debug and not config.background else 0).reshape((-1, ))
class Result:
def __init__(self, auc, mrr, ndcgv, ndcgx, pos, size, idx):
self.auc = auc
self.mrr = mrr
self.ndcgv = ndcgv
self.ndcgx = ndcgx
self.pos = pos
self.size = size
self.idx = idx
@property
def result(self):
return dict(auc=self.auc,
ndcgx=self.ndcgx,
ndcgv=self.ndcgv,
mrr=self.mrr)
@property
def info(self):
return dict(pos=self.pos, size=self.size, num=self.idx * 2 + 1)
def average(results):
return Result(np.mean([result.auc for result in results]),
np.mean([result.mrr for result in results]),
np.mean([result.ndcgv for result in results]),
np.mean([result.ndcgx for result in results]),
np.mean([result.pos for result in results]),
np.mean([result.size for result in results]),
np.mean([result.idx for result in results]))
present_user = users[0]
present_impr = imprs[0]
index = 0
impr_index = 0
user_results = []
impr_results = []
iv_user_results = []
oov_user_results = []
for i, user, impr in zip(range(1, len(y_pred)), users[1:], imprs[1:]):
if user != present_user or impr != present_impr:
try:
impr_results.append(
Result(
task.roc_auc_score(y_true[index:i], y_pred[index:i]),
utils.mrr_score(y_true[index:i], y_pred[index:i]),
utils.ndcg_score(y_true[index:i], y_pred[index:i], 5),
utils.ndcg_score(y_true[index:i], y_pred[index:i], 10),
sum(y_true[index:i]), i - index, len(impr_results)))
index = i
present_impr = impr
except Exception as e:
utils.interactive_console(locals())
if user != present_user:
avg = average(impr_results[impr_index:])
user_results.append(avg)
if mask[index] == 1:
iv_user_results.append(avg)
elif mask[index] == 0:
oov_user_results.append(avg)
impr_index = len(impr_results)
present_user = user
user_result = average(user_results)
impr_result = average(impr_results)
iv_user_result = average(iv_user_results)
oov_user_result = average(oov_user_results)
utils.logging_evaluation(user_result.result)
utils.logging_evaluation(user_result.info)
utils.logging_evaluation(impr_result.result)
utils.logging_evaluation(impr_result.info)
utils.logging_evaluation(iv_user_result.result)
utils.logging_evaluation(iv_user_result.info)
utils.logging_evaluation(oov_user_result.result)
utils.logging_evaluation(oov_user_result.info)
return 0
)
task_handler = task.get(config)
training_data = task_handler.train
#for i in range(100):
# next(training_data)
#print(task_handler.training_step)
for epoch in range(config.epochs):
logging.info('[+] start epoch {}'.format(epoch))
model = task_handler.build_model(epoch)
model.summary()
history = model.fit_generator(
training_data,
1,
# validation_data=task_handler.valid,
# validation_steps=task_handler.validation_step,
epochs=epoch + 1,
initial_epoch=epoch,
verbose=1 if config.debug and not config.background else 2)
utils.logging_history(history)
task_handler.callback(epoch)
evaluations = model.evaluate_generator(
task_handler.valid,
task_handler.validation_step,
verbose=1 if config.debug and not config.background else 2)
utils.logging_evaluation(dict(zip(model.metrics_names, evaluations)))
logging.info('[-] finish epoch {}'.format(epoch))
task_handler.save_model()