def run(simulator, dataset, run):
click_model = DFBNCM(256, 1024, 10240, 700, 700, dataset,
model=load_model('../click_model_results/DFBNCM_model/{}/train_set{}.h5'.format(simulator.name, run)))
click_log_path = "../click_logs/{}/train_set{}.txt".format(simulator.name, run)
click_log = rf.read_click_log(click_log_path)
click_model.initial_representation(click_log)
# click_model.train_tfrecord('../click_logs/{}/train_set{}_DFBNCM.tfrecord'.format(simulator.name, run),
# batch_size=64,
# epoch=20,
# steps_per_epoch=1)
#
# click_model.model.save("../click_model_results/DFBNCM_model/{}/train_set{}.h5".format(simulator.name, run))
test_click_log_path = "../click_logs/{}/seen_set{}.txt".format(simulator.name, run)
query_frequency_path = "../click_logs/{}/query_frequency{}.txt".format(simulator.name, run)
test_click_log = rf.read_click_log(test_click_log_path)
query_frequency = rf.read_query_frequency(query_frequency_path)
f = open("../click_model_results/{}/seen_set{}_{}_result.txt".format(simulator.name, run, "DFBNCM")
, "w+")
test_logs = {'10': [],
'100': [],
'1000': [],
'10000': []
}
for i in range(test_click_log.shape[0]):
qid = test_click_log[i][0]
test_logs[query_frequency[qid]].append(test_click_log[i])
frequencies = ['10', '100', '1000', '10000']
# i = 0
f.write("Click Model:" + "DFBNCM" + "\n")
for freq in frequencies:
perplexities = click_model.get_perplexity(np.array(test_logs[freq]))
MSEs = click_model.get_MSE(np.array(test_logs[freq]), dataset, simulator)
perplexity_line = "Frequency " + freq + " perplexities:"
MSEs_line = "Frequency " + freq + " MSE:"
for perp in perplexities:
perplexity_line += " " + str(perp)
for MSE in MSEs:
MSEs_line += " " + str(MSE)
f.write(perplexity_line + "\n")
f.write(MSEs_line + "\n")
f.close()
def job(click_log_path, output_path, simulator, dataset):
model = FBNCM(64, 700, 700, dataset)
click_log = rf.read_click_log(click_log_path)
model.initial_representation(click_log)
model.save_training_tfrecord(click_log, output_path, simulator)
from dataset import LetorDataset
import numpy as np
from clickModel.LSTMv2 import LSTMv2
from utils import read_file as rf
from clickModel.DCTR import DCTR
train_path = "../datasets/ltrc_yahoo/test_set.txt"
print("loading training set.......")
train_set = LetorDataset(train_path, 700)
click_log_path = "../datasets/ltrc_yahoo/test_click_log.txt"
test_click_log_path = "../datasets/ltrc_yahoo/test_click_log_test.txt"
click_log = rf.read_click_log(click_log_path)
test_click_log = rf.read_click_log(test_click_log_path)
pc = [0.05, 0.3, 0.5, 0.7, 0.95]
ps = [0.2, 0.3, 0.5, 0.7, 0.9]
simulator = DCTR(pc)
print(click_log.shape)
print(test_click_log.shape)
#
click_model = LSTMv2(700, 1024, train_set)
click_model.train(click_log)
print(
click_model.get_MSE(
test_click_log[np.random.choice(test_click_log.shape[0], 100)],
train_set, simulator))
train_path = "../datasets/ltrc_yahoo/set1.train.txt"
print("loading training set.......")
dataset = LetorDataset(train_path, 700)
model = NCM(64, 1024, 10240)
pc = [0.05, 0.3, 0.5, 0.7, 0.95]
ps = [0.2, 0.3, 0.5, 0.7, 0.9]
simulator = SDBN(pc, ps)
click_log_path = "../feature_click_datasets/{}/train_set_test.txt".format(
"SDBN", "1")
# click_log_path = "../click_logs/{}/train_set{}_small.txt".format("SDBN", "1")
click_log = rf.read_click_log(click_log_path)
model.initial_representation(click_log)
# model.save_training_set_numpy(click_log, "test", "SDBN")
# model.save_training_set(click_log, "../click_logs/{}/train_set{}_small_NCM.tfrecord".format("SDBN", "1"), "SDBN")
model.save_training_tfrecord(click_log, "NCM_test.tfrecord", "SDBN")
# model.save_training_set_numpy(click_log, "../click_logs/{}/train_set{}_NCM".format("SDBN", "1"), "SDBN")
# model.save_training_set_numpy(click_log, "test", "SDBN")
# data = np.load("test.npz")
# X = data["input"]
# Y = data["label"]
#
# data = np.load("../click_logs/{}/train_set{}_NCM.npy.npz".format("SDBN", "1"))
#
def job(click_log_path, output_path, simulator):
model = NCM(64, 1024, 10240)
click_log = rf.read_click_log(click_log_path)
model.initial_representation(click_log)
model.save_training_tfrecord(click_log, output_path, simulator)