def test_model_dkn(resource_path):
data_path = os.path.join(resource_path, "..", "resources", "deeprec",
"dkn")
yaml_file = os.path.join(data_path, "dkn.yaml")
train_file = os.path.join(data_path, "final_test_with_entity.txt")
valid_file = os.path.join(data_path, "final_test_with_entity.txt")
wordEmb_file = os.path.join(data_path, "word_embeddings_100.npy")
entityEmb_file = os.path.join(data_path, "TransE_entity2vec_100.npy")
if not os.path.exists(yaml_file):
download_deeprec_resources(
"https://recodatasets.blob.core.windows.net/deeprec/",
data_path,
"dknresources.zip",
)
hparams = prepare_hparams(
yaml_file,
wordEmb_file=wordEmb_file,
entityEmb_file=entityEmb_file,
epochs=1,
learning_rate=0.0001,
)
input_creator = DKNTextIterator
model = DKN(hparams, input_creator)
assert isinstance(model.fit(train_file, valid_file), BaseModel)
assert model.run_eval(valid_file) is not None
def test_model_dkn(resource_path):
data_path = os.path.join(resource_path, "..", "resources", "deeprec",
"dkn")
yaml_file = os.path.join(data_path, r'dkn.yaml')
train_file = os.path.join(data_path, r'train_mind_demo.txt')
valid_file = os.path.join(data_path, r'valid_mind_demo.txt')
test_file = os.path.join(data_path, r'test_mind_demo.txt')
news_feature_file = os.path.join(data_path, r'doc_feature.txt')
user_history_file = os.path.join(data_path, r'user_history.txt')
wordEmb_file = os.path.join(data_path, r'word_embeddings_100.npy')
entityEmb_file = os.path.join(data_path, r'TransE_entity2vec_100.npy')
contextEmb_file = os.path.join(data_path, r'TransE_context2vec_100.npy')
download_deeprec_resources(
"https://recodatasets.blob.core.windows.net/deeprec/",
data_path,
"mind-demo.zip",
)
hparams = prepare_hparams(yaml_file,
news_feature_file=news_feature_file,
user_history_file=user_history_file,
wordEmb_file=wordEmb_file,
entityEmb_file=entityEmb_file,
contextEmb_file=contextEmb_file,
epochs=1,
learning_rate=0.0001)
input_creator = DKNTextIterator
model = DKN(hparams, input_creator)
assert isinstance(model.fit(train_file, valid_file), BaseModel)
assert model.run_eval(valid_file) is not None
def test_model_dkn(resource_path):
data_path = os.path.join(resource_path, "..", "resources", "deeprec", "dkn")
yaml_file = os.path.join(data_path, "dkn.yaml")
train_file = os.path.join(data_path, "final_test_with_entity.txt")
valid_file = os.path.join(data_path, "final_test_with_entity.txt")
wordEmb_file = os.path.join(data_path, "word_embeddings_100.npy")
entityEmb_file = os.path.join(data_path, "TransE_entity2vec_100.npy")
if not os.path.exists(yaml_file):
download_deeprec_resources(
"https://recodatasets.blob.core.windows.net/deeprec/",
data_path,
"dknresources.zip",
)
hparams = prepare_hparams(
yaml_file,
wordEmb_file=wordEmb_file,
entityEmb_file=entityEmb_file,
epochs=1,
learning_rate=0.0001,
)
input_creator = DKNTextIterator
model = DKN(hparams, input_creator)
assert isinstance(model.fit(train_file, valid_file), BaseModel)
assert model.run_eval(valid_file) is not None
def test_dkn_component_definition(resource_path):
data_path = os.path.join(resource_path, "..", "resources", "deeprec",
"dkn")
yaml_file = os.path.join(data_path, "dkn.yaml")
wordEmb_file = os.path.join(data_path, "word_embeddings_100.npy")
entityEmb_file = os.path.join(data_path, "TransE_entity2vec_100.npy")
if not os.path.exists(yaml_file):
download_deeprec_resources(
"https://recodatasets.blob.core.windows.net/deeprec/",
data_path,
"dknresources.zip",
)
hparams = prepare_hparams(
yaml_file,
wordEmb_file=wordEmb_file,
entityEmb_file=entityEmb_file,
epochs=1,
learning_rate=0.0001,
)
assert hparams is not None
model = DKN(hparams, DKNTextIterator)
assert model.logit is not None
assert model.update is not None
assert model.iterator is not None
def test_dkn_component_definition(resource_path):
data_path = os.path.join(resource_path, "..", "resources", "deeprec",
"dkn")
yaml_file = os.path.join(data_path, "dkn.yaml")
news_feature_file = os.path.join(data_path, r'doc_feature.txt')
user_history_file = os.path.join(data_path, r'user_history.txt')
wordEmb_file = os.path.join(data_path, r'word_embeddings_100.npy')
entityEmb_file = os.path.join(data_path, r'TransE_entity2vec_100.npy')
contextEmb_file = os.path.join(data_path, r'TransE_context2vec_100.npy')
download_deeprec_resources(
"https://recodatasets.blob.core.windows.net/deeprec/",
data_path,
"mind-demo.zip",
)
hparams = prepare_hparams(yaml_file,
news_feature_file=news_feature_file,
user_history_file=user_history_file,
wordEmb_file=wordEmb_file,
entityEmb_file=entityEmb_file,
contextEmb_file=contextEmb_file,
epochs=1,
learning_rate=0.0001)
assert hparams is not None
model = DKN(hparams, DKNTextIterator)
assert model.logit is not None
assert model.update is not None
assert model.iterator is not None
def test_dkn_component_definition(deeprec_resource_path):
data_path = os.path.join(deeprec_resource_path, "dkn")
yaml_file = os.path.join(data_path, "dkn.yaml")
news_feature_file = os.path.join(data_path, r"doc_feature.txt")
user_history_file = os.path.join(data_path, r"user_history.txt")
wordEmb_file = os.path.join(data_path, r"word_embeddings_100.npy")
entityEmb_file = os.path.join(data_path, r"TransE_entity2vec_100.npy")
contextEmb_file = os.path.join(data_path, r"TransE_context2vec_100.npy")
download_deeprec_resources(
"https://recodatasets.z20.web.core.windows.net/deeprec/",
data_path,
"mind-demo.zip",
)
hparams = prepare_hparams(
yaml_file,
news_feature_file=news_feature_file,
user_history_file=user_history_file,
wordEmb_file=wordEmb_file,
entityEmb_file=entityEmb_file,
contextEmb_file=contextEmb_file,
epochs=1,
learning_rate=0.0001,
)
assert hparams is not None
model = DKN(hparams, DKNTextIterator)
assert model.logit is not None
assert model.update is not None
assert model.iterator is not None
### test DKN's item2item version
hparams = prepare_hparams(
yaml_file,
news_feature_file=news_feature_file,
wordEmb_file=wordEmb_file,
entityEmb_file=entityEmb_file,
contextEmb_file=contextEmb_file,
epochs=1,
is_clip_norm=True,
max_grad_norm=0.5,
his_size=20,
MODEL_DIR=os.path.join(data_path, "save_models"),
use_entity=True,
use_context=True,
)
hparams.neg_num = 9
assert hparams is not None
model_item2item = DKNItem2Item(hparams, DKNItem2itemTextIterator)
assert model_item2item.pred_logits is not None
assert model_item2item.update is not None
assert model_item2item.iterator is not None
def test_model_dkn(resource_path):
data_path = os.path.join(resource_path, '../resources/deeprec/dkn')
yaml_file = os.path.join(data_path, r'dkn.yaml')
train_file = os.path.join(data_path, r'final_test_with_entity.txt')
valid_file = os.path.join(data_path, r'final_test_with_entity.txt')
wordEmb_file = os.path.join(data_path, r'word_embeddings_100.npy')
entityEmb_file = os.path.join(data_path, r'TransE_entity2vec_100.npy')
if not os.path.exists(yaml_file):
download_deeprec_resources(
r'https://recodatasets.blob.core.windows.net/deeprec/', data_path,
'dknresources.zip')
hparams = prepare_hparams(yaml_file,
wordEmb_file=wordEmb_file,
entityEmb_file=entityEmb_file,
epochs=1,
learning_rate=0.0001)
input_creator = DKNTextIterator
model = DKN(hparams, input_creator)
assert (isinstance(model.fit(train_file, valid_file), BaseModel))
assert model.run_eval(valid_file) is not None
# news_entities,
# train_entities,
# valid_entities,
# max_sentence=10,
# word_embedding_dim=100,
# )
news_feature_file = os.path.join(data_path, 'doc_feature.txt')
word_embeddings_file = os.path.join(data_path, 'word_embeddings_5w_100.npy')
user_history_file = os.path.join(data_path, 'user_history.txt')
entity_embeddings_file = os.path.join(data_path,
'entity_embeddings_5w_100.npy')
yaml_file = os.path.join(data_path, 'dkn_MINDlarge.yaml')
# yaml_file = maybe_download(url="https://recodatasets.blob.core.windows.net/deeprec/deeprec/dkn/dkn_MINDsmall.yaml",
# work_directory=data_path)
hparams = prepare_hparams(yaml_file,
news_feature_file=news_feature_file,
user_history_file=user_history_file,
wordEmb_file=word_embeddings_file,
entityEmb_file=entity_embeddings_file,
epochs=epochs,
history_size=history_size,
MODEL_DIR=os.path.join(data_path, 'save_models'),
batch_size=batch_size)
model = DKN(hparams, DKNTextIterator)
model.fit(train_file, valid_file)
res = model.run_eval(valid_file)
print(res)
word_embeddings_file = os.path.join(data_path, "word_embeddings_5w_100.npy")
entity_embeddings_file = os.path.join(data_path,
"entity_embeddings_5w_100.npy")
train_path = os.path.join(data_path, "train")
valid_path = os.path.join(data_path, "valid")
test_path = os.path.join(data_path, "test")
yaml_file = maybe_download(
url=
"https://recodatasets.blob.core.windows.net/deeprec/deeprec/dkn/dkn_MINDsmall.yaml",
work_directory=data_path)
hparams = prepare_hparams(yaml_file,
news_feature_file=news_feature_file,
user_history_file=user_history_file,
wordEmb_file=word_embeddings_file,
entityEmb_file=entity_embeddings_file,
epochs=epochs,
history_size=history_size,
batch_size=batch_size)
hparams.save_model = True
hparams.show_step = 5000
hparams.MODEL_DIR = 'para'
model = DKN(hparams, DKNTextIterator)
model.load_model('./para/epoch_5')
# model.run_test(valid_file, 14085557, save_model=True, validate=True)
model.run_test(test_file, 10388965, save_model=False, validate=True)
news_words,
news_entities,
train_entities,
valid_entities,
test_entities=test_entities,
max_sentence=10,
word_embedding_dim=100,
)
yaml_file = maybe_download(
url=
"https://recodatasets.blob.core.windows.net/deeprec/deeprec/dkn/dkn_MINDsmall.yaml",
work_directory=data_path)
hparams = prepare_hparams(yaml_file,
news_feature_file=news_feature_file,
user_history_file=user_history_file,
wordEmb_file=word_embeddings_file,
entityEmb_file=entity_embeddings_file,
epochs=epochs,
history_size=history_size,
batch_size=batch_size)
hparams.save_model = True
hparams.show_step = 20000
hparams.MODEL_DIR = 'para'
hparams.save_epoch = 1
hparams.write_tfevents = False
model = DKN(hparams, DKNTextIterator)
model.fit(train_file, valid_file, 16918280)
news_words,
news_entities,
train_entities,
valid_entities,
test_entities=test_entities,
max_sentence=10,
word_embedding_dim=100,
)
yaml_file = maybe_download(
url=
"https://recodatasets.blob.core.windows.net/deeprec/deeprec/dkn/dkn_MINDsmall.yaml",
work_directory=data_path)
hparams = prepare_hparams(yaml_file,
news_feature_file=news_feature_file,
user_history_file=user_history_file,
wordEmb_file=word_embeddings_file,
entityEmb_file=entity_embeddings_file,
epochs=epochs,
history_size=history_size,
batch_size=batch_size)
hparams.save_model = True
hparams.show_step = 20000
hparams.MODEL_DIR = 'para'
hparams.save_epoch = 1
hparams.write_tfevents = False
model = DKN(hparams, DKNTextIterator)
model.fit(train_file, valid_file, 2853385)
entity_embeddings_file = os.path.join(data_path,
"entity_embeddings_5w_100.npy")
train_path = os.path.join(data_path, "train")
valid_path = os.path.join(data_path, "valid")
test_path = os.path.join(data_path, "test")
yaml_file = maybe_download(
url=
"https://recodatasets.blob.core.windows.net/deeprec/deeprec/dkn/dkn_MINDsmall.yaml",
work_directory=data_path)
hparams = prepare_hparams(yaml_file,
news_feature_file=news_feature_file,
user_history_file=user_history_file,
wordEmb_file=word_embeddings_file,
entityEmb_file=entity_embeddings_file,
epochs=epochs,
history_size=history_size,
batch_size=batch_size)
hparams.save_model = True
hparams.show_step = 5000
hparams.MODEL_DIR = 'para'
model = DKN(hparams, DKNTextIterator)
model.load_model('./para/epoch_4')
# model.run_test(valid_file, 14085557, save_model=True, validate=True)
model.run_test(test_file, 93115001, save_model=True, validate=False)
# model.run_eval(test_file)