# input: training set, validation set, test set
# output: part of test set such that each left hand side entity appear no more
# than FREQ times in the training set.
import sys
import io
import numpy as np
import batch
# split the data into training set, validation set and testing set
if __name__ == "__main__":
max_freq = 2 # selecting lhs that appear <= max_freq times in the training set
lhs, rel, rhs = batch.load_labeled_entities(io.open("../data/prescription-sparse2-train.txt"))
lhs_dict, lhs_count = batch.build_entity_dictionary(lhs)
lhs, rel, rhs = batch.load_labeled_entities(io.open("../data/prescription-sparse2-test.txt"))
buf = []
for i in range(len(lhs)):
if lhs[i] not in lhs_count or lhs_count[lhs[i]] <= max_freq:
buf.append("{}\t{}\t{}\n".format(lhs[i], rel[i], rhs[i]))
lhs, rel, rhs = batch.load_labeled_entities(io.open("../data/prescription-sparse2-valid.txt"))
for i in range(len(lhs)):
if lhs[i] not in lhs_count or lhs_count[lhs[i]] <= max_freq:
buf.append("{}\t{}\t{}\n".format(lhs[i], rel[i], rhs[i]))
with open("../data/prescription-sparse2-rare-{}-test.txt".format(max_freq), "w") as f_out:
f_out.writelines(buf)
#lhs, rel, rhs = batch.load_labeled_entities(io.open(sys.argv[1],'r'))
#lhs_v, rel_v, rhs_v = batch.load_labeled_entities(io.open(sys.argv[2],'r'))
#lhs_s, rel_s, rhs_s = batch.load_labeled_entities(io.open("../data/prescription-freq-test.txt"))
#lhs, rel, rhs = batch.load_labeled_entities(io.open("../data/prescription-sparse2-train.txt")) # sparse 2 is by different train,valid,test ratio
#lhs_v, rel_v, rhs_v = batch.load_labeled_entities(io.open("../data/prescription-sparse2-valid.txt"))
#lhs_s, rel_s, rhs_s = batch.load_labeled_entities(io.open("../data/prescription-sparse2-test.txt"))
lhs, rel, rhs = batch.load_labeled_entities(io.open("../data/yago-sparse-entity-train.txt")) # sparse 2 is by different train,valid,test ratio
lhs_v, rel_v, rhs_v = batch.load_labeled_entities(io.open("../data/yago-sparse-entity-valid.txt"))
lhs_s, rel_s, rhs_s = batch.load_labeled_entities(io.open("../data/yago-sparse-entity-test.txt"))
# left hand side dictionaries, both character and entity
chardict, charcount = batch.build_char_dictionary(lhs)
n_char = len(chardict.keys()) + 1
batch.save_dictionary(chardict,charcount,'%s/dict.pkl' % save_path)
lhs_dict, lhs_count = batch.build_entity_dictionary(lhs)
n_lhs = len(lhs_dict.keys())
batch.save_dictionary(lhs_dict,lhs_count,'%s/lhs_dict.pkl' % save_path)
# build dictionary for relations
rel_dict, rel_count = batch.build_entity_dictionary(rel)
batch.save_dictionary(rel_dict, rel_count, '%s/rel_dict.pkl' % save_path)
n_rel = len(rel_dict.keys())
# this tells number of triples in different relations
# build dictionary for right hand side entities
rhs_dict, rhs_count = batch.build_entity_dictionary(rhs)
batch.save_dictionary(rhs_dict, rhs_count, '%s/rhs_dict.pkl' % save_path)
n_rhs = len(rhs_dict.keys())
# batches
# input: training set, validation set, test set
# output: part of test set such that each left hand side entity appear no more
# than FREQ times in the training set.
import sys
import io
import numpy as np
import batch
# split the data into training set, validation set and testing set
if __name__ == "__main__":
max_freq = 2 # selecting lhs that appear <= max_freq times in the training set
lhs, rel, rhs = batch.load_labeled_entities(
io.open("../data/prescription-sparse2-train.txt"))
lhs_dict, lhs_count = batch.build_entity_dictionary(lhs)
lhs, rel, rhs = batch.load_labeled_entities(
io.open("../data/prescription-sparse2-test.txt"))
buf = []
for i in range(len(lhs)):
if lhs[i] not in lhs_count or lhs_count[lhs[i]] <= max_freq:
buf.append("{}\t{}\t{}\n".format(lhs[i], rel[i], rhs[i]))
lhs, rel, rhs = batch.load_labeled_entities(
io.open("../data/prescription-sparse2-valid.txt"))
for i in range(len(lhs)):
if lhs[i] not in lhs_count or lhs_count[lhs[i]] <= max_freq:
buf.append("{}\t{}\t{}\n".format(lhs[i], rel[i], rhs[i]))
with open("../data/prescription-sparse2-rare-{}-test.txt".format(max_freq),
"w") as f_out: