def valid_test_get_batch(wd_fact2id_path, y_path, batch_path):
print('loading facts and ys.', save_path + wd_fact2id_path,
save_path + y_path)
facts = np.load(save_path + wd_fact2id_path)
y = np.load(save_path + y_path)
p = Pool()
X = np.asarray(list(p.map(pad_X200_same, facts)), dtype=np.int64)
p.close()
p.join()
train_batch(X, y, batch_path, batch_size)
def train_get_batch(wd_fact2id_path, y_path, batch_path):
print('loading facts and ys.', save_path + wd_fact2id_path,
save_path + y_path)
facts = np.load(save_path + wd_fact2id_path)
y = np.load(save_path + y_path)
p = Pool()
X = np.asarray(list(p.map(pad_X200_same, facts)), dtype=np.int64)
p.close()
p.join()
sample_num = X.shape[0]
np.random.seed(13)
new_index = np.random.permutation(sample_num)
X = X[new_index]
y = y[new_index]
train_batch(X, y, batch_path, batch_size)
def valid_get_batch(word_label, batch_path):
print('loading words and ys.',
save_path + word_label)
word_y = np.load(save_path + word_label)
words = []
ys = []
for word, y in word_y:
# print(word)
# print(y)
words.append(word)
ys.append(y)
p = Pool()
X = np.asarray(list(p.map(pad_X400_same, words)), dtype=np.int64)
p.close()
p.join()
train_batch(X, ys, batch_path, batch_size)
def valid_test_get_batch(wd_fact2id_path, y_path, batch_path):
print('loading facts and ys.',
save_path + wd_fact2id_path,
save_path + y_path)
facts = np.load(save_path + wd_fact2id_path)
print(facts[0:2])
p = Pool()
word2id = np.asarray(list(p.map(get_id4words, facts)))
y = np.load(save_path + y_path)
print(y[0:10])
y_id = np.asarray(list(p.map(get_id4accus, y)))
print(y_id[0:50])
p = Pool()
X = np.asarray(list(p.map(pad_X200_same, word2id)), dtype=np.int64)
print(X[0:2])
train_batch(X, y_id, batch_path, batch_size)
def train_get_batch(word_label, batch_path):
print('loading words and ys.', save_path + word_label)
word_y = np.load(save_path + word_label)
words = []
ys = []
for word, y in word_y:
words.append(word)
ys.append(y)
words = np.asarray(words)
ys = np.asarray(ys)
p = Pool()
X = np.asarray(list(p.map(pad_X400_same, words)), dtype=np.int64)
p.close()
p.join()
sample_num = X.shape[0]
np.random.seed(13)
new_index = np.random.permutation(sample_num)
X = X[new_index]
y = ys[new_index]
train_batch(X, y, batch_path, batch_size)