def _check(config):
word2idx = Counter(
json.load(open("data/word2idx_new.json", "r"))["word2idx"])
vocab_size = len(word2idx)
#word2vec = {} # or get_word2vec(word2idx)
word2vec = Counter(
json.load(
open("data/word2vec_{}.json".format(config.pretrain_from),
"r"))["word2vec"])
idx2vec = {
word2idx[word]: vec
for word, vec in word2vec.items() if word in word2idx and word != "UNK"
}
unk_embedding = np.random.multivariate_normal(
np.zeros(config.word_embedding_size),
np.eye(config.word_embedding_size))
config.emb_mat = np.array([
idx2vec[idx] if idx in idx2vec else unk_embedding
for idx in range(vocab_size)
])
config.vocab_size = vocab_size
print("emb_mat:", config.emb_mat.shape)
if config.data_from == "reuters":
train_data = read_reuters(config, data_type="train", word2idx=word2idx)
dev_data = read_reuters(config, data_type="test", word2idx=word2idx)
elif config.data_from == "20newsgroup":
train_data = read_news(config, data_type="train", word2idx=word2idx)
dev_data = read_newss(config, data_type="test", word2idx=word2idx)
config.train_size = train_data.get_data_size()
config.dev_size = dev_data.get_data_size()
pprint(config.__flags, indent=2)
model = get_model(config)
graph_handler = GraphHandler(config, model)
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
graph_handler.initialize(sess)
num_batches = config.num_batches or int(
math.ceil(
train_data.num_examples / config.batch_size)) * config.num_epochs
global_step = 0
for batch in tqdm(train_data.get_batches(config.batch_size,
num_batches=num_batches,
shuffle=True,
cluster=config.cluster),
total=num_batches):
batch_idx, batch_ds = batch
global_step = sess.run(model.global_step) + 1
# print("global_step:", global_step)
get_summary = global_step % config.log_period
feed_dict = model.get_feed_dict(batch, config)
check, xx_final, xx_context = sess.run(
[model.check, model.xx_final, model.xx_context],
feed_dict=feed_dict)
print("check:", check.shape, type(check), xx_final.shape,
xx_context.shape)
def _test(config):
if config.data_from == "20newsgroup": config.test_batch_size = 281
word2idx = Counter(json.load(open("../data/{}/word2idx_{}.json".format(config.data_from, config.data_from), "r"))["word2idx"])
idx2word = json.load(open("../data/{}/word2idx_{}.json".format(config.data_from, config.data_from), "r"))["idx2word"]
assert len(word2idx) == len(idx2word)
for i in range(10): assert word2idx[idx2word[i]] == i
vocab_size = len(word2idx)
word2vec = Counter(json.load(open("../data/{}/word2vec_{}.json".format(config.data_from, config.pretrain_from), "r"))["word2vec"])
# word2vec = {} if config.debug or config.load else get_word2vec(config, word2idx)
idx2vec = {word2idx[word]: vec for word, vec in word2vec.items() if word in word2idx}
unk_embedding = np.random.multivariate_normal(np.zeros(config.word_embedding_size), np.eye(config.word_embedding_size))
config.emb_mat = np.array([idx2vec[idx] if idx in idx2vec else unk_embedding for idx in range(vocab_size)])
config.vocab_size = vocab_size
test_dict = {}
if os.path.exists("../data/{}/{}_{}{}.json".format(config.data_from, config.data_from, config.dev_type, config.clftype)):
test_dict = json.load(open("../data/{}/{}_{}{}.json".format(config.data_from, config.data_from, config.dev_type, config.clftype), "r"))
if config.data_from == "reuters":
dev_data = DataSet(test_dict, "test") if len(test_dict)>0 else read_reuters(config, data_type="test", word2idx=word2idx)
elif config.data_from == "20newsgroup":
dev_data = DataSet(test_dict, "test") if len(test_dict)>0 else read_news(config, data_type="test", word2idx=word2idx)
elif config.data_from == "ice":
dev_data = DataSet(test_dict, config.dev_type)
config.dev_size = dev_data.get_data_size()
# if config.use_glove_for_unk:
pprint(config.__flags, indent=2)
model = get_model(config)
graph_handler = GraphHandler(config, model)
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
graph_handler.initialize(sess)
# check
#w_embeddings = sess.run(model.word_embeddings)
#print("w_embeddings:", w_embeddings.shape, w_embeddings)
dev_evaluate = Evaluator(config, model)
num_steps = math.floor(dev_data.num_examples / config.test_batch_size)
if 0 < config.val_num_batches < num_steps:
num_steps = config.val_num_batches
# print("num_steps:", num_steps)
e_dev = dev_evaluate.get_evaluation_from_batches(
sess, tqdm(dev_data.get_batches(config.test_batch_size, num_batches=num_steps), total=num_steps))
def _train(config):
word2idx = Counter(json.load(open("../data/{}/word2idx_{}.json".format(config.data_from, config.data_from), "r"))["word2idx"])
idx2word = json.load(open("../data/{}/word2idx_{}.json".format(config.data_from, config.data_from), "r"))["idx2word"]
assert len(word2idx) == len(idx2word)
for i in range(10): assert word2idx[idx2word[i]] == i
vocab_size = len(word2idx)
print("vocab_size", vocab_size, idx2word[:10])
word2vec = Counter(json.load(open("../data/{}/word2vec_{}.json".format(config.data_from, config.pretrain_from), "r"))["word2vec"])
# word2vec = {} if config.debug or config.load else get_word2vec(config, word2idx)
idx2vec = {word2idx[word]: vec for word, vec in word2vec.items() if word in word2idx}
print("no unk words:", len(idx2vec))
unk_embedding = np.random.multivariate_normal(np.zeros(config.word_embedding_size), np.eye(config.word_embedding_size))
config.emb_mat = np.array([idx2vec[idx] if idx in idx2vec else unk_embedding for idx in range(vocab_size)])
config.vocab_size = vocab_size
print("emb_mat:", config.emb_mat.shape)
test_type = "test"
if config.data_from == "ice":
test_type = "dev"
else:
test_type = "test"
train_dict, test_dict = {}, {}
ice_flat = ""
if config.data_from == "ice" and config.model_name.endswith("flat"):
ice_flat = "_flat"
if os.path.exists("../data/{}/{}_{}{}{}.json".format(config.data_from, config.data_from, "train", ice_flat, config.clftype)):
train_dict = json.load(open("../data/{}/{}_{}{}{}.json".format(config.data_from, config.data_from, "train", ice_flat, config.clftype), "r"))
if os.path.exists("../data/{}/{}_{}{}{}.json".format(config.data_from, config.data_from, test_type, ice_flat, config.clftype)):
test_dict = json.load(open("../data/{}/{}_{}{}{}.json".format(config.data_from, config.data_from, test_type, ice_flat, config.clftype), "r"))
# check
for key, val in train_dict.items():
if isinstance(val[0], list) and len(val[0])>10: print(key, val[0][:50])
else: print(key, val[0:4])
print("train:", len(train_dict))
print("test:", len(test_dict))
if config.data_from == "reuters":
train_data = DataSet(train_dict, "train") if len(train_dict)>0 else read_reuters(config, data_type="train", word2idx=word2idx)
dev_data = DataSet(test_dict, "test") if len(test_dict)>0 else read_reuters(config, data_type="test", word2idx=word2idx)
elif config.data_from == "20newsgroup":
train_data = DataSet(train_dict, "train") if len(train_dict)>0 else read_news(config, data_type="train", word2idx=word2idx)
dev_data = DataSet(test_dict, "test") if len(test_dict)>0 else read_news(config, data_type="test", word2idx=word2idx)
elif config.data_from == "ice":
train_data = DataSet(train_dict, "train")
dev_data = DataSet(test_dict, "dev")
config.train_size = train_data.get_data_size()
config.dev_size = dev_data.get_data_size()
print("train/dev:", config.train_size, config.dev_size)
# calculate doc length
# TO CHECK
avg_len = 0
for d_l in train_dict["x_len"]:
avg_len += d_l/config.train_size
print("avg_len at train:", avg_len)
if config.max_docs_length > 2000: config.max_docs_length = 2000
pprint(config.__flags, indent=2)
model = get_model(config)
trainer = Trainer(config, model)
graph_handler = GraphHandler(config, model)
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
graph_handler.initialize(sess)
num_batches = config.num_batches or int(math.ceil(train_data.num_examples / config.batch_size)) * config.num_epochs
global_step = 0
dev_evaluate = Evaluator(config, model)
best_f1 = 0.50
for batch in tqdm(train_data.get_batches(config.batch_size, num_batches=num_batches, shuffle=True, cluster=config.cluster), total=num_batches):
global_step = sess.run(model.global_step) + 1
# print("global_step:", global_step)
get_summary = global_step % config.log_period
loss, summary, train_op = trainer.step(sess, batch, get_summary)
if get_summary:
graph_handler.add_summary(summary, global_step)
# occasional saving
# if global_step % config.save_period == 0 :
# graph_handler.save(sess, global_step=global_step)
if not config.eval:
continue
# Occasional evaluation
if global_step % config.eval_period == 0:
#config.test_batch_size = config.dev_size/3
num_steps = math.ceil(dev_data.num_examples / config.test_batch_size)
if 0 < config.val_num_batches < num_steps:
num_steps = config.val_num_batches
# print("num_steps:", num_steps)
e_dev = dev_evaluate.get_evaluation_from_batches(
sess, tqdm(dev_data.get_batches(config.test_batch_size, num_batches=num_steps), total=num_steps))
if e_dev.fv > best_f1:
best_f1 = e_dev.fv
#if global_step % config.save_period == 0:
graph_handler.save(sess, global_step=global_step)
graph_handler.add_summaries(e_dev.summaries, global_step)
print("f1:", best_f1)
def _train(config):
word2idx = Counter(
json.load(
open(
"data/{}/word2idx_{}.json".format(config.data_from,
config.data_from),
"r"))["word2idx"])
vocab_size = len(word2idx)
print("vocab_size", vocab_size)
word2vec = Counter(
json.load(
open(
"data/{}/word2vec_{}.json".format(config.data_from,
config.pretrain_from),
"r"))["word2vec"])
# word2vec = {} if config.debug or config.load else get_word2vec(config, word2idx)
idx2vec = {
word2idx[word]: vec
for word, vec in word2vec.items() if word in word2idx and word != "UNK"
}
unk_embedding = np.random.multivariate_normal(
np.zeros(config.word_embedding_size),
np.eye(config.word_embedding_size))
config.emb_mat = np.array([
idx2vec[idx] if idx in idx2vec else unk_embedding
for idx in range(vocab_size)
])
config.vocab_size = vocab_size
print("emb_mat:", config.emb_mat.shape)
train_dict, test_dict = {}, {}
if os.path.exists("data/{}/{}_{}.json".format(config.data_from,
config.data_from, "train")):
train_dict = json.load(
open(
"data/{}/{}_{}.json".format(config.data_from, config.data_from,
"train"), "r"))
if os.path.exists("data/{}/{}_{}.json".format(config.data_from,
config.data_from, "test")):
test_dict = json.load(
open(
"data/{}/{}_{}.json".format(config.data_from, config.data_from,
"test"), "r"))
# check
if config.data_from == "reuters":
train_data = DataSet(train_dict,
"train") if len(train_dict) > 0 else read_reuters(
config, data_type="train", word2idx=word2idx)
dev_data = DataSet(test_dict,
"test") if len(test_dict) > 0 else read_reuters(
config, data_type="test", word2idx=word2idx)
elif config.data_from == "20newsgroup":
train_data = DataSet(train_dict,
"train") if len(train_dict) > 0 else read_news(
config, data_type="train", word2idx=word2idx)
dev_data = DataSet(test_dict,
"test") if len(test_dict) > 0 else read_news(
config, data_type="test", word2idx=word2idx)
config.train_size = train_data.get_data_size()
config.dev_size = dev_data.get_data_size()
print("train/dev:", config.train_size, config.dev_size)
if config.max_docs_length > 2000: config.max_docs_length = 2000
pprint(config.__flags, indent=2)
model = get_model(config)
graph_handler = GraphHandler(config, model)
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
graph_handler.initialize(sess)
num_batches = config.num_batches or int(
math.ceil(
train_data.num_examples / config.batch_size)) * config.num_epochs
global_step = 0
dev_evaluate = Evaluator(config, model)
for batch in tqdm(train_data.get_batches(config.batch_size,
num_batches=num_batches,
shuffle=True,
cluster=config.cluster),
total=num_batches):
batch_idx, batch_ds = batch
'''
if config.debug:
for key, value in batch_ds.data.items():
if not key.startswith("x"):
print(key, value)
continue
'''
global_step = sess.run(model.global_step) + 1
# print("global_step:", global_step)
get_summary = global_step % config.log_period
feed_dict = model.get_feed_dict(batch, config)
logits, y, y_len, loss, summary, train_op = sess.run(
[
model.logits, model.y, model.y_seq_length, model.loss,
model.summary, model.train_op
],
feed_dict=feed_dict)
#print("logits:", logits[0:3], y[0:3], y_len[0:3], logits.shape, y.shape, y_len.shape)
print("loss:", loss)
if get_summary:
graph_handler.add_summary(summary, global_step)
# occasional saving
if global_step % config.save_period == 0:
graph_handler.save(sess, global_step=global_step)
if not config.eval:
continue
# Occasional evaluation
if global_step % config.eval_period == 0:
#config.test_batch_size = config.dev_size/3
num_steps = math.ceil(dev_data.num_examples /
config.test_batch_size)
if 0 < config.val_num_batches < num_steps:
num_steps = config.val_num_batches
# print("num_steps:", num_steps)
e_dev = dev_evaluate.get_evaluation_from_batches(
sess,
tqdm(dev_data.get_batches(config.test_batch_size,
num_batches=num_steps),
total=num_steps))
graph_handler.add_summaries(e_dev.summaries, global_step)