def __init__(self, hidden_size, word_emb_size):
super(NZSigmoidLoss, self).__init__()
require_type_lst = utils.get_ontoNotes_train_types()
self.weight = nn.Parameter(
torch.zeros(len(require_type_lst),
hidden_size * 2 + word_emb_size))
utils.init_weight(self.weight)
def get_type_lst(depth, data):
type_lst = None
if data == 'onto':
type_lst = utils.get_ontoNotes_train_types(depth)
elif data == 'wiki':
type_lst = utils.get_wiki_types(depth)
elif data == 'bbn':
type_lst = utils.get_bbn_types(depth)
return type_lst
def get_down_type_lst(depth, data):
type_lst = []
if data == 'onto':
for i in range(1, depth+1):
type_lst.extend(utils.get_ontoNotes_train_types(i))
elif data == 'wiki':
for i in range(1, depth + 1):
type_lst.extend(utils.get_wiki_types(i))
elif data == 'bbn':
for i in range(1, depth + 1):
type_lst.extend(utils.get_bbn_types(i))
return type_lst
def refine(labels, tier, maxDepth=3):
keep = [""] * maxDepth
short2full = get_short2full_map(utils.get_ontoNotes_train_types())
for l in labels:
path = l.split('/')[1:]
path = [short2full[k] for k in path]
for i in range(len(path)):
if keep[i] == "":
keep[i] = path[i]
elif keep[i] != path[i]:
break
return [l for l in keep if l != ""]
def evaluate_all(my_arg, pr=True):
emb = LoadEmbedding('res/onto_emb.txt')
print('finish loading embedding')
batch_size = 1000
batch_getter = OntoNotesFGGetter('data/OntoNotes/test.json',
utils.get_ontoNotes_train_types(),
batch_size, True)
print('finish loading train data')
ctx_lstm = CtxLSTM(emb.get_emb_size())
embedding_layer = EmbeddingLayer(emb)
ctx_att = CtxAtt(fg_config['hidden_size'], emb.get_emb_size())
sigmoid_loss = SigmoidLoss(fg_config['hidden_size'], emb.get_emb_size())
if fg_config['USE_CUDA']:
embedding_layer.cuda(fg_config['cuda_num'])
ctx_lstm.cuda(fg_config['cuda_num'])
ctx_att.cuda(fg_config['cuda_num'])
sigmoid_loss.cuda(fg_config['cuda_num'])
model_dir = 'et_model' + str(my_arg)
embedding_layer.load_state_dict(
torch.load(model_dir + '/embedding_layer.pkl'))
ctx_lstm.load_state_dict(torch.load(model_dir + '/ctx_lstm.pkl'))
ctx_att.load_state_dict(torch.load(model_dir + '/ctx_att.pkl'))
sigmoid_loss.load_state_dict(torch.load(model_dir + '/sigmoid_loss.pkl'))
embedding_layer.eval()
ctx_lstm.eval()
ctx_att.eval()
sigmoid_loss.eval()
ex_iterations = 0
evaluator = BoundaryPerformance()
for iteration, this_batch in enumerate(batch_getter):
pred, label = evaluate_one(ex_iterations + iteration, embedding_layer,
ctx_lstm, ctx_att, sigmoid_loss, this_batch)
evaluator.evaluate(label, pred)
if (iteration + 1) * batch_size % 100 == 0:
print('{} sentences processed'.format(
(iteration + 1) * batch_size))
evaluator.get_performance()
return evaluator.get_performance()
def __init__(self,
hidden_size,
word_emb_size,
dropout_p=fg_config['dropout']):
super(WARPLoss, self).__init__()
require_type_lst = None
if fg_config['data'] == 'onto':
require_type_lst = utils.get_ontoNotes_train_types()
elif fg_config['data'] == 'wiki':
require_type_lst = utils.get_wiki_types()
elif fg_config['data'] == 'bbn':
require_type_lst = utils.get_bbn_types()
num_labels = len(require_type_lst)
self.weight = nn.Parameter(
torch.zeros(hidden_size * 2 + word_emb_size, word_emb_size))
utils.init_weight(self.weight)
self.rank_weights = [1.0 / 1]
for i in range(1, num_labels):
self.rank_weights.append(self.rank_weights[i - 1] + (1.0 / i + 1))
self.trans = nn.Linear(hidden_size * 2 + word_emb_size, word_emb_size)
utils.init_linear(self.trans)
self.activate = nn.ReLU()
self.dropout = nn.Dropout(dropout_p)
def main(my_arg):
log_dir = 'et_logs' + str(my_arg)
logger = Logger(log_dir)
emb = LoadEmbedding('res/onto_embedding.txt')
print('finish loading embedding')
batch_getter = OntoNotesFGGetter('data/OntoNotes/train.json',
utils.get_ontoNotes_train_types(),
fg_config['batch_size'], True)
# batch_getter = OntoNotesNZGetter('data/OntoNotes/train.json', utils.get_ontoNotes_train_types(),
# fg_config['batch_size'], True)
print('finish loading train data')
ctx_lstm = CtxLSTM(emb.get_emb_size())
embedding_layer = EmbeddingLayer(emb)
ctx_att = CtxAtt(fg_config['hidden_size'], emb.get_emb_size())
sigmoid_loss = SigmoidLoss(fg_config['hidden_size'], emb.get_emb_size())
warp_loss = WARPLoss(89)
if fg_config['USE_CUDA']:
embedding_layer.cuda(fg_config['cuda_num'])
ctx_lstm.cuda(fg_config['cuda_num'])
ctx_att.cuda(fg_config['cuda_num'])
sigmoid_loss.cuda(fg_config['cuda_num'])
ctx_lstm_opt = torch.optim.Adam(ctx_lstm.parameters())
ctx_att_opt = torch.optim.Adam(ctx_att.parameters())
sig_opt = torch.optim.Adam(sigmoid_loss.parameters())
log_file = open('{}/log_file'.format(log_dir), 'w')
f_max = 0
low_epoch = 0
ex_iterations = 0
model_dir = 'et_model' + str(my_arg)
time0 = time.time()
for epoch in range(fg_config['max_epoch']):
embedding_layer.train()
ctx_lstm.train()
ctx_att.train()
sigmoid_loss.train()
# f, p, r = evaluate_all(my_arg, False)
for iteration, this_batch in enumerate(batch_getter):
if (ex_iterations + iteration) % 100 == 0:
print('epoch: {}, iteraton: {}'.format(
epoch, ex_iterations + iteration))
train_iteration(logger, ex_iterations + iteration, embedding_layer,
ctx_lstm, ctx_att, sigmoid_loss, ctx_lstm_opt,
ctx_att_opt, sig_opt, this_batch)
if (ex_iterations + iteration) % 100 == 0:
time1 = time.time()
print('this iteration time: ', time1 - time0, '\n')
time0 = time1
if (ex_iterations + iteration) % fg_config['save_freq'] == 0:
torch.save(embedding_layer.state_dict(),
model_dir + '/embedding_layer.pkl')
torch.save(ctx_lstm.state_dict(), model_dir + '/ctx_lstm.pkl')
torch.save(ctx_att.state_dict(), model_dir + '/ctx_att.pkl')
torch.save(sigmoid_loss.state_dict(),
model_dir + '/sigmoid_loss.pkl')
ex_iterations += iteration + 1
batch_getter.reset()
fg_config['use_dropout'] = False
f, p, r = evaluate_all(my_arg, False)
fg_config['use_dropout'] = True
log_file.write('epoch: {} f: {} p: {} r: {}\n'.format(epoch, f, p, r))
log_file.flush()
if f >= f_max:
f_max = f
low_epoch = 0
os.system('cp {}/embedding_layer.pkl {}/early_embedding_layer.pkl'.
format(model_dir, model_dir))
os.system('cp {}/ctx_lstm.pkl {}/early_ctx_lstm.pkl'.format(
model_dir, model_dir))
os.system('cp {}/ctx_att.pkl {}/early_ctx_att.pkl'.format(
model_dir, model_dir))
os.system(
'cp {}/sigmoid_loss.pkl {}/early_sigmoid_loss.pkl'.format(
model_dir, model_dir))
else:
low_epoch += 1
log_file.write('low' + str(low_epoch) + '\n')
log_file.flush()
if low_epoch >= fg_config['early_stop']:
break
log_file.close()
def evaluate_free(my_arg, pr=True):
word_emb = LoadEmbedding('res/glove_840B_emb.txt')
type_emb = LoadEmbedding('res/{}/zero_type_emb.txt'.format(fg_config['data']))
print('finish loading embedding')
batch_size = 100
depth = None
if fg_config['zero_shot']:
depth = 2
elif fg_config['no_zero'] == 'all':
depth = None
elif fg_config['no_zero'] == 'one':
depth = 1
type_lst = get_down_type_lst(depth, fg_config['data'])
batch_getter = OntoNotesNZGetter('data/{}/test.json'.format(fg_config['data']),
type_lst, batch_size, True, depth)
print('finish loading train data')
ctx_lstm = CtxLSTM(word_emb.get_emb_size())
word_embedding_layer = EmbeddingLayer(word_emb)
type_embedding_layer = EmbeddingLayer(type_emb)
ctx_att = NZCtxAtt(fg_config['hidden_size'], word_emb.get_emb_size())
warp_loss = WARPLoss(fg_config['hidden_size'], word_emb.get_emb_size())
if fg_config['USE_CUDA']:
word_embedding_layer.cuda(fg_config['cuda_num'])
type_embedding_layer.cuda(fg_config['cuda_num'])
ctx_lstm.cuda(fg_config['cuda_num'])
ctx_att.cuda(fg_config['cuda_num'])
warp_loss.cuda(fg_config['cuda_num'])
model_dir = '{}/et_model{}'.format(fg_config['data'], str(my_arg))
word_embedding_layer.load_state_dict(torch.load(model_dir+'/early_embedding_layer.pkl'))
type_embedding_layer.load_state_dict(torch.load(model_dir+'/early_type_embedding_layer.pkl'))
ctx_lstm.load_state_dict(torch.load(model_dir+'/early_ctx_lstm.pkl'))
ctx_att.load_state_dict(torch.load(model_dir+'/early_ctx_att.pkl'))
warp_loss.load_state_dict(torch.load(model_dir+'/early_sigmoid_loss.pkl'))
word_embedding_layer.eval()
type_embedding_layer.eval()
ctx_lstm.eval()
ctx_att.eval()
warp_loss.eval()
ex_iterations = 0
evaluator = BoundaryPerformance()
short2full = None
if fg_config['data'] == 'onto':
short2full = get_short2full_map(utils.get_ontoNotes_train_types())
elif fg_config['data'] == 'wiki':
short2full = get_short2full_map(utils.get_wiki_types())
patch = utils.wiki_short2full_patch()
short2full.update(patch)
elif fg_config['data'] == 'bbn':
short2full = get_short2full_map(utils.get_bbn_types())
for iteration, this_batch in enumerate(batch_getter):
pred, label = evaluate_one(ex_iterations + iteration, word_embedding_layer, type_embedding_layer,
ctx_lstm, ctx_att, warp_loss, this_batch)
# evaluator.evaluate(label, pred, type_lst, short2full)
evaluator.evaluate(this_batch['types_str'], pred, type_lst, short2full)
if (iteration+1)*batch_size % 100 == 0:
print('{} sentences processed'.format((iteration+1)*batch_size))
evaluator.get_performance()
return evaluator.get_performance()
src_tokens = []
targets = []
elif blank_line == 1:
if line.strip():
parts = line.strip().split(' ')
src_tokens.append(parts[0])
targets.append(parts[3])
train_file.close()
self.all_samples = all_samples
self.sample_num = len(self.all_samples)
if __name__ == '__main__':
# config['Tags'] = {'<PADDING>': 0, '<START>': 1, 'B': 2, 'I': 3, 'O': 4, 'E': 5, 'S': 6}
# config['misc'] = False
# config['bioes'] = True
# config['use_gaz'] = False
# onto_notes = OntoNotesGetter('data/OntoNotes/test.json', '/organization/company', 1, False)
# p = ConllBatchGetter('data/conll2003/bioes_eng.train', 'PER', 1, True)
# pernam_batch_getter = TrainConllBatchGetter('data/conll2003/bioes_eng.train', 'PER', 1, True)
# pass
batch_getter = OntoNotesFGGetter('data/OntoNotes/test.json',
utils.get_ontoNotes_train_types(),
fg_config['batch_size'], True)
print('finish load')
for this_batch in batch_getter:
pass
print('finish read')