def main(my_arg):
log_dir = 'ner_logs' + str(my_arg)
logger = Logger(log_dir)
emb = LoadEmbedding('res/embedding.txt')
if config['label_emb'] or config['question_alone']:
onto_emb = LoadEmbedding('res/onto_embedding.txt')
print('finish loading embedding')
# batch_getter = BatchGetter('data/train', 'GPE_NAM', config['batch_size'])
batch_getter_lst = []
if config['bioes']:
if config['data'] == 'conll':
# pernam_batch_getter = ConllBatchGetter('data/conll2003/bioes_eng.train', 'PER', 1, True)
pernam_batch_getter = ConllBatchGetter(
'data/conll2003/bioes_eng.train', 'PER', 1, True)
batch_getter_lst.append(pernam_batch_getter)
# loc_batch_getter = ConllBatchGetter('data/conll2003/bioes_eng.train', 'LOC', 1, True)
loc_batch_getter = ConllBatchGetter(
'data/conll2003/bioes_eng.train', 'LOC', 1, True)
batch_getter_lst.append(loc_batch_getter)
if not config['drop_misc']:
# misc_batch_getter = ConllBatchGetter('data/conll2003/bioes_eng.train', 'MISC', 1, True)
misc_batch_getter = ConllBatchGetter(
'data/conll2003/bioes_eng.train', 'MISC', 1, True)
batch_getter_lst.append(misc_batch_getter)
# org_batch_getter = ConllBatchGetter('data/conll2003/bioes_eng.train', 'ORG', 1, True)
org_batch_getter = ConllBatchGetter(
'data/conll2003/bioes_eng.train', 'ORG', 1, True)
batch_getter_lst.append(org_batch_getter)
elif config['data'] == 'OntoNotes':
# onto_notes_data = TrainOntoNotesGetter('data/OntoNotes/train.json', 1, True)
onto_notes_data = OntoNotesGetter(
'data/OntoNotes/leaf_train.json',
['/person', '/organization', '/location', '/other'], 1, True)
batch_getter_lst.append(onto_notes_data)
else:
pernam_batch_getter = ConllBatchGetter('data/conll2003/bio_eng.train',
'PER', 1, True)
batch_getter_lst.append(pernam_batch_getter)
loc_batch_getter = ConllBatchGetter('data/conll2003/bio_eng.train',
'LOC', 1, True)
batch_getter_lst.append(loc_batch_getter)
if not config['drop_misc']:
misc_batch_getter = ConllBatchGetter(
'data/conll2003/bio_eng.train', 'MISC', 1, True)
batch_getter_lst.append(misc_batch_getter)
org_batch_getter = ConllBatchGetter('data/conll2003/bio_eng.train',
'ORG', 1, True)
batch_getter_lst.append(org_batch_getter)
batch_getter = MergeBatchGetter(batch_getter_lst,
config['batch_size'],
True,
data_name=config['data'])
print('finish loading train data')
# if config['data'] == 'OntoNotes':
# emb_onto = True
# else:
# emb_onto = False
embedding_layer = EmbeddingLayer(emb)
if config['label_emb']:
q_word_embedding = nn.Embedding(onto_emb.get_voc_size(),
onto_emb.get_emb_size())
q_word_embedding.weight.data.copy_(onto_emb.get_embedding_tensor())
q_word_embedding.weight.requires_grad = False
else:
q_word_embedding = None
d = config['hidden_size']
if config['question_alone']:
q_emb_layer = QLabel(onto_emb)
else:
q_emb_layer = None
att_layer = AttentionFlowLayer(2 * d)
model_layer = ModelingLayer(8 * d, d, 2)
ner_hw_layer = NerHighway(2 * d, 8 * d, 1)
ner_out_layer = NerOutLayer(10 * d, len(config['Tags']))
crf = CRF(config, config['Tags'], 10 * d)
if config['USE_CUDA']:
att_layer.cuda(config['cuda_num'])
embedding_layer.cuda(config['cuda_num'])
if config['label_emb']:
q_word_embedding.cuda(config['cuda_num'])
model_layer.cuda(config['cuda_num'])
ner_hw_layer.cuda(config['cuda_num'])
ner_out_layer.cuda(config['cuda_num'])
crf.cuda(config['cuda_num'])
if config['question_alone']:
q_emb_layer.cuda(config['cuda_num'])
squad_model_dir = 'mr_model1'
if not config['not_pretrain']:
att_layer.load_state_dict(
torch.load(squad_model_dir + '/early_att_layer.pkl',
map_location=lambda storage, loc: storage))
model_layer.load_state_dict(
torch.load(squad_model_dir + '/early_model_layer.pkl',
map_location=lambda storage, loc: storage))
embedding_layer.load_state_dict(
torch.load(squad_model_dir + '/early_embedding_layer.pkl',
map_location=lambda storage, loc: storage))
if config['freeze']:
for param in att_layer.parameters():
param.requires_grad = False
for param in model_layer.parameters():
param.requires_grad = False
for param in embedding_layer.parameters():
param.requires_grad = False
embedding_layer.eval()
model_layer.eval()
att_layer.eval()
emb_opt = None
att_opt = None
model_opt = None
else:
if config['not_pretrain']:
emb_opt = torch.optim.Adam(
filter(lambda param: param.requires_grad,
embedding_layer.parameters()))
att_opt = torch.optim.Adam(
filter(lambda param: param.requires_grad,
att_layer.parameters()))
model_opt = torch.optim.Adam(
filter(lambda param: param.requires_grad,
model_layer.parameters()))
else:
emb_opt = torch.optim.Adam(filter(
lambda param: param.requires_grad,
embedding_layer.parameters()),
lr=1e-4)
att_opt = torch.optim.Adam(filter(
lambda param: param.requires_grad, att_layer.parameters()),
lr=1e-4)
model_opt = torch.optim.Adam(filter(
lambda param: param.requires_grad, model_layer.parameters()),
lr=1e-4)
# model_opt = torch.optim.Adam(filter(lambda param: param.requires_grad, model_layer.parameters()))
ner_hw_opt = torch.optim.Adam(
filter(lambda param: param.requires_grad, ner_hw_layer.parameters()))
ner_out_opt = torch.optim.Adam(
filter(lambda param: param.requires_grad, ner_out_layer.parameters()))
crf_opt = torch.optim.Adam(
filter(lambda param: param.requires_grad, crf.parameters()))
if config['question_alone']:
q_emb_opt = torch.optim.Adam(
filter(lambda param: param.requires_grad,
q_emb_layer.parameters()))
else:
q_emb_opt = None
log_file = open('{}/log_file'.format(log_dir), 'w')
f_max = 0
low_epoch = 0
ex_iterations = 0
model_dir = 'ner_model' + str(my_arg)
time0 = time.time()
for epoch in range(config['max_epoch']):
embedding_layer.train()
att_layer.train()
model_layer.train()
ner_hw_layer.train()
ner_out_layer.train()
crf.train()
if config['question_alone']:
q_emb_layer.train()
# f, p, r = evaluate_all(my_arg, False)
for iteration, this_batch in enumerate(batch_getter):
if (ex_iterations + iteration) % 100 == 0:
print('epoch: {}, iteraton: {}'.format(
epoch, ex_iterations + iteration))
train_iteration(logger, ex_iterations + iteration, embedding_layer,
q_word_embedding, q_emb_layer, att_layer,
model_layer, ner_hw_layer, ner_out_layer, crf,
emb_opt, q_emb_opt, att_opt, model_opt, ner_hw_opt,
ner_out_opt, crf_opt, this_batch)
if (ex_iterations + iteration) % 100 == 0:
time1 = time.time()
print('this iteration time: ', time1 - time0, '\n')
time0 = time1
if (ex_iterations + iteration) % config['save_freq'] == 0:
torch.save(embedding_layer.state_dict(),
model_dir + '/embedding_layer.pkl')
torch.save(att_layer.state_dict(),
model_dir + '/att_layer.pkl')
torch.save(model_layer.state_dict(),
model_dir + '/model_layer.pkl')
torch.save(ner_hw_layer.state_dict(),
model_dir + '/ner_hw_layer.pkl')
torch.save(ner_out_layer.state_dict(),
model_dir + '/ner_out_layer.pkl')
torch.save(crf.state_dict(), model_dir + '/crf.pkl')
if config['question_alone']:
torch.save(q_emb_layer.state_dict(),
model_dir + '/q_emb_layer.pkl')
ex_iterations += iteration + 1
batch_getter.reset()
config['use_dropout'] = False
f, p, r = evaluate_all(my_arg, False)
config['use_dropout'] = True
log_file.write('epoch: {} f: {} p: {} r: {}\n'.format(epoch, f, p, r))
log_file.flush()
if f >= f_max:
f_max = f
low_epoch = 0
os.system('cp {}/embedding_layer.pkl {}/early_embedding_layer.pkl'.
format(model_dir, model_dir))
os.system('cp {}/att_layer.pkl {}/early_att_layer.pkl'.format(
model_dir, model_dir))
os.system('cp {}/model_layer.pkl {}/early_model_layer.pkl'.format(
model_dir, model_dir))
os.system(
'cp {}/ner_hw_layer.pkl {}/early_ner_hw_layer.pkl'.format(
model_dir, model_dir))
os.system(
'cp {}/ner_out_layer.pkl {}/early_ner_out_layer.pkl'.format(
model_dir, model_dir))
os.system('cp {}/crf.pkl {}/early_crf.pkl'.format(
model_dir, model_dir))
if config['question_alone']:
os.system(
'cp {}/q_emb_layer.pkl {}/early_q_emb_layer.pkl'.format(
model_dir, model_dir))
else:
low_epoch += 1
log_file.write('low' + str(low_epoch) + '\n')
log_file.flush()
if low_epoch >= config['early_stop']:
break
log_file.close()
def main(my_arg):
log_dir = 'mr_logs' + str(my_arg)
logger = Logger(log_dir)
emb = LoadEmbedding('res/embedding.txt')
print('finish loading embedding')
batch_getter = SquadLoader('data/SQuAD/train-v1.1.json',
config['batch_size'], True)
# batch_getter = SquadLoader('data/SQuAD/dev-v1.1.json', config['batch_size'], True)
print('finish loading train data')
embedding_layer = EmbeddingLayer(emb)
d = config['hidden_size']
att_layer = AttentionFlowLayer(2 * d)
model_layer = ModelingLayer(8 * d, d, 2)
start_layer = StartProbLayer(10 * d)
end_layer = EndProbLayer(2 * d, d)
if config['USE_CUDA']:
att_layer.cuda(config['cuda_num'])
embedding_layer.cuda(config['cuda_num'])
model_layer.cuda(config['cuda_num'])
start_layer.cuda(config['cuda_num'])
end_layer.cuda(config['cuda_num'])
emb_opt = torch.optim.Adam(embedding_layer.parameters())
att_opt = torch.optim.Adam(att_layer.parameters())
model_opt = torch.optim.Adam(model_layer.parameters())
start_opt = torch.optim.Adam(start_layer.parameters())
end_opt = torch.optim.Adam(end_layer.parameters())
model_dir = 'mr_model' + str(my_arg)
check_epoch = 0
check_ex_iteration = 0
if config['resume']:
check = torch.load(model_dir + '/opt.pkl')
emb_opt.load_state_dict(check['emb_opt'])
att_opt.load_state_dict(check['att_opt'])
model_opt.load_state_dict(check['model_opt'])
start_opt.load_state_dict(check['start_opt'])
end_opt.load_state_dict(check['end_opt'])
check_epoch = check['epoch']
check_ex_iteration = check['iteration']
embedding_layer.load_state_dict(
torch.load(model_dir + '/embedding_layer.pkl'))
att_layer.load_state_dict(torch.load(model_dir + '/att_layer.pkl'))
model_layer.load_state_dict(torch.load(model_dir + '/model_layer.pkl'))
start_layer.load_state_dict(torch.load(model_dir + '/start_layer.pkl'))
end_layer.load_state_dict(torch.load(model_dir + '/end_layer.pkl'))
log_file = open('{}/log_file'.format(log_dir), 'w')
f_max = 0
low_epoch = 0
ex_iterations = check_ex_iteration + 1
for epoch in range(check_epoch, config['epochs']):
embedding_layer.train()
att_layer.train()
model_layer.train()
start_layer.train()
end_layer.train()
# exact_match, f = evaluate_all(my_arg, False)
for iteration, this_batch in enumerate(batch_getter):
time0 = time.time()
print('epoch: {}, iteraton: {}'.format(epoch,
ex_iterations + iteration))
train_iteration(logger, ex_iterations + iteration, embedding_layer,
att_layer, model_layer, start_layer, end_layer,
emb_opt, att_opt, model_opt, start_opt, end_opt,
this_batch)
time1 = time.time()
print('this iteration time: ', time1 - time0, '\n')
if (ex_iterations + iteration) % config['save_freq'] == 0:
torch.save(embedding_layer.state_dict(),
model_dir + '/embedding_layer.pkl')
torch.save(att_layer.state_dict(),
model_dir + '/att_layer.pkl')
torch.save(model_layer.state_dict(),
model_dir + '/model_layer.pkl')
torch.save(start_layer.state_dict(),
model_dir + '/start_layer.pkl')
torch.save(end_layer.state_dict(),
model_dir + '/end_layer.pkl')
check_point = {
'epoch': epoch,
'iteration': ex_iterations + iteration,
'emb_opt': emb_opt.state_dict(),
'att_opt': att_opt.state_dict(),
'model_opt': model_opt.state_dict(),
'start_opt': start_opt.state_dict(),
'end_opt': end_opt.state_dict()
}
torch.save(check_point, model_dir + '/opt.pkl')
if epoch == 11:
torch.save(embedding_layer.state_dict(),
model_dir + '/12_embedding_layer.pkl')
torch.save(att_layer.state_dict(), model_dir + '/12_att_layer.pkl')
torch.save(model_layer.state_dict(),
model_dir + '/12_model_layer.pkl')
torch.save(start_layer.state_dict(),
model_dir + '/12_start_layer.pkl')
torch.save(end_layer.state_dict(), model_dir + '/12_end_layer.pkl')
check_point = {
'epoch': epoch,
'iteration': ex_iterations + iteration,
'emb_opt': emb_opt.state_dict(),
'att_opt': att_opt.state_dict(),
'model_opt': model_opt.state_dict(),
'start_opt': start_opt.state_dict(),
'end_opt': end_opt.state_dict()
}
torch.save(check_point, model_dir + '/opt.pkl')
ex_iterations += iteration + 1
batch_getter.reset()
config['use_dropout'] = False
exact_match, f = evaluate_all(my_arg, False)
config['use_dropout'] = True
log_file.write('epoch: {} exact_match: {} f: {}\n'.format(
epoch, exact_match, f))
log_file.flush()
if f >= f_max:
f_max = f
low_epoch = 0
os.system('cp {}/embedding_layer.pkl {}/early_embedding_layer.pkl'.
format(model_dir, model_dir))
os.system('cp {}/att_layer.pkl {}/early_att_layer.pkl'.format(
model_dir, model_dir))
os.system('cp {}/model_layer.pkl {}/early_model_layer.pkl'.format(
model_dir, model_dir))
os.system('cp {}/start_layer.pkl {}/early_start_layer.pkl'.format(
model_dir, model_dir))
os.system('cp {}/end_layer.pkl {}/early_end_layer.pkl'.format(
model_dir, model_dir))
os.system('cp {}/opt.pkl {}/early_opt.pkl'.format(
model_dir, model_dir))
else:
low_epoch += 1
log_file.write('low' + str(low_epoch) + '\n')
log_file.flush()
if low_epoch >= config['early_stop']:
break
log_file.close()
def main(my_arg):
logger = Logger('./logs' + str(my_arg))
emb = LoadEmbedding('res/embedding.txt')
print 'finish loading embedding'
# batch_getter = BatchGetter('data/train', 'GPE_NAM', config['batch_size'])
batch_getter_lst = []
if my_arg == 0:
pernam_batch_getter = ConllBatchGetter('data/conll2003/bio_eng.train',
'PER', 1, True)
batch_getter_lst.append(pernam_batch_getter)
loc_batch_getter = ConllBatchGetter('data/conll2003/bio_eng.train',
'LOC', 1, True)
batch_getter_lst.append(loc_batch_getter)
misc_batch_getter = ConllBatchGetter('data/conll2003/bio_eng.train',
'MISC', 1, True)
batch_getter_lst.append(misc_batch_getter)
org_batch_getter = ConllBatchGetter('data/conll2003/bio_eng.train',
'ORG', 1, True)
batch_getter_lst.append(org_batch_getter)
if my_arg == 1:
pernam_batch_getter = BatchGetter('data/train', 'PER_NAM', 1)
batch_getter_lst.append(pernam_batch_getter)
fac_batch_getter = BatchGetter('data/train', 'FAC_NAM', 1)
batch_getter_lst.append(fac_batch_getter)
loc_batch_getter = BatchGetter('data/train', 'LOC_NAM', 1)
batch_getter_lst.append(loc_batch_getter)
gpe_batch_getter = BatchGetter('data/train', 'GPE_NAM', 1)
batch_getter_lst.append(gpe_batch_getter)
# if my_arg == 1:
org_batch_getter = BatchGetter('data/train', 'ORG_NAM', 1)
batch_getter_lst.append(org_batch_getter)
if my_arg == 2:
pernam_batch_getter = ConllBatchGetter('data/conll2003/bio_eng.train',
'PER', 1, True)
batch_getter_lst.append(pernam_batch_getter)
loc_batch_getter = ConllBatchGetter('data/conll2003/bio_eng.train',
'LOC', 1, True)
batch_getter_lst.append(loc_batch_getter)
misc_batch_getter = ConllBatchGetter('data/conll2003/bio_eng.train',
'MISC', 1, True)
batch_getter_lst.append(misc_batch_getter)
org_batch_getter = ConllBatchGetter('data/conll2003/bio_eng.train',
'ORG', 1, True)
batch_getter_lst.append(org_batch_getter)
batch_getter = MergeBatchGetter(batch_getter_lst, config['batch_size'],
True)
print 'finish loading train data'
embedding_layer = EmbeddingLayer(emb)
d = embedding_layer.get_out_dim()
att_layer = AttentionFlowLayer(2 * d)
# if my_arg == 2:
model_out_layer = ModelingOutLayer(8 * d, d, 2, 3)
# else:
# model_out_layer = ModelingOutLayer(8*d, d, 2, 2)
# models = [embedding_layer, att_layer, model_out_layer]
# opts = [emb_opt, att_opt, model_out_opt]
if config['USE_CUDA']:
att_layer.cuda(config['cuda_num'])
embedding_layer.cuda(config['cuda_num'])
model_out_layer.cuda(config['cuda_num'])
emb_opt = torch.optim.Adadelta(embedding_layer.parameters())
att_opt = torch.optim.Adadelta(att_layer.parameters())
model_out_opt = torch.optim.Adadelta(model_out_layer.parameters())
log_file = open('log_file' + str(my_arg), 'w')
f_max = 0
low_epoch = 0
ex_iterations = 0
model_dir = 'model' + str(my_arg)
for epoch in range(config['max_epoch']):
for iteration, this_batch in enumerate(batch_getter):
time0 = time.time()
print 'epoch: {}, iteraton: {}'.format(epoch,
ex_iterations + iteration)
train_iteration(logger, ex_iterations + iteration, embedding_layer,
att_layer, model_out_layer, emb_opt, att_opt,
model_out_opt, this_batch)
time1 = time.time()
print 'this iteration time: ', time1 - time0, '\n'
if (ex_iterations + iteration) % config['save_freq'] == 0:
torch.save(embedding_layer.state_dict(),
model_dir + '/embedding_layer.pkl')
torch.save(att_layer.state_dict(),
model_dir + '/att_layer.pkl')
torch.save(model_out_layer.state_dict(),
model_dir + '/model_out_layer.pkl')
ex_iterations += iteration + 1
batch_getter.reset()
f, p, r = evaluate_all(my_arg, False)
log_file.write('epoch: {} f: {} p: {} r: {}\n'.format(epoch, f, p, r))
log_file.flush()
if f >= f_max:
f_max = f
low_epoch = 0
# torch.save(embedding_layer.state_dict(), model_dir+'/early_embedding_layer.pkl')
# torch.save(att_layer.state_dict(), model_dir+'/early_att_layer.pkl')
# torch.save(model_out_layer.state_dict(), model_dir+'/early_model_out_layer.pkl')
os.system('cp {}/embedding_layer.pkl {}/early_embedding_layer.pkl'.
format(model_dir, model_dir))
os.system('cp {}/att_layer.pkl {}/early_att_layer.pkl'.format(
model_dir, model_dir))
os.system('cp {}/model_out_layer.pkl {}/early_model_out_layer.pkl'.
format(model_dir, model_dir))
else:
low_epoch += 1
log_file.write('low' + str(low_epoch) + '\n')
log_file.flush()
if low_epoch >= config['early_stop']:
break
log_file.close()