def __init__(self):
self.MAX_SENTENCE_LENGTH = 250
self.MAX_WORD_LENGTH = -1
self.number_normalized = True
self.norm_char_emb = True
self.norm_word_emb = True
self.norm_biword_emb = True
self.norm_gaz_emb = False
self.word_alphabet = Alphabet('word')
self.biword_alphabet = Alphabet('biword')
self.char_alphabet = Alphabet('character')
self.label_alphabet = Alphabet('label', True)
self.gaz_lower = False
self.gaz = Gazetteer(self.gaz_lower)
self.gaz_alphabet = Alphabet('gaz')
self.HP_fix_gaz_emb = False
self.HP_use_gaz = True
self.tagScheme = "NoSeg"
self.char_features = "LSTM"
self.train_texts = []
self.dev_texts = []
self.test_texts = []
self.raw_texts = []
## for evaluate word ner labels
self.train_char_labels = []
self.dev_char_labels = []
self.test_char_labels = []
self.train_Ids = []
self.dev_Ids = []
self.test_Ids = []
self.raw_Ids = []
self.use_bigram = True
self.word_emb_dim = 200
self.biword_emb_dim = 50
self.char_emb_dim = 50
self.gaz_emb_dim = 200
self.gaz_dropout = 0.5
self.pretrain_word_embedding = None
self.pretrain_biword_embedding = None
self.pretrain_char_embedding = None
self.label_size = 0
self.word_alphabet_size = 0
self.biword_alphabet_size = 0
self.char_alphabet_size = 0
self.label_alphabet_size = 0
### hyperparameters
self.HP_iteration = 100
self.HP_batch_size = 10
self.HP_char_hidden_dim = 100
self.HP_hidden_dim = 100
self.HP_dropout = 0.5
self.HP_lstm_layer = 1
self.HP_bilstm = True
self.HP_use_char = False
self.HP_gpu = False
self.HP_lr = 0.015
self.HP_lr_decay = 0.05
self.HP_clip = 5.0
self.HP_momentum = 0
self.unknow_index = {}
class Data:
def __init__(self):
self.MAX_SENTENCE_LENGTH = 5000
self.MAX_WORD_LENGTH = -1
self.number_normalized = True
self.norm_word_emb = True
self.norm_biword_emb = True
self.norm_gaz_emb = False
self.word_alphabet = Alphabet('word')
self.biword_alphabet = Alphabet('biword')
self.char_alphabet = Alphabet('character')
# self.word_alphabet.add(START)
# self.word_alphabet.add(UNKNOWN)
# self.char_alphabet.add(START)
# self.char_alphabet.add(UNKNOWN)
# self.char_alphabet.add(PADDING)
self.label_alphabet = Alphabet('label', True)
self.gaz_lower = False
self.gaz = Gazetteer(self.gaz_lower)
self.gaz_alphabet = Alphabet('gaz')
self.HP_fix_gaz_emb = False
self.HP_use_gaz = True
self.tagScheme = "NoSeg"
self.char_features = "LSTM"
self.train_texts = []
self.dev_texts = []
self.test_texts = []
self.raw_texts = []
self.train_Ids = []
self.dev_Ids = []
self.test_Ids = []
self.raw_Ids = []
self.use_bigram = True
self.word_emb_dim = 100
self.biword_emb_dim = 50
self.char_emb_dim = 30
self.gaz_emb_dim = 200
self.gaz_dropout = 0.5
self.pretrain_word_embedding = None
self.pretrain_biword_embedding = None
self.pretrain_gaz_embedding = None
self.label_size = 0
self.word_alphabet_size = 0
self.biword_alphabet_size = 0
self.char_alphabet_size = 0
self.label_alphabet_size = 0
### hyperparameters
self.HP_iteration = 100
self.HP_batch_size = 10
self.HP_char_hidden_dim = 100
self.HP_hidden_dim = 200
self.HP_dropout = 0.5
self.HP_lstm_layer = 1
self.HP_bilstm = True
self.HP_use_char = False
self.HP_gpu = False
self.HP_lr = 0.0035
self.HP_lr_decay = 0.05
self.HP_clip = 5.0
self.HP_momentum = 0
self.unknow_index = {}
def show_data_summary(self):
print("DATA SUMMARY START:")
print(" Tag scheme: %s" % (self.tagScheme))
print(" MAX SENTENCE LENGTH: %s" % (self.MAX_SENTENCE_LENGTH))
print(" MAX WORD LENGTH: %s" % (self.MAX_WORD_LENGTH))
print(" Number normalized: %s" % (self.number_normalized))
print(" Use bigram: %s" % (self.use_bigram))
print(" Word alphabet size: %s" % (self.word_alphabet_size))
print(" Biword alphabet size: %s" % (self.biword_alphabet_size))
print(" Char alphabet size: %s" % (self.char_alphabet_size))
print(" Gaz alphabet size: %s" % (self.gaz_alphabet.size()))
print(" Label alphabet size: %s" % (self.label_alphabet_size))
print(" Word embedding size: %s" % (self.word_emb_dim))
print(" Biword embedding size: %s" % (self.biword_emb_dim))
print(" Char embedding size: %s" % (self.char_emb_dim))
print(" Gaz embedding size: %s" % (self.gaz_emb_dim))
print(" Norm word emb: %s" % (self.norm_word_emb))
print(" Norm biword emb: %s" % (self.norm_biword_emb))
print(" Norm gaz emb: %s" % (self.norm_gaz_emb))
print(" Norm gaz dropout: %s" % (self.gaz_dropout))
print(" Train instance number: %s" % (len(self.train_texts)))
print(" Dev instance number: %s" % (len(self.dev_texts)))
print(" Test instance number: %s" % (len(self.test_texts)))
print(" Raw instance number: %s" % (len(self.raw_texts)))
print(" Hyperpara iteration: %s" % (self.HP_iteration))
print(" Hyperpara batch size: %s" % (self.HP_batch_size))
print(" Hyperpara lr: %s" % (self.HP_lr))
print(" Hyperpara lr_decay: %s" % (self.HP_lr_decay))
print(" Hyperpara HP_clip: %s" % (self.HP_clip))
print(" Hyperpara momentum: %s" % (self.HP_momentum))
print(" Hyperpara hidden_dim: %s" % (self.HP_hidden_dim))
print(" Hyperpara dropout: %s" % (self.HP_dropout))
print(" Hyperpara lstm_layer: %s" % (self.HP_lstm_layer))
print(" Hyperpara bilstm: %s" % (self.HP_bilstm))
print(" Hyperpara GPU: %s" % (self.HP_gpu))
print(" Hyperpara use_gaz: %s" % (self.HP_use_gaz))
print(" Hyperpara fix gaz emb: %s" % (self.HP_fix_gaz_emb))
print(" Hyperpara use_char: %s" % (self.HP_use_char))
if self.HP_use_char:
print(" Char_features: %s" % (self.char_features))
print("DATA SUMMARY END.")
sys.stdout.flush()
def refresh_label_alphabet(self, input_file):
old_size = self.label_alphabet_size
self.label_alphabet.clear(True)
in_lines = open(input_file, 'r', encoding='utf-8').readlines()
for line in in_lines:
if len(line) > 2:
pairs = line.strip().split()
label = pairs[-1]
self.label_alphabet.add(label)
self.label_alphabet_size = self.label_alphabet.size()
startS = False
startB = False
for label, _ in self.label_alphabet.iteritems():
if "S-" in label.upper():
startS = True
elif "B-" in label.upper():
startB = True
if startB:
if startS:
self.tagScheme = "BMES"
else:
self.tagScheme = "BIO"
self.fix_alphabet()
print("Refresh label alphabet finished: old:%s -> new:%s" % (old_size, self.label_alphabet_size))
def build_alphabet(self, input_file):
in_lines = open(input_file, 'r', encoding='utf-8').readlines()
for idx in range(len(in_lines)):
line = in_lines[idx]
if len(line) > 2:
pairs = line.strip().split()
word = pairs[0]
if self.number_normalized:
word = normalize_word(word)
label = pairs[-1]
self.label_alphabet.add(label)
self.word_alphabet.add(word)
if idx < len(in_lines) - 1 and len(in_lines[idx + 1]) > 2:
biword = word + in_lines[idx + 1].strip().split()[0]
else:
biword = word + NULLKEY
self.biword_alphabet.add(biword)
for char in word:
self.char_alphabet.add(char)
self.word_alphabet_size = self.word_alphabet.size()
self.biword_alphabet_size = self.biword_alphabet.size()
self.char_alphabet_size = self.char_alphabet.size()
self.label_alphabet_size = self.label_alphabet.size()
startS = False
startB = False
for label, _ in self.label_alphabet.iteritems():
if "S-" in label.upper():
startS = True
elif "B-" in label.upper():
startB = True
if startB:
if startS:
self.tagScheme = "BMES"
else:
self.tagScheme = "BIO"
def build_gaz_file(self, vocab_file='data/vocab.txt', train_seg='data/train_seg_vocab.txt'):
"""
we use a vocab.txt file to build gaz file, we do not use embedding file, because it too large.
"""
# vocab = set()
with open(vocab_file, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if line:
self.gaz.insert(line, 'one_source')
print("Load gaz file: ", vocab_file, " total size:", self.gaz.size())
def build_gaz_alphabet(self, input_file):
"""
based on the train, dev, test file, we only save the seb-sequence word that my be appear
"""
in_lines = open(input_file, 'r', encoding='utf-8').readlines()
word_list = []
for line in in_lines:
if len(line) > 3:
word = line.split()[0]
if self.number_normalized:
word = normalize_word(word)
word_list.append(word)
else:
### a black line
w_length = len(word_list)
## Travser from [0: n], [1: n] to [n-1: n]
for idx in range(w_length):
matched_entity = self.gaz.enumerateMatchList(word_list[idx:])
for entity in matched_entity:
# print entity, self.gaz.searchId(entity),self.gaz.searchType(entity)
self.gaz_alphabet.add(entity)
word_list = []
print("gaz alphabet size:", self.gaz_alphabet.size())
def add_segs(self, file='data/seg_vocab.txt'):
"""
we add the seg to gaz after generate the gaz_alphabet
"""
# a 'unknow' word for word not in embeddings
self.gaz_alphabet.add("unknow")
self.unknow_index['unknow'] = self.gaz_alphabet.next_index - 1
for i in range(1, 9):
self.gaz_alphabet.add("unknow" + str(i))
self.unknow_index['unknow' + str(i)] = self.gaz_alphabet.next_index - 1
with open(file, 'r', encoding='utf-8') as f:
words = f.readlines()
for word in words:
word = word.strip()
if word:
self.gaz.insert(word, 'one_source')
print('total gaz size after add seg: ', self.gaz.size())
def fix_alphabet(self):
self.word_alphabet.close()
self.biword_alphabet.close()
self.char_alphabet.close()
self.label_alphabet.close()
self.gaz_alphabet.close()
def build_word_pretrain_emb(self, emb_path):
print("build word pretrain emb...")
self.pretrain_word_embedding, self.word_emb_dim = build_pretrain_embedding(emb_path, self.word_alphabet,
self.word_emb_dim,
self.norm_word_emb)
def build_biword_pretrain_emb(self, emb_path):
print("build biword pretrain emb...")
self.pretrain_biword_embedding, self.biword_emb_dim = build_pretrain_embedding(emb_path, self.biword_alphabet,
self.biword_emb_dim,
self.norm_biword_emb)
def build_gaz_pretrain_emb(self, emb_path):
print("build gaz pretrain emb...")
self.pretrain_gaz_embedding, self.gaz_emb_dim = build_pretrain_embedding_for_gaz(self.gaz_alphabet,
embedding_dir='data/small_embeddings',
embedding_name='embed',
file_num=30,
embedd_dim=self.gaz_emb_dim,
norm=self.norm_gaz_emb)
# self.pretrain_gaz_embedding, self.gaz_emb_dim = build_pretrain_embedding(emb_path, self.gaz_alphabet,
# self.gaz_emb_dim, self.norm_gaz_emb)
def generate_instance_with_gaz_no_char(self, input_file, name):
"""
every instance include:
words, biwords, gazs, labels
word_Ids, biword_Ids, gazs_Ids, label_Ids
"""
self.fix_alphabet()
if name == "train":
self.train_texts, self.train_Ids = read_instance_with_gaz_no_char(input_file, self.gaz,
self.word_alphabet,
self.biword_alphabet,
self.gaz_alphabet,
self.label_alphabet,
self.number_normalized,
self.MAX_SENTENCE_LENGTH, self.unknow_index)
elif name == "dev":
self.dev_texts, self.dev_Ids = read_instance_with_gaz_no_char(input_file, self.gaz, self.word_alphabet,
self.biword_alphabet, self.gaz_alphabet,
self.label_alphabet,
self.number_normalized,
self.MAX_SENTENCE_LENGTH, self.unknow_index)
elif name == "test":
self.test_texts, self.test_Ids = read_instance_with_gaz_no_char(input_file, self.gaz,
self.word_alphabet,
self.biword_alphabet,
self.gaz_alphabet, self.label_alphabet,
self.number_normalized,
self.MAX_SENTENCE_LENGTH, self.unknow_index)
elif name == "raw":
self.raw_texts, self.raw_Ids = read_instance_with_gaz_no_char_ls(input_file, self.gaz,
self.word_alphabet,
self.biword_alphabet,
self.gaz_alphabet,
self.label_alphabet,
self.number_normalized,
self.MAX_SENTENCE_LENGTH)
else:
print("Error: you can only generate train/dev/test instance! Illegal input:%s" % (name))
def write_decoded_results(self, output_file, predict_results, name):
fout = open(output_file, 'w', encoding='utf-8')
sent_num = len(predict_results)
content_list = []
if name == 'raw':
content_list = self.raw_texts
elif name == 'test':
content_list = self.test_texts
elif name == 'dev':
content_list = self.dev_texts
elif name == 'train':
content_list = self.train_texts
else:
print("Error: illegal name during writing predict result, name should be within train/dev/test/raw !")
assert (sent_num == len(content_list))
for idx in range(sent_num):
sent_length = len(predict_results[idx])
for idy in range(sent_length):
## content_list[idx] is a list with [word, char, label]
fout.write(content_list[idx][0][idy].encode('utf-8') + " " + predict_results[idx][idy] + '\n')
fout.write('\n')
fout.close()
print("Predict %s result has been written into file. %s" % (name, output_file))