def __init__(self, model, options, vocab, nnvecs=1):
self.word_counts, words, chars, pos, cpos, rels, treebanks, langs = vocab
self.model = model
self.nnvecs = nnvecs
# Load ELMo if the option is set
if options.elmo is not None:
from elmo import ELMo
self.elmo = ELMo(options.elmo, options.elmo_gamma,
options.elmo_learn_gamma)
self.elmo.init_weights(model)
else:
self.elmo = None
extra_words = 2 # MLP padding vector and OOV vector
self.words = {word: ind for ind, word in enumerate(words, extra_words)}
self.word_lookup = self.model.add_lookup_parameters(
(len(self.words) + extra_words, options.word_emb_size))
extra_pos = 2 # MLP padding vector and OOV vector
self.pos = {pos: ind for ind, pos in enumerate(cpos, extra_pos)}
self.pos_lookup = self.model.add_lookup_parameters(
(len(cpos) + extra_pos, options.pos_emb_size))
self.irels = rels
self.rels = {rel: ind for ind, rel in enumerate(rels)}
extra_chars = 1 # OOV vector
self.chars = {char: ind for ind, char in enumerate(chars, extra_chars)}
self.char_lookup = self.model.add_lookup_parameters(
(len(chars) + extra_chars, options.char_emb_size))
extra_treebanks = 1 # Padding vector
self.treebanks = {
treebank: ind
for ind, treebank in enumerate(treebanks, extra_treebanks)
}
self.treebank_lookup = self.model.add_lookup_parameters(
(len(treebanks) + extra_treebanks, options.tbank_emb_size))
# initialise word vectors with external embeddings where they exist
# This part got ugly - TODO: refactor
if not options.predict:
self.external_embedding = defaultdict(lambda: {})
if options.ext_word_emb_file and options.word_emb_size > 0:
# Load pre-trained word embeddings
for lang in langs:
embeddings = utils.get_external_embeddings(
options,
emb_file=options.ext_word_emb_file,
lang=lang,
words=self.words.viewkeys())
self.external_embedding["words"].update(embeddings)
if options.ext_char_emb_file and options.char_emb_size > 0:
# Load pre-trained character embeddings
for lang in langs:
embeddings = utils.get_external_embeddings(
options,
emb_file=options.ext_char_emb_file,
lang=lang,
words=self.chars,
chars=True)
self.external_embedding["chars"].update(embeddings)
if options.ext_emb_dir:
# For every language, load the data for the word and character
# embeddings from a directory.
for lang in langs:
if options.word_emb_size > 0:
embeddings = utils.get_external_embeddings(
options,
emb_dir=options.ext_emb_dir,
lang=lang,
words=self.words.viewkeys())
self.external_embedding["words"].update(embeddings)
if options.char_emb_size > 0:
embeddings = utils.get_external_embeddings(
options,
emb_dir=options.ext_emb_dir,
lang=lang,
words=self.chars,
chars=True)
self.external_embedding["chars"].update(embeddings)
self.init_lookups(options)
elmo_emb_size = self.elmo.emb_dim if self.elmo else 0
self.lstm_input_size = (
options.word_emb_size + elmo_emb_size + options.pos_emb_size +
options.tbank_emb_size + 2 *
(options.char_lstm_output_size if options.char_emb_size > 0 else 0)
)
print "Word-level LSTM input size: " + str(self.lstm_input_size)
self.bilstms = []
if options.no_bilstms > 0:
self.bilstms.append(
BiLSTM(self.lstm_input_size,
options.lstm_output_size,
self.model,
dropout_rate=0.33))
for i in range(1, options.no_bilstms):
self.bilstms.append(
BiLSTM(2 * options.lstm_output_size,
options.lstm_output_size,
self.model,
dropout_rate=0.33))
#used in the PaddingVec
self.word2lstm = self.model.add_parameters(
(options.lstm_output_size * 2, self.lstm_input_size))
self.word2lstmbias = self.model.add_parameters(
(options.lstm_output_size * 2))
else:
self.word2lstm = self.model.add_parameters(
(self.lstm_input_size, self.lstm_input_size))
self.word2lstmbias = self.model.add_parameters(
(self.lstm_input_size))
self.char_bilstm = BiLSTM(options.char_emb_size,
options.char_lstm_output_size,
self.model,
dropout_rate=0.33)
self.charPadding = self.model.add_parameters(
(options.char_lstm_output_size * 2))
def __init__(self, model, options, vocab, nnvecs):
self.word_counts, words, chars, pos, cpos, self.irels, treebanks, langs = vocab
self.model = model
self.nnvecs = nnvecs
extra_words = 2 # MLP padding vector and OOV vector
self.words = {word: ind for ind, word in enumerate(words, extra_words)}
self.word_lookup = self.model.add_lookup_parameters(
(len(self.words) + extra_words, options.word_emb_size))
extra_pos = 2 # MLP padding vector and OOV vector
self.pos = {pos: ind for ind, pos in enumerate(cpos, extra_pos)}
self.pos_lookup = self.model.add_lookup_parameters(
(len(cpos) + extra_pos, options.pos_emb_size))
extra_chars = 1 # OOV vector
self.chars = {char: ind for ind, char in enumerate(chars, extra_chars)}
self.char_lookup = self.model.add_lookup_parameters(
(len(chars) + extra_chars, options.char_emb_size))
extra_treebanks = 1 # Padding vector
self.treebanks = {
treebank: ind
for ind, treebank in enumerate(treebanks, extra_treebanks)
}
self.treebank_lookup = self.model.add_lookup_parameters(
(len(treebanks) + extra_treebanks, options.tbank_emb_size))
# initialise word vectors with external embeddings where they exist
if (options.ext_emb_dir
or options.ext_emb_file) and not options.predict:
self.external_embedding = defaultdict(lambda: {})
for lang in langs:
if options.word_emb_size > 0:
self.external_embedding["words"].update(
utils.get_external_embeddings(options, lang,
self.words.viewkeys()))
if options.char_emb_size > 0:
self.external_embedding["chars"].update(
utils.get_external_embeddings(options,
lang,
self.chars,
chars=True))
self.init_lookups(options)
self.lstm_input_size = options.word_emb_size + options.pos_emb_size + options.tbank_emb_size +\
2* (options.char_lstm_output_size if options.char_emb_size > 0 else 0)
print "Word-level LSTM input size: " + str(self.lstm_input_size)
self.bilstms = []
if options.no_bilstms > 0:
self.bilstms.append(
BiLSTM(self.lstm_input_size,
options.lstm_output_size,
self.model,
dropout_rate=0.33))
for i in range(1, options.no_bilstms):
self.bilstms.append(
BiLSTM(2 * options.lstm_output_size,
options.lstm_output_size,
self.model,
dropout_rate=0.33))
#used in the PaddingVec
self.word2lstm = self.model.add_parameters(
(options.lstm_output_size * 2, self.lstm_input_size))
self.word2lstmbias = self.model.add_parameters(
(options.lstm_output_size * 2))
else:
self.word2lstm = self.model.add_parameters(
(self.lstm_input_size, self.lstm_input_size))
self.word2lstmbias = self.model.add_parameters(
(self.lstm_input_size))
self.char_bilstm = BiLSTM(options.char_emb_size,
options.char_lstm_output_size,
self.model,
dropout_rate=0.33)
self.charPadding = self.model.add_parameters(
(options.char_lstm_output_size * 2))
def __init__(self, model, options, vocab, nnvecs):
self.word_counts, words, chars, pos, cpos, self.irels, treebanks, langs = vocab
self.model = model
self.nnvecs = nnvecs
extra_words = 2 # MLP padding vector and OOV vector
self.words = {word: ind for ind, word in enumerate(words, extra_words)}
#why not just len(self.words)?
self.word_lookup = self.model.add_lookup_parameters(
(len(self.words) + extra_words, options.word_emb_size))
extra_pos = 2 # MLP padding vector and OOV vector
self.pos = {pos: ind for ind, pos in enumerate(cpos, extra_pos)}
self.pos_lookup = self.model.add_lookup_parameters(
(len(cpos) + extra_pos, options.pos_emb_size))
extra_chars = 1 # OOV vector
self.chars = {char: ind for ind, char in enumerate(chars, extra_chars)}
self.char_lookup = self.model.add_lookup_parameters(
(len(chars) + extra_chars, options.char_emb_size))
extra_treebanks = 1 # Padding vector
self.treebanks = {
treebank: ind
for ind, treebank in enumerate(treebanks, extra_treebanks)
}
self.treebank_lookup = self.model.add_lookup_parameters(
(len(treebanks) + extra_treebanks, options.tbank_emb_size))
# initialise word vectors with external embeddings where they exist
if (options.ext_emb_dir
or options.ext_emb_file) and not options.predict:
self.external_embedding = defaultdict(lambda: {})
for lang in langs:
if options.word_emb_size > 0:
self.external_embedding["words"].update(
utils.get_external_embeddings(options, lang,
self.words.viewkeys()))
if options.char_emb_size > 0:
self.external_embedding["chars"].update(
utils.get_external_embeddings(options,
lang,
self.chars,
chars=True))
self.init_lookups(options)
self.lstm_input_size = options.word_emb_size + options.pos_emb_size + options.tbank_emb_size +\
2* (options.char_lstm_output_size if options.char_emb_size > 0 else 0)
print "Word-level LSTM input size: " + str(self.lstm_input_size)
self.bilstms = []
if options.no_bilstms > 0:
if options.unidir_lstm is not None:
#replace the BiLSTMs with unidirectional ones
#it's ugly to still call it bilstm but easier
if options.unidir_lstm:
self.bilstms.append(
LSTM(self.lstm_input_size,
2 * options.lstm_output_size,
self.model,
dropout_rate=0.33,
direction=options.unidir_lstm,
layers=options.no_bilstms))
else:
self.bilstms.append(
BiLSTM(self.lstm_input_size,
options.lstm_output_size,
self.model,
dropout_rate=0.33))
for i in range(1, options.no_bilstms):
self.bilstms.append(
BiLSTM(2 * options.lstm_output_size,
options.lstm_output_size,
self.model,
dropout_rate=0.33))
#used in the PaddingVec
self.word2lstm = self.model.add_parameters(
(options.lstm_output_size * 2, self.lstm_input_size))
self.word2lstmbias = self.model.add_parameters(
(options.lstm_output_size * 2))
else:
self.word2lstm = self.model.add_parameters(
(self.lstm_input_size, self.lstm_input_size))
self.word2lstmbias = self.model.add_parameters(
(self.lstm_input_size))
self.char_bilstm = BiLSTM(options.char_emb_size,
options.char_lstm_output_size,
self.model,
dropout_rate=0.33)
self.charPadding = self.model.add_parameters(
(options.char_lstm_output_size * 2))
#recursive composition things
if options.use_recursive_composition:
deprel_dir = [(rel, direction) for rel in self.irels
for direction in [0, 1]]
extra_deprel = 1 # padding rel vec
#this does not work
self.ideprel_dir = {
val: ind
for ind, val in enumerate(deprel_dir, extra_deprel)
}
self.deprel_lookup = self.model.add_lookup_parameters(
(len(self.ideprel_dir) + extra_deprel, options.deprel_size))
lstm_out_dim = options.lstm_output_size * 2
if options.use_recursive_composition == 'RecNN':
self.hCompos = self.model.add_parameters(
(lstm_out_dim, lstm_out_dim))
self.dCompos = self.model.add_parameters(
(lstm_out_dim, lstm_out_dim))
self.rCompos = self.model.add_parameters(
(lstm_out_dim, options.deprel_size))
self.biasCompos = self.model.add_parameters((lstm_out_dim))
else:
compos_in_dim = lstm_out_dim * 2 + options.deprel_size
self.composLSTM = dy.VanillaLSTMBuilder(
1, compos_in_dim, lstm_out_dim, self.model)
def Predict(self, treebanks, datasplit, options):
char_map = {}
if options.char_map_file:
char_map_fh = codecs.open(options.char_map_file,encoding='utf-8')
char_map = json.loads(char_map_fh.read())
# should probably use a namedtuple in get_vocab to make this prettier
_, test_words, test_chars, _, _, _, test_treebanks, test_langs = utils.get_vocab(treebanks,datasplit,char_map)
# get external embeddings for the set of words and chars in the
# test vocab but not in the training vocab
test_embeddings = defaultdict(lambda: {})
if options.word_emb_size > 0 and options.ext_word_emb_file:
new_test_words = \
set(test_words) - self.feature_extractor.words.viewkeys()
print "Number of OOV word types at test time: %i (out of %i)" % (
len(new_test_words), len(test_words))
if len(new_test_words) > 0:
# no point loading embeddings if there are no words to look for
for lang in test_langs:
embeddings = utils.get_external_embeddings(
options,
emb_file=options.ext_word_emb_file,
lang=lang,
words=new_test_words
)
test_embeddings["words"].update(embeddings)
if len(test_langs) > 1 and test_embeddings["words"]:
print "External embeddings found for %i words "\
"(out of %i)" % \
(len(test_embeddings["words"]), len(new_test_words))
if options.char_emb_size > 0:
new_test_chars = \
set(test_chars) - self.feature_extractor.chars.viewkeys()
print "Number of OOV char types at test time: %i (out of %i)" % (
len(new_test_chars), len(test_chars))
if len(new_test_chars) > 0:
for lang in test_langs:
embeddings = utils.get_external_embeddings(
options,
emb_file=options.ext_char_emb_file,
lang=lang,
words=new_test_chars,
chars=True
)
test_embeddings["chars"].update(embeddings)
if len(test_langs) > 1 and test_embeddings["chars"]:
print "External embeddings found for %i chars "\
"(out of %i)" % \
(len(test_embeddings["chars"]), len(new_test_chars))
data = utils.read_conll_dir(treebanks,datasplit,char_map=char_map)
for iSentence, osentence in enumerate(data,1):
sentence = deepcopy(osentence)
self.feature_extractor.Init(options)
conll_sentence = [entry for entry in sentence if isinstance(entry, utils.ConllEntry)]
self.feature_extractor.getWordEmbeddings(conll_sentence, False, options, test_embeddings)
scores, exprs = self.__evaluate(conll_sentence, True)
if self.proj:
heads = decoder.parse_proj(scores)
#LATTICE solution to multiple roots
# see https://github.com/jujbob/multilingual-bist-parser/blob/master/bist-parser/bmstparser/src/mstlstm.py
## ADD for handling multi-roots problem
rootHead = [head for head in heads if head==0]
if len(rootHead) != 1:
print "it has multi-root, changing it for heading first root for other roots"
rootHead = [seq for seq, head in enumerate(heads) if head == 0]
for seq in rootHead[1:]:heads[seq] = rootHead[0]
## finish to multi-roots
else:
heads = chuliu_edmonds_one_root(scores.T)
for entry, head in zip(conll_sentence, heads):
entry.pred_parent_id = head
entry.pred_relation = '_'
if self.labelsFlag:
for modifier, head in enumerate(heads[1:]):
scores, exprs = self.__evaluateLabel(conll_sentence, head, modifier+1)
conll_sentence[modifier+1].pred_relation = self.feature_extractor.irels[max(enumerate(scores), key=itemgetter(1))[0]]
dy.renew_cg()
#keep in memory the information we need, not all the vectors
oconll_sentence = [entry for entry in osentence if isinstance(entry, utils.ConllEntry)]
for tok_o, tok in zip(oconll_sentence, conll_sentence):
tok_o.pred_relation = tok.pred_relation
tok_o.pred_parent_id = tok.pred_parent_id
yield osentence
def Predict(self, treebanks, datasplit, options):
reached_max_swap = 0
char_map = {}
if options.char_map_file:
char_map_fh = codecs.open(options.char_map_file,encoding='utf-8')
char_map = json.loads(char_map_fh.read())
# should probably use a namedtuple in get_vocab to make this prettier
print "Collecting test data vocab"
_, test_words, test_chars, _, _, _, test_treebanks, test_langs = utils.get_vocab(treebanks,datasplit,char_map)
# get external embeddings for the set of words and chars in the test vocab but not in the training vocab
test_embeddings = defaultdict(lambda:{})
if options.word_emb_size > 0:
new_test_words = set(test_words) - self.feature_extractor.words.viewkeys()
print "Number of OOV word types at test time: %i (out of %i)"%(len(new_test_words),len(test_words))
if len(new_test_words) > 0: # no point loading embeddings if there are no words to look for
for lang in test_langs:
test_embeddings["words"].update(utils.get_external_embeddings(options,lang,new_test_words))
if len(test_langs) > 1 and test_embeddings["words"]:
print "External embeddings found for %i words (out of %i)"%(len(test_embeddings["words"]),len(new_test_words))
if options.char_emb_size > 0:
new_test_chars = set(test_chars) - self.feature_extractor.chars.viewkeys()
print "Number of OOV char types at test time: %i (out of %i)"%(len(new_test_chars),len(test_chars))
if len(new_test_chars) > 0:
for lang in test_langs:
test_embeddings["chars"].update(utils.get_external_embeddings(options,lang,new_test_chars,chars=True))
if len(test_langs) > 1 and test_embeddings["chars"]:
print "External embeddings found for %i chars (out of %i)"%(len(test_embeddings["chars"]),len(new_test_chars))
ts = time()
data = utils.read_conll_dir(treebanks,datasplit,char_map=char_map)
for iSentence, osentence in enumerate(data,1):
sentence = deepcopy(osentence)
reached_swap_for_i_sentence = False
max_swap = 2*len(sentence)
iSwap = 0
self.feature_extractor.Init(options)
conll_sentence = [entry for entry in sentence if isinstance(entry, utils.ConllEntry)]
conll_sentence = conll_sentence[1:] + [conll_sentence[0]]
self.feature_extractor.getWordEmbeddings(conll_sentence, False, options, test_embeddings)
stack = ParseForest([])
buf = ParseForest(conll_sentence)
hoffset = 1 if self.headFlag else 0
for root in conll_sentence:
root.lstms = [root.vec] if self.headFlag else []
if not self.recursive_composition:
root.lstms += [self.feature_extractor.paddingVec for _ in range(self.nnvecs - hoffset)]
else:
root.lstms += [root.vec]
root.lstm = None #only necessary for treeLSTM case
root.composed_rep = root.vec.value()
while not (len(buf) == 1 and len(stack) == 0):
scores = self.__evaluate(stack, buf, False)
best = max(chain(*(scores if iSwap < max_swap else scores[:3] )), key = itemgetter(2) )
if iSwap == max_swap and not reached_swap_for_i_sentence:
reached_max_swap += 1
reached_swap_for_i_sentence = True
print "reached max swap in %d out of %d sentences"%(reached_max_swap, iSentence)
self.apply_transition(best,stack,buf,hoffset)
if best[1] == SWAP:
iSwap += 1
#keep in memory the information we need, not all the vectors
oconll_sentence = [entry for entry in osentence if isinstance(entry, utils.ConllEntry)]
oconll_sentence = oconll_sentence[1:] + [oconll_sentence[0]]
for tok_o, tok in zip(oconll_sentence, conll_sentence):
tok_o.pred_relation = tok.pred_relation
tok_o.pred_parent_id = tok.pred_parent_id
if self.recursive_composition:
tok_o.composed_rep = tok.composed_rep
yield osentence
dy.renew_cg()
print "Total prediction time: %.2fs"%(time()-ts)
def Predict(self, treebanks, datasplit, options):
reached_max_swap = 0
char_map = {}
if options.char_map_file:
char_map_fh = open(options.char_map_file, encoding='utf-8')
char_map = json.loads(char_map_fh.read())
# should probably use a namedtuple in get_vocab to make this prettier
_, test_words, test_chars, _, _, _, test_treebanks, test_langs = utils.get_vocab(
treebanks, datasplit, char_map)
# get external embeddings for the set of words and chars in the
# test vocab but not in the training vocab
test_embeddings = defaultdict(lambda: {})
if options.word_emb_size > 0 and options.ext_word_emb_file:
new_test_words = \
set(test_words) - self.feature_extractor.words.keys()
print("Number of OOV word types at test time: %i (out of %i)" %
(len(new_test_words), len(test_words)))
if len(new_test_words) > 0:
# no point loading embeddings if there are no words to look for
for lang in test_langs:
embeddings = utils.get_external_embeddings(
options,
emb_file=options.ext_word_emb_file,
lang=lang,
words=new_test_words)
test_embeddings["words"].update(embeddings)
if len(test_langs) > 1 and test_embeddings["words"]:
print("External embeddings found for %i words "\
"(out of %i)" % \
(len(test_embeddings["words"]), len(new_test_words)))
if options.char_emb_size > 0:
new_test_chars = \
set(test_chars) - self.feature_extractor.chars.keys()
print("Number of OOV char types at test time: %i (out of %i)" %
(len(new_test_chars), len(test_chars)))
if len(new_test_chars) > 0:
for lang in test_langs:
embeddings = utils.get_external_embeddings(
options,
emb_file=options.ext_char_emb_file,
lang=lang,
words=new_test_chars,
chars=True)
test_embeddings["chars"].update(embeddings)
if len(test_langs) > 1 and test_embeddings["chars"]:
print("External embeddings found for %i chars "\
"(out of %i)" % \
(len(test_embeddings["chars"]), len(new_test_chars)))
data = utils.read_conll_dir(treebanks, datasplit, char_map=char_map)
for iSentence, osentence in enumerate(data, 1):
sentence = deepcopy(osentence)
reached_swap_for_i_sentence = False
max_swap = 2 * len(sentence)
iSwap = 0
self.feature_extractor.Init(options)
conll_sentence = [
entry for entry in sentence
if isinstance(entry, utils.ConllEntry)
]
conll_sentence = conll_sentence[1:] + [conll_sentence[0]]
self.feature_extractor.getWordEmbeddings(conll_sentence, False,
options, test_embeddings)
stack = ParseForest([])
buf = ParseForest(conll_sentence)
hoffset = 1 if self.headFlag else 0
for root in conll_sentence:
#empty = dy.zeros(2*options.lstm_output_size)
root.lstms = [root.vec] if self.headFlag else []
root.lstms += [root.vec for _ in range(self.nnvecs - hoffset)]
root.relation = root.relation if root.relation in self.irels else 'runk'
while not (len(buf) == 1 and len(stack) == 0):
scores = self.__evaluate(stack, buf, False)
best = max(
chain(*(scores if iSwap < max_swap else scores[:3])),
key=itemgetter(2))
if iSwap == max_swap and not reached_swap_for_i_sentence:
reached_max_swap += 1
reached_swap_for_i_sentence = True
print("reached max swap in %d out of %d sentences" %
(reached_max_swap, iSentence))
self.apply_transition(best, stack, buf, hoffset)
if best[1] == SWAP:
iSwap += 1
dy.renew_cg()
#keep in memory the information we need, not all the vectors
oconll_sentence = [
entry for entry in osentence
if isinstance(entry, utils.ConllEntry)
]
oconll_sentence = oconll_sentence[1:] + [oconll_sentence[0]]
for tok_o, tok in zip(oconll_sentence, conll_sentence):
tok_o.pred_relation = tok.pred_relation
tok_o.pred_parent_id = tok.pred_parent_id
yield osentence