def char_mapping(sentences):
"""
Create a dictionary and mapping of characters, sorted by frequency.
"""
chars = ["".join([w[0] for w in s]) for s in sentences]
dico = create_dico(chars)
dico['<PAD>'] = 10000000
# dico[';'] = 0
char_to_id, id_to_char = create_mapping(dico)
print("Found %i unique characters" % len(dico))
return dico, char_to_id, id_to_char
def tag_mapping(sentences):
"""
Create a dictionary and a mapping of tags, sorted by frequency.
"""
tags = [[word[-1] for word in s] for s in sentences]
dico = create_dico(tags)
dico[model.START_TAG] = -1
dico[model.STOP_TAG] = -2
tag_to_id, id_to_tag = create_mapping(dico)
print("Found %i unique named entity tags" % len(dico))
# print(dico)
return dico, tag_to_id, id_to_tag
def word_mapping(sentences, lower):
"""
Create a dictionary and a mapping of words, sorted by frequency.
"""
words = [[x[0].lower() if lower else x[0] for x in s] for s in sentences]
dico = create_dico(words) #dict with word frequency
# print(dico)
dico['<PAD>'] = 10000001
dico['<UNK>'] = 10000000
dico = {k:v for k,v in dico.items() if v>=3} #prune words which has occureced less than 3 times
word_to_id, id_to_word = create_mapping(dico)
print("Found %i unique words (%i in total)" % (
len(dico), sum(len(x) for x in words)
))
return dico, word_to_id, id_to_word