def build_utterance_vocab(dialogues,
special_symbols=[],
entity_forms=[],
except_words=[]):
vocab = Vocabulary(offset=0, unk=True, except_words=except_words)
def _add_entity(entity):
for entity_form in entity_forms:
word = get_entity_form(entity, entity_form)
vocab.add_word(word)
# Add words
for dialogue in dialogues:
assert dialogue.is_int is False
for turn in dialogue.token_turns:
for token in turn:
if is_entity(token):
_add_entity(token)
else:
vocab.add_word(token)
# Add special symbols
vocab.add_words(special_symbols, special=True)
vocab.finish(size_threshold=10000)
print('Utterance vocab size:', vocab.size)
return vocab
def build_lf_vocab(dialogues):
vocab = Vocabulary(offset=0, unk=True)
for dialogue in dialogues:
assert dialogue.is_int is False
for lf in dialogue.lfs:
vocab.add_words(lf)
vocab.add_words(sequence_markers, special=True)
vocab.finish()
print 'LF vocabulary size:', vocab.size
return vocab
def build_kb_vocab(dialogues, special_symbols=[]):
kb_vocab = Vocabulary(offset=0, unk=False)
for dialogue in dialogues:
assert dialogue.is_int is False
kb_vocab.add_words(dialogue.scenario)
kb_vocab.add_words(special_symbols, special=True)
kb_vocab.finish()
print 'KB vocab size:', kb_vocab.size
return kb_vocab
def build_vocab(dialogues, special_symbols=[], entity_forms=[]):
vocab = Vocabulary(offset=0, unk=True)
def _add_entity(entity):
for entity_form in entity_forms:
# If copy entity embedding from the graph embedding, don't need entity in vocab
if entity_form != 'graph':
word = Preprocessor.get_entity_form(entity, entity_form)
vocab.add_word(word)
# Add words
for dialogue in dialogues:
assert dialogue.is_int is False
for turns in dialogue.token_turns:
for turn in turns:
for token in chain.from_iterable(turn):
if is_entity(token):
_add_entity(token)
else:
vocab.add_word(token)
# Add special symbols
vocab.add_words(special_symbols)
print('Vocabulary size:', vocab.size)
return vocab
def build_lf_vocab_simple(dialogues):
vocab = Vocabulary(offset=0, unk=True)
for dialogue in dialogues:
assert dialogue.is_int is False
for lf in dialogue.lf_tokens:
vocab.add_words(lf)
vocab.add_words(sequence_markers, special=True)
print('LF vocabulary size:', vocab.size)
return vocab
def build_lf_vocab(dialogues, special_symbols=[], entity_forms=[]):
vocab = Vocabulary(offset=0, unk=True)
def _add_entity(entity):
for entity_form in entity_forms:
word = get_entity_form(entity, entity_form)
vocab.add_word(word)
for dialogue in dialogues:
assert dialogue.is_int is False
for lf in dialogue.lfs:
for token in lf:
if is_entity(token):
_add_entity(token)
else:
vocab.add_word(token)
vocab.add_words(sequence_markers, special=True)
vocab.finish(size_threshold=10000)
print('LF vocabulary size:', vocab.size)
return vocab
def build_schema_mappings(schema, num_items):
entity_map = Vocabulary(unk=True)
for type_, values in schema.values.iteritems():
entity_map.add_words(((value.lower(), type_) for value in values))
# Add item nodes
for i in range(num_items):
entity_map.add_word(item_to_entity(i)[1])
# Add attr nodes
#for attr in schema.attributes:
# entity_map.add_word((attr.name.lower(), 'attr'))
relation_map = Vocabulary(unk=False)
attribute_types = schema.get_attributes() # {attribute_name: value_type}
relation_map.add_words((a.lower() for a in attribute_types.keys()))
relation_map.add_word('has')
# Inverse relation
relation_map.add_words([inv_rel(r) for r in relation_map.word_to_ind])
return entity_map, relation_map
def build_kb_vocab(dialogues, special_symbols=[]):
kb_vocab = Vocabulary(offset=0, unk=True)
cat_vocab = Vocabulary(offset=0, unk=False)
for dialogue in dialogues:
assert dialogue.is_int is False
kb_vocab.add_words(dialogue.title)
kb_vocab.add_words(dialogue.description)
cat_vocab.add_word(dialogue.category)
kb_vocab.add_words(special_symbols, special=True)
kb_vocab.finish(freq_threshold=5)
cat_vocab.add_words(
['bike', 'car', 'electronics', 'furniture', 'housing', 'phone'],
special=True)
cat_vocab.finish()
print('KB vocab size:', kb_vocab.size)
print('Category vocab size:', cat_vocab.size)
return kb_vocab, cat_vocab