def create_fields_tuples():
fields_tuples = [
('words', fields.WordsField()),
('words_hyp', fields.WordsField()),
('marks', fields.MarkIndexesField()),
]
return fields_tuples
def create_fields_tuples():
hyp_target_field = fields.WordsField(lower=True)
fields_tuples = [('words', fields.WordsField(lower=True)),
('words_hyp', hyp_target_field),
('target', hyp_target_field),
('attn', fields.AttentionField())]
return fields_tuples
def create_fields_tuples():
words_field = fields.WordsField()
fields_tuples = [('words', words_field), ('words_hyp', words_field),
('words_expl', words_field),
('marks', fields.MarkIndexesField()),
('marks_hyp', fields.MarkIndexesField()),
('target', fields.TagsField())]
return fields_tuples
def create_fields_tuples():
# if you choose tokenizer='spacy', please install the en package:
# python3 -m spacy download en
tokenizer = nltk.WordPunctTokenizer()
# tokenizer = nltk.TreebankWordTokenizer()
fields_tuples = [('words',
fields.WordsField(tokenize=tokenizer.tokenize)),
('target', fields.TagsField())]
return fields_tuples
def create_fields_tuples():
fields_tuples = [('words', fields.WordsField())]
return fields_tuples
def create_fields_tuples():
fields_tuples = [('words', fields.WordsField()),
('target', fields.TagsField())]
return fields_tuples
def create_fields_tuples():
tokenizer = nltk.WordPunctTokenizer()
fields_tuples = [('words',
fields.WordsField(tokenize=tokenizer.tokenize)),
('target', fields.TagsField())]
return fields_tuples