def _calc_mwe_field_sets(self, field_name: str):
r'''Return a Set[Counter[field_value: str]]
Example for field_name=="LEMMA":
{{take, bath}, {look, up}, {there, be}, {to(2x), be(2x), or, not}}
'''
return set(
tsvlib.FrozenCounter(field for (field, index) in fieldindex_set)
for fieldindex_set in self.mwe_fieldindex_sets[field_name])
def _seen_in_train(self, mweinfo: tsvlib.MWEInfo, field_name: str):
r'''Return True iff `mweinfo` was seen in train.'''
field_counter = tsvlib.FrozenCounter(
mweinfo.sentence.words[i].get(field_name, "_")
for i in mweinfo.token_indexes)
return field_counter in self.mwe_field_sets[field_name]