def _load_frequency_data(letter, include_subentries):
frequencies = defaultdict(list)
subfrequencies = defaultdict(list)
iterator = OedContentIterator(letter=letter,
include_entries=True,
include_subentries=include_subentries)
for wordclass_set in iterator.iterate():
if wordclass_set.has_frequency_table():
oed_id = wordclass_set.link(target='oed', defragment=True)
node_id = wordclass_set.link(target='oed', as_tuple=True)[1]
wcdata = WordclassData(wordclass_set.wordclass(),
wordclass_set.frequency_table(),
[],)
for type_unit in [t for t in wordclass_set.types()
if t.has_frequency_table()]:
typedata = TypeData(type_unit.form,
type_unit.wordclass(),
type_unit.frequency_table())
wcdata.types.append(typedata)
if wordclass_set.oed_entry_type() == 'entry':
frequencies[oed_id].append(wcdata)
else:
subfrequencies[(oed_id, node_id)].append(wcdata)
return frequencies, subfrequencies