def process_each_estep(tup):
expected = defaultdict(int)
objective = 0
num_tokens = 0
(items, pieces, trie) = tup
L = Lattice()
for item in items:
if item is None:
continue
(key, freq) = item
L.set_sentence(key)
L.populate_nodes(pieces, trie)
Z, ret_expected = L.populate_marginal(freq)
for key, val in ret_expected.items():
expected[key] += val
N = len(L.Viterbi())
num_tokens += N
objective -= Z
return (expected, objective, num_tokens)
