def finalize(self, inputs):
if len(inputs) == self.out_width:
if hasattr(self, 'outputs'):
raise RuntimeError("Tried to finalize '%s' twice" % self.name)
inputs = [queue_to_iter(q) for q in inputs]
output_threads = [[iter_to_queue(i) for i in itertools.tee(iterator, self.num_output_uses)]
for iterator in inputs]
self.outputs = zip(*output_threads)
self.uses = 0
elif len(inputs) == 0:
return
else:
raise RuntimeError("Tried to finalize '%s' twice with different widths" % self.name)
def writer(idx, stream):
with self.tempfiles[idx] as tmp:
for val in queue_to_iter(stream):
pickle.dump(val, tmp)
