def get_output(self):
def read_stream(file):
with open(file.name, 'rb') as reader:
try:
while True:
yield pickle.load(reader)
except EOFError:
pass
return [iter_to_queue(read_stream(tmp)) for tmp in self.tempfiles] # iterator_to_queue handles threading
def finalize(self, inputs):
if len(inputs) == self.out_width:
if hasattr(self, 'outputs'):
raise RuntimeError("Tried to finalize '%s' twice" % self.name)
inputs = [queue_to_iter(q) for q in inputs]
output_threads = [[iter_to_queue(i) for i in itertools.tee(iterator, self.num_output_uses)]
for iterator in inputs]
self.outputs = zip(*output_threads)
self.uses = 0
elif len(inputs) == 0:
return
else:
raise RuntimeError("Tried to finalize '%s' twice with different widths" % self.name)