def concat_input(cls, task, output_label, replicas):
output = AtomicFile(task.output_path(output_label))
BUFFER_SIZE = 1024*1024
for reps in replicas:
# Use only the first replica for now, since a set of one
# is the most common case.
# TODO: handle falling back to alternative replicas.
inp = open_url(reps[0])
buf = inp.read(BUFFER_SIZE)
while (len(buf) > 0):
output.write(buf)
buf = inp.read(BUFFER_SIZE)
inp.close()
output.close()
return output.path, output.size()
def concat_input(cls, task, output_label, replicas):
output = AtomicFile(task.output_path(output_label))
BUFFER_SIZE = 1024 * 1024
for reps in replicas:
# Use only the first replica for now, since a set of one
# is the most common case.
# TODO: handle falling back to alternative replicas.
inp = open_url(reps[0])
buf = inp.read(BUFFER_SIZE)
while (len(buf) > 0):
output.write(buf)
buf = inp.read(BUFFER_SIZE)
inp.close()
output.close()
return output.path, output.size()
