def test_composite_reader_builder(self): ws = workspace.C.Workspace() session = LocalSession(ws) num_srcs = 3 names = ["src_{}".format(i) for i in range(num_srcs)] size = 100 offsets = [i * size for i in range(num_srcs)] src_ds_builders = [ TestReaderBuilder(offset=offset, size=size, name=name) for (name, offset) in zip(names, offsets) ] # Make an identically-sized empty destnation dataset dst_ds_schema = schema.Struct( *[(name, src_ds_builder.schema()) for name, src_ds_builder in zip(names, src_ds_builders)]) dst_ds = make_destination_dataset(ws, dst_ds_schema) with TaskGroup() as tg: reader_builder = CompositeReaderBuilder(names, src_ds_builders) reader_builder.setup(ws=ws) pipe(reader_builder.new_reader(), dst_ds.writer(), num_runtime_threads=3) session.run(tg) for name, offset in zip(names, offsets): written_data = sorted( ws.fetch_blob(str(dst_ds.content()[name].label()))) npt.assert_array_equal(range(offset, offset + size), written_data, "name: {}".format(name))
def test_composite_reader_builder(self): ws = workspace.C.Workspace() session = LocalSession(ws) num_srcs = 3 names = ["src_{}".format(i) for i in range(num_srcs)] size = 100 offsets = [i * size for i in range(num_srcs)] src_ds_builders = [ TestReaderBuilder(offset=offset, size=size, name=name) for (name, offset) in zip(names, offsets) ] # Make an identically-sized empty destnation dataset dst_ds_schema = schema.Struct( *[ (name, src_ds_builder.schema()) for name, src_ds_builder in zip(names, src_ds_builders) ] ) dst_ds = make_destination_dataset(ws, dst_ds_schema) with TaskGroup() as tg: reader_builder = CompositeReaderBuilder( names, src_ds_builders) reader_builder.setup(ws=ws) pipe(reader_builder.new_reader(), dst_ds.writer(), num_runtime_threads=3) session.run(tg) for name, offset in zip(names, offsets): written_data = sorted( ws.fetch_blob(str(dst_ds.content()[name].label()))) npt.assert_array_equal(range(offset, offset + size), written_data, "name: {}".format(name))