def write_querydata(args: ProcessQueryArgs) -> None:
log.info("Query data is strip {} of {}".format(args.strip_idx,
args.total_strips))
log.info("Writing query data to tfrecord in {}-point batches".format(
args.batchsize))
reader_src = IdReader()
it, n_total = indices_strip(args.image_spec, args.strip_idx,
args.total_strips, args.batchsize)
worker = _QueryDataProcessor(args.feature_path, args.image_spec,
args.halfwidth)
tasks = list(it)
out_it = task_list(tasks, reader_src, worker, args.nworkers)
tfwrite.query(out_it, n_total, args.directory, args.tag)
def read(
self,
npoints: Optional[int] = None,
shuffle: bool = False,
random_seed: int = 220,
) -> Iterator[XData]:
"""Read N points of (optionally random) query data (in batches)."""
npoints = min(npoints or self.size, self.size)
if shuffle:
it, _ = random_indices(self.meta.image,
npoints,
self.batchsize,
random_seed=random_seed)
else:
it_all, _ = indices_strip(self.meta.image, 1, 1, self.batchsize)
it = _islice_batched(it_all, npoints)
return self._run(list(it))
def test_indices_strip(nstrips, rows, cols):
# coords are 1 past last pixel
x_coords = np.arange(cols + 1)
y_coords = np.arange(rows + 1)
crs = {"init": "egs123"}
spec = image.ImageSpec(x_coords, y_coords, crs)
batchsize = 10
xy_inds = []
n = 0
for i in range(nstrips):
it_i, n_i = image.indices_strip(spec, i + 1, nstrips, batchsize)
n += n_i
for xy in it_i:
assert xy.shape[0] <= batchsize
assert xy.shape[1] == 2
x, y = xy.T
xy_inds.append(xy)
xy_inds = np.concatenate(xy_inds, axis=0)
assert n == rows * cols
ans = np.fliplr(np.array(list(product(range(rows), range(cols)))))
assert np.all(xy_inds == ans)