def block_aligned_roi(self, desired_roi):
'''
Returns the block aligned pixel region to read in a Rectangle format
to get the requested data region while respecting block boundaries.
'''
self.__asert_open()
bounds = rectangle.Rectangle(0, 0, width=self.width(), height=self.height())
if not bounds.contains_rect(desired_roi):
raise Exception('desired_roi ' + str(desired_roi)
+ ' is outside the bounds of image with size' + str(self.size()))
(block_size, unused_num_blocks) = self.block_info(0)
start_block_x = int(math.floor(desired_roi.min_x / block_size[0]))
start_block_y = int(math.floor(desired_roi.min_y / block_size[1]))
# Rect max is exclusive
stop_block_x = int(math.floor((desired_roi.max_x-1) / block_size[0]))
# The stops are inclusive
stop_block_y = int(math.floor((desired_roi.max_y-1) / block_size[1]))
start_x = start_block_x * block_size[0]
start_y = start_block_y * block_size[1]
w = (stop_block_x - start_block_x + 1) * block_size[0]
h = (stop_block_y - start_block_y + 1) * block_size[1]
# Restrict the output region to the bounding box of the image.
# - Needed to handle images with partial tiles at the boundaries.
ans = rectangle.Rectangle(start_x, start_y, width=w, height=h)
bounds = rectangle.Rectangle(0, 0, width=self.width(), height=self.height())
return ans.get_intersection(bounds)
def read_image(img, rect):
lock = image_locks[img]
preprocess = image_preprocesses[img]
buf = np.zeros(shape=(img.num_bands(), rect.height(),
rect.width()),
dtype=img.dtype())
mod_r = rectangle.Rectangle(min_x=rect.min_x,
min_y=rect.min_y,
max_x=rect.max_x,
max_y=rect.max_y)
mod_r.shift(rand_offset[0], rand_offset[1])
request_r = mod_r.get_intersection(
rectangle.Rectangle(min_x=0,
min_y=0,
width=img.width(),
height=img.height()))
lock.acquire()
partial_buf = buf[:, request_r.min_y - mod_r.min_y:mod_r.height() +
request_r.max_y - mod_r.max_y,
request_r.min_x - mod_r.min_x:mod_r.width() +
request_r.max_x - mod_r.max_x]
img.read(request_r, buf=partial_buf)
lock.release()
# preprocess outside of lock for concurrency
buf = np.transpose(buf, [1, 2, 0])
if preprocess:
buf = preprocess(buf, rect, None)
return buf
def block_aligned_roi(self, desired_roi):
self.__asert_open()
bounds = rectangle.Rectangle(0,
0,
width=self.width(),
height=self.height())
if not bounds.contains_rect(desired_roi):
raise Exception('desired_roi ' + str(desired_roi) +
' is outside the bounds of image with size' +
str(self.size()))
block_height, block_width = self.block_size()
start_block_x = int(math.floor(desired_roi.min_x / block_width))
start_block_y = int(math.floor(desired_roi.min_y / block_height))
# Rect max is exclusive
stop_block_x = int(math.floor((desired_roi.max_x - 1) / block_width))
# The stops are inclusive
stop_block_y = int(math.floor((desired_roi.max_y - 1) / block_height))
start_x = start_block_x * block_width
start_y = start_block_y * block_height
w = (stop_block_x - start_block_x + 1) * block_width
h = (stop_block_y - start_block_y + 1) * block_height
# Restrict the output region to the bounding box of the image.
# - Needed to handle images with partial tiles at the boundaries.
ans = rectangle.Rectangle(start_x, start_y, width=w, height=h)
bounds = rectangle.Rectangle(0,
0,
width=self.width(),
height=self.height())
return ans.get_intersection(bounds)
def test_rectangle():
"""
Tests the Rectangle class basics.
"""
r = rectangle.Rectangle(5, 10, 15, 30)
assert r.min_x == 5
assert r.min_y == 10
assert r.max_x == 15
assert r.max_y == 30
assert r.bounds() == (5, 15, 10, 30)
assert r.has_area()
assert r.get_min_coord() == (5, 10)
assert r.perimeter() == 60
assert r.area() == 200
r.shift(-5, -10)
assert r.bounds() == (0, 10, 0, 20)
r.scale_by_constant(2, 1)
assert r.bounds() == (0, 20, 0, 20)
r.expand(0, 0, -10, -5)
assert r.bounds() == (0, 10, 0, 15)
r.expand_to_contain_pt(14, 14)
assert r.bounds() == (0, 15, 0, 15)
r2 = rectangle.Rectangle(-5, -5, 5, 10)
assert r.get_intersection(r2).bounds() == (0, 5, 0, 10)
assert not r.contains_rect(r2)
assert r.overlaps(r2)
assert not r.contains_pt(-1, -1)
assert r2.contains_pt(-1, -1)
r.expand_to_contain_rect(r2)
assert r.bounds() == (-5, 15, -5, 15)
def callback_function(roi, data):
pred_image = self._predict_array(data, image.nodata_value())
block_x = (roi.min_x - input_bounds.min_x)
block_y = (roi.min_y - input_bounds.min_y)
(sx, sy) = (block_x, block_y)
labels = None
if label:
label_x = roi.min_x + (roi.width() - pred_image.shape[0]) // 2
label_y = roi.min_y + (roi.height() - pred_image.shape[1]) // 2
label_roi = rectangle.Rectangle(label_x, label_y,
label_x + pred_image.shape[0],
label_y + pred_image.shape[1])
labels = np.squeeze(label.read(label_roi))
tl = [0, 0]
tl = (overlap[0] // 2 if block_x > 0 else 0,
overlap[1] // 2 if block_y > 0 else 0)
br = (roi.max_x - roi.min_x, roi.max_y - roi.min_y)
br = (br[0] -
(overlap[0] // 2 if roi.max_x < input_bounds.max_x else 0),
br[1] -
(overlap[1] // 2 if roi.max_x < input_bounds.max_x else 0))
if len(pred_image.shape) == 2:
input_block = pred_image[tl[0]:br[0], tl[1]:br[1]]
else:
input_block = pred_image[tl[0]:br[0], tl[1]:br[1], :]
self._process_block(
input_block, sx + tl[0], sy + tl[1],
None if labels is None else labels[tl[0]:br[0],
tl[1]:br[1]], label_nodata)
def save(self, path, tile_size=(0,0), nodata_value=None, show_progress=False):
"""
Save a TiffImage to the file output_path, optionally overwriting the tile_size.
"""
if nodata_value is None:
nodata_value = self.nodata_value()
# Use the input tile size for the block size unless the user specified one.
(bs, _) = self.block_info()
block_size_x = bs[0]
block_size_y = bs[1]
if tile_size[0] > 0:
block_size_x = tile_size[0]
if tile_size[1] > 0:
block_size_y = tile_size[1]
# Set up the output image
with TiffWriter(path, self.width(), self.height(), self.num_bands(),
self.data_type(), block_size_x, block_size_y,
nodata_value, self.metadata()) as writer:
input_bounds = rectangle.Rectangle(0, 0, width=self.width(), height=self.height())
output_rois = input_bounds.make_tile_rois(block_size_x, block_size_y, include_partials=True)
def callback_function(output_roi, data):
"""Callback function to write the first channel to the output file."""
# Figure out some ROI positioning values
block_x = output_roi.min_x / block_size_x
block_y = output_roi.min_y / block_size_y
# Loop on bands
for band in range(data.shape[2]):
writer.write_block(data[:, :, band], block_x, block_y, band)
self.process_rois(output_rois, callback_function, show_progress=show_progress)
def check_landsat_tiff(filename):
'''
Checks reading landsat tiffs.
'''
input_reader = TiffImage(filename)
assert input_reader.size() == (37, 37)
assert input_reader.num_bands() == 8
for i in range(0, input_reader.num_bands()):
(bsize, (blocks_x, blocks_y)) = input_reader.block_info(i)
assert bsize == (6, 37)
assert blocks_x == 7
assert blocks_y == 1
assert input_reader.numpy_type(i) == np.float32
assert input_reader.nodata_value(i) is None
meta = input_reader.metadata()
geo = meta['geotransform']
assert geo[0] == pytest.approx(-122.3, abs=0.01)
assert geo[1] == pytest.approx(0.0, abs=0.01)
assert geo[2] == pytest.approx(0.0, abs=0.01)
assert geo[3] == pytest.approx(37.5, abs=0.01)
assert geo[4] == pytest.approx(0.0, abs=0.01)
assert geo[5] == pytest.approx(0.0, abs=0.01)
assert 'gcps' in meta
assert 'gcpproj' in meta
assert 'projection' in meta
assert 'metadata' in meta
r = rectangle.Rectangle(0, 0, width=input_reader.size()[0],
height=input_reader.size()[0])
d1 = input_reader.read(roi=r)
assert d1.shape == (input_reader.height(), input_reader.width(), input_reader.num_bands())
def read(self, roi: rectangle.Rectangle=None, bands: List[int]=None, buf: np.ndarray=None) -> np.ndarray:
"""
Reads the image in [row, col, band] indexing.
If `roi` is not specified, reads the entire image.
If `buf` is specified, writes the image to buf.
If `bands` is not specified, reads all bands, otherwise
only the listed bands are read.
If bands is a single integer, drops the band dimension.
"""
if roi is None:
roi = rectangle.Rectangle(0, 0, width=self.width(), height=self.height())
else:
if roi.min_x < 0 or roi.min_y < 0 or roi.max_x > self.width() or roi.max_y > self.height():
raise IndexError('Rectangle (%d, %d, %d, %d) outside of bounds (%d, %d).' %
(roi.min_x, roi.min_y, roi.max_x, roi.max_y, self.width(), self.height()))
if bands is None:
bands = range(self.num_bands())
if isinstance(bands, int):
result = self._read(roi, [bands], buf)
result = result[:, :, 0] # reduce dimensions
else:
result = self._read(roi, bands, buf)
if self.__preprocess_function:
return self.__preprocess_function(result, roi, bands)
return result
def _load_tensor_imagery(self, is_labels, image_index, bbox):
"""Loads a single image as a tensor."""
data = self._labels if is_labels else self._images
if not is_labels: # Record each time we write a tile
file_path = data[image_index.numpy()]
log_path = self._get_image_read_log_path(file_path)
if log_path:
with portalocker.Lock(log_path, 'a', timeout=300) as f:
f.write(str(bbox) + '\n')
# TODO: What to write and when to clear it?
try:
image = loader.load_image(data, image_index.numpy())
w = int(bbox[2])
h = int(bbox[3])
rect = rectangle.Rectangle(int(bbox[0]), int(bbox[1]), w, h)
r = image.read(rect)
except Exception as e: #pylint: disable=W0703
print('Caught exception loading tile from image: ' + data[image_index.numpy()] + ' -> ' + str(e)
+ '\nSkipping tile: ' + str(bbox))
if config.general.stop_on_input_error():
print('Aborting processing, set --bypass-input-errors to bypass this error.')
raise
# Else just skip this tile
r = np.zeros(shape=(0,0,0), dtype=np.float32)
return r
def check_landsat_tiff(filename):
'''
Checks reading landsat tiffs.
'''
input_reader = TiffImage(filename)
assert input_reader.size() == (37, 37)
assert input_reader.num_bands() == 8
assert input_reader.dtype() == np.float32
assert input_reader.block_size() == (6, 37)
meta = input_reader.metadata()
geo = meta['geotransform']
assert geo[0] == pytest.approx(-122.3, abs=0.01)
assert geo[1] == pytest.approx(0.0, abs=0.01)
assert geo[2] == pytest.approx(0.0, abs=0.01)
assert geo[3] == pytest.approx(37.5, abs=0.01)
assert geo[4] == pytest.approx(0.0, abs=0.01)
assert geo[5] == pytest.approx(0.0, abs=0.01)
assert 'gcps' in meta
assert 'gcpproj' in meta
assert 'projection' in meta
assert 'metadata' in meta
r = rectangle.Rectangle(0,
0,
width=input_reader.size()[0],
height=input_reader.size()[0])
d1 = input_reader.read(roi=r)
assert d1.shape == (input_reader.height(), input_reader.width(),
input_reader.num_bands())
def _load_tensor_imagery(self, is_labels, image_index, bbox):
"""Loads a single image as a tensor."""
image = loader.load_image(self._labels if is_labels else self._images,
image_index.numpy())
w = int(bbox[2])
h = int(bbox[3])
rect = rectangle.Rectangle(int(bbox[0]), int(bbox[1]), w, h)
r = image.read(rect)
return r
def predict(self, image, label=None, input_bounds=None):
"""
Runs the model on `image`, comparing the results to `label` if specified.
Results are limited to `input_bounds`. Returns output, the meaning of which
depends on the subclass.
"""
net_input_shape = self._model.get_input_shape_at(0)[1:]
net_output_shape = self._model.get_output_shape_at(0)[1:]
offset_r = -net_input_shape[0] + net_output_shape[0]
offset_c = -net_input_shape[1] + net_output_shape[1]
block_size_x = net_input_shape[0] * (_TILE_SIZE // net_input_shape[0])
block_size_y = net_input_shape[1] * (_TILE_SIZE // net_input_shape[1])
# Set up the output image
if not input_bounds:
input_bounds = rectangle.Rectangle(0,
0,
width=image.width(),
height=image.height())
self._initialize((input_bounds.width() + offset_r,
input_bounds.height() + offset_c), label, image)
def callback_function(roi, data):
pred_image = self._predict_array(data)
block_x = (roi.min_x - input_bounds.min_x)
block_y = (roi.min_y - input_bounds.min_y)
(sx, sy) = (block_x, block_y)
labels = None
if label:
label_x = roi.min_x + (roi.width() - pred_image.shape[0]) // 2
label_y = roi.min_y + (roi.height() - pred_image.shape[1]) // 2
label_roi = rectangle.Rectangle(label_x, label_y,
label_x + pred_image.shape[0],
label_y + pred_image.shape[1])
labels = np.squeeze(label.read(label_roi))
self._process_block(pred_image, sx, sy, labels)
output_rois = input_bounds.make_tile_rois(block_size_x - offset_r,
block_size_y - offset_c,
include_partials=False,
overlap_amount=-offset_r)
try:
image.process_rois(output_rois,
callback_function,
show_progress=self._show_progress)
except KeyboardInterrupt:
self._abort()
raise
return self._complete()
def save(self,
path,
tile_size=None,
nodata_value=None,
show_progress=False):
"""
Save to file, with preprocessing applied.
Parameters
----------
path: str
Filename to save to.
tile_size: (int, int)
If specified, overwrite block size
nodata_value: image dtype
If specified, overwrite nodata value
show_progress: bool
Write progress bar to stdout
"""
if nodata_value is None:
nodata_value = self.nodata_value()
# Use the input tile size for the block size unless the user specified one.
block_size_y, block_size_x = self.block_size()
if tile_size is not None:
block_size_x = tile_size[0]
block_size_y = tile_size[1]
# Set up the output image
with _TiffWriter(path, self.width(), self.height(), self.num_bands(),
self._gdal_type(), block_size_x, block_size_y,
nodata_value, self.metadata()) as writer:
input_bounds = rectangle.Rectangle(0,
0,
width=self.width(),
height=self.height())
output_rois = input_bounds.make_tile_rois(
(block_size_x, block_size_y), include_partials=True)
def callback_function(output_roi, data):
"""Callback function to write the first channel to the output file."""
# Figure out some ROI positioning values
block_x = output_roi.min_x / block_size_x
block_y = output_roi.min_y / block_size_y
# Loop on bands
for band in range(data.shape[2]):
writer.write_block(data[:, :, band], block_x, block_y,
band)
self.process_rois(output_rois,
callback_function,
show_progress=show_progress)
def roi_generator(self, requested_rois: Iterator[rectangle.Rectangle]) -> Iterator[rectangle.Rectangle]:
"""
Generator that yields ROIs of blocks in the requested region.
"""
block_rois = copy.copy(requested_rois)
whole_bounds = rectangle.Rectangle(0, 0, width=self.width(), height=self.height())
for roi in requested_rois:
if not whole_bounds.contains_rect(roi):
raise Exception('Roi outside image bounds: ' + str(roi) + str(whole_bounds))
# gdal doesn't work reading multithreading. But this let's a thread
# take care of IO input while we do computation.
jobs = []
total_rois = len(block_rois)
while block_rois:
# For the next (output) block, figure out the (input block) aligned
# data read that we need to perform to get it.
read_roi = self.block_aligned_roi(block_rois[0])
applicable_rois = []
# Loop through the remaining ROIs and apply the callback function to each
# ROI that is contained in the section we read in.
index = 0
while index < len(block_rois):
if not read_roi.contains_rect(block_rois[index]):
index += 1
continue
applicable_rois.append(block_rois.pop(index))
jobs.append((read_roi, applicable_rois))
# only do a few reads ahead since otherwise we will exhaust our memory
pending = []
exe = concurrent.futures.ThreadPoolExecutor(1)
NUM_AHEAD = 2
for i in range(min(NUM_AHEAD, len(jobs))):
pending.append(exe.submit(functools.partial(self.read, jobs[i][0])))
num_remaining = total_rois
for (i, (read_roi, rois)) in enumerate(jobs):
buf = pending.pop(0).result()
for roi in rois:
x0 = roi.min_x - read_roi.min_x
y0 = roi.min_y - read_roi.min_y
num_remaining -= 1
yield (roi, buf[x0:x0 + roi.width(), y0:y0 + roi.height(), :],
(total_rois - num_remaining, total_rois))
if i + NUM_AHEAD < len(jobs):
pending.append(exe.submit(functools.partial(self.read, jobs[i + NUM_AHEAD][0])))
def callback_function(roi, data):
pred_image = self._predict_array(data)
block_x = (roi.min_x - input_bounds.min_x)
block_y = (roi.min_y - input_bounds.min_y)
(sx, sy) = (block_x , block_y)
labels = None
if label:
label_x = roi.min_x + (roi.width() - pred_image.shape[0]) // 2
label_y = roi.min_y + (roi.height() - pred_image.shape[1]) // 2
label_roi = rectangle.Rectangle(label_x, label_y,
label_x + pred_image.shape[0], label_y + pred_image.shape[1])
labels = np.squeeze(label.read(label_roi))
self._process_block(pred_image, sx, sy, labels)
def read(self,
roi: rectangle.Rectangle = None,
bands: List[int] = None,
buf: np.ndarray = None) -> np.ndarray:
"""
Reads the image in [row, col, band] indexing.
Subclasses should generally not overwrite this method--- they will likely want to implement
`_read`.
Parameters
----------
roi: `rectangle.Rectangle`
The bounding box to read from the image. If None, read the entire image.
bands: List[int]
Bands to load (zero-indexed). If None, read all bands.
buf: np.ndarray
If specified, reads the image into this buffer. Must be sufficiently large.
Returns
-------
np.ndarray:
A buffer containing the requested part of the image.
"""
if roi is None:
roi = rectangle.Rectangle(0,
0,
width=self.width(),
height=self.height())
else:
if roi.min_x < 0 or roi.min_y < 0 or roi.max_x > self.width(
) or roi.max_y > self.height():
raise IndexError(
'Rectangle (%d, %d, %d, %d) outside of bounds (%d, %d).' %
(roi.min_x, roi.min_y, roi.max_x, roi.max_y, self.width(),
self.height()))
if bands is None:
bands = range(self.num_bands())
if isinstance(bands, int):
result = self._read(roi, [bands], buf)
result = result[:, :, 0] # reduce dimensions
else:
result = self._read(roi, bands, buf)
if self.__preprocess_function:
return self.__preprocess_function(result, roi, bands)
return result
def tiles(self,
width: int,
height: int,
min_width: int = 0,
min_height: int = 0,
overlap: int = 0) -> Iterator[rectangle.Rectangle]:
"""Generator to yield ROIs for the image."""
input_bounds = rectangle.Rectangle(0,
0,
width=self.width(),
height=self.height())
return input_bounds.make_tile_rois(width,
height,
min_width=min_width,
min_height=min_height,
include_partials=True,
overlap_amount=overlap)
def tiles(self,
shape,
overlap_shape=(0, 0),
partials: bool = True,
min_shape=(0, 0),
partials_overlap: bool = False,
by_block=False) -> List:
"""
Splits the image into tiles with the given properties.
Parameters
----------
shape: (int, int)
Shape of each tile
overlap_shape: (int, int)
Amount to overlap tiles in y and x direction
partials: bool
If true, include partial tiles at the edge of the image.
min_shape: (int, int)
If true and `partials` is true, keep partial tiles of this minimum size.
partials_overlap: bool
If `partials` is false, and this is true, expand partial tiles
to the desired size. Tiles may overlap in some areas.
by_block: bool
If true, changes the returned generator to group tiles by block.
This is intended to optimize disk reads by reading the entire block at once.
Returns
-------
List[Rectangle] or List[(Rectangle, List[Rectangle])]
List of ROIs. If `by_block` is true, returns a list of (Rectangle, List[Rectangle])
instead, where the first rectangle is a larger block containing multiple tiles in a list.
"""
input_bounds = rectangle.Rectangle(0,
0,
max_x=self.width(),
max_y=self.height())
return input_bounds.make_tile_rois_yx(
shape,
overlap_shape=overlap_shape,
include_partials=partials,
min_shape=min_shape,
partials_overlap=partials_overlap,
by_block=by_block)[0]
def test_rectangle_rois():
"""
Tests make_tile_rois.
"""
r = rectangle.Rectangle(0, 0, 10, 10)
tiles = r.make_tile_rois((5, 5), include_partials=False)[0]
assert len(tiles) == 4
for t in tiles:
assert t.width() == 5 and t.height() == 5
tiles = r.make_tile_rois((5, 10), include_partials=False)[0]
assert len(tiles) == 2
tiles, valid_tiles = r.make_tile_rois((11, 11), include_partials=False)
assert len(tiles) == 0
assert len(valid_tiles) == 0
tiles = r.make_tile_rois((11, 11), include_partials=True)[0]
assert len(tiles) == 1
assert tiles[0].bounds() == (0, 10, 0, 10)
tiles = r.make_tile_rois((20, 20),
include_partials=True,
min_shape=(11, 11))[0]
assert len(tiles) == 0
tiles = r.make_tile_rois((20, 20),
include_partials=True,
min_shape=(10, 10))[0]
assert len(tiles) == 1
tiles = r.make_tile_rois((6, 6), include_partials=False)[0]
assert len(tiles) == 1
tiles, valid_tiles = r.make_tile_rois((6, 6),
include_partials=False,
overlap_shape=(2, 2))
assert len(tiles) == 4
assert len(valid_tiles) == 4
assert valid_tiles[0].width() == 5 and valid_tiles[0].height() == 5
assert valid_tiles[3].width() == 5 and valid_tiles[3].height() == 5
tiles, valid_tiles = r.make_tile_rois((6, 6),
include_partials=False,
partials_overlap=True)
assert len(tiles) == 4
assert len(valid_tiles) == 4
for t in tiles:
assert t.width() == 6 and t.height() == 6
assert valid_tiles[1].width() == 4 and valid_tiles[1].height() == 6
assert valid_tiles[2].width() == 6 and valid_tiles[2].height() == 4
assert valid_tiles[3].width() == 4 and valid_tiles[3].height() == 4
tiles = r.make_tile_rois((5, 5), include_partials=False, by_block=True)[0]
assert len(tiles) == 2
for row in tiles:
assert len(row) == 2
r = rectangle.Rectangle(0, 0, 12, 10)
tiles, valid_tiles = r.make_tile_rois((6, 6),
include_partials=False,
overlap_shape=(2, 2))
assert len(tiles) == 4
assert len(valid_tiles) == 4
for t in tiles:
assert t.width() == 6 and t.height() == 6
for t in valid_tiles:
assert t.width() == 5 and t.height() == 5
c = rectangle.Rectangle(0, 0, 20, 20)
tiles = r.make_tile_rois((7, 7), include_partials=False,
containing_rect=c)[0]
assert len(tiles) == 4
def roi_generator(
self,
requested_rois: Iterator[rectangle.Rectangle],
roi_extra_data=None
) -> Iterator[Tuple[rectangle.Rectangle, np.ndarray, int, int]]:
"""
Generator that yields image blocks of the requested rois.
Parameters
----------
requested_rois: Iterator[Rectangle]
Regions of interest to read.
Returns
-------
Iterator[Tuple[Rectangle, numpy.ndarray, int, int]]
A generator with read image regions. In each tuple, the first item
is the region of interest, the second is a numpy array of the image contents,
the third is the index of the current region of interest, and the fourth is the total
number of rois.
"""
if roi_extra_data and len(roi_extra_data) != len(requested_rois):
raise Exception(
'Number of ROIs and extra ROI data must be the same!')
block_rois = copy.copy(requested_rois)
block_roi_extra_data = copy.copy(roi_extra_data)
whole_bounds = rectangle.Rectangle(0,
0,
width=self.width(),
height=self.height())
for roi in requested_rois:
if not whole_bounds.contains_rect(roi):
raise Exception('Roi outside image bounds: ' + str(roi) +
str(whole_bounds))
# gdal doesn't work reading multithreading. But this let's a thread
# take care of IO input while we do computation.
jobs = []
total_rois = len(block_rois)
while block_rois:
# For the next (output) block, figure out the (input block) aligned
# data read that we need to perform to get it.
read_roi = self.block_aligned_roi(block_rois[0])
applicable_rois = []
applicable_rois_extra_data = []
# Loop through the remaining ROIs and apply the callback function to each
# ROI that is contained in the section we read in.
index = 0
while index < len(block_rois):
if not read_roi.contains_rect(block_rois[index]):
index += 1
continue
applicable_rois.append(block_rois.pop(index))
if block_roi_extra_data:
applicable_rois_extra_data.append(
block_roi_extra_data.pop(index))
else:
applicable_rois_extra_data.append(None)
jobs.append(
(read_roi, applicable_rois, applicable_rois_extra_data))
# only do a few reads ahead since otherwise we will exhaust our memory
pending = []
exe = concurrent.futures.ThreadPoolExecutor(1)
NUM_AHEAD = 2
for i in range(min(NUM_AHEAD, len(jobs))):
pending.append(exe.submit(functools.partial(self.read,
jobs[i][0])))
num_remaining = total_rois
for (i, (read_roi, rois, rois_extra_data)) in enumerate(jobs):
buf = pending.pop(0).result()
for roi, extra_data in zip(rois, rois_extra_data):
x0 = roi.min_x - read_roi.min_x
y0 = roi.min_y - read_roi.min_y
num_remaining -= 1
if len(buf.shape) == 2:
b = buf[y0:y0 + roi.height(), x0:x0 + roi.width()]
else:
b = buf[y0:y0 + roi.height(), x0:x0 + roi.width(), :]
yield (roi, b, extra_data, (total_rois - num_remaining,
total_rois))
if i + NUM_AHEAD < len(jobs):
pending.append(
exe.submit(
functools.partial(self.read, jobs[i + NUM_AHEAD][0])))
assert len(sys.argv) == 3, 'Please specify two tiff files of the same size.'
img1 = TiffImage(sys.argv[1])
img2 = TiffImage(sys.argv[2])
output_image = TiffWriter('diff.tiff')
output_image.initialize((img1.width(), img1.height(), 3), np.uint8,
img1.metadata())
assert img1.width()== img2.width() and img1.height() == img2.height() and \
img1.num_bands() == img2.num_bands(), 'Images must be same size.'
def callback_function(roi, data):
data2 = img2.read(roi)
diff = np.mean((data - data2)**2, axis=-1)
diff = np.uint8(np.clip(diff * 128.0, 0.0, 255.0))
out = np.stack([diff, diff, diff], axis=-1)
output_image.write(out, roi.min_x, roi.min_y)
input_bounds = rectangle.Rectangle(0,
0,
width=img1.width(),
height=img1.height())
output_rois = input_bounds.make_tile_rois((2048, 2048), include_partials=True)
img1.process_rois(output_rois, callback_function, show_progress=True)
output_image.close()
def predict(self, image, label=None, input_bounds=None, overlap=(0, 0)):
"""
Runs the model on an image. The behavior is specific to the subclass.
Parameters
----------
image: delta.imagery.delta_image.DeltaImage
Image to evalute.
label: delta.imagery.delta_image.DeltaImage
Optional label to compare to.
input_bounds: delta.imagery.rectangle.Rectangle
If specified, only evaluate the given portion of the image.
overlap: (int, int)
`predict` evaluates the image by selecting tiles, dependent on the tile_shape
provided in the subclass. If an overlap is specified, the tiles will be overlapped
by the given amounts in the x and y directions. Subclasses may select or interpolate
to favor tile interior pixels for improved classification.
Returns
-------
The result of the `_complete` function, which depends on the sublcass.
"""
net_input_shape = self._model.input_shape[1:]
net_output_shape = self._model.output_shape[1:]
# Set up the output image
if not input_bounds:
input_bounds = rectangle.Rectangle(0,
0,
width=image.width(),
height=image.height())
output_shape = (input_bounds.width(), input_bounds.height())
ts = self._tile_shape if self._tile_shape else (image.width(),
image.height())
if net_input_shape[0] is None and net_input_shape[1] is None:
assert net_output_shape[0] is None and net_output_shape[1] is None
out_shape = self._model.compute_output_shape(
(0, ts[0], ts[1], net_input_shape[2]))
tiles = input_bounds.make_tile_rois(
ts,
include_partials=False,
overlap_shape=(ts[0] - out_shape[1] + overlap[0],
ts[1] - out_shape[2] + overlap[1]),
partials_overlap=True)
else:
offset_r = -net_input_shape[0] + net_output_shape[0] + overlap[0]
offset_c = -net_input_shape[1] + net_output_shape[1] + overlap[1]
output_shape = (output_shape[0] + offset_r,
output_shape[1] + offset_c)
block_size_x = net_input_shape[0] * max(
1, ts[0] // net_input_shape[0])
block_size_y = net_input_shape[1] * max(
1, ts[1] // net_input_shape[1])
tiles = input_bounds.make_tile_rois(
(block_size_x - offset_r, block_size_y - offset_c),
include_partials=False,
overlap_shape=(-offset_r, -offset_c))
self._initialize(output_shape, image, label)
label_nodata = label.nodata_value() if label else None
def callback_function(roi, data):
pred_image = self._predict_array(data, image.nodata_value())
block_x = (roi.min_x - input_bounds.min_x)
block_y = (roi.min_y - input_bounds.min_y)
(sx, sy) = (block_x, block_y)
labels = None
if label:
label_x = roi.min_x + (roi.width() - pred_image.shape[0]) // 2
label_y = roi.min_y + (roi.height() - pred_image.shape[1]) // 2
label_roi = rectangle.Rectangle(label_x, label_y,
label_x + pred_image.shape[0],
label_y + pred_image.shape[1])
labels = np.squeeze(label.read(label_roi))
tl = [0, 0]
tl = (overlap[0] // 2 if block_x > 0 else 0,
overlap[1] // 2 if block_y > 0 else 0)
br = (roi.max_x - roi.min_x, roi.max_y - roi.min_y)
br = (br[0] -
(overlap[0] // 2 if roi.max_x < input_bounds.max_x else 0),
br[1] -
(overlap[1] // 2 if roi.max_x < input_bounds.max_x else 0))
if len(pred_image.shape) == 2:
input_block = pred_image[tl[0]:br[0], tl[1]:br[1]]
else:
input_block = pred_image[tl[0]:br[0], tl[1]:br[1], :]
self._process_block(
input_block, sx + tl[0], sy + tl[1],
None if labels is None else labels[tl[0]:br[0],
tl[1]:br[1]], label_nodata)
try:
image.process_rois(tiles,
callback_function,
show_progress=self._show_progress)
except KeyboardInterrupt:
self._abort()
raise
return self._complete()
def write_tiff(output_path: str,
data: np.ndarray = None,
image: delta_image.DeltaImage = None,
nodata=None,
metadata: dict = None,
block_size=None,
show_progress: bool = False):
"""
Write a numpy array to a file as a tiff.
Parameters
----------
output_path: str
Filename to save tiff file to
data: numpy.ndarray
Image data to save.
image: delta_image.DeltaImage
Image data to save (specify one of this or data).
nodata: Any
Nodata value.
metadata: dict
Optional metadata to include.
block_size: Tuple[int]
Optionally override block size for writing.
show_progress: bool
Display command line progress bar.
"""
if data is not None:
assert image is None, 'Must specify one of data or image.'
image = NumpyImage(data, nodata_value=nodata)
assert image is not None, 'Must specify either data or image.'
num_bands = image.num_bands()
data_type = _numpy_dtype_to_gdal_type(image.dtype())
size = image.size()
# gdal requires block size of 16 when writing...
ts = image.block_size()
if block_size:
ts = block_size
if nodata is None:
nodata = image.nodata_value()
if metadata is None:
metadata = image.metadata()
with _TiffWriter(output_path,
height=size[0],
width=size[1],
num_bands=num_bands,
data_type=data_type,
metadata=metadata,
nodata_value=nodata,
tile_height=ts[0],
tile_width=ts[1]) as writer:
input_bounds = rectangle.Rectangle(0, 0, width=size[1], height=size[0])
ts = writer.tile_shape()
output_rois = input_bounds.make_tile_rois_yx(ts,
include_partials=True)[0]
def callback_function(output_roi, data, _):
"""Callback function to write the first channel to the output file."""
# Figure out some ROI positioning values
block_x = output_roi.min_x // ts[1]
block_y = output_roi.min_y // ts[0]
# Loop on bands
if len(data.shape) == 2:
writer.write_block(data[:, :], block_y, block_x, 0)
else:
for band in range(num_bands):
writer.write_block(data[:, :, band], block_y, block_x,
band)
image.process_rois(output_rois,
callback_function,
show_progress=show_progress)
