def test_window_fromslices(col_off, row_off, col_stop, row_stop):
"""Empty and non-empty absolute windows from slices, tuples, or lists
are valid"""
# Constrain windows to >= 0 in each dimension
assume(col_stop >= col_off)
assume(row_stop >= row_off)
rows = (row_off, row_stop)
cols = (col_off, col_stop)
expected = (col_off, row_off, col_stop - col_off, row_stop - row_off)
assert np.allclose(
Window.from_slices(rows=slice(*rows), cols=slice(*cols)).flatten(),
expected
)
assert np.allclose(
Window.from_slices(rows=rows, cols=cols).flatten(),
expected
)
assert np.allclose(
Window.from_slices(rows=list(rows), cols=list(cols)).flatten(),
expected
)
def test_window_fromslices_negative_start_missing_dim_err():
"""Should raise error if width or height are not provided"""
with pytest.raises(WindowError):
Window.from_slices(rows=(-10, 4), cols=(0, 4))
with pytest.raises(WindowError):
Window.from_slices(rows=(0, 4), cols=(-10, 4))
def test_window_fromslices_implicit_err():
""" height and width are required if stop index is None; failing to
provide them will result in error"""
with pytest.raises(WindowError):
Window.from_slices(rows=(1, None), cols=(1, 4))
with pytest.raises(WindowError):
Window.from_slices(rows=(1, 4), cols=(1, None))
def test_window_fromslices_negative_stop():
# TODO: Should negative stops even allowed?? Limited to boundless case?
assert np.allclose(
Window.from_slices(rows=(-4, -1), cols=(0, 4), height=10).flatten(),
(0, 6, 4, 3)
)
assert np.allclose(
Window.from_slices(rows=(0, 4), cols=(-4, -1), width=10).flatten(),
(6, 0, 3, 4)
)
def test_window_fromslices_stops_lt_starts():
"""Should produce empty windows if stop indexes are less than start
indexes"""
assert np.allclose(
Window.from_slices(rows=(4, 2), cols=(0, 4)).flatten(),
(0, 4, 4, 0)
)
assert np.allclose(
Window.from_slices(rows=(0, 4), cols=(4, 2)).flatten(),
(4, 0, 0, 4)
)
def test_window_fromslices_invalid_rows_cols():
"""Should raise error if rows or cols are not slices, lists, or tuples
of length 2"""
invalids = (
np.array([0, 4]), # wrong type, but close
'04', # clearly the wrong type but right length
(1, 2, 3) # wrong length
)
for invalid in invalids:
with pytest.raises(WindowError):
Window.from_slices(rows=invalid, cols=(0, 4))
with pytest.raises(WindowError):
Window.from_slices(rows=(0, 4), cols=invalid)
def test_data_window_maskedarray():
"""Get window of masked arr."""
arr = np.ones((3, 3))
arr[0, :] = 0
arr = np.ma.masked_array(arr, arr == 0)
window = get_data_window(arr)
assert window == Window.from_slices((1, 3), (0, 3))
def test_window_fromslices_negative_start():
# TODO: if passing negative start, what are valid values for stop?
assert np.allclose(
Window.from_slices(rows=(-4, None), cols=(0, 4), height=10).flatten(),
(0, 6, 4, 4)
)
assert np.allclose(
Window.from_slices(rows=(0, 4), cols=(-4, None), width=10).flatten(),
(6, 0, 4, 4)
)
assert np.allclose(
Window.from_slices(rows=(-6, None), cols=(-4, None),
height=8, width=10).flatten(),
(6, 2, 4, 6)
)
def test_window_fromslices_implicit(abs_off, imp_off, stop, dim):
""" providing None for start index will default to 0
and providing None for stop index will default to width or height """
assume(stop >= abs_off)
assume(dim >= imp_off)
absolute = (abs_off, stop)
implicit_start = (None, stop) # => (0, stop)
implicit_stop = (imp_off, None) # => (implicit_offset, dim)
implicit_both = (None, None) # => (implicit_offset, dim)
# Implicit start indexes resolve to 0
assert np.allclose(
Window.from_slices(rows=implicit_start, cols=absolute).flatten(),
(abs_off, 0, stop - abs_off, stop)
)
assert np.allclose(
Window.from_slices(rows=absolute, cols=implicit_start).flatten(),
(0, abs_off, stop, stop - abs_off)
)
# Implicit stop indexes resolve to dim (height or width)
assert np.allclose(
Window.from_slices(
rows=implicit_stop, cols=absolute, height=dim).flatten(),
(abs_off, imp_off, stop - abs_off, dim - imp_off)
)
assert np.allclose(
Window.from_slices(
rows=absolute, cols=implicit_stop, width=dim).flatten(),
(imp_off, abs_off, dim - imp_off, stop - abs_off)
)
# Both can be implicit
assert np.allclose(
Window.from_slices(
rows=implicit_both, cols=implicit_both,
width=dim, height=dim).flatten(),
(0, 0, dim, dim)
)
def test_window_float(path_rgb_byte_tif):
"""Test window float values"""
with rasterio.open(path_rgb_byte_tif) as src:
left, bottom, right, top = src.bounds
dx, dy = src.res
height = src.height
width = src.width
assert_window_almost_equals(from_bounds(
left, top - 400, left + 400, top, src.transform,
height, width), Window.from_slices((0, 400 / src.res[1]), (0, 400 / src.res[0])))
def test_window_fromslices_boundless(col_off, row_off, col_stop, row_stop):
# Constrain windows to >= 0 in each dimension
assume(col_stop >= col_off)
assume(row_stop >= row_off)
assert np.allclose(
Window.from_slices(
rows=(-row_off, row_stop), cols=(col_off, col_stop),
boundless=True).flatten(),
(col_off, -row_off, col_stop - col_off, row_stop + row_off)
)
def test_window_from_bounds(path_rgb_byte_tif):
# TODO: break this test up.
with rasterio.open(path_rgb_byte_tif) as src:
left, bottom, right, top = src.bounds
dx, dy = src.res
height = src.height
width = src.width
assert_window_almost_equals(from_bounds(
left + EPS, bottom + EPS, right - EPS, top - EPS, src.transform,
height, width), Window.from_slices((0, height), (0, width)))
assert_window_almost_equals(from_bounds(
left, top - 2 * dy - EPS, left + 2 * dx - EPS, top, src.transform,
height, width), Window.from_slices((0, 2), (0, 2)))
# boundless
assert_window_almost_equals(
from_bounds(left - 2 * dx, top - 2 * dy, left + 2 * dx,
top + 2 * dy, src.transform, height=height,
width=width),
Window.from_slices((-2, 2), (-2, 2), boundless=True, height=height,
width=width))
name = forest + year #reduce form name
rast = os.path.join(wdir, x) #path to raster
rast_lag = os.path.join(wdir_updated, forest + str(int(year) - 1) +
".tif") #path to lagged raster
rast_updated = os.path.join(wdir_updated, name + ".tif")
# get rows and cols
data = rasterio.open(rast)
nrow, ncol = data.shape
del (data)
row_batch_size = floor(nrow / n_block)
col_batch_size = floor(ncol / n_block)
row_list = list(range(0, nrow - row_batch_size + 1, row_batch_size))
col_list = list(range(0, ncol - col_batch_size + 1, col_batch_size))
if int(year) == 1985:
for row_idx, col_idx in tqdm(combinaion_2lists(row_list, col_list)):
window = Window.from_slices((row_idx, row_idx + row_batch_size),
(col_idx, col_idx + col_batch_size))
with rasterio.open(rast) as data:
arr = data.read(1, window=window)
result = (arr >= 3) & (arr <= 5)
with rasterio.Env():
profile = data.profile
profile.update(dtype=rasterio.uint8,
count=1,
compress='lzw')
write_tiff(rast_updated, result, window, profile)
elif int(year) > 1985:
for row_idx, col_idx in tqdm(combinaion_2lists(row_list, col_list)):
window = Window.from_slices((row_idx, row_idx + row_batch_size),
(col_idx, col_idx + col_batch_size))
with rasterio.open(rast) as data, rasterio.open(
rast_lag) as data_lag:
def to_window(w):
w = ast.literal_eval(w)
return Window.from_slices(*w)
def test_window_from_offlen():
"""from_offlen classmethod works."""
with pytest.warns(RasterioDeprecationWarning):
assert Window.from_offlen(2, 0, 1, 1) == Window.from_slices((0, 1),
(2, 3))
print("Full of zero, pass!")
continue
# 对应参考数据的窗口,设置一定的缓冲距离
window_trans = rasterio.windows.transform(window, src.transform)
res = window_trans.a
buffer_dis = buffer * res
# 左上角在参考数据位置
ul_col, ul_row = ~ref_trans * (window_trans.c - buffer_dis,
window_trans.f + buffer_dis)
# 右下角在参考数据位置
lr_col, lr_row = ~ref_trans * (
window_trans.c + window.width * res + buffer_dis,
window_trans.f - window.height * res - buffer_dis)
window_ref = Window.from_slices((int(ul_row), int(lr_row)),
(int(ul_col), int(lr_col)))
# 参考数据数组
trans_ref = rasterio.windows.transform(window_ref, ref_trans)
arr_ref = ds.read(window=window_ref)
arr_ref = rgb2gray(arr_ref)
arr_ref = bytscl_std(arr_ref, n=2.5, nodata=0)
arr_ref = gamma(arr_ref,
gamma=auto_gamma(arr_ref, mean_v=0.5, nodata=0))
try:
kp_ori, des_ori = cal_keypoint_and_descriptor(arr_ori)
kp_ref, des_ref = cal_keypoint_and_descriptor(arr_ref)
except Exception as e:
print(e)
continue
model.to(DEVICE)
model.eval()
print("Now using:" + "WholeModel_id_{}".format(element) +
PTH_NAME + ".pth")
slices = make_grid(dataset.shape,
window=TEST_WINDOW,
min_overlap=TEST_MIN_OVERLAP)
preds = np.zeros(dataset.shape, dtype=np.float32)
if dataset.count != 3:
# print('Image file with subdatasets as channels')
layers = [rasterio.open(subd) for subd in dataset.subdatasets]
for (x1, x2, y1, y2) in (slices):
if dataset.count == 3: # normal
image = dataset.read([1, 2, 3],
window=Window.from_slices((x1, x2),
(y1, y2)))
image = np.moveaxis(image, 0, -1)
else: # with subdatasets/layers
image = np.zeros((TEST_WINDOW, TEST_WINDOW, 3),
dtype=np.uint8)
for fl in range(3):
image[:, :, fl] = layers[fl].read(
window=Window.from_slices((x1, x2), (y1, y2)))
# print("Test {}-{}:Shape is:{}".format(filename,index,image.shape))
image = cv2.resize(image, (TEST_NEW_SIZE, TEST_NEW_SIZE))
image = trfm(image)
with torch.no_grad():
image = image.to(DEVICE)[
None] # 这里加入的是batch维度 这里的测试是每张图的测试
score = model(image)[0][0]
def window(cols, rows):
return Window.from_slices((np.min(rows), np.max(rows) + 1),
(np.min(cols), np.max(cols) + 1))
def test_data_window_nodata_3d():
"""Get window of 3d arr with nodata."""
arr = np.ones((3, 3, 3))
arr[:, 0, :] = 0
window = get_data_window(arr, nodata=0)
assert window == Window.from_slices((1, 3), (0, 3))
def test_data_window_novalid():
"""Get window of arr with nodata."""
arr = np.ones((3, 3))
arr[:, :] = 0
window = get_data_window(arr, nodata=0)
assert window == Window.from_slices((0, 0), (0, 0))
def test_window_from_slices():
"""from_slices classmethod works."""
assert Window.from_slices((0, 1), (2, 3)) == Window.from_slices((0, 1), (2, 3))
def test_data_window_novalid():
"""Get window of arr with nodata."""
arr = np.ones((3, 3))
arr[:, :] = 0
window = get_data_window(arr, nodata=0)
assert window == Window.from_slices((0, 0), (0, 0))
def test_data_window_full():
"""Get window of entirely valid data array."""
arr = np.ones((3, 3))
window = get_data_window(arr)
assert window == Window.from_slices((0, 3), (0, 3))
def test_window_from_slices():
"""from_slices classmethod works."""
assert Window.from_slices((0, 1), (2, 3)) == Window.from_slices((0, 1),
(2, 3))
def clip(self, im, geometry, newname, resize=False, write=True):
"""Clip image & update metadata of output image. Option to write
output image to disk.
Args:
im (string): Path to image.
geometry (geoDataframe): Geometry dataframe used as bounding box to clip image.
newname (string): Piece of string added to the end of the new filename.
write (bool (opt)): Whether to save output raster to disk. Defaults to True.
Returns:
out_img (array): clipped array.
out_meta (dictionary): updated metadata for clipped raster.
"""
# New name for output image. Split on second occurence of dot.
out_tif = im.split('.')[0] + '.' + im.split('.')[1] + str(
newname) + '.tif'
if os.path.exists(out_tif) == True and os.stat(out_tif).st_size != 0:
# Pass if file already exists & it's size is not zero.
return
with rasterio.open(im) as src:
# Image metadata.
metadata = src.meta
# # It doesn't work well. It changes orinal minx & maxy.
# # Convert window's CRS to image's CRS.
# geom = geometry.to_crs(crs=metadata['crs'])
# Convert x,y to row, col.
row_start, col_start = src.index(geometry.bounds['minx'][0],
geometry.bounds['maxy'][0])
row_stop, col_stop = src.index(geometry.bounds['maxx'][0],
geometry.bounds['miny'][0])
# Parse pixel size from metadata.
pixelSize = list(metadata['transform'])[0]
# Create the new transformation.
transf = rasterio.transform.from_origin(geometry.bounds['minx'][0],
geometry.bounds['maxy'][0],
pixelSize, pixelSize)
# Update metadata.
metadata.update(driver='GTiff',
transform=transf,
height=(row_stop - row_start),
width=(col_stop - col_start))
# Construct a window by image coordinates.
win = Window.from_slices(slice(row_start, row_stop),
slice(col_start, col_stop))
# Clip image.
out_img = src.read(1, window=win)
if resize == True:
# Create the new transformation.
transf = rasterio.transform.from_origin(geometry.bounds['minx'][0],
geometry.bounds['maxy'][0],
pixelSize // 2,
pixelSize // 2)
# Update metadata for output image
metadata.update({
"height": out_img.shape[0] * 2,
"width": out_img.shape[1] * 2,
"transform": transf
})
# Upsample.
out_img = cv2.resize(out_img,
(2 * out_img.shape[0], 2 * out_img.shape[1]),
interpolation=cv2.INTER_LINEAR)
if write == True:
# Reshape as rasterio needs the shape.
temp = out_img.reshape(1, out_img.shape[0], out_img.shape[1])
# Write output image to disk
with rasterio.open(out_tif, "w", **metadata) as dest:
dest.write(temp)
"""
# Plot output image
clipped = rasterio.open(out_tif)
#show((clipped, 1), cmap='terrain')
"""
return out_img, metadata
def build_slices(self):
# self.masks = []
self.files = []
self.slices = []
need_shift = {
"e79de561c": [5, 10],
"095bf7a1f": [20, 15],
"54f2eec69": [-10, -15]
}
for i, filename in enumerate(self.csv['id']):
# filepath = (self.path /(filename+'.tiff')).as_posix()
filepath = "{}/train/{}.tiff".format(self.path, filename)
if self.valid_mode:
if i not in self.valid_index:
continue
else:
if i in self.valid_index:
continue
# if not os.path.exists(filepath):
# continue
self.files.append(filepath)
print('{}/{}, Transform {}'.format(i + 1, len(self.csv['id']),
filename))
with rasterio.open(filepath, transform=self.identity) as dataset:
mask = rle_decode(
self.csv.loc[self.csv['id'] == filename]
['encoding'].values[0], dataset.shape)
if filename in need_shift:
print(filename, "需要修正mask")
shift_horizontal, shift_vertical = need_shift[filename]
h, w = mask.shape
mask_temp = np.zeros_like(mask)
mask_temp[max(0, -shift_vertical):min(h, h-shift_vertical), max(0, -shift_horizontal):min(w, w-shift_horizontal)] = \
mask[max(0, shift_vertical):min(h, h+shift_vertical), max(0, shift_horizontal):min(w, w+shift_horizontal)]
mask = mask_temp.copy()
del mask_temp
mask_re = cv2.resize(mask, (1024, 1024))
props = regionprops(label(mask_re, connectivity=mask_re.ndim))
# self.masks.append(mask)
slices = make_grid(dataset.shape,
window=self.window,
min_overlap=self.overlap)
# slices = list(slices)
print("props 处理")
for p in tqdm(props):
bbox = list(p.bbox)
bbox[0] = int(bbox[0] * mask.shape[0] / 1024)
bbox[2] = int(bbox[2] * mask.shape[0] / 1024)
bbox[1] = int(bbox[1] * mask.shape[1] / 1024)
bbox[3] = int(bbox[3] * mask.shape[1] / 1024)
margin = int(
((bbox[2] - bbox[0]) + (bbox[3] - bbox[1])) // 6)
x1, x2, y1, y2 = bbox[0] - margin, bbox[2] + margin, bbox[
1] - margin, bbox[3] + margin
self.slices.append([i, x1, x2, y1, y2])
image = dataset.read([1, 2, 3],
window=Window.from_slices((x1, x2),
(y1, y2)))
image = np.moveaxis(image, 0, -1)
augments = self.resizefunc(image=image,
mask=mask[x1:x2, y1:y2])
self.x.append(augments['image'])
self.y.append(augments['mask'])
print("slices 处理")
for slc in tqdm(slices):
x1, x2, y1, y2 = slc
if mask[x1:x2, y1:y2].sum() > self.threshold and (
mask[x1:x2, y1:y2].sum() /
(abs(x2 - x1) * abs(y2 - y1))) > 0.015:
self.slices.append([i, x1, x2, y1, y2])
image = dataset.read([1, 2, 3],
window=Window.from_slices(
(x1, x2), (y1, y2)))
# if image.std().mean() < 10:
# continue
# print(image.std().mean(), mask[x1:x2,y1:y2].sum())
image = np.moveaxis(image, 0, -1)
augments = self.resizefunc(image=image,
mask=mask[x1:x2, y1:y2])
self.x.append(augments['image'])
self.y.append(augments['mask'])
def test_intersection():
"""Window intersection works."""
window = intersection(Window(0, 0, 10, 10), Window(8, 8, 12, 12))
assert window == Window.from_slices((8, 10), (8, 10))
def read_from_slice(dataset, x1, x2, y1, y2):
image = dataset.read([1, 2, 3],
window=Window.from_slices((x1, x2), (y1, y2)))
image = np.moveaxis(image, 0, -1)
return image
temp1 = r'C:\Dimos\database\Koppen\wc2.1_30s_tavg\wc2.1_30s_tavg_0{}.tif'
temp2 = r'C:\Dimos\database\Koppen\wc2.1_30s_tavg\wc2.1_30s_tavg_{}.tif'
elev_raster = r'C:\Dimos\database\Koppen\wc2.1_30s_elev\wc2.1_30s_elev.tif'
start_row = 10000
end_row = 10050
start_col = 25000
end_col = 25050
#Raster read
for im in range(1, 10):
w1 = (rasterio.open(prec1.format(im)))
prec.append(
np.array(
w1.read(
window=Window.from_slices((start_row, end_row), (start_col,
end_col)))))
for im in range(10, 13):
w1 = (rasterio.open(prec2.format(im)))
prec.append(
w1.read(window=Window.from_slices((start_row, end_row), (start_col,
end_col))))
for im in range(1, 10):
w0 = (rasterio.open(temp1.format(im)))
T.append(
w0.read(window=Window.from_slices((start_row, end_row), (start_col,
end_col))))
for im in range(10, 13):
w0 = (rasterio.open(temp2.format(im)))
T.append(
w0.read(window=Window.from_slices((start_row, end_row), (start_col,
end_col))))
def test_data_window_nodata_3d():
"""Get window of 3d arr with nodata."""
arr = np.ones((3, 3, 3))
arr[:, 0, :] = 0
window = get_data_window(arr, nodata=0)
assert window == Window.from_slices((1, 3), (0, 3))
img[img == -999999] = np.nan
img[img == -0] = 0
plt.imshow(img > 100)
# rasterio.plot.plotting_extent(ds)
# fig, ax = plt.subplots(figsize=(8, 8))
# rasterio.plot.show(ds, ax=ax, with_bounds=True)
# ======================================================================================================================
# View all features in bounding box
y_top = 950
x_left = 1250
y_bottom = 650
x_right = 900
# Read only a portion of the image
window = Window.from_slices((y_top, y_bottom), (x_left, x_right))
# Read only a portion of the image
window = Window.from_slices((y_top, y_bottom), (x_left, x_right))
# window = Window.from_slices((950, 1250), (1400, 1900))
with rasterio.open(stack_path, 'r') as src:
w = src.read(window=window)
w[w == -999999] = np.nan
w[np.isneginf(w)] = np.nan
w[15, ((w[14, :, :] == 1) & (w[15, :, :] == 1))] = 0
w = w[1:, :, :]
titles = feat_list_fancy
plt.figure(figsize=(6, 4))
axes = [plt.subplot(4, 4, i + 1) for i in range(15)]
for i, ax in enumerate(axes):
def test_data_window_full():
"""Get window of entirely valid data array."""
arr = np.ones((3, 3))
window = get_data_window(arr)
assert window == Window.from_slices((0, 3), (0, 3))
def test_window_union():
"""Window union works."""
window = union(Window(0, 0, 1, 1), Window(1, 1, 2, 2))
assert window == Window.from_slices((0, 3), (0, 3))
def test_intersection():
"""Window intersection works."""
window = intersection(Window(0, 0, 10, 10), Window(8, 8, 12, 12))
assert window == Window.from_slices((8, 10), (8, 10))
def test_round_window_already_at_edge(path_alpha_tif):
with rasterio.open(path_alpha_tif) as src:
block_shapes = src.block_shapes
test_window = ((256, 512), (512, 768))
rounded_window = round_window_to_full_blocks(test_window, block_shapes)
assert rounded_window == Window.from_slices(*test_window)
def test_window_union():
"""Window union works."""
window = union(Window(0, 0, 1, 1), Window(1, 1, 2, 2))
assert window == Window.from_slices((0, 3), (0, 3))
def test_window_from_offlen():
"""from_offlen classmethod works."""
with pytest.warns(RasterioDeprecationWarning):
assert Window.from_offlen(2, 0, 1, 1) == Window.from_slices((0, 1), (2, 3))
def test_round_window_already_at_edge(path_alpha_tif):
with rasterio.open(path_alpha_tif) as src:
block_shapes = src.block_shapes
test_window = ((256, 512), (512, 768))
rounded_window = round_window_to_full_blocks(test_window, block_shapes)
assert rounded_window == Window.from_slices(*test_window)
def flip_window(w):
# Turn a slice into a window, and also reflect it vertically
# in the list of rows of windows (e.g., the top left slice becomes
# the bottom left window):
m = Window.from_slices((height - w[0][1], height - w[0][0]), (w[1][0], w[1][1]))
return m
