def slope_tif_to_slope_shapefile(slope_tif,slope_bin_path,slope_threshold):
if os.path.isfile(slope_bin_path):
print('%s exist'%slope_bin_path)
else:
slope_data, nodata = raster_io.read_raster_one_band_np(slope_tif)
bin_slope = np.zeros_like(slope_data,dtype=np.uint8)
bin_slope[slope_data > slope_threshold] = 1
bin_slope[slope_data > 88] = 0 # if slope is too large, it may caused by artifacts, so remove them
# # Dilation or opening
# # https://opencv-python-tutroals.readthedocs.io/en/latest/py_tutorials/py_imgproc/py_morphological_ops/py_morphological_ops.html
# kernel = np.ones((3, 3), np.uint8) # if kernal is 5 or larger, will remove some narrow parts.
# # bin_slope = cv2.dilate(bin_slope,kernel,iterations = 1)
# bin_slope = cv2.morphologyEx(bin_slope, cv2.MORPH_OPEN, kernel) # use opening to remove some noise
# # bin_slope = cv2.morphologyEx(bin_slope, cv2.MORPH_CLOSE, kernel) # closing small holes inside
# save
slope_bin = bin_slope*255
raster_io.save_numpy_array_to_rasterfile(slope_bin,slope_bin_path,slope_tif,nodata=0,compress='lzw',tiled='yes',bigtiff='if_safer') # set nodata as 0
# to shapefile
slope_bin_shp = vector_gpd.raster2shapefile(slope_bin_path,connect8=True)
if slope_bin_shp is None:
return False
return slope_bin_shp
def sum_matchtag(input_tifs, save_path):
if len(input_tifs) < 1:
return False
# check band, with, height
height, width, count, dtype = raster_io.get_height_width_bandnum_dtype(
input_tifs[0])
for idx in range(1, len(input_tifs)):
h, w, c, type = raster_io.get_height_width_bandnum_dtype(
input_tifs[idx])
if h != height or w != width or c != count or type != dtype:
raise ValueError(
'size or data type is different between %s and %s' %
(input_tifs[0], input_tifs[idx]))
if count != 1:
raise ValueError('Matchtag should only have one band')
sum_data = np.zeros((height, width), dtype=np.uint8)
for tif in input_tifs:
data, nodata = raster_io.read_raster_one_band_np(tif)
# print(data.shape)
sum_data += data
# save to file
raster_io.save_numpy_array_to_rasterfile(sum_data,
save_path,
input_tifs[0],
compress='lzw',
tiled='yes',
bigtiff='if_safer')
return True
def segment_changes_on_dem_diff(dem_diff_tif, save_dir):
out_pre = os.path.splitext(os.path.basename(dem_diff_tif))[0]
# read images
one_band_img, nodata = raster_io.read_raster_one_band_np(dem_diff_tif)
# segmentation algorithm (the output of these algorithms is not alway good, need to chose the parameters carafully)
# out_labels = watershed_segmentation(one_band_img)
# out_labels = k_mean_cluster_segmentation(one_band_img)
# out_labels = quickshift_segmentaion(one_band_img)
out_labels = mean_shift_segmentation(one_band_img)
# segmentation by threshold (may have too many noise)
# mean = np.nanmean(one_band_img)
# print("mean value is: %.4f"%mean)
# one_band_img = one_band_img - mean
# out_labels = np.zeros_like(one_band_img,dtype=np.uint8)
# out_labels[ np.abs(one_band_img) > 2 ] = 1
# save the label
label_path = os.path.join(save_dir, out_pre + '_label.tif')
raster_io.save_numpy_array_to_rasterfile(out_labels,
label_path,
dem_diff_tif,
nodata=0)
# convert the label to shapefile
out_shp = os.path.join(save_dir, out_pre + '.shp')
command_string = 'gdal_polygonize.py -8 %s -b 1 -f "ESRI Shapefile" %s' % (
label_path, out_shp)
res = os.system(command_string)
if res != 0:
sys.exit(1)
def segment_subsidence_on_dem_diff(dem_diff_tif, save_dir):
out_pre = os.path.splitext(os.path.basename(dem_diff_tif))[0]
# read images
one_band_img, nodata = raster_io.read_raster_one_band_np(dem_diff_tif)
# segmentation by threshold (may have too many noise)
# mean = np.nanmean(one_band_img)
# print("mean value is: %.4f"%mean)
# one_band_img = one_band_img - mean # cannot use mean which may affect by some Outliers
out_labels = np.zeros_like(one_band_img,dtype=np.uint8)
out_labels[ one_band_img < -2 ] = 1 # end in a lot of noise, change to -2, -1 results in a lot of polygons
# apply median filter
out_labels = cv2.medianBlur(out_labels, 3) # with kernal=3
# save the label
if os.path.isdir(save_dir) is False:
io_function.mkdir(save_dir)
label_path = os.path.join(save_dir, out_pre + '_label.tif')
raster_io.save_numpy_array_to_rasterfile(out_labels, label_path, dem_diff_tif, nodata=0)
# convert the label to shapefile
out_shp = os.path.join(save_dir, out_pre + '.shp')
command_string = 'gdal_polygonize.py -8 %s -b 1 -f "ESRI Shapefile" %s' % (label_path, out_shp)
res = os.system(command_string)
if res != 0:
sys.exit(1)
# post-processing
post_processing_subsidence(out_shp)
def dem_to_relative_dem(input, output, patch_width, patch_height, process_num):
if os.path.isfile(output):
basic.outputlogMessage('%s exists, skip' % output)
return True
height, width, _, _ = raster_io.get_height_width_bandnum_dtype(input)
dst_nodata = 255
# divide the image the many small patches, then calcuate one by one, solving memory issues.
image_patches = split_image.sliding_window(width,
height,
patch_width,
patch_height,
adj_overlay_x=0,
adj_overlay_y=0)
patch_count = len(image_patches)
# get the difference
dem_relative_8bit_np = np.zeros((height, width), dtype=np.uint8)
if process_num == 1:
for idx, patch in enumerate(image_patches):
_, patch_rel_dem_8bit = dem_to_relative_8bit_a_patch(
idx, patch, patch_count, input, dst_nodata)
# copy to the entire image
row_s = patch[1]
row_e = patch[1] + patch[3]
col_s = patch[0]
col_e = patch[0] + patch[2]
dem_relative_8bit_np[row_s:row_e, col_s:col_e] = patch_rel_dem_8bit
else:
theadPool = Pool(process_num)
parameters_list = [(idx, patch, patch_count, input, dst_nodata)
for idx, patch in enumerate(image_patches)]
results = theadPool.starmap(dem_to_relative_8bit_a_patch,
parameters_list)
for res in results:
patch, patch_rel_dem_8bit = res
# copy to the entire image
row_s = patch[1]
row_e = patch[1] + patch[3]
col_s = patch[0]
col_e = patch[0] + patch[2]
dem_relative_8bit_np[row_s:row_e, col_s:col_e] = patch_rel_dem_8bit
theadPool.close()
# save date diff to tif (16 bit)
raster_io.save_numpy_array_to_rasterfile(dem_relative_8bit_np,
output,
input,
nodata=dst_nodata,
compress='lzw',
tiled='yes',
bigtiff='if_safer')
return True
def test_to_unique_label_for_superpixels():
label_img = os.path.expanduser(
'~/Data/Arctic/canada_arctic/DEM/WR_dem_diff/segment_parallel_sub/WR_dem_diff_DEM_diff_prj_8bit_sub_label.tif'
)
out_labels, nodata = raster_io.read_raster_one_band_np(label_img)
print('nodata', nodata)
print('min and max labels of out_labels', np.min(out_labels),
np.max(out_labels))
new_labels = image_segment.to_unique_label_for_superpixels(out_labels)
save_new_label = io_function.get_name_by_adding_tail(label_img, 'new')
raster_io.save_numpy_array_to_rasterfile(new_labels, save_new_label,
label_img)
def mask_by_surface_water(map_raster, surface_water_crop):
# save mask result to current folder
save_mask_result = io_function.get_name_by_adding_tail(
os.path.basename(map_raster), 'WaterMask')
if os.path.isfile(save_mask_result):
print('warning, %s exists' % save_mask_result)
return save_mask_result
# read
map_array_2d, nodata = raster_io.read_raster_one_band_np(map_raster)
water_array_2d, _ = raster_io.read_raster_one_band_np(surface_water_crop)
print(map_array_2d.shape)
if map_array_2d.shape != water_array_2d.shape:
raise ValueError('size inconsistent: %s and %s' %
(str(map_array_2d.shape), str(water_array_2d.shape)))
# mask out pixel, original is water or others
map_array_2d[np.logical_or(water_array_2d == 1, water_array_2d == 255)] = 0
if raster_io.save_numpy_array_to_rasterfile(map_array_2d,
save_mask_result,
map_raster,
compress='lzw',
tiled='Yes',
bigtiff='if_safer'):
return save_mask_result
def mask_by_elevation(map_raster_path, elevation_crop_path, threashold):
# save mask result to current folder
save_mask_result = io_function.get_name_by_adding_tail(
os.path.basename(map_raster_path), 'DEMMask')
if os.path.isfile(save_mask_result):
print('warning, %s exists' % save_mask_result)
return save_mask_result
# read
map_array_2d, nodata = raster_io.read_raster_one_band_np(map_raster_path)
dem_array_2d, _ = raster_io.read_raster_one_band_np(elevation_crop_path)
print(map_array_2d.shape)
if map_array_2d.shape != dem_array_2d.shape:
raise ValueError('size inconsistent: %s and %s' %
(str(map_array_2d.shape), str(dem_array_2d.shape)))
# mask out pixel with high elevation
map_array_2d[dem_array_2d > threashold] = 0
if raster_io.save_numpy_array_to_rasterfile(map_array_2d,
save_mask_result,
map_raster_path,
compress='lzw',
tiled='Yes',
bigtiff='if_safer'):
return save_mask_result
def mask_dem_by_matchtag(input_dem, mask_tif, save_path):
# check band, with, height
height, width, count, dtype = raster_io.get_height_width_bandnum_dtype(input_dem)
height_mask, width_mask, count_mask, dtype_mask = raster_io.get_height_width_bandnum_dtype(mask_tif)
if height_mask!=height or width_mask!=width or count_mask!=count:
raise ValueError('size different between %s and %s'%(input_dem, mask_tif))
if count != 1:
raise ValueError('DEM and Matchtag should only have one band')
dem_data, nodata = raster_io.read_raster_one_band_np(input_dem)
matchdata, mask_nodata = raster_io.read_raster_one_band_np(mask_tif)
# mask as nodata
dem_data[ matchdata == 0 ] = nodata
# save to file
raster_io.save_numpy_array_to_rasterfile(dem_data,save_path,input_dem,compress='lzw',tiled='yes',bigtiff='if_safer')
return save_path
def test_projection_epsg_2163():
# read a patch from iamge with epsg_2163, then save, see what's the projection
# path on my Mac
# folder = os.path.expanduser('~/Data/flooding_area/Houston/Houston_SAR_GRD_FLOAT_gee/S1_Houston_prj_8bit')
# path on tesia
folder = os.path.expanduser(
'~/Bhaltos2/lingcaoHuang/flooding_area/Houston/Houston_SAR_GRD_FLOAT_gee/S1_Houston_prj_8bit_select'
)
img_path = os.path.join(
folder,
'S1A_IW_GRDH_1SDV_20170829T002620_20170829T002645_018131_01E74D_D734_prj_8bit.tif'
)
xoff, yoff, xsize, ysize = 10000, 10000, 500, 500
boundary = (xoff, yoff, xsize, ysize)
img_data, nodata = raster_io.read_raster_one_band_np(img_path,
boundary=boundary)
raster_io.save_numpy_array_to_rasterfile(img_data,
'test_projection.tif',
img_path,
boundary=boundary)
def main(options, args):
img_path = args[0]
save_path = args[1]
io_function.is_file_exist(img_path)
if os.path.isfile(save_path):
print('%s exists, remove it if want to re-generate it' % save_path)
return
img_np_allbands, src_nodata = raster_io.read_raster_all_bands_np(img_path)
if options.src_nodata is not None:
src_nodata = options.src_nodata
scales = options.scale
if scales is not None:
print('input scale (src_min src_max dst_min dst_max): ' + str(scales))
img_array_8bit = raster_io.image_numpy_allBands_to_8bit(
img_np_allbands,
scales,
src_nodata=src_nodata,
dst_nodata=options.dst_nodata)
else:
min_percent = options.hist_min_percent
max_percent = options.hist_max_percent
min_max_value = options.min_max_value
img_array_8bit = raster_io.image_numpy_allBands_to_8bit_hist(
img_np_allbands,
min_max_value,
per_min=min_percent,
per_max=max_percent,
src_nodata=src_nodata,
dst_nodata=options.dst_nodata)
# save to file
if options.dst_nodata is None:
nodata = src_nodata
else:
nodata = options.dst_nodata
return raster_io.save_numpy_array_to_rasterfile(img_array_8bit,
save_path,
img_path,
nodata=nodata,
compress='lzw',
tiled='yes',
bigtiff='if_safer')
def mask_strip_dem_outlier_by_ArcticDEM_mosaic(crop_strip_dem_list, extent_poly, extent_id, crop_tif_dir, o_res, process_num):
# get list of the ArcticDEM mosaic
arcticDEM_mosaic_reg_tifs = io_function.get_file_list_by_ext('.tif',arcticDEM_tile_reg_tif_dir,bsub_folder=False)
mosaic_dem_ext_polys = get_dem_tif_ext_polygons(arcticDEM_mosaic_reg_tifs)
overlap_index = vector_gpd.get_poly_index_within_extent(mosaic_dem_ext_polys,extent_poly)
#### crop and mosaic mosaic_reg_tifs
sub_mosaic_dem_tifs = [arcticDEM_mosaic_reg_tifs[item] for item in overlap_index]
mosaic_crop_tif_list = []
for tif in sub_mosaic_dem_tifs:
save_crop_path = os.path.join(crop_tif_dir, os.path.basename(io_function.get_name_by_adding_tail(tif, 'sub_poly_%d' % extent_id)) )
if os.path.isfile(save_crop_path):
basic.outputlogMessage('%s exists, skip cropping' % save_crop_path)
mosaic_crop_tif_list.append(save_crop_path)
else:
crop_tif = subset_image_by_polygon_box(tif, save_crop_path, extent_poly, resample_m='near',
o_format='VRT', out_res=o_res,same_extent=True,thread_num=process_num)
if crop_tif is False:
raise ValueError('warning, crop %s failed' % tif)
mosaic_crop_tif_list.append(crop_tif)
if len(mosaic_crop_tif_list) < 1:
basic.outputlogMessage('No mosaic version of ArcticDEM for %d grid, skip mask_strip_dem_outlier_by_ArcticDEM_mosaic'%extent_id)
return False
# create mosaic, can handle only input one file, but is slow
save_dem_mosaic = os.path.join(crop_tif_dir, 'ArcticDEM_tiles_grid%d.tif'%extent_id)
result = RSImageProcess.mosaic_crop_images_gdalwarp(mosaic_crop_tif_list, save_dem_mosaic, resampling_method='average',o_format='GTiff',
compress='lzw', tiled='yes', bigtiff='if_safer',thread_num=process_num)
if result is False:
return False
height_tileDEM, width_tileDEM, count_tileDEM, dtype_tileDEM = raster_io.get_height_width_bandnum_dtype(save_dem_mosaic)
tileDEM_data, tileDEM_nodata = raster_io.read_raster_one_band_np(save_dem_mosaic)
# masking the strip version of DEMs
mask_strip_dem_list = []
for idx, strip_dem in enumerate(crop_strip_dem_list):
save_path = io_function.get_name_by_adding_tail(strip_dem, 'maskOutlier')
if os.path.isfile(save_path):
basic.outputlogMessage('%s exist, skip'%save_path)
mask_strip_dem_list.append(save_path)
continue
# check band, with, height
height, width, count, dtype = raster_io.get_height_width_bandnum_dtype(strip_dem)
if height_tileDEM != height or width_tileDEM != width or count_tileDEM != count:
raise ValueError('size different between %s and %s' % (strip_dem, save_dem_mosaic))
if count != 1:
raise ValueError('DEM and Matchtag should only have one band')
try:
dem_data, nodata = raster_io.read_raster_one_band_np(strip_dem)
except:
basic.outputlogMessage(' invalid tif file: %s'%strip_dem)
continue
nodata_loc = np.where(dem_data == nodata)
diff = dem_data - tileDEM_data
# mask as nodata
dem_data[np.abs(diff) > 50 ] = nodata # ignore greater than 50 m
dem_data[ nodata_loc ] = nodata # may change some nodata pixel, change them back
# save to file
raster_io.save_numpy_array_to_rasterfile(dem_data, save_path, strip_dem, compress='lzw', tiled='yes',
bigtiff='if_safer')
mask_strip_dem_list.append(save_path)
return mask_strip_dem_list
def segment_a_patch(idx, patch, patch_count, img_path, org_raster,
b_save_patch_label):
print('tile: %d / %d' % (idx + 1, patch_count))
image_name_no_ext = io_function.get_name_no_ext(img_path)
patch_dir = image_name_no_ext + '_patch%d' % idx
patch_label_path = os.path.join(
patch_dir, image_name_no_ext + '_patch%d_label.tif' % idx)
if b_save_patch_label is True:
if os.path.isdir(patch_dir) is False:
io_function.mkdir(patch_dir)
if os.path.isfile(patch_label_path):
print('%s exists, skip' % patch_label_path)
return patch, patch_label_path, None, None
# read imag
one_band_img, nodata = raster_io.read_raster_one_band_np(img_path,
boundary=patch)
# # apply median filter to image (remove some noise)
one_band_img = cv2.medianBlur(one_band_img,
3) # with kernal=3, cannot accept int32
# segmentation algorithm (the output of these algorithms is not alway good, need to chose the parameters carafully)
# out_labels = watershed_segmentation(one_band_img)
# out_labels = k_mean_cluster_segmentation(one_band_img)
out_labels = quickshift_segmentaion(one_band_img,
ratio=0.3,
kernel_size=5,
max_dist=10,
sigma=1,
convert2lab=False)
#
#
# out_labels = mean_shift_segmentation(one_band_img)
# print('min and max labels of out_labels', np.min(out_labels), np.max(out_labels))
if b_save_patch_label is True:
# save the label
raster_io.save_numpy_array_to_rasterfile(
out_labels, patch_label_path, img_path,
boundary=patch) # it copy nodata, need to unset it later
return patch, patch_label_path, None, None
# calculate the attributes based on orginal data for original data
object_attributes = {} # object id (label) and attributes (list)
if org_raster is not None:
org_img_b1, org_nodata = raster_io.read_raster_one_band_np(
org_raster, boundary=patch)
# get regions (the labels output by segmentation is not unique for superpixels)
# regions = measure.regionprops(out_labels, intensity_image=org_img_b1) # regions is based on out_labels, so it has the same issue.
# print('region count from sk-image measure:',len(regions))
label_list = np.unique(out_labels)
# get statistics for each segmented object (label)
for label in label_list:
in_array = org_img_b1[out_labels == label]
object_attributes[label] = get_stastics_from_array(
in_array, org_nodata)
return patch, out_labels, nodata, object_attributes
return patch, out_labels, nodata, None
def segment_a_grey_image(img_path,
save_dir,
process_num,
org_raster=None,
b_save_patch_label=False):
out_pre = os.path.splitext(os.path.basename(img_path))[0]
label_path = os.path.join(save_dir, out_pre + '_label.tif')
if os.path.isfile(label_path):
basic.outputlogMessage('%s exist, skip segmentation' % label_path)
return label_path
height, width, band_num, date_type = raster_io.get_height_width_bandnum_dtype(
img_path)
print('input image: height, width, band_num, date_type', height, width,
band_num, date_type)
# if the original data is available, then calculate the attributes based on that
if org_raster is not None:
org_height, org_width, org_band_num, org_date_type = raster_io.get_height_width_bandnum_dtype(
org_raster)
if org_height != height or org_width != width:
raise ValueError('%s and %s do not have the same size' %
(img_path, org_raster))
save_labes = np.zeros((height, width), dtype=np.int32)
# divide the image the many small patches, then calcuate one by one, solving memory issues.
image_patches = split_image.sliding_window(width,
height,
1024,
1024,
adj_overlay_x=0,
adj_overlay_y=0)
patch_count = len(image_patches)
# for idx, patch in enumerate(image_patches):
# out_patch,out_labels = segment_a_patch(idx, patch, patch_count,img_path)
# # copy to the entire image
# row_s = patch[1]
# row_e = patch[1] + patch[3]
# col_s = patch[0]
# col_e = patch[0] + patch[2]
# save_labes[row_s:row_e, col_s:col_e] = out_labels
theadPool = Pool(process_num)
parameters_list = [(idx, patch, patch_count, img_path, org_raster,
b_save_patch_label)
for idx, patch in enumerate(image_patches)]
results = theadPool.starmap(segment_a_patch, parameters_list)
patch_label_path_list = []
patch_label_id_range = []
object_attributes = {} # object id (label) and attributes (list)
for res in results:
patch, out_labels, nodata, attributes = res
if isinstance(
out_labels,
str) and os.path.isfile(out_labels): #if it's a label file
patch_label_path_list.append(out_labels)
else:
# copy to the entire image
row_s = patch[1]
row_e = patch[1] + patch[3]
col_s = patch[0]
col_e = patch[0] + patch[2]
current_min = np.max(save_labes)
print('current_max', current_min)
patch_label_id_range.append(current_min)
save_labes[row_s:row_e, col_s:col_e] = out_labels + current_min + 1
if attributes is not None:
update_label_attr = {}
for key in attributes:
update_label_attr[key + current_min] = attributes[key]
# add to the attributes
object_attributes.update(update_label_attr)
# # apply median filter (remove some noise), # we should not use median filter, because it's labels, not images.
# label_blurs = cv2.medianBlur(np.float32(save_labes), 3) # with kernal=3, cannot accept int32
# # print(label_blurs, label_blurs.dtype)
# save_labes = label_blurs.astype(np.int32)
# return a list of labels saved in current working folder.
if b_save_patch_label:
return patch_label_path_list
if os.path.isdir(save_dir) is False:
io_function.mkdir(save_dir)
# save attributes (if not empty)
if object_attributes:
attribute_path = os.path.join(save_dir, out_pre + '_attributes.txt')
io_function.save_dict_to_txt_json(attribute_path, object_attributes)
# save the label
raster_io.save_numpy_array_to_rasterfile(save_labes, label_path,
img_path) # do not set nodata
# save id ranges to txt
label_id_range_txt = os.path.splitext(label_path)[0] + '_IDrange.txt'
patch_label_id_range = [str(item) for item in patch_label_id_range]
io_function.save_list_to_txt(label_id_range_txt, patch_label_id_range)
return label_path
def dem_diff_newest_oldest(dem_tif_list,
out_dem_diff,
out_date_diff,
process_num,
b_max_subsidence=False,
b_save_cm=False):
'''
get DEM difference, for each pixel, newest vaild value - oldest valid value
:param dem_list:
:param output:
:return:
'''
if len(dem_tif_list) < 2:
basic.outputlogMessage('error, the count of DEM is smaller than 2')
return False
# groups DEM with original images acquired at the same year months
dem_groups_date = group_demTif_yearmonthDay(dem_tif_list, diff_days=0)
# sort based on yeardate in accending order : operator.itemgetter(0)
dem_groups_date = dict(
sorted(dem_groups_date.items(), key=operator.itemgetter(0)))
txt_save_path = os.path.splitext(out_date_diff)[0] + '.txt'
# change the key to integer number after sorting and save to txt file
dem_groups_date_sort_idx = {}
for idx, key in enumerate(dem_groups_date.keys()):
dem_groups_date_sort_idx[idx] = dem_groups_date[key]
io_function.save_dict_to_txt_json(txt_save_path, dem_groups_date_sort_idx)
date_list = list(dem_groups_date.keys())
dem_tif_list = [dem_groups_date[key][0] for key in dem_groups_date.keys()
] # each date, only have one tif
tif_obj_list = [raster_io.open_raster_read(tif) for tif in dem_tif_list]
height, width, _ = raster_io.get_width_heigth_bandnum(tif_obj_list[0])
# check them have the width and height
for tif, obj in zip(dem_tif_list[1:], tif_obj_list[1:]):
h, w, _ = raster_io.get_width_heigth_bandnum(obj)
if h != height or w != width:
raise ValueError(
'the height and width of %s is different from others' % tif)
# divide the image the many small patches, then calcuate one by one, solving memory issues.
image_patches = split_image.sliding_window(width,
height,
1024,
1024,
adj_overlay_x=0,
adj_overlay_y=0)
patch_count = len(image_patches)
tif_obj_list = None
# read all and their date
date_pair_list = list(combinations(date_list, 2))
date_diff_list = [(item[1] - item[0]).days for item in date_pair_list]
# sort based on day difference (from max to min)
date_pair_list_sorted = [
x for _, x in sorted(zip(date_diff_list, date_pair_list), reverse=True)
] # descending
# get the difference
date_diff_np = np.zeros((height, width), dtype=np.uint16)
old_date_index = np.zeros((height, width), dtype=np.uint8)
new_date_index = np.zeros((height, width), dtype=np.uint8)
dem_diff_np = np.empty((height, width), dtype=np.float32)
dem_diff_np[:] = np.nan
if process_num == 1:
for idx, patch in enumerate(image_patches):
_,patch_dem_diff,patch_date_diff, patch_old_date_idx,patch_new_date_idx = \
dem_diff_newest_oldest_a_patch(idx, patch, patch_count,date_pair_list_sorted,dem_groups_date)
# copy to the entire image
row_s = patch[1]
row_e = patch[1] + patch[3]
col_s = patch[0]
col_e = patch[0] + patch[2]
dem_diff_np[row_s:row_e, col_s:col_e] = patch_dem_diff
date_diff_np[row_s:row_e, col_s:col_e] = patch_date_diff
old_date_index[row_s:row_e, col_s:col_e] = patch_old_date_idx
new_date_index[row_s:row_e, col_s:col_e] = patch_new_date_idx
else:
theadPool = Pool(process_num)
parameters_list = [(idx, patch, patch_count, date_pair_list_sorted,
dem_groups_date)
for idx, patch in enumerate(image_patches)]
if b_max_subsidence is False:
results = theadPool.starmap(dem_diff_newest_oldest_a_patch,
parameters_list)
else:
results = theadPool.starmap(dem_diff_new_old_min_neg_diff_patch,
parameters_list)
for res in results:
patch, patch_dem_diff, patch_date_diff, patch_old_date_idx, patch_new_date_idx = res
# copy to the entire image
row_s = patch[1]
row_e = patch[1] + patch[3]
col_s = patch[0]
col_e = patch[0] + patch[2]
dem_diff_np[row_s:row_e, col_s:col_e] = patch_dem_diff
date_diff_np[row_s:row_e, col_s:col_e] = patch_date_diff
old_date_index[row_s:row_e, col_s:col_e] = patch_old_date_idx
new_date_index[row_s:row_e, col_s:col_e] = patch_new_date_idx
theadPool.close()
# save date diff to tif (16 bit)
raster_io.save_numpy_array_to_rasterfile(date_diff_np,
out_date_diff,
dem_tif_list[0],
nodata=0,
compress='lzw',
tiled='yes',
bigtiff='if_safer')
# save old and new date index to tif (8 bit)
out_old_date_idx = io_function.get_name_by_adding_tail(
out_date_diff, 'oldIndex')
out_new_date_idx = io_function.get_name_by_adding_tail(
out_date_diff, 'newIndex')
raster_io.save_numpy_array_to_rasterfile(old_date_index,
out_old_date_idx,
dem_tif_list[0],
nodata=255,
compress='lzw',
tiled='yes',
bigtiff='if_safer')
raster_io.save_numpy_array_to_rasterfile(new_date_index,
out_new_date_idx,
dem_tif_list[0],
nodata=255,
compress='lzw',
tiled='yes',
bigtiff='if_safer')
# # stretch the DEM difference, save to 8 bit.
# dem_diff_np_8bit = raster_io.image_numpy_to_8bit(dem_diff_np,10,-10,dst_nodata=0)
# out_dem_diff_8bit = io_function.get_name_by_adding_tail(out_dem_diff, '8bit')
# raster_io.save_numpy_array_to_rasterfile(dem_diff_np_8bit, out_dem_diff_8bit, dem_tif_list[0], nodata=0)
# if possible, save to 16 bit, to save the disk storage.
# dem_diff_np[0:5,0] = -500
# dem_diff_np[0,0:5] = 500
# print(np.nanmin(dem_diff_np))
# print(np.nanmax(dem_diff_np))
# if np.nanmin(dem_diff_np_cm) < range.min or np.nanmax(dem_diff_np_cm) > range.max:
# save dem diff to files (float), meter
if b_save_cm is False:
raster_io.save_numpy_array_to_rasterfile(dem_diff_np,
out_dem_diff,
dem_tif_list[0],
nodata=-9999,
compress='lzw',
tiled='yes',
bigtiff='if_safer')
else:
# save dem diff to 16bit, centimeter, only handle diff from -327.67 to 327.67 meters
bit16_nodata = 32767
range = np.iinfo(np.int16)
dem_diff_np_cm = dem_diff_np * 100
dem_diff_np_cm[dem_diff_np_cm < range.min] = range.min
dem_diff_np_cm[dem_diff_np_cm > range.max] = range.max
dem_diff_np_cm[np.isnan(
dem_diff_np_cm)] = bit16_nodata # set the nodata for int16
dem_diff_np_cm = dem_diff_np_cm.astype(np.int16) # save to int16
out_dem_diff_cm = out_dem_diff
basic.outputlogMessage(
'note, save DEM difference (%s) to centimeter, int16, range: -327.68 to 327.67 m'
% os.path.basename(out_dem_diff_cm))
raster_io.save_numpy_array_to_rasterfile(dem_diff_np_cm,
out_dem_diff_cm,
dem_tif_list[0],
nodata=bit16_nodata,
compress='lzw',
tiled='yes',
bigtiff='if_safer')
return True
def dem_diff_newest_oldest(dem_tif_list, out_dem_diff, out_date_diff):
'''
get DEM difference, for each pixel, newest vaild value - oldest valid value
:param dem_list:
:param output:
:return:
'''
if len(dem_tif_list) < 2:
basic.outputlogMessage('error, the count of DEM is smaller than 2')
return False
# groups DEM with original images acquired at the same year months
dem_groups_date = group_demTif_yearmonthDay(dem_tif_list, diff_days=0)
# sort based on yeardate in accending order : operator.itemgetter(0)
dem_groups_date = dict(
sorted(dem_groups_date.items(), key=operator.itemgetter(0)))
txt_save_path = os.path.splitext(out_date_diff)[0] + '.txt'
io_function.save_dict_to_txt_json(txt_save_path, dem_groups_date)
date_list = list(dem_groups_date.keys())
dem_tif_list = [dem_groups_date[key][0] for key in dem_groups_date.keys()
] # each date, only have one tif
tif_obj_list = [raster_io.open_raster_read(tif) for tif in dem_tif_list]
height, width, _ = raster_io.get_width_heigth_bandnum(tif_obj_list[0])
# check them have the width and height
for tif, obj in zip(dem_tif_list[1:], tif_obj_list[1:]):
h, w, _ = raster_io.get_width_heigth_bandnum(obj)
if h != height or w != width:
raise ValueError(
'the height and width of %s is different from others' % tif)
# divide the image the many small patches, then calcuate one by one, solving memory issues.
image_patches = split_image.sliding_window(width,
height,
1024,
1024,
adj_overlay_x=0,
adj_overlay_y=0)
patch_count = len(image_patches)
tif_obj_list = None
# read all and their date
date_pair_list = list(combinations(date_list, 2))
date_diff_list = [(item[1] - item[0]).days for item in date_pair_list]
# sort based on day difference (from max to min)
date_pair_list_sorted = [
x for _, x in sorted(zip(date_diff_list, date_pair_list), reverse=True)
] # descending
# get the difference
date_diff_np = np.zeros((height, width), dtype=np.uint16)
dem_diff_np = np.empty((height, width), dtype=np.float32)
dem_diff_np[:] = np.nan
for idx, patch in enumerate(image_patches):
print('tile: %d / %d' % (idx + 1, patch_count))
patch_w = patch[2]
patch_h = patch[3]
patch_date_diff = np.zeros((patch_h, patch_w), dtype=np.uint16)
patch_dem_diff = np.empty((patch_h, patch_w), dtype=np.float32)
patch_dem_diff[:] = np.nan
# use dict to read data from disk (only need)
dem_data_dict = {}
for p_idx, pair in enumerate(date_pair_list_sorted):
diff_days = (pair[1] - pair[0]).days
basic.outputlogMessage(
'Getting DEM difference using the one on %s and %s, total day diff: %d'
% (timeTools.date2str(pair[1]), timeTools.date2str(
pair[0]), diff_days))
# print(pair,':',(pair[1] - pair[0]).days)
data_old, data_new = read_date_dem_to_memory(p_idx,
pair,
date_pair_list_sorted,
dem_data_dict,
dem_groups_date,
boundary=patch)
# print('data_old shape:',data_old.shape)
# print('data_new shape:',data_new.shape)
diff_two = data_new - data_old
# print(diff_two)
# fill the element
new_ele = np.where(
np.logical_and(np.isnan(patch_dem_diff), ~np.isnan(diff_two)))
patch_dem_diff[new_ele] = diff_two[new_ele]
patch_date_diff[new_ele] = diff_days
# check if all have been filled ( nan pixels)
diff_remain_hole = np.where(np.isnan(patch_dem_diff))
# basic.outputlogMessage(' remain %.4f percent pixels need to be filled'% (100.0*diff_remain_hole[0].size/patch_dem_diff.size) )
if diff_remain_hole[0].size < 1:
break
# copy to the entire image
row_s = patch[1]
row_e = patch[1] + patch[3]
col_s = patch[0]
col_e = patch[0] + patch[2]
dem_diff_np[row_s:row_e, col_s:col_e] = patch_dem_diff
date_diff_np[row_s:row_e, col_s:col_e] = patch_date_diff
# save date diff to tif (16 bit)
raster_io.save_numpy_array_to_rasterfile(date_diff_np,
out_date_diff,
dem_tif_list[0],
nodata=0,
compress='lzw',
tiled='yes',
bigtiff='if_safer')
# # stretch the DEM difference, save to 8 bit.
# dem_diff_np_8bit = raster_io.image_numpy_to_8bit(dem_diff_np,10,-10,dst_nodata=0)
# out_dem_diff_8bit = io_function.get_name_by_adding_tail(out_dem_diff, '8bit')
# raster_io.save_numpy_array_to_rasterfile(dem_diff_np_8bit, out_dem_diff_8bit, dem_tif_list[0], nodata=0)
# if possible, save to 16 bit, to save the disk storage.
# dem_diff_np[0:5,0] = -500
# dem_diff_np[0,0:5] = 500
# print(np.nanmin(dem_diff_np))
# print(np.nanmax(dem_diff_np))
range = np.iinfo(np.int16)
# dem_diff_np_cm = dem_diff_np*100
# if np.nanmin(dem_diff_np_cm) < range.min or np.nanmax(dem_diff_np_cm) > range.max:
# save dem diff to files (float), meter
raster_io.save_numpy_array_to_rasterfile(dem_diff_np,
out_dem_diff,
dem_tif_list[0],
nodata=-9999,
compress='lzw',
tiled='yes',
bigtiff='if_safer')
# else:
# # save dem diff to 16bit, centimeter, only handle diff from -327.67 to 327.67 meters
# dem_diff_np_cm = dem_diff_np_cm.astype(np.int16) # save to int16
# raster_io.save_numpy_array_to_rasterfile(dem_diff_np_cm, out_dem_diff_cm, dem_tif_list[0],nodata=32767,compress='lzw',tiled='yes',bigtiff='if_safer')
return True
