def slope_tif_to_slope_shapefile(slope_tif,slope_bin_path,slope_threshold): if os.path.isfile(slope_bin_path): print('%s exist'%slope_bin_path) else: slope_data, nodata = raster_io.read_raster_one_band_np(slope_tif) bin_slope = np.zeros_like(slope_data,dtype=np.uint8) bin_slope[slope_data > slope_threshold] = 1 bin_slope[slope_data > 88] = 0 # if slope is too large, it may caused by artifacts, so remove them # # Dilation or opening # # https://opencv-python-tutroals.readthedocs.io/en/latest/py_tutorials/py_imgproc/py_morphological_ops/py_morphological_ops.html # kernel = np.ones((3, 3), np.uint8) # if kernal is 5 or larger, will remove some narrow parts. # # bin_slope = cv2.dilate(bin_slope,kernel,iterations = 1) # bin_slope = cv2.morphologyEx(bin_slope, cv2.MORPH_OPEN, kernel) # use opening to remove some noise # # bin_slope = cv2.morphologyEx(bin_slope, cv2.MORPH_CLOSE, kernel) # closing small holes inside # save slope_bin = bin_slope*255 raster_io.save_numpy_array_to_rasterfile(slope_bin,slope_bin_path,slope_tif,nodata=0,compress='lzw',tiled='yes',bigtiff='if_safer') # set nodata as 0 # to shapefile slope_bin_shp = vector_gpd.raster2shapefile(slope_bin_path,connect8=True) if slope_bin_shp is None: return False return slope_bin_shp
def sum_matchtag(input_tifs, save_path): if len(input_tifs) < 1: return False # check band, with, height height, width, count, dtype = raster_io.get_height_width_bandnum_dtype( input_tifs[0]) for idx in range(1, len(input_tifs)): h, w, c, type = raster_io.get_height_width_bandnum_dtype( input_tifs[idx]) if h != height or w != width or c != count or type != dtype: raise ValueError( 'size or data type is different between %s and %s' % (input_tifs[0], input_tifs[idx])) if count != 1: raise ValueError('Matchtag should only have one band') sum_data = np.zeros((height, width), dtype=np.uint8) for tif in input_tifs: data, nodata = raster_io.read_raster_one_band_np(tif) # print(data.shape) sum_data += data # save to file raster_io.save_numpy_array_to_rasterfile(sum_data, save_path, input_tifs[0], compress='lzw', tiled='yes', bigtiff='if_safer') return True
def segment_changes_on_dem_diff(dem_diff_tif, save_dir): out_pre = os.path.splitext(os.path.basename(dem_diff_tif))[0] # read images one_band_img, nodata = raster_io.read_raster_one_band_np(dem_diff_tif) # segmentation algorithm (the output of these algorithms is not alway good, need to chose the parameters carafully) # out_labels = watershed_segmentation(one_band_img) # out_labels = k_mean_cluster_segmentation(one_band_img) # out_labels = quickshift_segmentaion(one_band_img) out_labels = mean_shift_segmentation(one_band_img) # segmentation by threshold (may have too many noise) # mean = np.nanmean(one_band_img) # print("mean value is: %.4f"%mean) # one_band_img = one_band_img - mean # out_labels = np.zeros_like(one_band_img,dtype=np.uint8) # out_labels[ np.abs(one_band_img) > 2 ] = 1 # save the label label_path = os.path.join(save_dir, out_pre + '_label.tif') raster_io.save_numpy_array_to_rasterfile(out_labels, label_path, dem_diff_tif, nodata=0) # convert the label to shapefile out_shp = os.path.join(save_dir, out_pre + '.shp') command_string = 'gdal_polygonize.py -8 %s -b 1 -f "ESRI Shapefile" %s' % ( label_path, out_shp) res = os.system(command_string) if res != 0: sys.exit(1)
def segment_subsidence_on_dem_diff(dem_diff_tif, save_dir): out_pre = os.path.splitext(os.path.basename(dem_diff_tif))[0] # read images one_band_img, nodata = raster_io.read_raster_one_band_np(dem_diff_tif) # segmentation by threshold (may have too many noise) # mean = np.nanmean(one_band_img) # print("mean value is: %.4f"%mean) # one_band_img = one_band_img - mean # cannot use mean which may affect by some Outliers out_labels = np.zeros_like(one_band_img,dtype=np.uint8) out_labels[ one_band_img < -2 ] = 1 # end in a lot of noise, change to -2, -1 results in a lot of polygons # apply median filter out_labels = cv2.medianBlur(out_labels, 3) # with kernal=3 # save the label if os.path.isdir(save_dir) is False: io_function.mkdir(save_dir) label_path = os.path.join(save_dir, out_pre + '_label.tif') raster_io.save_numpy_array_to_rasterfile(out_labels, label_path, dem_diff_tif, nodata=0) # convert the label to shapefile out_shp = os.path.join(save_dir, out_pre + '.shp') command_string = 'gdal_polygonize.py -8 %s -b 1 -f "ESRI Shapefile" %s' % (label_path, out_shp) res = os.system(command_string) if res != 0: sys.exit(1) # post-processing post_processing_subsidence(out_shp)
def dem_to_relative_dem(input, output, patch_width, patch_height, process_num): if os.path.isfile(output): basic.outputlogMessage('%s exists, skip' % output) return True height, width, _, _ = raster_io.get_height_width_bandnum_dtype(input) dst_nodata = 255 # divide the image the many small patches, then calcuate one by one, solving memory issues. image_patches = split_image.sliding_window(width, height, patch_width, patch_height, adj_overlay_x=0, adj_overlay_y=0) patch_count = len(image_patches) # get the difference dem_relative_8bit_np = np.zeros((height, width), dtype=np.uint8) if process_num == 1: for idx, patch in enumerate(image_patches): _, patch_rel_dem_8bit = dem_to_relative_8bit_a_patch( idx, patch, patch_count, input, dst_nodata) # copy to the entire image row_s = patch[1] row_e = patch[1] + patch[3] col_s = patch[0] col_e = patch[0] + patch[2] dem_relative_8bit_np[row_s:row_e, col_s:col_e] = patch_rel_dem_8bit else: theadPool = Pool(process_num) parameters_list = [(idx, patch, patch_count, input, dst_nodata) for idx, patch in enumerate(image_patches)] results = theadPool.starmap(dem_to_relative_8bit_a_patch, parameters_list) for res in results: patch, patch_rel_dem_8bit = res # copy to the entire image row_s = patch[1] row_e = patch[1] + patch[3] col_s = patch[0] col_e = patch[0] + patch[2] dem_relative_8bit_np[row_s:row_e, col_s:col_e] = patch_rel_dem_8bit theadPool.close() # save date diff to tif (16 bit) raster_io.save_numpy_array_to_rasterfile(dem_relative_8bit_np, output, input, nodata=dst_nodata, compress='lzw', tiled='yes', bigtiff='if_safer') return True
def test_to_unique_label_for_superpixels(): label_img = os.path.expanduser( '~/Data/Arctic/canada_arctic/DEM/WR_dem_diff/segment_parallel_sub/WR_dem_diff_DEM_diff_prj_8bit_sub_label.tif' ) out_labels, nodata = raster_io.read_raster_one_band_np(label_img) print('nodata', nodata) print('min and max labels of out_labels', np.min(out_labels), np.max(out_labels)) new_labels = image_segment.to_unique_label_for_superpixels(out_labels) save_new_label = io_function.get_name_by_adding_tail(label_img, 'new') raster_io.save_numpy_array_to_rasterfile(new_labels, save_new_label, label_img)
def mask_by_surface_water(map_raster, surface_water_crop): # save mask result to current folder save_mask_result = io_function.get_name_by_adding_tail( os.path.basename(map_raster), 'WaterMask') if os.path.isfile(save_mask_result): print('warning, %s exists' % save_mask_result) return save_mask_result # read map_array_2d, nodata = raster_io.read_raster_one_band_np(map_raster) water_array_2d, _ = raster_io.read_raster_one_band_np(surface_water_crop) print(map_array_2d.shape) if map_array_2d.shape != water_array_2d.shape: raise ValueError('size inconsistent: %s and %s' % (str(map_array_2d.shape), str(water_array_2d.shape))) # mask out pixel, original is water or others map_array_2d[np.logical_or(water_array_2d == 1, water_array_2d == 255)] = 0 if raster_io.save_numpy_array_to_rasterfile(map_array_2d, save_mask_result, map_raster, compress='lzw', tiled='Yes', bigtiff='if_safer'): return save_mask_result
def mask_by_elevation(map_raster_path, elevation_crop_path, threashold): # save mask result to current folder save_mask_result = io_function.get_name_by_adding_tail( os.path.basename(map_raster_path), 'DEMMask') if os.path.isfile(save_mask_result): print('warning, %s exists' % save_mask_result) return save_mask_result # read map_array_2d, nodata = raster_io.read_raster_one_band_np(map_raster_path) dem_array_2d, _ = raster_io.read_raster_one_band_np(elevation_crop_path) print(map_array_2d.shape) if map_array_2d.shape != dem_array_2d.shape: raise ValueError('size inconsistent: %s and %s' % (str(map_array_2d.shape), str(dem_array_2d.shape))) # mask out pixel with high elevation map_array_2d[dem_array_2d > threashold] = 0 if raster_io.save_numpy_array_to_rasterfile(map_array_2d, save_mask_result, map_raster_path, compress='lzw', tiled='Yes', bigtiff='if_safer'): return save_mask_result
def mask_dem_by_matchtag(input_dem, mask_tif, save_path): # check band, with, height height, width, count, dtype = raster_io.get_height_width_bandnum_dtype(input_dem) height_mask, width_mask, count_mask, dtype_mask = raster_io.get_height_width_bandnum_dtype(mask_tif) if height_mask!=height or width_mask!=width or count_mask!=count: raise ValueError('size different between %s and %s'%(input_dem, mask_tif)) if count != 1: raise ValueError('DEM and Matchtag should only have one band') dem_data, nodata = raster_io.read_raster_one_band_np(input_dem) matchdata, mask_nodata = raster_io.read_raster_one_band_np(mask_tif) # mask as nodata dem_data[ matchdata == 0 ] = nodata # save to file raster_io.save_numpy_array_to_rasterfile(dem_data,save_path,input_dem,compress='lzw',tiled='yes',bigtiff='if_safer') return save_path
def test_projection_epsg_2163(): # read a patch from iamge with epsg_2163, then save, see what's the projection # path on my Mac # folder = os.path.expanduser('~/Data/flooding_area/Houston/Houston_SAR_GRD_FLOAT_gee/S1_Houston_prj_8bit') # path on tesia folder = os.path.expanduser( '~/Bhaltos2/lingcaoHuang/flooding_area/Houston/Houston_SAR_GRD_FLOAT_gee/S1_Houston_prj_8bit_select' ) img_path = os.path.join( folder, 'S1A_IW_GRDH_1SDV_20170829T002620_20170829T002645_018131_01E74D_D734_prj_8bit.tif' ) xoff, yoff, xsize, ysize = 10000, 10000, 500, 500 boundary = (xoff, yoff, xsize, ysize) img_data, nodata = raster_io.read_raster_one_band_np(img_path, boundary=boundary) raster_io.save_numpy_array_to_rasterfile(img_data, 'test_projection.tif', img_path, boundary=boundary)
def main(options, args): img_path = args[0] save_path = args[1] io_function.is_file_exist(img_path) if os.path.isfile(save_path): print('%s exists, remove it if want to re-generate it' % save_path) return img_np_allbands, src_nodata = raster_io.read_raster_all_bands_np(img_path) if options.src_nodata is not None: src_nodata = options.src_nodata scales = options.scale if scales is not None: print('input scale (src_min src_max dst_min dst_max): ' + str(scales)) img_array_8bit = raster_io.image_numpy_allBands_to_8bit( img_np_allbands, scales, src_nodata=src_nodata, dst_nodata=options.dst_nodata) else: min_percent = options.hist_min_percent max_percent = options.hist_max_percent min_max_value = options.min_max_value img_array_8bit = raster_io.image_numpy_allBands_to_8bit_hist( img_np_allbands, min_max_value, per_min=min_percent, per_max=max_percent, src_nodata=src_nodata, dst_nodata=options.dst_nodata) # save to file if options.dst_nodata is None: nodata = src_nodata else: nodata = options.dst_nodata return raster_io.save_numpy_array_to_rasterfile(img_array_8bit, save_path, img_path, nodata=nodata, compress='lzw', tiled='yes', bigtiff='if_safer')
def mask_strip_dem_outlier_by_ArcticDEM_mosaic(crop_strip_dem_list, extent_poly, extent_id, crop_tif_dir, o_res, process_num): # get list of the ArcticDEM mosaic arcticDEM_mosaic_reg_tifs = io_function.get_file_list_by_ext('.tif',arcticDEM_tile_reg_tif_dir,bsub_folder=False) mosaic_dem_ext_polys = get_dem_tif_ext_polygons(arcticDEM_mosaic_reg_tifs) overlap_index = vector_gpd.get_poly_index_within_extent(mosaic_dem_ext_polys,extent_poly) #### crop and mosaic mosaic_reg_tifs sub_mosaic_dem_tifs = [arcticDEM_mosaic_reg_tifs[item] for item in overlap_index] mosaic_crop_tif_list = [] for tif in sub_mosaic_dem_tifs: save_crop_path = os.path.join(crop_tif_dir, os.path.basename(io_function.get_name_by_adding_tail(tif, 'sub_poly_%d' % extent_id)) ) if os.path.isfile(save_crop_path): basic.outputlogMessage('%s exists, skip cropping' % save_crop_path) mosaic_crop_tif_list.append(save_crop_path) else: crop_tif = subset_image_by_polygon_box(tif, save_crop_path, extent_poly, resample_m='near', o_format='VRT', out_res=o_res,same_extent=True,thread_num=process_num) if crop_tif is False: raise ValueError('warning, crop %s failed' % tif) mosaic_crop_tif_list.append(crop_tif) if len(mosaic_crop_tif_list) < 1: basic.outputlogMessage('No mosaic version of ArcticDEM for %d grid, skip mask_strip_dem_outlier_by_ArcticDEM_mosaic'%extent_id) return False # create mosaic, can handle only input one file, but is slow save_dem_mosaic = os.path.join(crop_tif_dir, 'ArcticDEM_tiles_grid%d.tif'%extent_id) result = RSImageProcess.mosaic_crop_images_gdalwarp(mosaic_crop_tif_list, save_dem_mosaic, resampling_method='average',o_format='GTiff', compress='lzw', tiled='yes', bigtiff='if_safer',thread_num=process_num) if result is False: return False height_tileDEM, width_tileDEM, count_tileDEM, dtype_tileDEM = raster_io.get_height_width_bandnum_dtype(save_dem_mosaic) tileDEM_data, tileDEM_nodata = raster_io.read_raster_one_band_np(save_dem_mosaic) # masking the strip version of DEMs mask_strip_dem_list = [] for idx, strip_dem in enumerate(crop_strip_dem_list): save_path = io_function.get_name_by_adding_tail(strip_dem, 'maskOutlier') if os.path.isfile(save_path): basic.outputlogMessage('%s exist, skip'%save_path) mask_strip_dem_list.append(save_path) continue # check band, with, height height, width, count, dtype = raster_io.get_height_width_bandnum_dtype(strip_dem) if height_tileDEM != height or width_tileDEM != width or count_tileDEM != count: raise ValueError('size different between %s and %s' % (strip_dem, save_dem_mosaic)) if count != 1: raise ValueError('DEM and Matchtag should only have one band') try: dem_data, nodata = raster_io.read_raster_one_band_np(strip_dem) except: basic.outputlogMessage(' invalid tif file: %s'%strip_dem) continue nodata_loc = np.where(dem_data == nodata) diff = dem_data - tileDEM_data # mask as nodata dem_data[np.abs(diff) > 50 ] = nodata # ignore greater than 50 m dem_data[ nodata_loc ] = nodata # may change some nodata pixel, change them back # save to file raster_io.save_numpy_array_to_rasterfile(dem_data, save_path, strip_dem, compress='lzw', tiled='yes', bigtiff='if_safer') mask_strip_dem_list.append(save_path) return mask_strip_dem_list
def segment_a_patch(idx, patch, patch_count, img_path, org_raster, b_save_patch_label): print('tile: %d / %d' % (idx + 1, patch_count)) image_name_no_ext = io_function.get_name_no_ext(img_path) patch_dir = image_name_no_ext + '_patch%d' % idx patch_label_path = os.path.join( patch_dir, image_name_no_ext + '_patch%d_label.tif' % idx) if b_save_patch_label is True: if os.path.isdir(patch_dir) is False: io_function.mkdir(patch_dir) if os.path.isfile(patch_label_path): print('%s exists, skip' % patch_label_path) return patch, patch_label_path, None, None # read imag one_band_img, nodata = raster_io.read_raster_one_band_np(img_path, boundary=patch) # # apply median filter to image (remove some noise) one_band_img = cv2.medianBlur(one_band_img, 3) # with kernal=3, cannot accept int32 # segmentation algorithm (the output of these algorithms is not alway good, need to chose the parameters carafully) # out_labels = watershed_segmentation(one_band_img) # out_labels = k_mean_cluster_segmentation(one_band_img) out_labels = quickshift_segmentaion(one_band_img, ratio=0.3, kernel_size=5, max_dist=10, sigma=1, convert2lab=False) # # # out_labels = mean_shift_segmentation(one_band_img) # print('min and max labels of out_labels', np.min(out_labels), np.max(out_labels)) if b_save_patch_label is True: # save the label raster_io.save_numpy_array_to_rasterfile( out_labels, patch_label_path, img_path, boundary=patch) # it copy nodata, need to unset it later return patch, patch_label_path, None, None # calculate the attributes based on orginal data for original data object_attributes = {} # object id (label) and attributes (list) if org_raster is not None: org_img_b1, org_nodata = raster_io.read_raster_one_band_np( org_raster, boundary=patch) # get regions (the labels output by segmentation is not unique for superpixels) # regions = measure.regionprops(out_labels, intensity_image=org_img_b1) # regions is based on out_labels, so it has the same issue. # print('region count from sk-image measure:',len(regions)) label_list = np.unique(out_labels) # get statistics for each segmented object (label) for label in label_list: in_array = org_img_b1[out_labels == label] object_attributes[label] = get_stastics_from_array( in_array, org_nodata) return patch, out_labels, nodata, object_attributes return patch, out_labels, nodata, None
def segment_a_grey_image(img_path, save_dir, process_num, org_raster=None, b_save_patch_label=False): out_pre = os.path.splitext(os.path.basename(img_path))[0] label_path = os.path.join(save_dir, out_pre + '_label.tif') if os.path.isfile(label_path): basic.outputlogMessage('%s exist, skip segmentation' % label_path) return label_path height, width, band_num, date_type = raster_io.get_height_width_bandnum_dtype( img_path) print('input image: height, width, band_num, date_type', height, width, band_num, date_type) # if the original data is available, then calculate the attributes based on that if org_raster is not None: org_height, org_width, org_band_num, org_date_type = raster_io.get_height_width_bandnum_dtype( org_raster) if org_height != height or org_width != width: raise ValueError('%s and %s do not have the same size' % (img_path, org_raster)) save_labes = np.zeros((height, width), dtype=np.int32) # divide the image the many small patches, then calcuate one by one, solving memory issues. image_patches = split_image.sliding_window(width, height, 1024, 1024, adj_overlay_x=0, adj_overlay_y=0) patch_count = len(image_patches) # for idx, patch in enumerate(image_patches): # out_patch,out_labels = segment_a_patch(idx, patch, patch_count,img_path) # # copy to the entire image # row_s = patch[1] # row_e = patch[1] + patch[3] # col_s = patch[0] # col_e = patch[0] + patch[2] # save_labes[row_s:row_e, col_s:col_e] = out_labels theadPool = Pool(process_num) parameters_list = [(idx, patch, patch_count, img_path, org_raster, b_save_patch_label) for idx, patch in enumerate(image_patches)] results = theadPool.starmap(segment_a_patch, parameters_list) patch_label_path_list = [] patch_label_id_range = [] object_attributes = {} # object id (label) and attributes (list) for res in results: patch, out_labels, nodata, attributes = res if isinstance( out_labels, str) and os.path.isfile(out_labels): #if it's a label file patch_label_path_list.append(out_labels) else: # copy to the entire image row_s = patch[1] row_e = patch[1] + patch[3] col_s = patch[0] col_e = patch[0] + patch[2] current_min = np.max(save_labes) print('current_max', current_min) patch_label_id_range.append(current_min) save_labes[row_s:row_e, col_s:col_e] = out_labels + current_min + 1 if attributes is not None: update_label_attr = {} for key in attributes: update_label_attr[key + current_min] = attributes[key] # add to the attributes object_attributes.update(update_label_attr) # # apply median filter (remove some noise), # we should not use median filter, because it's labels, not images. # label_blurs = cv2.medianBlur(np.float32(save_labes), 3) # with kernal=3, cannot accept int32 # # print(label_blurs, label_blurs.dtype) # save_labes = label_blurs.astype(np.int32) # return a list of labels saved in current working folder. if b_save_patch_label: return patch_label_path_list if os.path.isdir(save_dir) is False: io_function.mkdir(save_dir) # save attributes (if not empty) if object_attributes: attribute_path = os.path.join(save_dir, out_pre + '_attributes.txt') io_function.save_dict_to_txt_json(attribute_path, object_attributes) # save the label raster_io.save_numpy_array_to_rasterfile(save_labes, label_path, img_path) # do not set nodata # save id ranges to txt label_id_range_txt = os.path.splitext(label_path)[0] + '_IDrange.txt' patch_label_id_range = [str(item) for item in patch_label_id_range] io_function.save_list_to_txt(label_id_range_txt, patch_label_id_range) return label_path
def dem_diff_newest_oldest(dem_tif_list, out_dem_diff, out_date_diff, process_num, b_max_subsidence=False, b_save_cm=False): ''' get DEM difference, for each pixel, newest vaild value - oldest valid value :param dem_list: :param output: :return: ''' if len(dem_tif_list) < 2: basic.outputlogMessage('error, the count of DEM is smaller than 2') return False # groups DEM with original images acquired at the same year months dem_groups_date = group_demTif_yearmonthDay(dem_tif_list, diff_days=0) # sort based on yeardate in accending order : operator.itemgetter(0) dem_groups_date = dict( sorted(dem_groups_date.items(), key=operator.itemgetter(0))) txt_save_path = os.path.splitext(out_date_diff)[0] + '.txt' # change the key to integer number after sorting and save to txt file dem_groups_date_sort_idx = {} for idx, key in enumerate(dem_groups_date.keys()): dem_groups_date_sort_idx[idx] = dem_groups_date[key] io_function.save_dict_to_txt_json(txt_save_path, dem_groups_date_sort_idx) date_list = list(dem_groups_date.keys()) dem_tif_list = [dem_groups_date[key][0] for key in dem_groups_date.keys() ] # each date, only have one tif tif_obj_list = [raster_io.open_raster_read(tif) for tif in dem_tif_list] height, width, _ = raster_io.get_width_heigth_bandnum(tif_obj_list[0]) # check them have the width and height for tif, obj in zip(dem_tif_list[1:], tif_obj_list[1:]): h, w, _ = raster_io.get_width_heigth_bandnum(obj) if h != height or w != width: raise ValueError( 'the height and width of %s is different from others' % tif) # divide the image the many small patches, then calcuate one by one, solving memory issues. image_patches = split_image.sliding_window(width, height, 1024, 1024, adj_overlay_x=0, adj_overlay_y=0) patch_count = len(image_patches) tif_obj_list = None # read all and their date date_pair_list = list(combinations(date_list, 2)) date_diff_list = [(item[1] - item[0]).days for item in date_pair_list] # sort based on day difference (from max to min) date_pair_list_sorted = [ x for _, x in sorted(zip(date_diff_list, date_pair_list), reverse=True) ] # descending # get the difference date_diff_np = np.zeros((height, width), dtype=np.uint16) old_date_index = np.zeros((height, width), dtype=np.uint8) new_date_index = np.zeros((height, width), dtype=np.uint8) dem_diff_np = np.empty((height, width), dtype=np.float32) dem_diff_np[:] = np.nan if process_num == 1: for idx, patch in enumerate(image_patches): _,patch_dem_diff,patch_date_diff, patch_old_date_idx,patch_new_date_idx = \ dem_diff_newest_oldest_a_patch(idx, patch, patch_count,date_pair_list_sorted,dem_groups_date) # copy to the entire image row_s = patch[1] row_e = patch[1] + patch[3] col_s = patch[0] col_e = patch[0] + patch[2] dem_diff_np[row_s:row_e, col_s:col_e] = patch_dem_diff date_diff_np[row_s:row_e, col_s:col_e] = patch_date_diff old_date_index[row_s:row_e, col_s:col_e] = patch_old_date_idx new_date_index[row_s:row_e, col_s:col_e] = patch_new_date_idx else: theadPool = Pool(process_num) parameters_list = [(idx, patch, patch_count, date_pair_list_sorted, dem_groups_date) for idx, patch in enumerate(image_patches)] if b_max_subsidence is False: results = theadPool.starmap(dem_diff_newest_oldest_a_patch, parameters_list) else: results = theadPool.starmap(dem_diff_new_old_min_neg_diff_patch, parameters_list) for res in results: patch, patch_dem_diff, patch_date_diff, patch_old_date_idx, patch_new_date_idx = res # copy to the entire image row_s = patch[1] row_e = patch[1] + patch[3] col_s = patch[0] col_e = patch[0] + patch[2] dem_diff_np[row_s:row_e, col_s:col_e] = patch_dem_diff date_diff_np[row_s:row_e, col_s:col_e] = patch_date_diff old_date_index[row_s:row_e, col_s:col_e] = patch_old_date_idx new_date_index[row_s:row_e, col_s:col_e] = patch_new_date_idx theadPool.close() # save date diff to tif (16 bit) raster_io.save_numpy_array_to_rasterfile(date_diff_np, out_date_diff, dem_tif_list[0], nodata=0, compress='lzw', tiled='yes', bigtiff='if_safer') # save old and new date index to tif (8 bit) out_old_date_idx = io_function.get_name_by_adding_tail( out_date_diff, 'oldIndex') out_new_date_idx = io_function.get_name_by_adding_tail( out_date_diff, 'newIndex') raster_io.save_numpy_array_to_rasterfile(old_date_index, out_old_date_idx, dem_tif_list[0], nodata=255, compress='lzw', tiled='yes', bigtiff='if_safer') raster_io.save_numpy_array_to_rasterfile(new_date_index, out_new_date_idx, dem_tif_list[0], nodata=255, compress='lzw', tiled='yes', bigtiff='if_safer') # # stretch the DEM difference, save to 8 bit. # dem_diff_np_8bit = raster_io.image_numpy_to_8bit(dem_diff_np,10,-10,dst_nodata=0) # out_dem_diff_8bit = io_function.get_name_by_adding_tail(out_dem_diff, '8bit') # raster_io.save_numpy_array_to_rasterfile(dem_diff_np_8bit, out_dem_diff_8bit, dem_tif_list[0], nodata=0) # if possible, save to 16 bit, to save the disk storage. # dem_diff_np[0:5,0] = -500 # dem_diff_np[0,0:5] = 500 # print(np.nanmin(dem_diff_np)) # print(np.nanmax(dem_diff_np)) # if np.nanmin(dem_diff_np_cm) < range.min or np.nanmax(dem_diff_np_cm) > range.max: # save dem diff to files (float), meter if b_save_cm is False: raster_io.save_numpy_array_to_rasterfile(dem_diff_np, out_dem_diff, dem_tif_list[0], nodata=-9999, compress='lzw', tiled='yes', bigtiff='if_safer') else: # save dem diff to 16bit, centimeter, only handle diff from -327.67 to 327.67 meters bit16_nodata = 32767 range = np.iinfo(np.int16) dem_diff_np_cm = dem_diff_np * 100 dem_diff_np_cm[dem_diff_np_cm < range.min] = range.min dem_diff_np_cm[dem_diff_np_cm > range.max] = range.max dem_diff_np_cm[np.isnan( dem_diff_np_cm)] = bit16_nodata # set the nodata for int16 dem_diff_np_cm = dem_diff_np_cm.astype(np.int16) # save to int16 out_dem_diff_cm = out_dem_diff basic.outputlogMessage( 'note, save DEM difference (%s) to centimeter, int16, range: -327.68 to 327.67 m' % os.path.basename(out_dem_diff_cm)) raster_io.save_numpy_array_to_rasterfile(dem_diff_np_cm, out_dem_diff_cm, dem_tif_list[0], nodata=bit16_nodata, compress='lzw', tiled='yes', bigtiff='if_safer') return True
def dem_diff_newest_oldest(dem_tif_list, out_dem_diff, out_date_diff): ''' get DEM difference, for each pixel, newest vaild value - oldest valid value :param dem_list: :param output: :return: ''' if len(dem_tif_list) < 2: basic.outputlogMessage('error, the count of DEM is smaller than 2') return False # groups DEM with original images acquired at the same year months dem_groups_date = group_demTif_yearmonthDay(dem_tif_list, diff_days=0) # sort based on yeardate in accending order : operator.itemgetter(0) dem_groups_date = dict( sorted(dem_groups_date.items(), key=operator.itemgetter(0))) txt_save_path = os.path.splitext(out_date_diff)[0] + '.txt' io_function.save_dict_to_txt_json(txt_save_path, dem_groups_date) date_list = list(dem_groups_date.keys()) dem_tif_list = [dem_groups_date[key][0] for key in dem_groups_date.keys() ] # each date, only have one tif tif_obj_list = [raster_io.open_raster_read(tif) for tif in dem_tif_list] height, width, _ = raster_io.get_width_heigth_bandnum(tif_obj_list[0]) # check them have the width and height for tif, obj in zip(dem_tif_list[1:], tif_obj_list[1:]): h, w, _ = raster_io.get_width_heigth_bandnum(obj) if h != height or w != width: raise ValueError( 'the height and width of %s is different from others' % tif) # divide the image the many small patches, then calcuate one by one, solving memory issues. image_patches = split_image.sliding_window(width, height, 1024, 1024, adj_overlay_x=0, adj_overlay_y=0) patch_count = len(image_patches) tif_obj_list = None # read all and their date date_pair_list = list(combinations(date_list, 2)) date_diff_list = [(item[1] - item[0]).days for item in date_pair_list] # sort based on day difference (from max to min) date_pair_list_sorted = [ x for _, x in sorted(zip(date_diff_list, date_pair_list), reverse=True) ] # descending # get the difference date_diff_np = np.zeros((height, width), dtype=np.uint16) dem_diff_np = np.empty((height, width), dtype=np.float32) dem_diff_np[:] = np.nan for idx, patch in enumerate(image_patches): print('tile: %d / %d' % (idx + 1, patch_count)) patch_w = patch[2] patch_h = patch[3] patch_date_diff = np.zeros((patch_h, patch_w), dtype=np.uint16) patch_dem_diff = np.empty((patch_h, patch_w), dtype=np.float32) patch_dem_diff[:] = np.nan # use dict to read data from disk (only need) dem_data_dict = {} for p_idx, pair in enumerate(date_pair_list_sorted): diff_days = (pair[1] - pair[0]).days basic.outputlogMessage( 'Getting DEM difference using the one on %s and %s, total day diff: %d' % (timeTools.date2str(pair[1]), timeTools.date2str( pair[0]), diff_days)) # print(pair,':',(pair[1] - pair[0]).days) data_old, data_new = read_date_dem_to_memory(p_idx, pair, date_pair_list_sorted, dem_data_dict, dem_groups_date, boundary=patch) # print('data_old shape:',data_old.shape) # print('data_new shape:',data_new.shape) diff_two = data_new - data_old # print(diff_two) # fill the element new_ele = np.where( np.logical_and(np.isnan(patch_dem_diff), ~np.isnan(diff_two))) patch_dem_diff[new_ele] = diff_two[new_ele] patch_date_diff[new_ele] = diff_days # check if all have been filled ( nan pixels) diff_remain_hole = np.where(np.isnan(patch_dem_diff)) # basic.outputlogMessage(' remain %.4f percent pixels need to be filled'% (100.0*diff_remain_hole[0].size/patch_dem_diff.size) ) if diff_remain_hole[0].size < 1: break # copy to the entire image row_s = patch[1] row_e = patch[1] + patch[3] col_s = patch[0] col_e = patch[0] + patch[2] dem_diff_np[row_s:row_e, col_s:col_e] = patch_dem_diff date_diff_np[row_s:row_e, col_s:col_e] = patch_date_diff # save date diff to tif (16 bit) raster_io.save_numpy_array_to_rasterfile(date_diff_np, out_date_diff, dem_tif_list[0], nodata=0, compress='lzw', tiled='yes', bigtiff='if_safer') # # stretch the DEM difference, save to 8 bit. # dem_diff_np_8bit = raster_io.image_numpy_to_8bit(dem_diff_np,10,-10,dst_nodata=0) # out_dem_diff_8bit = io_function.get_name_by_adding_tail(out_dem_diff, '8bit') # raster_io.save_numpy_array_to_rasterfile(dem_diff_np_8bit, out_dem_diff_8bit, dem_tif_list[0], nodata=0) # if possible, save to 16 bit, to save the disk storage. # dem_diff_np[0:5,0] = -500 # dem_diff_np[0,0:5] = 500 # print(np.nanmin(dem_diff_np)) # print(np.nanmax(dem_diff_np)) range = np.iinfo(np.int16) # dem_diff_np_cm = dem_diff_np*100 # if np.nanmin(dem_diff_np_cm) < range.min or np.nanmax(dem_diff_np_cm) > range.max: # save dem diff to files (float), meter raster_io.save_numpy_array_to_rasterfile(dem_diff_np, out_dem_diff, dem_tif_list[0], nodata=-9999, compress='lzw', tiled='yes', bigtiff='if_safer') # else: # # save dem diff to 16bit, centimeter, only handle diff from -327.67 to 327.67 meters # dem_diff_np_cm = dem_diff_np_cm.astype(np.int16) # save to int16 # raster_io.save_numpy_array_to_rasterfile(dem_diff_np_cm, out_dem_diff_cm, dem_tif_list[0],nodata=32767,compress='lzw',tiled='yes',bigtiff='if_safer') return True