def classify_image(in_image_path, model_path, out_image_path, num_chunks=10, rescale_predict_image=None, ref_img_for_linear_shift=None, generate_mask=True): print("Using model: " + model_path) model = joblib.load(model_path) if generate_mask: image, image_array = general_functions.read_tif(in_image_path) print("Generating extent mask for mosaicing: ") mask_array = image_array[6, :, :] mask_array[mask_array != 0] = 1 general_functions.create_tif(filename=in_image_path[:-4] + '_mask.msk', g=image, Nx=mask_array.shape[0], Ny=mask_array.shape[1], new_array=mask_array, data_type=gdal.GDT_UInt16, noData=0) else: image = gdal.Open(in_image_path) image_array = image.GetVirtualMemArray() features_to_classify = reshape_raster_for_ml(image_array) width = image.RasterXSize height = image.RasterYSize if ref_img_for_linear_shift is not None: diff_array = cal_linear_shift_array(ref_tif=ref_img_for_linear_shift, tobe_shift_tif=in_image_path) features_to_classify_filtered = features_to_classify + diff_array if rescale_predict_image is not None: features_to_classify_norm = rescale_predict_image.transform( features_to_classify_filtered) else: features_to_classify_norm = features_to_classify_filtered print("Classifying image") out_chunks = [] for i, chunk in enumerate( np.array_split(features_to_classify_norm, num_chunks)): print("Classifying {0}".format(i)) chunk_copy = np.copy(chunk) chunk_copy = np.where(np.isfinite(chunk_copy), chunk_copy, 0) # this is a slower line out_chunks.append(model.predict(chunk_copy)) out_classes = np.concatenate(out_chunks) image = gdal.Open(in_image_path) out_image = create_matching_dataset(image, out_image_path) image_array = None image = None out_image_array = out_image.GetVirtualMemArray(eAccess=gdal.GA_Update) out_image_array[...] = reshape_ml_out_to_raster(out_classes, width, height) out_image_array = None out_image = None
def generate_20m_6bands(in_20m_tif): g, arr = general_functions.read_tif(in_20m_tif) out_arr = arr[3:, :, :] filename = in_20m_tif[:-4] + '_6bands.tif' general_functions.create_tif(filename=filename, g=g, Nx=arr.shape[1], Ny=arr.shape[2], new_array=out_arr, data_type=gdal.GDT_Int16, noData=0)
def generate_veg_index_tif(tif_10m, tif_20m, out_tif): g_10m, arr_10m = general_functions.read_tif(intif=tif_10m, type=np.int32) g_20m, arr_20m = general_functions.read_tif(intif=tif_20m, type=np.int32) veg_arr = cal_vegIndex(arr_10m, arr_20m) veg_array_trans = np.transpose(veg_arr, (2, 0, 1)) general_functions.create_tif(filename=out_tif, g=g_10m, Nx=arr_20m.shape[1], Ny=arr_20m.shape[2], new_array=veg_array_trans, noData=0, data_type=gdal.GDT_Int32)
def generate_seg_rst(): ''' :param s0:array read from s2 20m 9 bands data .tif or 10m 4 bands data.tif for s2 20m, the band sequence are: band 2, 3, 4, 5, 6, 7, 8a, 11, 12 for s2 10m, the band sequence are: band 2, 3, 4, 8 :return: ''' merge_10m_dir = "/media/ubuntu/Data/Ghana/north_region/s2/composites/10m/" merge_20m_dir = "/media/ubuntu/Data/Ghana/north_region/s2/composites/20m/" tif_10m_list = s2_functions.search_files_fulldir(input_path=merge_10m_dir, search_type='end', search_key='NWM.tif') tif_20m_list = s2_functions.search_files_fulldir(input_path=merge_20m_dir, search_type='end', search_key='NWM.tif') for n in range(len(tif_10m_list)): tif_10m = tif_10m_list[n] tif_20m = tif_20m_list[n] print(tif_10m) print(tif_20m) out_tif = tif_10m[:-4] + "_NIR_SWIR_red.tif" g_10m, a_10m = general_functions.read_tif(tif_10m) a_10m_trans = np.transpose(a_10m, (1, 2, 0)) g_20m, a_20m = general_functions.read_tif(tif_20m) a_20m_trans = np.transpose(a_20m, (1, 2, 0)) a_3band = np.zeros((a_10m_trans.shape[0], a_10m_trans.shape[1], 3), dtype=int) # # # version 3: NIR, SWIR, and red a_3band[:, :, 0] = a_10m_trans[:, :, 3] a_3band[:, :, 1] = a_20m_trans[:, :, 7] a_3band[:, :, 2] = a_10m_trans[:, :, 2] a_3band_trans = np.transpose(a_3band, (2, 0, 1)) a_3band_out = tif_10m[:-4] + '_NIR_SWIR_red.tif' general_functions.create_tif(filename=a_3band_out, g=g_20m, Nx=a_20m.shape[1], Ny=a_20m.shape[2], new_array=a_3band_trans, noData=0, data_type=gdal.GDT_UInt16)
def cal_seg_mean(in_value_ras, seg_ras, out_value_ras, output_filtered_value_ras=True): g_seg, seg_arr = general_functions.read_tif(seg_ras, type=np.int64) g_value, value_arr = general_functions.read_tif(in_value_ras, type=np.int64) out_array = np.zeros(value_arr.shape) unique_segvals = np.unique(seg_arr) band_number = value_arr.shape[0] for n in range(band_number): print('working on band: ' + str(n + 1)) band_val = value_arr[n, :, :] mean_arr = scipy.ndimage.measurements.mean(band_val, labels=seg_arr, index=unique_segvals) # new_arr = np.zeros(band_val.shape) lookup_array = np.zeros(unique_segvals.max() + 1) lookup_array[unique_segvals] = mean_arr new_arr = lookup_array[seg_arr] out_array[n, :, :] = new_arr if output_filtered_value_ras: general_functions.create_tif(filename=out_value_ras, g=g_seg, Nx=seg_arr.shape[0], Ny=seg_arr.shape[1], new_array=out_array, data_type=gdal.GDT_UInt32, noData=0) else: print('Caculating brightness layer...') brightness = np.mean(out_array, axis=0) print(brightness.shape) general_functions.create_tif(filename=out_value_ras, g=g_seg, Nx=seg_arr.shape[0], Ny=seg_arr.shape[1], new_array=brightness, data_type=gdal.GDT_UInt32, noData=0)
def scale_s1_tif(in_tif, out_tif, scale=10000): g, array = read_tif(in_tif) array[array < 0] = 0 array_int = array * scale if array_int.ndim == 2: general_functions.create_tif(filename=out_tif, g=g, Nx=array.shape[0], Ny=array.shape[1], new_array=array_int, data_type=gdal.GDT_UInt32, noData=0) elif array_int.ndim == 3: general_functions.create_tif(filename=out_tif, g=g, Nx=array.shape[1], Ny=array.shape[2], new_array=array_int, data_type=gdal.GDT_UInt32, noData=0) else: print('wrong')
def build_cocoa_map(working_dir, path_to_aoi, start_date, end_date, path_to_s1_image, path_to_config, epsg_for_map, path_to_model, cloud_cover=20, log_path="build_cocoa_map.log", use_sen2cor=False, sen2cor_path=None, skip_download_and_preprocess=False, skip_composite=False): # Step 0: Get things ready. Folder structure for downloads, load credentials from config. fu.init_log(log_path) os.chdir(working_dir) # fu.create_file_structure(os.getcwd()) # # make_directory("images/merged/10m") # make_directory("images/merged/20m") # # make_directory("images/stacked/with_indices") # make_directory("images/stacked/with_s1_seg") # make_directory("images/stacked/all_19bands") # # make_directory("segmentation") # make_directory("composites") # make_directory("composites/10m") # make_directory("composites/20m") # # make_directory("composites/10m_full") # make_directory("composites/20m_full") general_functions.make_all_dirs(working_dir) # Step 1: Download S2 3imagery for the timescale if not skip_download_and_preprocess: config = configparser.ConfigParser() config.read(path_to_config) images_to_download = query.check_for_s2_data_by_date( path_to_aoi, start_date, end_date, config, cloud_cover) if not use_sen2cor: images_to_download = query.filter_non_matching_s2_data( images_to_download) else: images_to_download = query.filter_to_l1_data(images_to_download) query.download_s2_data(images_to_download, "images/L1", "images/L2") # Step 2: Preprocess S2 imagery. Perform atmospheric correction if needed, stack and mask 10 and 20m bands. if use_sen2cor: ras.atmospheric_correction("images/L1" "images/L2", sen2cor_path=sen2cor_path) ras.preprocess_sen2_images("images/L2", "images/merged/10m", "images/L1", cloud_threshold=0, epsg=epsg_for_map, bands=("B02", "B03", "B04", "B08"), out_resolution=10) ras.preprocess_sen2_images("images/L2", "images/merged/20m", "images/L1", cloud_threshold=0, epsg=epsg_for_map, bands=("B02", "B03", "B04", "B05", "B06", "B07", "B8A", "B11", "B12"), out_resolution=20) if not skip_composite: # Step 2.5: Build a pair of cloud-free composites sort_into_tile("images/merged/10m") sort_into_tile("images/merged/20m") for tile in os.listdir("images/merged/10m"): tile_path = os.path.join("images/merged/10m", tile) this_composite_path = ras.composite_directory( tile_path, "composites/10m") new_composite_path = "{}_{}.tif".format( this_composite_path.rsplit('.')[0], tile) os.rename(this_composite_path, new_composite_path) for tile in os.listdir("images/merged/20m"): tile_path = os.path.join("images/merged/20m", tile) this_composite_path = ras.composite_directory( tile_path, "composites/20m") new_composite_path = "{}_{}.tif".format( this_composite_path.rsplit('.')[0], tile) os.rename(this_composite_path, new_composite_path) # Step 3: Generate the bands. Time for the New Bit. clip_to_aoi = False if clip_to_aoi: for image in os.listdir("composites/10m_full"): if image.endswith(".tif"): image_path_10m_full = os.path.join("composites/10m_full", image) image_path_20m_full = os.path.join("composites/20m_full", image) image_path_10m_clipped = os.path.join("composites/10m", image) image_path_20m_clipped = os.path.join("composites/20m", image) # config = configparser.ConfigParser() # conf = config.read(path_to_config) # print(conf) # aoi = config['cocoa_mapping']['path_to_aoi'] aoi = "/media/ubuntu/Data/Ghana/cocoa_big/shp/cocoa_big.shp" ras.clip_raster(raster_path=image_path_10m_full, aoi_path=aoi, out_path=image_path_10m_clipped, srs_id=32630) ras.clip_raster(raster_path=image_path_20m_full, aoi_path=aoi, out_path=image_path_20m_clipped, srs_id=32630) do_segmentation = True if do_segmentation == True: for image in os.listdir("composites/10m"): if image.endswith(".tif"): with TemporaryDirectory() as td: image_path_10m = os.path.join("composites/10m", image) image_path_20m = os.path.join("composites/20m", image) resample_path_20m_v1 = os.path.join( td, image) # You'll probably regret this later, roberts. shutil.copy(image_path_20m, resample_path_20m_v1) ras.resample_image_in_place(resample_path_20m_v1, 10) index_image_path = os.path.join(td, "index_image.tif") temp_pre_seg_path = os.path.join(td, "pre_seg.tif") temp_seg_path = os.path.join(td, "seg.tif") temp_shp_path = os.path.join(td, "outline.shp") temp_clipped_seg_path = os.path.join(td, "seg_clip.tif") # This bit's your show, Qing temp_s1_outline_path = os.path.join(td, "s1_outline.shp") ras.get_extent_as_shp(in_ras_path=image_path_10m, out_shp_path=temp_s1_outline_path) resample_path_20m = os.path.join( td, image[:-4] + '_to_10moutline.tif') general_functions.clip_rst( in_tif=resample_path_20m_v1, outline_shp=temp_s1_outline_path, out_tif=resample_path_20m, keep_rst_extent=False) generate_veg_index_tif(image_path_10m, resample_path_20m, index_image_path) ras.stack_images([index_image_path, image_path_10m], "images/stacked/with_indices/" + image) # Now, we do Highly Experimental Image Segmentation. Please put on your goggles. # SAGA, please. # Meatball science time vis_10m = gdal.Open(image_path_10m) vis_20m_resampled = gdal.Open(resample_path_20m) vis_10m_array = vis_10m.GetVirtualMemArray() vis_20m_array = vis_20m_resampled.GetVirtualMemArray() # NIR, SWIR, red array_to_classify = np.stack([ vis_10m_array[3, ...], vis_20m_array[7, ...], vis_10m_array[2, ...] ]) g, arr = general_functions.read_tif(intif=image_path_10m) general_functions.create_tif(filename=temp_pre_seg_path, g=g, Nx=arr.shape[1], Ny=arr.shape[2], new_array=array_to_classify, noData=0, data_type=gdal.GDT_UInt32) out_segment_tif = os.path.join("segmentation", image) segment_image(temp_pre_seg_path, out_segment_tif) print('Generate brighness raster from the segments') make_directory("segmentation/brightness") out_brightness_value_ras = os.path.join( "segmentation/brightness", image) output_filtered_value_ras = False ras.get_extent_as_shp(in_ras_path=temp_pre_seg_path, out_shp_path=temp_shp_path) general_functions.clip_rst(in_tif=out_segment_tif, outline_shp=temp_shp_path, out_tif=temp_clipped_seg_path, keep_rst_extent=False) cal_seg_mean( temp_pre_seg_path, temp_clipped_seg_path, out_brightness_value_ras, output_filtered_value_ras=output_filtered_value_ras) # image_20m_6bands_array = vis_20m_array[3:,:,:] # try: # os.mkdir("composites/20m/20m_6bands") # except FileExistsError: # pass # # out_20m_tif_for_stack = os.path.join("composites/20m/20m_6bands", image) # general_functions.create_tif(filename=out_20m_tif_for_stack,g=g,Nx=arr.shape[1],Ny=arr.shape[2], # new_array=image_20m_6bands_array,data_type=gdal.GDT_UInt16,noData=0) do_stack = True if do_stack == True: # Step 4: Stack the new bands with the S1, seg, and 6 band 20m rasters for image in os.listdir("images/stacked/with_indices"): if image.endswith(".tif"): path_to_image = os.path.join("images/stacked/with_indices", image) path_to_brightness_image = os.path.join( "segmentation/brightness", image) # path_to_20m_image = os.path.join("composites/20m/20m_6bands", image) # ras.stack_images([path_to_image, path_to_s1_image,path_to_20m_image,out_brightness_value_ras], os.path.join("images/stacked/all_19bands", image)) ras.stack_images([ path_to_image, path_to_s1_image, path_to_brightness_image ], os.path.join("images/stacked/with_s1_seg", image)) #sys.exit() # # Step 5: Classify with trained model for image in os.listdir("images/stacked/with_s1_seg"): if image.endswith(".tif"): path_to_image = os.path.join("images/stacked/with_s1_seg", image) path_to_out = os.path.join("output", image) PYEO_model.classify_image(path_to_image, path_to_model, path_to_out)