예제 #1
0
def classify_image(in_image_path,
                   model_path,
                   out_image_path,
                   num_chunks=10,
                   rescale_predict_image=None,
                   ref_img_for_linear_shift=None,
                   generate_mask=True):
    print("Using model: " + model_path)
    model = joblib.load(model_path)
    if generate_mask:
        image, image_array = general_functions.read_tif(in_image_path)
        print("Generating extent mask for mosaicing: ")
        mask_array = image_array[6, :, :]
        mask_array[mask_array != 0] = 1
        general_functions.create_tif(filename=in_image_path[:-4] + '_mask.msk',
                                     g=image,
                                     Nx=mask_array.shape[0],
                                     Ny=mask_array.shape[1],
                                     new_array=mask_array,
                                     data_type=gdal.GDT_UInt16,
                                     noData=0)
    else:
        image = gdal.Open(in_image_path)
        image_array = image.GetVirtualMemArray()

    features_to_classify = reshape_raster_for_ml(image_array)
    width = image.RasterXSize
    height = image.RasterYSize

    if ref_img_for_linear_shift is not None:
        diff_array = cal_linear_shift_array(ref_tif=ref_img_for_linear_shift,
                                            tobe_shift_tif=in_image_path)
        features_to_classify_filtered = features_to_classify + diff_array
    if rescale_predict_image is not None:
        features_to_classify_norm = rescale_predict_image.transform(
            features_to_classify_filtered)
    else:
        features_to_classify_norm = features_to_classify_filtered

    print("Classifying image")
    out_chunks = []
    for i, chunk in enumerate(
            np.array_split(features_to_classify_norm, num_chunks)):
        print("Classifying {0}".format(i))
        chunk_copy = np.copy(chunk)
        chunk_copy = np.where(np.isfinite(chunk_copy), chunk_copy,
                              0)  # this is a slower line
        out_chunks.append(model.predict(chunk_copy))
    out_classes = np.concatenate(out_chunks)

    image = gdal.Open(in_image_path)
    out_image = create_matching_dataset(image, out_image_path)
    image_array = None
    image = None

    out_image_array = out_image.GetVirtualMemArray(eAccess=gdal.GA_Update)
    out_image_array[...] = reshape_ml_out_to_raster(out_classes, width, height)
    out_image_array = None
    out_image = None
예제 #2
0
def generate_20m_6bands(in_20m_tif):
    g, arr = general_functions.read_tif(in_20m_tif)
    out_arr = arr[3:, :, :]
    filename = in_20m_tif[:-4] + '_6bands.tif'
    general_functions.create_tif(filename=filename,
                                 g=g,
                                 Nx=arr.shape[1],
                                 Ny=arr.shape[2],
                                 new_array=out_arr,
                                 data_type=gdal.GDT_Int16,
                                 noData=0)
예제 #3
0
def generate_veg_index_tif(tif_10m, tif_20m, out_tif):
    g_10m, arr_10m = general_functions.read_tif(intif=tif_10m, type=np.int32)
    g_20m, arr_20m = general_functions.read_tif(intif=tif_20m, type=np.int32)
    veg_arr = cal_vegIndex(arr_10m, arr_20m)

    veg_array_trans = np.transpose(veg_arr, (2, 0, 1))

    general_functions.create_tif(filename=out_tif,
                                 g=g_10m,
                                 Nx=arr_20m.shape[1],
                                 Ny=arr_20m.shape[2],
                                 new_array=veg_array_trans,
                                 noData=0,
                                 data_type=gdal.GDT_Int32)
예제 #4
0
def generate_seg_rst():
    '''
    :param s0:array read from s2 20m 9 bands data .tif or 10m 4 bands data.tif
    for s2 20m, the band sequence are: band 2, 3, 4, 5, 6, 7, 8a, 11, 12
    for s2 10m, the band sequence are: band 2, 3, 4, 8
    :return:
    '''
    merge_10m_dir = "/media/ubuntu/Data/Ghana/north_region/s2/composites/10m/"
    merge_20m_dir = "/media/ubuntu/Data/Ghana/north_region/s2/composites/20m/"
    tif_10m_list = s2_functions.search_files_fulldir(input_path=merge_10m_dir,
                                                     search_type='end',
                                                     search_key='NWM.tif')
    tif_20m_list = s2_functions.search_files_fulldir(input_path=merge_20m_dir,
                                                     search_type='end',
                                                     search_key='NWM.tif')

    for n in range(len(tif_10m_list)):
        tif_10m = tif_10m_list[n]
        tif_20m = tif_20m_list[n]
        print(tif_10m)
        print(tif_20m)
        out_tif = tif_10m[:-4] + "_NIR_SWIR_red.tif"

        g_10m, a_10m = general_functions.read_tif(tif_10m)
        a_10m_trans = np.transpose(a_10m, (1, 2, 0))

        g_20m, a_20m = general_functions.read_tif(tif_20m)
        a_20m_trans = np.transpose(a_20m, (1, 2, 0))

        a_3band = np.zeros((a_10m_trans.shape[0], a_10m_trans.shape[1], 3),
                           dtype=int)

        # # # version 3: NIR, SWIR, and red
        a_3band[:, :, 0] = a_10m_trans[:, :, 3]
        a_3band[:, :, 1] = a_20m_trans[:, :, 7]
        a_3band[:, :, 2] = a_10m_trans[:, :, 2]

        a_3band_trans = np.transpose(a_3band, (2, 0, 1))
        a_3band_out = tif_10m[:-4] + '_NIR_SWIR_red.tif'

        general_functions.create_tif(filename=a_3band_out,
                                     g=g_20m,
                                     Nx=a_20m.shape[1],
                                     Ny=a_20m.shape[2],
                                     new_array=a_3band_trans,
                                     noData=0,
                                     data_type=gdal.GDT_UInt16)
예제 #5
0
def cal_seg_mean(in_value_ras,
                 seg_ras,
                 out_value_ras,
                 output_filtered_value_ras=True):
    g_seg, seg_arr = general_functions.read_tif(seg_ras, type=np.int64)
    g_value, value_arr = general_functions.read_tif(in_value_ras,
                                                    type=np.int64)

    out_array = np.zeros(value_arr.shape)

    unique_segvals = np.unique(seg_arr)

    band_number = value_arr.shape[0]
    for n in range(band_number):
        print('working on band: ' + str(n + 1))
        band_val = value_arr[n, :, :]
        mean_arr = scipy.ndimage.measurements.mean(band_val,
                                                   labels=seg_arr,
                                                   index=unique_segvals)

        # new_arr = np.zeros(band_val.shape)
        lookup_array = np.zeros(unique_segvals.max() + 1)
        lookup_array[unique_segvals] = mean_arr
        new_arr = lookup_array[seg_arr]
        out_array[n, :, :] = new_arr

    if output_filtered_value_ras:
        general_functions.create_tif(filename=out_value_ras,
                                     g=g_seg,
                                     Nx=seg_arr.shape[0],
                                     Ny=seg_arr.shape[1],
                                     new_array=out_array,
                                     data_type=gdal.GDT_UInt32,
                                     noData=0)
    else:
        print('Caculating brightness layer...')
        brightness = np.mean(out_array, axis=0)
        print(brightness.shape)
        general_functions.create_tif(filename=out_value_ras,
                                     g=g_seg,
                                     Nx=seg_arr.shape[0],
                                     Ny=seg_arr.shape[1],
                                     new_array=brightness,
                                     data_type=gdal.GDT_UInt32,
                                     noData=0)
예제 #6
0
def scale_s1_tif(in_tif, out_tif, scale=10000):
    g, array = read_tif(in_tif)
    array[array < 0] = 0
    array_int = array * scale
    if array_int.ndim == 2:
        general_functions.create_tif(filename=out_tif,
                                     g=g,
                                     Nx=array.shape[0],
                                     Ny=array.shape[1],
                                     new_array=array_int,
                                     data_type=gdal.GDT_UInt32,
                                     noData=0)
    elif array_int.ndim == 3:
        general_functions.create_tif(filename=out_tif,
                                     g=g,
                                     Nx=array.shape[1],
                                     Ny=array.shape[2],
                                     new_array=array_int,
                                     data_type=gdal.GDT_UInt32,
                                     noData=0)
    else:
        print('wrong')
예제 #7
0
def build_cocoa_map(working_dir,
                    path_to_aoi,
                    start_date,
                    end_date,
                    path_to_s1_image,
                    path_to_config,
                    epsg_for_map,
                    path_to_model,
                    cloud_cover=20,
                    log_path="build_cocoa_map.log",
                    use_sen2cor=False,
                    sen2cor_path=None,
                    skip_download_and_preprocess=False,
                    skip_composite=False):

    # Step 0: Get things ready. Folder structure for downloads, load credentials from config.
    fu.init_log(log_path)
    os.chdir(working_dir)
    # fu.create_file_structure(os.getcwd())
    #
    # make_directory("images/merged/10m")
    # make_directory("images/merged/20m")
    #
    # make_directory("images/stacked/with_indices")
    # make_directory("images/stacked/with_s1_seg")
    # make_directory("images/stacked/all_19bands")
    #
    # make_directory("segmentation")
    # make_directory("composites")
    # make_directory("composites/10m")
    # make_directory("composites/20m")
    #
    # make_directory("composites/10m_full")
    # make_directory("composites/20m_full")
    general_functions.make_all_dirs(working_dir)

    # Step 1: Download S2 3imagery for the timescale
    if not skip_download_and_preprocess:
        config = configparser.ConfigParser()
        config.read(path_to_config)

        images_to_download = query.check_for_s2_data_by_date(
            path_to_aoi, start_date, end_date, config, cloud_cover)
        if not use_sen2cor:
            images_to_download = query.filter_non_matching_s2_data(
                images_to_download)
        else:
            images_to_download = query.filter_to_l1_data(images_to_download)
        query.download_s2_data(images_to_download, "images/L1", "images/L2")

        # Step 2: Preprocess S2 imagery. Perform atmospheric correction if needed, stack and mask 10 and 20m bands.
        if use_sen2cor:
            ras.atmospheric_correction("images/L1"
                                       "images/L2",
                                       sen2cor_path=sen2cor_path)
        ras.preprocess_sen2_images("images/L2",
                                   "images/merged/10m",
                                   "images/L1",
                                   cloud_threshold=0,
                                   epsg=epsg_for_map,
                                   bands=("B02", "B03", "B04", "B08"),
                                   out_resolution=10)
        ras.preprocess_sen2_images("images/L2",
                                   "images/merged/20m",
                                   "images/L1",
                                   cloud_threshold=0,
                                   epsg=epsg_for_map,
                                   bands=("B02", "B03", "B04", "B05", "B06",
                                          "B07", "B8A", "B11", "B12"),
                                   out_resolution=20)

    if not skip_composite:
        # Step 2.5: Build a pair of cloud-free composites
        sort_into_tile("images/merged/10m")
        sort_into_tile("images/merged/20m")

        for tile in os.listdir("images/merged/10m"):
            tile_path = os.path.join("images/merged/10m", tile)
            this_composite_path = ras.composite_directory(
                tile_path, "composites/10m")
            new_composite_path = "{}_{}.tif".format(
                this_composite_path.rsplit('.')[0], tile)
            os.rename(this_composite_path, new_composite_path)

        for tile in os.listdir("images/merged/20m"):
            tile_path = os.path.join("images/merged/20m", tile)
            this_composite_path = ras.composite_directory(
                tile_path, "composites/20m")
            new_composite_path = "{}_{}.tif".format(
                this_composite_path.rsplit('.')[0], tile)
            os.rename(this_composite_path, new_composite_path)

    # Step 3: Generate the bands. Time for the New Bit.
    clip_to_aoi = False
    if clip_to_aoi:
        for image in os.listdir("composites/10m_full"):
            if image.endswith(".tif"):
                image_path_10m_full = os.path.join("composites/10m_full",
                                                   image)
                image_path_20m_full = os.path.join("composites/20m_full",
                                                   image)

                image_path_10m_clipped = os.path.join("composites/10m", image)
                image_path_20m_clipped = os.path.join("composites/20m", image)

                # config = configparser.ConfigParser()
                # conf = config.read(path_to_config)
                # print(conf)
                # aoi = config['cocoa_mapping']['path_to_aoi']

                aoi = "/media/ubuntu/Data/Ghana/cocoa_big/shp/cocoa_big.shp"

                ras.clip_raster(raster_path=image_path_10m_full,
                                aoi_path=aoi,
                                out_path=image_path_10m_clipped,
                                srs_id=32630)
                ras.clip_raster(raster_path=image_path_20m_full,
                                aoi_path=aoi,
                                out_path=image_path_20m_clipped,
                                srs_id=32630)

    do_segmentation = True
    if do_segmentation == True:
        for image in os.listdir("composites/10m"):
            if image.endswith(".tif"):
                with TemporaryDirectory() as td:
                    image_path_10m = os.path.join("composites/10m", image)
                    image_path_20m = os.path.join("composites/20m", image)
                    resample_path_20m_v1 = os.path.join(
                        td,
                        image)  # You'll probably regret this later, roberts.
                    shutil.copy(image_path_20m, resample_path_20m_v1)
                    ras.resample_image_in_place(resample_path_20m_v1, 10)

                    index_image_path = os.path.join(td, "index_image.tif")
                    temp_pre_seg_path = os.path.join(td, "pre_seg.tif")
                    temp_seg_path = os.path.join(td, "seg.tif")
                    temp_shp_path = os.path.join(td, "outline.shp")
                    temp_clipped_seg_path = os.path.join(td, "seg_clip.tif")

                    # This bit's your show, Qing
                    temp_s1_outline_path = os.path.join(td, "s1_outline.shp")
                    ras.get_extent_as_shp(in_ras_path=image_path_10m,
                                          out_shp_path=temp_s1_outline_path)

                    resample_path_20m = os.path.join(
                        td, image[:-4] + '_to_10moutline.tif')
                    general_functions.clip_rst(
                        in_tif=resample_path_20m_v1,
                        outline_shp=temp_s1_outline_path,
                        out_tif=resample_path_20m,
                        keep_rst_extent=False)

                    generate_veg_index_tif(image_path_10m, resample_path_20m,
                                           index_image_path)
                    ras.stack_images([index_image_path, image_path_10m],
                                     "images/stacked/with_indices/" + image)

                    # Now, we do Highly Experimental Image Segmentation. Please put on your goggles.
                    # SAGA, please.
                    # Meatball science time
                    vis_10m = gdal.Open(image_path_10m)
                    vis_20m_resampled = gdal.Open(resample_path_20m)
                    vis_10m_array = vis_10m.GetVirtualMemArray()
                    vis_20m_array = vis_20m_resampled.GetVirtualMemArray()
                    # NIR, SWIR, red
                    array_to_classify = np.stack([
                        vis_10m_array[3, ...], vis_20m_array[7, ...],
                        vis_10m_array[2, ...]
                    ])

                    g, arr = general_functions.read_tif(intif=image_path_10m)
                    general_functions.create_tif(filename=temp_pre_seg_path,
                                                 g=g,
                                                 Nx=arr.shape[1],
                                                 Ny=arr.shape[2],
                                                 new_array=array_to_classify,
                                                 noData=0,
                                                 data_type=gdal.GDT_UInt32)
                    out_segment_tif = os.path.join("segmentation", image)
                    segment_image(temp_pre_seg_path, out_segment_tif)

                    print('Generate brighness raster from the segments')
                    make_directory("segmentation/brightness")

                    out_brightness_value_ras = os.path.join(
                        "segmentation/brightness", image)
                    output_filtered_value_ras = False

                    ras.get_extent_as_shp(in_ras_path=temp_pre_seg_path,
                                          out_shp_path=temp_shp_path)

                    general_functions.clip_rst(in_tif=out_segment_tif,
                                               outline_shp=temp_shp_path,
                                               out_tif=temp_clipped_seg_path,
                                               keep_rst_extent=False)

                    cal_seg_mean(
                        temp_pre_seg_path,
                        temp_clipped_seg_path,
                        out_brightness_value_ras,
                        output_filtered_value_ras=output_filtered_value_ras)

                # image_20m_6bands_array = vis_20m_array[3:,:,:]
                # try:
                #     os.mkdir("composites/20m/20m_6bands")
                # except FileExistsError:
                #     pass
                #
                # out_20m_tif_for_stack = os.path.join("composites/20m/20m_6bands", image)
                # general_functions.create_tif(filename=out_20m_tif_for_stack,g=g,Nx=arr.shape[1],Ny=arr.shape[2],
                #                              new_array=image_20m_6bands_array,data_type=gdal.GDT_UInt16,noData=0)

    do_stack = True
    if do_stack == True:
        # Step 4: Stack the new bands with the S1, seg, and 6 band 20m rasters
        for image in os.listdir("images/stacked/with_indices"):
            if image.endswith(".tif"):
                path_to_image = os.path.join("images/stacked/with_indices",
                                             image)
                path_to_brightness_image = os.path.join(
                    "segmentation/brightness", image)
                # path_to_20m_image = os.path.join("composites/20m/20m_6bands", image)
                # ras.stack_images([path_to_image, path_to_s1_image,path_to_20m_image,out_brightness_value_ras], os.path.join("images/stacked/all_19bands", image))
                ras.stack_images([
                    path_to_image, path_to_s1_image, path_to_brightness_image
                ], os.path.join("images/stacked/with_s1_seg", image))

    #sys.exit()
    #
    # Step 5: Classify with trained model
    for image in os.listdir("images/stacked/with_s1_seg"):
        if image.endswith(".tif"):
            path_to_image = os.path.join("images/stacked/with_s1_seg", image)
            path_to_out = os.path.join("output", image)
            PYEO_model.classify_image(path_to_image, path_to_model,
                                      path_to_out)