Example #1
0
def gather_data():
    
    
    cloud_classifier = get_s2_pixel_cloud_detector(average_over=2, dilation_size=1, all_bands=False)
    add_clm = AddCloudMaskTask(cloud_classifier, 'BANDS-S2CLOUDLESS', cm_size_y='80m', cm_size_x='80m', 
                           cmask_feature='CLM', # cloud mask name
                           cprobs_feature='CLP' # cloud prob. map name
                          )
    ndvi = NormalizedDifferenceIndex('NDVI', 'BANDS/3', 'BANDS/2')
    add_sh_valmask = AddValidDataMaskTask(SentinelHubValidData(), 
                                      'IS_VALID')
    layer = 'BANDS-S2-L1C'
    custom_script = 'return [B02, B03];'
    input_task = S2L1CWCSInput(layer=layer,
                           feature=(FeatureType.DATA, 'BANDS'), 
    custom_url_params={CustomUrlParam.EVALSCRIPT: custom_script},
                           resx='10m', resy='10m',
                           maxcc=.8)
    add_ndvi = S2L1CWCSInput(layer='NDVI')
    save = SaveToDisk('io_example', overwrite_permission=2)#compress_level=1
    workflow = LinearWorkflow(
       input_task,
        add_clm,
        add_ndvi,
        add_sh_valmask,
    
        )
    time_interval = ('2017-01-01', '2017-12-31')
    result = workflow.execute({input_task: {'bbox': roi_bbox, 'time_interval': time_interval},
                           save: {'eopatch_folder': 'eopatch'}})
    return list(result.values())[0].data['NDVI'], list(result.values())[-1].mask['IS_VALID'],  np.array(list(result.values())[0].timestamp)
Example #2
0
def get_create_and_add_lpis_workflow(country, year, out_path):
    """
    Creates an EOWorkflow that:
    1. creates an empty patch
    2. add LPIS vector data fetched from Geopedia 
    3. calculate the ratio between area of all fields and area of EOPatch
    4. save newly created EOPatch to disk
    """
    print(f'Preparing EOWorkflow for {country} and year {year}:')

    # 1. create empty patch
    create = CreatePatch()
    print(f'   1. Creating empty EOPatch')

    # 2. Add LPIS vector data
    layer_id = GEOPEDIA_LPIS_LAYERS[f'{country}_LPIS_{year}']
    ftr_name = f'LPIS_{year}'
    year_filter = (
        GEOPEDIA_LPIS_YEAR_NAME[country],
        year) if GEOPEDIA_LPIS_YEAR_NAME[country] is not None else None

    add_lpis = AddGeopediaVectorFeature(
        (FeatureType.VECTOR_TIMELESS, ftr_name),
        layer=layer_id,
        year_filter=year_filter,
        drop_duplicates=True)
    print(f'   2. Adding LPIS vector data to feature {ftr_name}')
    print(f'      -> Geopedia layer ID: {layer_id}')
    if year_filter is not None:
        print(f'      -> with year filter applied: {year_filter}')

    # 3. Add Area Ratio
    area_ratio = AddAreaRatio(
        (FeatureType.VECTOR_TIMELESS, ftr_name),
        (FeatureType.SCALAR_TIMELESS, 'FIELD_AREA_RATIO'))
    print(f'   3. Calcuating the field area ratio')

    # 4. Save EOPatch to disk
    out_dir = str(out_path + '/' + country + '/' + str(year))
    save = SaveToDisk(out_dir,
                      overwrite_permission=OverwritePermission.OVERWRITE_PATCH)
    print(f'   4. Saving EOPatches to: {out_dir}')

    workflow = LinearWorkflow(create,
                              add_lpis,
                              area_ratio,
                              save,
                              task_names={
                                  create: 'create',
                                  add_lpis: 'add_lpis',
                                  area_ratio: ' area_ratio',
                                  save: 'save'
                              })

    return workflow
    def save_patch(self,
                   save_folder,
                   feature=None,
                   overwrite_permission=OverwritePermission.OVERWRITE_PATCH,
                   compress_level=0):
        """
        Save indexed EOPatches to a folder.

        :param save_folder: folder to save eopatches
        :type save_folder: str
        :param feature: Feature to be exported
        :type feature: (FeatureType, feature_name) or FeatureType
        :param overwrite_permission: Permissions to overwrite exist EOPatch.
            Permissions are in the following hierarchy:
            - `ADD_ONLY` - Only new features can be added, anything that is already saved cannot be changed.
            - `OVERWRITE_FEATURES` - Overwrite only data for features which have to be saved. The remaining content of
             saved EOPatch will stay unchanged.
            - `OVERWRITE_PATCH` - Overwrite entire content of saved EOPatch and replace it with the new content.
        :type overwrite_permission: OverwritePermission
        :param compress_level: A level of data compression and can be specified with an integer from 0 (no compression)
            to 9 (highest compression).
        :type compress_level: int
        """
        if not feature:
            feature = self.feature
        if not self._is_loaded():
            self._load_with_index(feature=feature)
        tile_rows, tile_columns = self._get_tile_rows_columns()
        self._assure_folder_exist(save_folder)
        save_task = SaveToDisk(save_folder,
                               features=[feature, FeatureType.BBOX],
                               overwrite_permission=overwrite_permission,
                               compress_level=compress_level)
        workflow = LinearWorkflow(save_task)
        execution_args = []
        for row in range(tile_rows):
            for column in range(tile_columns):
                execution_args.append({
                    save_task: {
                        'eopatch_folder':
                        'patch_{row}_{column}'.format(row=row, column=column),
                        'eopatch':
                        self.patch_index[row][column]
                    }
                })
        executor = EOExecutor(workflow, execution_args)
        executor.run(workers=1, multiprocess=False)
Example #4
0
    SentinelHubValidData(),
    'IS_VALID'  # name of output mask
)

# TASK FOR COUNTING VALID PIXELS
# count number of valid observations per pixel using valid data mask
count_val_sh = CountValid(
    'IS_VALID',  # name of existing mask
    'VALID_COUNT'  # name of output scalar
)

# TASK FOR SAVING TO OUTPUT (if needed)
path_out = './eopatches_large/'
if not os.path.isdir(path_out):
    os.makedirs(path_out)
save = SaveToDisk(path_out,
                  overwrite_permission=OverwritePermission.OVERWRITE_PATCH)

#%% Section 4
#Execution of workflow: Filling patches with data
print('Starting download')

workflow = LinearWorkflow(add_data, add_clm, ndvi, ndwi, norm, add_sh_valmask,
                          count_val_sh, save)

for idx, bbox in enumerate(bbox_list[patchIDs]):

    # define additional parameters of the workflow
    extra_param = {
        add_data: {
            'bbox': bbox,
            'time_interval': time_interval
    # Print selected EOPatches boundary
    idxs_x = [info['index_x'] for info in info_list[patchIDs]]
    idxs_y = [info['index_y'] for info in info_list[patchIDs]]
    for (x, y, geo) in zip(idxs_x, idxs_y, geometries):
        print("x:{} y:{} geometry:{}".format(x, y, geo))

    # Fill EOPatches with data from geogenius platform:
    # Define ImportFromGeogenius task
    add_data = ImportFromGeogenius(feature=(FeatureType.DATA, 'BANDS'),
                                   geogenius_image=img)
    # Define Save EOPatch Task
    path_out = get_current_folder("eopatches")
    if not os.path.isdir(path_out):
        os.makedirs(path_out)
    save = SaveToDisk(path_out,
                      overwrite_permission=OverwritePermission.OVERWRITE_PATCH)

    # patch = add_data.execute(bbox=bbox_list[patchIDs][0])
    # save.execute(patch, eopatch_folder="1")

    # Define workflow
    workflow = LinearWorkflow(add_data, save)

    # Execute the workflow
    # define additional parameters of the workflow
    execution_args = []
    for idx, bbox in enumerate(bbox_list[patchIDs]):
        execution_args.append({
            add_data: {
                'bbox': bbox
            },
def load_LPIS(country, year, path, no_patches):
    patch_location = path + '/{}/'.format(country)
    load = LoadFromDisk(patch_location)
    save_path_location = patch_location
    if not os.path.isdir(save_path_location):
        os.makedirs(save_path_location)
    save = SaveToDisk(save_path_location,
                      overwrite_permission=OverwritePermission.OVERWRITE_PATCH)

    # workflow_data = get_create_and_add_lpis_workflow(country, year, save_path_location)

    name_of_feature = 'LPIS_{}'.format(year)

    groups_to_number, crops_to_number = create_mapping(country)

    layer_id = GEOPEDIA_LPIS_LAYERS[f'{country}_LPIS_{year}']
    ftr_name = f'LPIS_{year}'
    year_filter = (
        GEOPEDIA_LPIS_YEAR_NAME[country],
        year) if GEOPEDIA_LPIS_YEAR_NAME[country] is not None else None
    add_lpis = AddGeopediaVectorFeature(
        (FeatureType.VECTOR_TIMELESS, ftr_name),
        layer=layer_id,
        year_filter=year_filter,
        drop_duplicates=True)
    area_ratio = AddAreaRatio(
        (FeatureType.VECTOR_TIMELESS, ftr_name),
        (FeatureType.SCALAR_TIMELESS, 'FIELD_AREA_RATIO'))
    fixlpis = FixLPIS(feature=name_of_feature, country=country)

    rasterize = VectorToRaster(vector_input=(FeatureType.VECTOR_TIMELESS,
                                             name_of_feature),
                               raster_feature=(FeatureType.MASK_TIMELESS,
                                               name_of_feature),
                               values=None,
                               values_column='GROUP',
                               raster_shape=(FeatureType.DATA, 'BANDS'),
                               raster_dtype=np.int16,
                               no_data_value=np.nan)

    add_group = AddGroup(crops_to_number, name_of_feature)
    remove_dtf = RemoveFeature(FeatureType.VECTOR_TIMELESS, name_of_feature)

    exclude = WorkflowExclude(area_ratio, fixlpis, add_group, rasterize,
                              remove_dtf)

    workflow = LinearWorkflow(load, add_lpis, exclude, save)

    execution_args = []
    for i in range(no_patches):
        execution_args.append({
            load: {
                'eopatch_folder': 'eopatch_{}'.format(i)
            },
            save: {
                'eopatch_folder': 'eopatch_{}'.format(i)
            }
        })
    ##### here you choose how many processes/threads you will run, workers=none is max of processors

    executor = EOExecutor(workflow,
                          execution_args,
                          save_logs=True,
                          logs_folder='ExecutionLogs')
    # executor.run(workers=None, multiprocess=True)
    executor.run()
Example #7
0
def download_data(path_save,
                  coords_top,
                  coords_bot,
                  patch_n,
                  s_date,
                  e_date,
                  debug=False):
    # before moving onto actual tasks, check setup
    check_sentinel_cfg()

    [lat_left_top, lon_left_top] = coords_top
    [lat_right_bot, lon_right_bot] = coords_bot
    # TASK FOR BAND DATA
    # add a request for B(B02), G(B03), R(B04), NIR (B08), SWIR1(B11), SWIR2(B12)
    # from default layer 'ALL_BANDS' at 10m resolution
    # Here we also do a simple filter of cloudy scenes. A detailed cloud cover
    # detection is performed in the next step
    custom_script = "return [B02, B03, B04, B08, B11, B12];"
    add_data = S2L1CWCSInput(
        layer="BANDS-S2-L1C",
        feature=(FeatureType.DATA, "BANDS"),  # save under name 'BANDS'
        # custom url for 6 specific bands
        custom_url_params={CustomUrlParam.EVALSCRIPT: custom_script},
        resx="10m",  # resolution x
        resy="10m",  # resolution y
        maxcc=0.1,  # maximum allowed cloud cover of original ESA tiles
    )

    # TASK FOR CLOUD INFO
    # cloud detection is performed at 80m resolution
    # and the resulting cloud probability map and mask
    # are scaled to EOPatch's resolution
    cloud_classifier = get_s2_pixel_cloud_detector(average_over=2,
                                                   dilation_size=1,
                                                   all_bands=False)
    add_clm = AddCloudMaskTask(
        cloud_classifier,
        "BANDS-S2CLOUDLESS",
        cm_size_y="80m",
        cm_size_x="80m",
        cmask_feature="CLM",  # cloud mask name
        cprobs_feature="CLP",  # cloud prob. map name
    )

    # TASKS FOR CALCULATING NEW FEATURES
    # NDVI: (B08 - B04)/(B08 + B04)
    # NDWI: (B03 - B08)/(B03 + B08)
    # NORM: sqrt(B02^2 + B03^2 + B04^2 + B08^2 + B11^2 + B12^2)
    ndvi = NormalizedDifferenceIndex("NDVI", "BANDS/3", "BANDS/2")
    ndwi = NormalizedDifferenceIndex("NDWI", "BANDS/1", "BANDS/3")
    norm = EuclideanNorm("NORM", "BANDS")

    # TASK FOR VALID MASK
    # validate pixels using SentinelHub's cloud detection mask and region of acquisition
    add_sh_valmask = AddValidDataMaskTask(
        SentinelHubValidData(),
        "IS_VALID"  # name of output mask
    )

    # TASK FOR COUNTING VALID PIXELS
    # count number of valid observations per pixel using valid data mask
    count_val_sh = CountValid(
        "IS_VALID",
        "VALID_COUNT"  # name of existing mask  # name of output scalar
    )

    # TASK FOR SAVING TO OUTPUT (if needed)
    path_save = Path(path_save)
    path_save.mkdir(exist_ok=True)
    # if not os.path.isdir(path_save):
    #     os.makedirs(path_save)
    save = SaveToDisk(path_save,
                      overwrite_permission=OverwritePermission.OVERWRITE_PATCH)

    # Define the workflow
    workflow = LinearWorkflow(add_data, add_clm, ndvi, ndwi, norm,
                              add_sh_valmask, count_val_sh, save)
    # Execute the workflow

    # time interval for the SH request
    # TODO: need to check if specified time interval is valid
    time_interval = [s_date, e_date]

    # define additional parameters of the workflow
    execution_args = []

    path_EOPatch = path_save / f"eopatch_{patch_n}"

    execution_args.append({
        add_data: {
            "bbox":
            BBox(
                ((lon_left_top, lat_left_top), (lon_right_bot, lat_right_bot)),
                crs=CRS.WGS84,
            ),
            "time_interval":
            time_interval,
        },
        save: {
            "eopatch_folder": path_EOPatch.stem
        },
    })

    executor = EOExecutor(workflow, execution_args, save_logs=True)
    if debug:
        print("Downloading Satellite data ...")

    executor.run(workers=2, multiprocess=False)
    if executor.get_failed_executions():
        raise RuntimeError("EOExecutor failed in finishing tasks!")

    if debug:
        executor.make_report()
    if debug:
        print("Satellite data is downloaded")
    return path_EOPatch
if __name__ == '__main__':

    # no_patches = 1085
    no_patches = 1

    path = '/home/beno/Documents/test'
    # path = 'E:/Data/PerceptiveSentinel'

    patch_location = path + '/Slovenija/'
    load = LoadFromDisk(patch_location)

    save_path_location = path + '/Slovenija/'
    if not os.path.isdir(save_path_location):
        os.makedirs(save_path_location)

    save = SaveToDisk(save_path_location,
                      overwrite_permission=OverwritePermission.OVERWRITE_PATCH)

    addStreamNDVI = AddStreamTemporalFeaturesTask(data_feature='NDVI')
    addStreamSAVI = AddStreamTemporalFeaturesTask(data_feature='SAVI')
    addStreamEVI = AddStreamTemporalFeaturesTask(data_feature='EVI')
    addStreamARVI = AddStreamTemporalFeaturesTask(data_feature='ARVI')
    addStreamSIPI = AddStreamTemporalFeaturesTask(data_feature='SIPI')
    addStreamNDWI = AddStreamTemporalFeaturesTask(data_feature='NDWI')
    '''
    lulc_cmap = mpl.colors.ListedColormap([entry.color for entry in LULC])
    lulc_norm = mpl.colors.BoundaryNorm(np.arange(-0.5, 11, 1), lulc_cmap.N)

    land_cover_path = path+'/shapefiles/slovenia.shp'

    land_cover = gpd.read_file(land_cover_path)
Example #9
0
def predict_raster_patch(path_EOPatch, patch_n, scale, debug=False):
    path_EOPatch = Path(path_EOPatch)
    model_path = path_module / "model.pkl"
    model = joblib.load(model_path)

    # TASK TO LOAD EXISTING EOPATCHES
    load = LoadFromDisk(path_EOPatch.parent)

    # TASK FOR CONCATENATION
    concatenate = ConcatenateData("FEATURES",
                                  ["BANDS", "NDVI", "NDWI", "NORM"])

    # TASK FOR FILTERING OUT TOO CLOUDY SCENES
    # keep frames with > 80 % valid coverage
    valid_data_predicate = ValidDataFractionPredicate(0.8)
    filter_task = SimpleFilterTask((FeatureType.MASK, 'IS_VALID'),
                                   valid_data_predicate)

    save = SaveToDisk(path_EOPatch.parent,
                      overwrite_permission=OverwritePermission.OVERWRITE_PATCH)

    workflow = LinearWorkflow(
        load,
        concatenate,
        filter_task,
        save,
    )

    execution_args = []
    for idx in range(0, 1):
        execution_args.append({
            load: {
                "eopatch_folder": path_EOPatch.stem
            },
            save: {
                "eopatch_folder": path_EOPatch.stem
            },
        })
    if debug:
        print("Saving the features ...")
    executor = EOExecutor(workflow, execution_args, save_logs=False)
    executor.run(workers=5, multiprocess=False)

    if debug:
        executor.make_report()

    # load from disk to determine number of valid pictures
    eopatch = EOPatch.load(path_EOPatch, lazy_loading=True)
    n_pics = eopatch.data["BANDS"].shape[0]

    print(f'Number of valid pictures detected: {n_pics}')

    list_path_raster = []
    for pic_n in range(n_pics):

        # TASK TO LOAD EXISTING EOPATCHES
        load = LoadFromDisk(path_EOPatch.parent)

        # TASK FOR PREDICTION
        predict = PredictPatch(model, (FeatureType.DATA, "FEATURES"), "LBL",
                               pic_n, "SCR")

        # TASK FOR SAVING
        save = SaveToDisk(
            str(path_EOPatch.parent),
            overwrite_permission=OverwritePermission.OVERWRITE_PATCH)

        # TASK TO EXPORT TIFF
        export_tiff = ExportToTiff((FeatureType.MASK_TIMELESS, "LBL"))
        tiff_location = (path_EOPatch.parent / f"predicted_tiff")

        if not os.path.isdir(tiff_location):
            os.makedirs(tiff_location)

        workflow = LinearWorkflow(load, predict, export_tiff, save)

        # create a list of execution arguments for each patch
        execution_args = []
        path_predict = tiff_location / f"prediction-eopatch_{patch_n}-pic_{pic_n}.tiff"
        for i in range(0, 1):
            execution_args.append({
                load: {
                    "eopatch_folder": path_EOPatch.stem
                },
                export_tiff: {
                    "filename": path_predict
                },
                save: {
                    "eopatch_folder": path_EOPatch.stem
                },
            })

        # run the executor on 2 cores
        executor = EOExecutor(workflow, execution_args)

        # uncomment below save the logs in the current directory and produce a report!
        # executor = EOExecutor(workflow, execution_args, save_logs=True)
        if debug:
            print("Predicting the land cover ...")
        executor.run(workers=5, multiprocess=False)
        if debug:
            executor.make_report()

        # PATH = path_out / "predicted_tiff" / f"patch{patch_n}"
        path_merged = tiff_location / f"merged_prediction-eopatch_{patch_n}-pic_{pic_n}.tiff"
        if path_merged.exists():
            path_merged.unlink()
        cmd = f"gdal_merge.py -o {path_merged} -co compress=LZW {path_predict}"
        os.system(cmd)

        # save path
        list_path_raster.append(path_merged)

        # Reference colormap things
        lulc_cmap = mpl.colors.ListedColormap([entry.color for entry in LULC])
        lulc_norm = mpl.colors.BoundaryNorm(np.arange(-0.5, 3, 1), lulc_cmap.N)

        size = 20
        fig, ax = plt.subplots(figsize=(2 * size * 1, 1 * size * scale),
                               nrows=1,
                               ncols=2)
        eopatch = EOPatch.load(path_EOPatch, lazy_loading=True)
        im = ax[0].imshow(eopatch.mask_timeless["LBL"].squeeze(),
                          cmap=lulc_cmap,
                          norm=lulc_norm)
        ax[0].set_xticks([])
        ax[0].set_yticks([])
        ax[0].set_aspect("auto")

        fig.subplots_adjust(wspace=0, hspace=0)
        for i in range(0, 1):
            eopatch = EOPatch.load(path_EOPatch, lazy_loading=True)
            ax = ax[1]
            plt.imshow(
                np.clip(
                    eopatch.data["BANDS"][pic_n, :, :, :][..., [2, 1, 0]] *
                    3.5, 0, 1))
            plt.xticks([])
            plt.yticks([])
            ax.set_aspect("auto")
            del eopatch

        if debug:
            print("saving the predicted image ...")
        plt.savefig(path_EOPatch.parent /
                    f"predicted_vs_real_{patch_n}-{pic_n}.png")

    return list_path_raster
Example #10
0
def get_add_l2a_data_workflow(data):
    """
    Creates an workflow that:
    1. loads existing EOPatch
    2. adds sen2cor scene classification map
    3. adds L2A data (all 12 bands)
    4. adds s2cloudless cloud masks
    5. determines `L2A_VALID` - map of valid observations (t,h,w,1) based on L2A SCL map
        * pixels are marked as valid, if they're tagged as
        `[DARK_AREA_PIXELS, VEGETATION, NOT_VEGETATED, WATER, UNCLASSIFIED]`
        * performs opening with disk with radius 11 on `L2A_VALID`
    6. determines `L1C_VALID` - map of valid observations (t,h,w,1) based on s2cloudless cloud map
        * pixels are marked as valid, if they're tagged as not cloud
    7. saves EOPatch to disk
    """
    # 1. loads existing EOPatch
    load = LoadFromDisk(str(data))

    # 2. add L2A
    add_l2a = S2L2AWCSInput(layer='BANDS-S2-L2A',
                            resx='10m',
                            resy='10m',
                            maxcc=0.8,
                            time_difference=timedelta(hours=2),
                            raise_download_errors=False)

    # 3. add sen2cor's scene classification map and snow probability map
    add_scl = AddSen2CorClassificationFeature(sen2cor_classification='SCL',
                                              layer='TRUE-COLOR-S2-L2A',
                                              image_format=MimeType.TIFF_d32f,
                                              raise_download_errors=False)

    # 4. add s2cloudless cloud mask
    cloud_classifier = get_s2_pixel_cloud_detector(average_over=2,
                                                   dilation_size=1,
                                                   all_bands=False)
    add_clm = AddCloudMaskTask(cloud_classifier,
                               'BANDS-S2CLOUDLESS',
                               cm_size_y='160m',
                               cm_size_x='160m',
                               cmask_feature='CLM')

    # create valid data masks
    scl_valid_classes = [2, 4, 5, 6, 7]

    # 5. and 6. add L2A and L1C valid data masks
    add_l1c_valmask = AddValidDataMaskTask(SentinelHubValidData(), 'L1C_VALID')
    add_l2a_valmask = AddValidDataMaskTask(
        Sen2CorValidData(scl_valid_classes, 6, 22), 'L2A_VALID')
    add_valmask = AddValidDataMaskTask(MergeMasks('L1C_VALID', 'L2A_VALID'),
                                       'VALID_DATA')

    # 3. keep only frames with valid data fraction over 70%
    valid_data_predicate = ValidDataFractionPredicate(0.7)
    filter_task = SimpleFilterTask((FeatureType.MASK, 'VALID_DATA'),
                                   valid_data_predicate)

    # save
    save = SaveToDisk(str(data),
                      compress_level=1,
                      overwrite_permission=OverwritePermission.OVERWRITE_PATCH)

    workflow = LinearWorkflow(load,
                              add_l2a,
                              add_scl,
                              add_clm,
                              add_l1c_valmask,
                              add_l2a_valmask,
                              add_valmask,
                              filter_task,
                              save,
                              task_names={
                                  load: 'load',
                                  add_l2a: 'add_L2A',
                                  add_scl: 'add_SCL',
                                  add_clm: 'add_clm',
                                  add_l1c_valmask: 'add_L1C_valmask',
                                  add_l2a_valmask: 'add_L2A_valmask',
                                  add_valmask: 'add_valmask',
                                  filter_task: ' filter_task',
                                  save: 'save'
                              })

    return workflow