def get_exec_args(workflow: LinearWorkflow, eopatch_list: List[str], config: PostProcessConfig) -> List[dict]:
""" Utility function to get execution arguments """
exec_args = []
tasks = workflow.get_tasks()
load_bbox = LoadTask(path=f's3://{config.bucket_name}/{config.eopatches_folder}', features=[FeatureType.BBOX])
for name in tqdm(eopatch_list):
single_exec_dict = {}
try:
eop = load_bbox.execute(eopatch_folder=name)
for task_name, task in tasks.items():
if isinstance(task, ExportToTiff):
single_exec_dict[task] = dict(filename=f'{name}-{eop.bbox.crs.epsg}.tiff')
if isinstance(task, (LoadTask, SaveTask)):
single_exec_dict[task] = dict(eopatch_folder=name)
exec_args.append(single_exec_dict)
except ResourceNotFound as exc:
print(f'{name} - {exc}')
return exec_args
def test_nonexistent_location(fs_loader):
path = "./folder/subfolder/new-eopatch/"
empty_eop = EOPatch()
with fs_loader() as temp_fs:
with pytest.raises(ResourceNotFound):
EOPatch.load(path, filesystem=temp_fs)
empty_eop.save(path, filesystem=temp_fs)
with TempFS() as temp_fs:
full_path = os.path.join(temp_fs.root_path, path)
with pytest.raises(CreateFailed):
EOPatch.load(full_path)
load_task = LoadTask(full_path)
with pytest.raises(CreateFailed):
load_task.execute()
empty_eop.save(full_path)
assert os.path.exists(full_path)
with TempFS() as temp_fs:
full_path = os.path.join(temp_fs.root_path, path)
save_task = SaveTask(full_path)
save_task.execute(empty_eop)
assert os.path.exists(full_path)
def test_nonexistent_location(self):
path = './folder/subfolder/new-eopatch/'
empty_eop = EOPatch()
for fs_loader in self.filesystem_loaders:
with fs_loader() as temp_fs:
with self.assertRaises(ResourceNotFound):
EOPatch.load(path, filesystem=temp_fs)
empty_eop.save(path, filesystem=temp_fs)
with TempFS() as temp_fs:
full_path = os.path.join(temp_fs.root_path, path)
with self.assertRaises(CreateFailed):
EOPatch.load(full_path)
load_task = LoadTask(full_path)
with self.assertRaises(CreateFailed):
load_task.execute()
empty_eop.save(full_path)
self.assertTrue(os.path.exists(full_path))
with TempFS() as temp_fs:
full_path = os.path.join(temp_fs.root_path, path)
save_task = SaveTask(full_path)
save_task.execute(empty_eop)
self.assertTrue(os.path.exists(full_path))
def tasks(self, bands_task=LoadTask(".")):
input_task = LoadTask(".")
return [
(input_task, [bands_task], f"Calculate index {self.input_feature[1]}"),
(self.basic_statistic, [input_task], f"Basic statistics {self.input_feature[1]}"),
(self.rolling_windows, [input_task], f"Rolling window statistics {self.input_feature[1]}"),
(self.max_mean_len_task, [self.rolling_windows], f"Max mean len statistic {self.input_feature[1]}"),
(self.positive_derivative_task, [input_task], f"Positive derivative statistics {self.input_feature[1]}"),
(self.negative_derivative_task, [input_task], f"Negative derivative statistics {self.input_feature[1]}"),
]
def to_workflow(self):
input_task = LoadTask(".") # Dummy to show correct graph
return EOWorkflow([
(input_task, [], "Download bands"),
*self.tasks(input_task),
]
)
def predict_using_model(patch_dir, model_file, method, window_size):
''' Defines a workflow that will perform the prediction step on a given EOPatch.
For a given EOPatch, use the specified model to apply prediction step.
Parameters:
- patch_dir: the directory that contains the patch
- model_file; the path to the model file.
- method: The local noramalization method, one of 'min', 'median' or 'mean'. This should be the same as the one used to train the model.
- window_size: The window_size used in the local normalization step. Should be the same as that used to train the model.
Returns:
Nothing. Updates the EOPatch on disk.
'''
path = patch_dir
if (type(path) != str):
path = str(path)
save = SaveTask(path=path,
overwrite_permission=OverwritePermission.OVERWRITE_PATCH)
load_task = LoadTask(path=path)
local_norm = LocalNormalization()
detect_plastics = DetectPlastics(model_file=model_file)
workflow = LinearWorkflow(load_task, local_norm, detect_plastics, save)
workflow.execute(
{local_norm: {
'method': method,
'window_size': window_size
}})
def get_gsaa_to_eopatch_workflow(config: GsaaToEopatchConfig) -> EOWorkflow:
# set up AWS credentials
sh_config = set_sh_config(config)
# load patch
load_task = LoadTask(path=f's3://{config.bucket_name}/{config.eopatches_folder}', config=sh_config)
# add original vectors to patch
vec2vec = DB2Vector(database=config.database,
user=config.user, password=config.password,
host=config.host, port=config.port, crs=config.crs,
vector_output_feature=config.vector_feature)
# get extent mask from vector
vec2ras = VectorToRaster(config.vector_feature,
config.extent_feature,
values=1, raster_shape=(config.width, config.height),
no_data_value=config.no_data_value,
buffer=config.buffer_poly, write_to_existing=False)
# get boundary mask from extent mask
ras2bound = Extent2Boundary(config.extent_feature,
config.boundary_feature,
structure=disk(config.disk_radius))
# get distance from extent mask
ras2dist = Extent2Distance(config.extent_feature,
config.distance_feature,
normalize=True)
# save new features
save_task = SaveTask(path=f's3://{config.bucket_name}/{config.eopatches_folder}',
features=[config.vector_feature,
config.extent_feature,
config.boundary_feature,
config.distance_feature],
overwrite_permission=OverwritePermission.OVERWRITE_FEATURES, config=sh_config)
return LinearWorkflow(load_task, vec2vec, vec2ras, ras2bound, ras2dist, save_task)
def plot_classifications(patchDir, features=None):
''' Method that will take a given patch plot the results of the model for that patch.
Parameters:
- patchDir: the directory of the EOPatch to visualize
- features: Features, could be the training dataset, to overlay on the scatter plots.
Returns
Nothing. Will create a file called classifications.png in the EOPatch folder.
'''
patch = LoadTask(path=str(patchDir)).execute()
classifcations = patch.data['CLASSIFICATION'][0, :, :, 0]
ndvi = patch.data['NDVI'][0, :, :, 0]
fdi = patch.data['FDI'][0, :, :, 0]
norm_ndvi = patch.data['NORM_NDVI'][0, :, :, 0]
norm_fdi = patch.data['NORM_FDI'][0, :, :, 0]
fig, axs = plt.subplots(nrows=2, ncols=3, figsize=(20, 30))
axs = axs.flatten()
fndvi = norm_ndvi.flatten()
ffdi = norm_fdi.flatten()
fclassifications = classifcations.flatten()
fclassifications[(ffdi < 0.007)] = 0
p_grid = np.array([cols_rgb[val] for val in fclassifications])
axs[0].set_title("Labels")
axs[0].imshow(
p_grid.reshape(classifcations.shape[0], classifcations.shape[1], 3))
axs[1].imshow(patch.data['NDVI'][0, :, :, 0])
axs[1].set_title('NDVI')
axs[2].imshow(patch.data['FDI'][0, :, :, 0])
axs[2].set_title('FDI')
for cat in colors.keys():
mask = classifcations == cat
axs[3].scatter(norm_ndvi[mask].flatten(),
norm_fdi[mask].flatten(),
c=colors[cat],
s=0.5,
alpha=0.2)
if (features):
features.plot.scatter(
x='normed_ndvi',
y='normed_fdi',
ax=axs[3],
color=features.label.apply(lambda l: colors[catMap[l]]))
axs[4].imshow(norm_ndvi)
axs[4].set_title('Normed NDVI')
axs[5].imshow(norm_fdi)
axs[5].set_title('Normed FDI')
plt.tight_layout()
plt.savefig(Path(patchDir) / 'classifications.png')
plt.close(fig)
def get_post_processing_workflow(config: PostProcessConfig) -> LinearWorkflow:
sh_config = set_sh_config(config)
load_task = LoadTask(path=f's3://{config.bucket_name}/{config.eopatches_folder}',
features=[config.feature_extent,
config.feature_boundary,
(FeatureType.MASK, 'CLM'),
(FeatureType.MASK, 'IS_DATA'),
FeatureType.TIMESTAMP,
FeatureType.META_INFO,
FeatureType.BBOX],
config=sh_config), 'Load EOPatch'
merge_extent_tasks = [(TemporalMerging(feature=config.feature_extent,
feature_merged=(FeatureType.DATA_TIMELESS,
f'{config.feature_extent[1]}_{month}'),
woy_start=woy_start, woy_end=woy_end,
percentile=config.percentile,
max_cloud_coverage=config.max_cloud_coverage), f'Merge EXTENT for {month}')
for month, (woy_start, woy_end) in config.time_intervals.items()]
merge_boundary_tasks = [(TemporalMerging(feature=config.feature_boundary,
feature_merged=(FeatureType.DATA_TIMELESS,
f'{config.feature_boundary[1]}_{month}'),
woy_start=woy_start, woy_end=woy_end,
percentile=config.percentile,
max_cloud_coverage=config.max_cloud_coverage),
f'Merge BOUNDARY for {month}')
for month, (woy_start, woy_end) in config.time_intervals.items()]
combine_tasks = [(CombineUpsample(
feature_extent=(FeatureType.DATA_TIMELESS, f'{config.feature_extent[1]}_{month}'),
feature_boundary=(FeatureType.DATA_TIMELESS, f'{config.feature_boundary[1]}_{month}'),
feature_output=(FeatureType.DATA_TIMELESS, f'PREDICTED_{config.model_version}_{month}'),
scale_factor=config.scale_factor, disk_size=config.disk_size), f'Combine masks for {month}')
for month in config.time_intervals]
save_task = SaveTask(path=f's3://{config.bucket_name}/{config.eopatches_folder}',
features=[(FeatureType.DATA_TIMELESS, f'{config.feature_extent[1]}_{month}')
for month in config.time_intervals] +
[(FeatureType.DATA_TIMELESS, f'{config.feature_boundary[1]}_{month}')
for month in config.time_intervals] +
[(FeatureType.DATA_TIMELESS, f'PREDICTED_{config.model_version}_{month}')
for month in config.time_intervals],
overwrite_permission=OverwritePermission.OVERWRITE_FEATURES,
config=sh_config), 'Save Task'
export_tasks = [(ExportToTiff(feature=(FeatureType.DATA_TIMELESS, f'PREDICTED_{config.model_version}_{month}'),
folder=f's3://{config.bucket_name}/{config.tiffs_folder}/{month}/',
image_dtype=np.float32), f'Export tiffs for {month}')
for month in config.time_intervals]
workflow = LinearWorkflow(load_task, *merge_extent_tasks, *merge_boundary_tasks,
*combine_tasks, save_task, *export_tasks)
return workflow
def test_output_task_in_workflow(test_eopatch_path, test_eopatch):
load = EONode(LoadTask(test_eopatch_path))
output = EONode(OutputTask(name="result-name"), inputs=[load])
workflow = EOWorkflow([load, output, EONode(DummyTask(), inputs=[load])])
results = workflow.execute()
assert len(results.outputs) == 1
assert results.outputs["result-name"] == test_eopatch
def extract_targets(patchDir):
path = patchDir
if (type(path) != str):
path = str(path)
patch = LoadTask(path=path).execute()
box = patch.bbox
classification = patch.data['CLASSIFICATION'][0, :, :, 0]
print(classification)
for coord in np.argwhere(classification == catMap['Debris']):
print(coord)
def test_save_and_load_tasks(eopatch, fs_loader):
folder = "foo-folder"
patch_folder = "patch-folder"
with fs_loader() as temp_fs:
temp_fs.makedir(folder)
save_task = SaveTask(folder, filesystem=temp_fs, compress_level=9)
load_task = LoadTask(folder, filesystem=temp_fs, lazy_loading=False)
saved_eop = save_task(eopatch, eopatch_folder=patch_folder)
bbox_path = fs.path.join(folder, patch_folder, "bbox.geojson.gz")
assert temp_fs.exists(bbox_path)
assert saved_eop == eopatch
eop = load_task(eopatch_folder=patch_folder)
assert eop == eopatch
def test_save_and_load_tasks(self):
folder = 'foo-folder'
patch_folder = 'patch-folder'
for fs_loader in self.filesystem_loaders:
with fs_loader() as temp_fs:
temp_fs.makedir(folder)
save_task = SaveTask(folder,
filesystem=temp_fs,
compress_level=9)
load_task = LoadTask(folder,
filesystem=temp_fs,
lazy_loading=False)
saved_eop = save_task(self.eopatch,
eopatch_folder=patch_folder)
bbox_path = fs.path.join(folder, patch_folder, 'bbox.pkl.gz')
self.assertTrue(temp_fs.exists(bbox_path))
self.assertEqual(saved_eop, self.eopatch)
eop = load_task(eopatch_folder=patch_folder)
self.assertEqual(eop, self.eopatch)
def load_and_apply_local_norm(feature_index,method, window_size):
'''A function to apply the local normalization step to each feature
Parameters:
feature (GeoSeries): A row from the GeoDataFrame produced by load_fetures_from_file
feature_index (int): The integer used in saving the EOPatch to disk.
method: One of 'min', 'median' or 'mean' indicating the type of averaging the window function should use.
window_size: The extent in pixles that averaging should carried out over.
Returns:
EOPatch including the normalized data
'''
load_task = LoadTask(path=f'data/Training/feature_{feature_index}/')
local_norm = LocalNormalization()
workflow = LinearWorkflow(load_task, local_norm)
patch = workflow.execute({
local_norm: {
'method' : method,
'window_size': window_size
}
})
return patch
def execute(self, eopatch):
elevation = eopatch[self.feature[0]][self.feature[1]].squeeze()
gradient = ndimage.gaussian_gradient_magnitude(elevation, 1)
eopatch.add_feature(self.result_feature[0], self.result_feature[1],
gradient[..., np.newaxis])
return eopatch
if __name__ == '__main__':
# path = 'E:/Data/PerceptiveSentinel'
path = '/home/beno/Documents/test/Slovenia/'
size_small = (337, 333)
size_big = (505, 500)
load = LoadTask(path, lazy_loading=True)
save_path_location = path
if not os.path.isdir(save_path_location):
os.makedirs(save_path_location)
save = SaveTask(save_path_location,
overwrite_permission=OverwritePermission.OVERWRITE_PATCH)
dem = SentinelHubDemTask((FeatureType.DATA_TIMELESS, 'DEM'), size=size_big)
grad = AddGradientTask((FeatureType.DATA_TIMELESS, 'DEM'),
(FeatureType.DATA_TIMELESS, 'INCLINATION'))
workflow = LinearWorkflow(load, dem, grad, save)
no_patches = 1061
execution_args = []
path = 'E:/Data/PerceptiveSentinel'
# path = '/home/beno/Documents/test'
gdf, bbox_list = generate_slo_shapefile(path)
broken_patches = [12]
download_patches(path, gdf, bbox_list, broken_patches)
# no_patches = 1085
no_patches = 1061
# path = '/home/beno/Documents/test'
# path = 'E:/Data/PerceptiveSentinel'
patch_location = path + '/Slovenia/'
load = LoadTask(patch_location, lazy_loading=True)
save_path_location = path + '/Slovenia/'
if not os.path.isdir(save_path_location):
os.makedirs(save_path_location)
save = SaveTask(save_path_location, overwrite_permission=OverwritePermission.OVERWRITE_PATCH)
addStreamNDVI = AddStreamTemporalFeaturesTask(data_feature='NDVI')
addStreamSAVI = AddStreamTemporalFeaturesTask(data_feature='SAVI')
addStreamEVI = AddStreamTemporalFeaturesTask(data_feature='EVI')
addStreamARVI = AddStreamTemporalFeaturesTask(data_feature='ARVI')
addStreamSIPI = AddStreamTemporalFeaturesTask(data_feature='SIPI')
addStreamNDWI = AddStreamTemporalFeaturesTask(data_feature='NDWI')
# add_data = S2L1CWCSInput(
def run_prediction_on_eopatch(
eopatch_name: str,
config: PredictionConfig,
model: ResUnetA = None,
normalisation_factors: pd.DataFrame = None) -> dict:
""" Run prediction workflow on one eopatch. Model and dataframe can be provided to avoid loading them every time """
sh_config = set_sh_config(config)
filesystem = prepare_filesystem(config)
if normalisation_factors is None:
normalisation_factors = load_metadata(filesystem, config)
if model is None:
model = load_model(filesystem, config)
load_task = LoadTask(
path=f's3://{config.bucket_name}/{config.eopatches_folder}',
features=[
config.feature_bands, config.reference_distance,
config.reference_extent, config.reference_boundary,
FeatureType.TIMESTAMP, FeatureType.META_INFO, FeatureType.BBOX
],
config=sh_config)
save_task = SaveTask(
path=f's3://{config.bucket_name}/{config.eopatches_folder}',
features=[
config.feature_extent, config.feature_boundary,
config.feature_distance, FeatureType.META_INFO
],
overwrite_permission=OverwritePermission.OVERWRITE_FEATURES,
config=sh_config)
try:
eop = load_task.execute(eopatch_folder=eopatch_name)
eop = prediction_fn(eop,
normalisation_factors=normalisation_factors,
normalise=config.normalise,
model=model,
model_name=config.model_name,
extent_feature=config.feature_extent,
boundary_feature=config.feature_boundary,
distance_feature=config.feature_distance,
suffix=config.model_version,
batch_size=config.batch_size,
n_classes=config.n_classes,
bands_feature=config.feature_bands,
reference_boundary=config.reference_boundary,
reference_distance=config.reference_distance,
reference_extent=config.reference_extent)
_ = save_task.execute(eop, eopatch_folder=eopatch_name)
del eop
return dict(name=eopatch_name, status='Success')
except Exception as exc:
return dict(name=eopatch_name, status=exc)