def test_nonexistent_location(self):
path = './folder/subfolder/new-eopatch/'
empty_eop = EOPatch()
for fs_loader in self.filesystem_loaders:
with fs_loader() as temp_fs:
with self.assertRaises(ResourceNotFound):
EOPatch.load(path, filesystem=temp_fs)
empty_eop.save(path, filesystem=temp_fs)
with TempFS() as temp_fs:
full_path = os.path.join(temp_fs.root_path, path)
with self.assertRaises(CreateFailed):
EOPatch.load(full_path)
load_task = LoadTask(full_path)
with self.assertRaises(CreateFailed):
load_task.execute()
empty_eop.save(full_path)
self.assertTrue(os.path.exists(full_path))
with TempFS() as temp_fs:
full_path = os.path.join(temp_fs.root_path, path)
save_task = SaveTask(full_path)
save_task.execute(empty_eop)
self.assertTrue(os.path.exists(full_path))
def test_nonexistent_location(fs_loader):
path = "./folder/subfolder/new-eopatch/"
empty_eop = EOPatch()
with fs_loader() as temp_fs:
with pytest.raises(ResourceNotFound):
EOPatch.load(path, filesystem=temp_fs)
empty_eop.save(path, filesystem=temp_fs)
with TempFS() as temp_fs:
full_path = os.path.join(temp_fs.root_path, path)
with pytest.raises(CreateFailed):
EOPatch.load(full_path)
load_task = LoadTask(full_path)
with pytest.raises(CreateFailed):
load_task.execute()
empty_eop.save(full_path)
assert os.path.exists(full_path)
with TempFS() as temp_fs:
full_path = os.path.join(temp_fs.root_path, path)
save_task = SaveTask(full_path)
save_task.execute(empty_eop)
assert os.path.exists(full_path)
def get_exec_args(workflow: LinearWorkflow, eopatch_list: List[str], config: PostProcessConfig) -> List[dict]:
""" Utility function to get execution arguments """
exec_args = []
tasks = workflow.get_tasks()
load_bbox = LoadTask(path=f's3://{config.bucket_name}/{config.eopatches_folder}', features=[FeatureType.BBOX])
for name in tqdm(eopatch_list):
single_exec_dict = {}
try:
eop = load_bbox.execute(eopatch_folder=name)
for task_name, task in tasks.items():
if isinstance(task, ExportToTiff):
single_exec_dict[task] = dict(filename=f'{name}-{eop.bbox.crs.epsg}.tiff')
if isinstance(task, (LoadTask, SaveTask)):
single_exec_dict[task] = dict(eopatch_folder=name)
exec_args.append(single_exec_dict)
except ResourceNotFound as exc:
print(f'{name} - {exc}')
return exec_args
def run_prediction_on_eopatch(
eopatch_name: str,
config: PredictionConfig,
model: ResUnetA = None,
normalisation_factors: pd.DataFrame = None) -> dict:
""" Run prediction workflow on one eopatch. Model and dataframe can be provided to avoid loading them every time """
sh_config = set_sh_config(config)
filesystem = prepare_filesystem(config)
if normalisation_factors is None:
normalisation_factors = load_metadata(filesystem, config)
if model is None:
model = load_model(filesystem, config)
load_task = LoadTask(
path=f's3://{config.bucket_name}/{config.eopatches_folder}',
features=[
config.feature_bands, config.reference_distance,
config.reference_extent, config.reference_boundary,
FeatureType.TIMESTAMP, FeatureType.META_INFO, FeatureType.BBOX
],
config=sh_config)
save_task = SaveTask(
path=f's3://{config.bucket_name}/{config.eopatches_folder}',
features=[
config.feature_extent, config.feature_boundary,
config.feature_distance, FeatureType.META_INFO
],
overwrite_permission=OverwritePermission.OVERWRITE_FEATURES,
config=sh_config)
try:
eop = load_task.execute(eopatch_folder=eopatch_name)
eop = prediction_fn(eop,
normalisation_factors=normalisation_factors,
normalise=config.normalise,
model=model,
model_name=config.model_name,
extent_feature=config.feature_extent,
boundary_feature=config.feature_boundary,
distance_feature=config.feature_distance,
suffix=config.model_version,
batch_size=config.batch_size,
n_classes=config.n_classes,
bands_feature=config.feature_bands,
reference_boundary=config.reference_boundary,
reference_distance=config.reference_distance,
reference_extent=config.reference_extent)
_ = save_task.execute(eop, eopatch_folder=eopatch_name)
del eop
return dict(name=eopatch_name, status='Success')
except Exception as exc:
return dict(name=eopatch_name, status=exc)
