def test_rct_stats(self):
"""Test the raster collection tile statistics UDF"""
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_statistics.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
temp = create_datacube(name="temp",
value=1,
dims=("t", "x", "y"),
shape=(3, 3, 3))
udf_data = UdfData(proj={"EPSG": 4326}, datacube_list=[temp])
run_user_code(code=udf_code.source, data=udf_data)
result = udf_data.to_dict()
self.assertEqual(len(result["datacubes"]), 0)
self.assertEqual(len(result["structured_data_list"]), 1)
self.assertEqual(result["structured_data_list"][0]["type"], "dict")
self.assertEqual(result["structured_data_list"][0]["data"]["temp"], {
'max': 1.0,
'mean': 1.0,
'min': 1.0,
'sum': 27.0
})
def run_model_test(self, model):
MachineLearningTestCase.train_sklearn_model(model=model)
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_sklearn_ml.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
red = create_datacube(name="red",
value=1,
dims=("t", "x", "y"),
shape=(2, 2, 2))
nir = create_datacube(name="nir",
value=1,
dims=("t", "x", "y"),
shape=(2, 2, 2))
ml = MachineLearnModelConfig(
framework="sklearn",
name="random_forest",
description=
"A sklearn model that adds two numbers in range of [1,1]",
path="/tmp/rf_add_model.pkl.xz")
udf_data = UdfData(proj={"EPSG": 4326},
datacube_list=[red, nir],
ml_model_list=[ml])
pprint.pprint(udf_data.to_dict())
run_user_code(code=udf_code.source, data=udf_data)
result = udf_data.to_dict()
self.assertAlmostEqual(2.0, result['datacubes'][0]['data'][0][0][0], 2)
def test_pytorch_linear_nn(self):
"""Test linear pytorch model training and UDF application"""
model = SimpleNetwork()
MachineLearningPytorchTestCase.train_pytorch_model(model=model)
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_pytorch_ml.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
temp = create_datacube(name="temp",
value=1,
dims=("x", "y"),
shape=(2, 2))
ml = MachineLearnModelConfig(
framework="pytorch",
name="linear_model",
description=
"A pytorch model that adds two numbers in range of [1,1]",
path="/tmp/simple_linear_nn_pytorch.pt")
udf_data = UdfData(proj={"EPSG": 4326},
datacube_list=[temp],
ml_model_list=[ml])
run_user_code(code=udf_code.source, data=udf_data)
pprint.pprint(udf_data.to_dict())
def test_sklearn_extra_tree_message_pack_md5_hash(self):
"""Test extra tree training and UDF application with message pack protocol and the machine learn model
uploaded to the UDF md5 hash based storage system"""
model = ExtraTreesRegressor(n_estimators=100,
max_depth=7,
max_features="log2",
min_samples_split=2,
min_samples_leaf=1,
verbose=0)
model_path = MachineLearningTestCase.train_sklearn_model(model=model)
request_model = RequestStorageModel(
uri=model_path,
title="This is a test model",
description="This is the test description.")
response = self.app.post('/storage', json=request_model.dict())
print(response.content)
self.assertEqual(response.status_code, 200)
md5_hash = response.content.decode("ascii").strip().replace("\"", "")
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_sklearn_ml.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
red = create_datacube(name="red",
value=1,
dims=("t", "x", "y"),
shape=(2, 2, 2))
nir = create_datacube(name="nir",
value=1,
dims=("t", "x", "y"),
shape=(2, 2, 2))
ml = MachineLearnModelConfig(
framework="sklearn",
name="random_forest",
description=
"A sklearn model that adds two numbers in range of [1,1]",
md5_hash=md5_hash)
udf_data = UdfData(proj={"EPSG": 4326},
datacube_list=[red, nir],
ml_model_list=[ml])
pprint.pprint(udf_data.to_dict())
run_user_code(code=udf_code.source, data=udf_data)
result = udf_data.to_dict()
self.assertAlmostEqual(2.0, result['datacubes'][0]['data'][0][0][0], 2)
#result = self.send_msgpack_request(data=udf_data, code=udf_code)
#self.assertAlmostEqual(2.0, result['datacubes'][0]['data'][0][0][0], 2)
response = self.app.delete(f'/storage/{md5_hash}')
self.assertEqual(response.status_code, 200)
def test_DataCube_map_fabs(self):
"""Test the DataCube mapping of the numpy fabs function"""
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_map_fabs.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
temp = create_datacube(name="temp",
value=1,
dims=("t", "x", "y"),
shape=(3, 3, 3))
udf_data = UdfData(proj={"EPSG": 4326}, datacube_list=[temp])
run_user_code(code=udf_code.source, data=udf_data)
self.checkDataCubeMapFabs(udf_data=udf_data)
def test_DataCube_reduce_min_median_max(self):
"""Test the DataCube min, median, max reduction"""
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_reduce_time_min_median_max.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
temp = create_datacube(name="temp",
value=1,
dims=("t", "y", "x"),
shape=(3, 3, 3))
udf_data = UdfData(proj={"EPSG": 4326}, datacube_list=[temp])
run_user_code(code=udf_code.source, data=udf_data)
self.check_DataCube_min_median_max(udf_data=udf_data)
def not_implemented_yet_test_sampling(self):
"""Test the feature collection sampling UDF"""
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_sampling.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
temp = create_datacube(name="temp", value=1, shape=(3, 3, 3))
udf_data = UdfData(proj={"EPSG": 4326}, datacube_list=[temp])
run_user_code(code=udf_code.source, data=udf_data)
result = udf_data.to_dict()
self.assertEqual(len(result["feature_collection_tiles"]), 1)
self.assertEqual(
len(result["feature_collection_tiles"][0]["data"]["features"]), 1)
self.assertEqual(
result["feature_collection_tiles"][0]["data"]["features"][0]
["properties"], {'temp': 4})
def test_DataCube_ndvi(self):
"""Test the DataCube NDVI computation"""
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_ndvi.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
hc_red = create_datacube(name="red",
value=1,
dims=("t", "y", "x"),
shape=(3, 3, 3))
hc_nir = create_datacube(name="nir",
value=3,
dims=("t", "y", "x"),
shape=(3, 3, 3))
udf_data = UdfData(proj={"EPSG": 4326}, datacube_list=[hc_red, hc_nir])
run_user_code(code=udf_code.source, data=udf_data)
self.checkDataCubeNdvi(udf_data=udf_data)
def run_udf(code: str, epsg_code: str,
datacube_list: List[DataCube]) -> UdfData:
"""Run the user defined code (udf) and create the required input for the function
:param code: The UDF code
:param epsg_code: The EPSG code of the projection
:param datacube: The id of the strds
:return: The resulting udf data object
"""
data = UdfData(proj={"EPSG": epsg_code}, datacube_list=datacube_list)
return run_user_code(code=code, data=data)