def run_model_test(self, model):
MachineLearningTestCase.train_sklearn_model(model=model)
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_sklearn_ml.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
red = create_datacube(name="red",
value=1,
dims=("t", "x", "y"),
shape=(2, 2, 2))
nir = create_datacube(name="nir",
value=1,
dims=("t", "x", "y"),
shape=(2, 2, 2))
ml = MachineLearnModelConfig(
framework="sklearn",
name="random_forest",
description=
"A sklearn model that adds two numbers in range of [1,1]",
path="/tmp/rf_add_model.pkl.xz")
udf_data = UdfData(proj={"EPSG": 4326},
datacube_list=[red, nir],
ml_model_list=[ml])
pprint.pprint(udf_data.to_dict())
run_user_code(code=udf_code.source, data=udf_data)
result = udf_data.to_dict()
self.assertAlmostEqual(2.0, result['datacubes'][0]['data'][0][0][0], 2)
def unused_test_DataCube_ndvi_message_pack(self):
"""Test the DataCube NDVI computation with the message pack protocol"""
# TODO: Reactivate this test
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_ndvi.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
hc_red = create_datacube(name="red",
value=1,
dims=("t", "y", "x"),
shape=(3, 3, 3))
hc_nir = create_datacube(name="nir",
value=3,
dims=("t", "y", "x"),
shape=(3, 3, 3))
udf_data = UdfData(proj={"EPSG": 4326}, datacube_list=[hc_red, hc_nir])
udf_request = UdfRequestModel(data=udf_data.to_dict(), code=udf_code)
udf_request = base64.b64encode(
msgpack.packb(udf_request.dict(), use_bin_type=True))
response = self.app.post(
'/udf_message_pack',
data=udf_request,
headers={"Content-Type": "application/base64"})
self.assertEqual(response.status_code, 200)
blob = base64.b64decode(response.content)
udf_data = msgpack.unpackb(blob, raw=False)
self.checkDataCubeNdvi(udf_data=udf_data)
def test_sklearn_extra_tree_message_pack_md5_hash(self):
"""Test extra tree training and UDF application with message pack protocol and the machine learn model
uploaded to the UDF md5 hash based storage system"""
model = ExtraTreesRegressor(n_estimators=100,
max_depth=7,
max_features="log2",
min_samples_split=2,
min_samples_leaf=1,
verbose=0)
model_path = MachineLearningTestCase.train_sklearn_model(model=model)
request_model = RequestStorageModel(
uri=model_path,
title="This is a test model",
description="This is the test description.")
response = self.app.post('/storage', json=request_model.dict())
print(response.content)
self.assertEqual(response.status_code, 200)
md5_hash = response.content.decode("ascii").strip().replace("\"", "")
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_sklearn_ml.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
red = create_datacube(name="red",
value=1,
dims=("t", "x", "y"),
shape=(2, 2, 2))
nir = create_datacube(name="nir",
value=1,
dims=("t", "x", "y"),
shape=(2, 2, 2))
ml = MachineLearnModelConfig(
framework="sklearn",
name="random_forest",
description=
"A sklearn model that adds two numbers in range of [1,1]",
md5_hash=md5_hash)
udf_data = UdfData(proj={"EPSG": 4326},
datacube_list=[red, nir],
ml_model_list=[ml])
pprint.pprint(udf_data.to_dict())
run_user_code(code=udf_code.source, data=udf_data)
result = udf_data.to_dict()
self.assertAlmostEqual(2.0, result['datacubes'][0]['data'][0][0][0], 2)
#result = self.send_msgpack_request(data=udf_data, code=udf_code)
#self.assertAlmostEqual(2.0, result['datacubes'][0]['data'][0][0][0], 2)
response = self.app.delete(f'/storage/{md5_hash}')
self.assertEqual(response.status_code, 200)
def test_rct_stats(self):
"""Test the raster collection tile statistics UDF"""
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_statistics.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
temp = create_datacube(name="temp",
value=1,
dims=("t", "x", "y"),
shape=(3, 3, 3))
udf_data = UdfData(proj={"EPSG": 4326}, datacube_list=[temp])
run_user_code(code=udf_code.source, data=udf_data)
result = udf_data.to_dict()
self.assertEqual(len(result["datacubes"]), 0)
self.assertEqual(len(result["structured_data_list"]), 1)
self.assertEqual(result["structured_data_list"][0]["type"], "dict")
self.assertEqual(result["structured_data_list"][0]["data"]["temp"], {
'max': 1.0,
'mean': 1.0,
'min': 1.0,
'sum': 27.0
})
def test_pytorch_linear_nn(self):
"""Test linear pytorch model training and UDF application"""
model = SimpleNetwork()
MachineLearningPytorchTestCase.train_pytorch_model(model=model)
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_pytorch_ml.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
temp = create_datacube(name="temp",
value=1,
dims=("x", "y"),
shape=(2, 2))
ml = MachineLearnModelConfig(
framework="pytorch",
name="linear_model",
description=
"A pytorch model that adds two numbers in range of [1,1]",
path="/tmp/simple_linear_nn_pytorch.pt")
udf_data = UdfData(proj={"EPSG": 4326},
datacube_list=[temp],
ml_model_list=[ml])
run_user_code(code=udf_code.source, data=udf_data)
pprint.pprint(udf_data.to_dict())
def test_timeseries_wrapper(self):
temp = create_datacube(name="temp", value=1, shape=(3, 3, 3))
udf_data = UdfData(proj={"EPSG": 4326}, datacube_list=[temp])
from openeo_udf.api.udf_wrapper import apply_timeseries_generic
rcts = udf_data.get_datacube_list
apply_timeseries_generic(udf_data)
self.assertEqual(rcts, udf_data.get_datacube_list)
def test_DataCube_reduce_mean(self):
"""Test the DataCube mean reduction"""
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_reduce_time_mean.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
temp = create_datacube(name="temp",
value=1,
dims=("t", "y", "x"),
shape=(3, 3, 3))
prec = create_datacube(name="prec",
value=3,
dims=("t", "y", "x"),
shape=(3, 3, 3))
udf_data = UdfData(proj={"EPSG": 4326}, datacube_list=[temp, prec])
run_user_code(code=udf_code.source, data=udf_data)
self.checkDataCubeMean(udf_data=udf_data)
def test_DataCube_ndvi(self):
"""Test the DataCube NDVI computation"""
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_ndvi.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
hc_red = create_datacube(name="red",
value=1,
dims=("t", "y", "x"),
shape=(3, 3, 3))
hc_nir = create_datacube(name="nir",
value=3,
dims=("t", "y", "x"),
shape=(3, 3, 3))
udf_data = UdfData(proj={"EPSG": 4326}, datacube_list=[hc_red, hc_nir])
run_user_code(code=udf_code.source, data=udf_data)
self.checkDataCubeNdvi(udf_data=udf_data)
def test_DataCube_map_fabs(self):
"""Test the DataCube mapping of the numpy fabs function"""
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_map_fabs.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
temp = create_datacube(name="temp",
value=1,
dims=("t", "x", "y"),
shape=(3, 3, 3))
udf_data = UdfData(proj={"EPSG": 4326}, datacube_list=[temp])
run_user_code(code=udf_code.source, data=udf_data)
self.checkDataCubeMapFabs(udf_data=udf_data)
def not_implemented_yet_test_sampling(self):
"""Test the feature collection sampling UDF"""
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_sampling.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
temp = create_datacube(name="temp", value=1, shape=(3, 3, 3))
udf_data = UdfData(proj={"EPSG": 4326}, datacube_list=[temp])
run_user_code(code=udf_code.source, data=udf_data)
result = udf_data.to_dict()
self.assertEqual(len(result["feature_collection_tiles"]), 1)
self.assertEqual(
len(result["feature_collection_tiles"][0]["data"]["features"]), 1)
self.assertEqual(
result["feature_collection_tiles"][0]["data"]["features"][0]
["properties"], {'temp': 4})
