class DataCubeStatisticsTestCase(unittest.TestCase):
create_storage_directory()
def setUp(self):
self.app = TestClient(app=app)
def test_rct_stats(self):
"""Test the raster collection tile statistics UDF"""
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_statistics.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
temp = create_datacube(name="temp",
value=1,
dims=("t", "x", "y"),
shape=(3, 3, 3))
udf_data = UdfData(proj={"EPSG": 4326}, datacube_list=[temp])
run_user_code(code=udf_code.source, data=udf_data)
result = udf_data.to_dict()
self.assertEqual(len(result["datacubes"]), 0)
self.assertEqual(len(result["structured_data_list"]), 1)
self.assertEqual(result["structured_data_list"][0]["type"], "dict")
self.assertEqual(result["structured_data_list"][0]["data"]["temp"], {
'max': 1.0,
'mean': 1.0,
'min': 1.0,
'sum': 27.0
})
class DataCubeMapFabsTestCase(unittest.TestCase):
create_storage_directory()
def setUp(self):
self.app = TestClient(app=app)
def test_DataCube_map_fabs(self):
"""Test the DataCube mapping of the numpy fabs function"""
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_map_fabs.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
temp = create_datacube(name="temp",
value=1,
dims=("t", "x", "y"),
shape=(3, 3, 3))
udf_data = UdfData(proj={"EPSG": 4326}, datacube_list=[temp])
run_user_code(code=udf_code.source, data=udf_data)
self.checkDataCubeMapFabs(udf_data=udf_data)
def checkDataCubeMapFabs(self, udf_data: UdfData):
"""Check the mapped fabs hyper cube data that was processed in the UDF server"""
hc_ndvi: DataCube = udf_data.datacube_list[0]
self.assertEqual(hc_ndvi.id, "temp_fabs")
self.assertEqual(hc_ndvi.array.name, "temp_fabs")
self.assertEqual(hc_ndvi.array.data.shape, (3, 3, 3))
self.assertEqual(hc_ndvi.array.data[0][0][0], 1)
self.assertEqual(hc_ndvi.array.data[2][2][2], 1)
class DataCubeApiTestCase(unittest.TestCase):
create_storage_directory()
def setUp(self):
self.app = TestClient(app=app)
def test_hypercube_api(self):
"""Test the hypercube mean reduction"""
dcm = create_data_collection_model_example()
dc = DataCube.from_data_collection(data_collection=dcm)
print(dc[0].get_array())
print(dc[1].get_array())
dc1: DataCube = dc[0]
dc2: DataCube = dc[1]
self.assertEqual(dc1.id,
dcm.variables_collections[0].variables[0].name)
self.assertEqual(dc2.id,
dcm.variables_collections[0].variables[1].name)
a1: xarray.DataArray = dc1.get_array()
a1 = numpy.asarray(a1).reshape([27])
v1 = dcm.variables_collections[0].variables[0].values
v1 = numpy.asarray(v1)
self.assertTrue(a1.all() == v1.all())
a2: xarray.DataArray = dc2.get_array()
a2 = numpy.asarray(a2).reshape([27])
v2 = dcm.variables_collections[0].variables[1].values
v2 = numpy.asarray(v2)
self.assertTrue(a2.all() == v2.all())
class HypercubeSamplingTestCase(unittest.TestCase):
create_storage_directory()
def setUp(self):
self.app = TestClient(app=app)
def not_implemented_yet_test_sampling(self):
"""Test the feature collection sampling UDF"""
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_sampling.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
temp = create_datacube(name="temp", value=1, shape=(3, 3, 3))
udf_data = UdfData(proj={"EPSG": 4326}, datacube_list=[temp])
run_user_code(code=udf_code.source, data=udf_data)
result = udf_data.to_dict()
self.assertEqual(len(result["feature_collection_tiles"]), 1)
self.assertEqual(
len(result["feature_collection_tiles"][0]["data"]["features"]), 1)
self.assertEqual(
result["feature_collection_tiles"][0]["data"]["features"][0]
["properties"], {'temp': 4})
class DataCubeMinMedianMaxTestCase(unittest.TestCase):
create_storage_directory()
def setUp(self):
self.app = TestClient(app=app)
def test_DataCube_reduce_min_median_max(self):
"""Test the DataCube min, median, max reduction"""
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_reduce_time_min_median_max.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
temp = create_datacube(name="temp",
value=1,
dims=("t", "y", "x"),
shape=(3, 3, 3))
udf_data = UdfData(proj={"EPSG": 4326}, datacube_list=[temp])
run_user_code(code=udf_code.source, data=udf_data)
self.check_DataCube_min_median_max(udf_data=udf_data)
def check_DataCube_min_median_max(self, udf_data):
"""Check the min, median, max hyper cube data that was processed in the UDF server"""
hc: DataCube = udf_data.datacube_list[0]
self.assertEqual(hc.id, "temp_min")
self.assertEqual(hc.array.name, "temp_min")
self.assertEqual(hc.array.data.shape, (3, 3))
self.assertEqual(hc.array.data[0][0], 1)
self.assertEqual(hc.array.data[2][2], 1)
hc: DataCube = udf_data.datacube_list[1]
self.assertEqual(hc.id, "temp_median")
self.assertEqual(hc.array.name, "temp_median")
self.assertEqual(hc.array.data.shape, (3, 3))
self.assertEqual(hc.array.data[0][0], 1)
self.assertEqual(hc.array.data[2][2], 1)
hc: DataCube = udf_data.datacube_list[2]
self.assertEqual(hc.id, "temp_max")
self.assertEqual(hc.array.name, "temp_max")
self.assertEqual(hc.array.data.shape, (3, 3))
self.assertEqual(hc.array.data[0][0], 1)
self.assertEqual(hc.array.data[2][2], 1)
class MachineLearningTestCase(unittest.TestCase):
create_storage_directory()
def setUp(self):
self.app = TestClient(app)
@staticmethod
def compute_efficiency(model_result, measurement):
diff = model_result - measurement
eff = 1 - sum(diff * diff) / ((measurement.var()) * len(measurement))
return (eff)
@staticmethod
def train_sklearn_model(model):
"""This method trains a sklearn random forest regressor to add two numbers that must be
in range [1,2,3]. The input arrays into the model must have the names *red* and *nir*.
Args:
model: The machine learn model to be used for training
Returns:
str:
The filename of the resulting
"""
# Train a value adder that represents the formula (a + b)
a = np.random.randint(1, 4, 1000)
b = np.random.randint(1, 4, 1000)
# Create the predicting data that is used for training
y = (a + b)
print("Train model ", model.__class__)
# This is the training data with two arrays
X = pd.DataFrame()
X["red"] = a
X["nir"] = b
# Fit the model and compute the model efficiency
model = model.fit(X, y)
# Predict values
predicted_values = model.predict(X)
# Compute the score of the model
score = model.score(X, y)
# Compute the mean square error
mse = mean_squared_error(predicted_values, y)
print("Model score", score, "MSE", mse)
print("Save the model as compressed joblib object")
# Save the model with compression
file_name = '/tmp/rf_add_model.pkl.xz'
joblib.dump(value=model, filename=file_name, compress=("xz", 3))
return file_name
def send_json_request(self, data: UdfData, code: UdfCodeModel) -> Dict:
udf_request = UdfRequestModel(data=data.to_dict(), code=code)
result = run_legacy_user_code(dict_data=udf_request.dict())
return result
def send_msgpack_request(self, data: UdfData, code: UdfCodeModel) -> Dict:
return self.send_json_request(data=data, code=code)
# TODO: Implement the code below
udf_request = UdfRequestModel(data=data.to_dict(), code=code)
udf_request = base64.b64encode(
msgpack.packb(udf_request.dict(), use_bin_type=True))
response = self.app.post(
'/udf_message_pack',
data=udf_request,
headers={"Content-Type": "application/base64"})
self.assertEqual(response.status_code, 200)
blob = base64.b64decode(response.content)
result = msgpack.unpackb(blob, raw=False)
return result
def run_model_test(self, model):
MachineLearningTestCase.train_sklearn_model(model=model)
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_sklearn_ml.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
red = create_datacube(name="red",
value=1,
dims=("t", "x", "y"),
shape=(2, 2, 2))
nir = create_datacube(name="nir",
value=1,
dims=("t", "x", "y"),
shape=(2, 2, 2))
ml = MachineLearnModelConfig(
framework="sklearn",
name="random_forest",
description=
"A sklearn model that adds two numbers in range of [1,1]",
path="/tmp/rf_add_model.pkl.xz")
udf_data = UdfData(proj={"EPSG": 4326},
datacube_list=[red, nir],
ml_model_list=[ml])
pprint.pprint(udf_data.to_dict())
run_user_code(code=udf_code.source, data=udf_data)
result = udf_data.to_dict()
self.assertAlmostEqual(2.0, result['datacubes'][0]['data'][0][0][0], 2)
#result = self.send_msgpack_request(data=udf_data, code=udf_code)
#self.assertAlmostEqual(2.0, result['datacubes'][0]['data'][0][0][0], 2)
def test_sklearn_random_forest_with_msgpack(self):
"""Test random forest model training and UDF application"""
model = RandomForestRegressor(n_estimators=100,
max_depth=7,
max_features="log2",
n_jobs=16,
min_samples_split=2,
min_samples_leaf=1,
verbose=0)
self.run_model_test(model=model)
def test_sklearn_gradient_boost(self):
"""Test gradent boost model training and UDF application"""
model = GradientBoostingRegressor(n_estimators=100,
max_depth=7,
max_features="log2",
min_samples_split=2,
min_samples_leaf=1,
verbose=0)
self.run_model_test(model=model)
def test_sklearn_extra_tree(self):
"""Test extra tree training and UDF application"""
model = ExtraTreesRegressor(n_estimators=100,
max_depth=7,
max_features="log2",
min_samples_split=2,
min_samples_leaf=1,
verbose=0)
self.run_model_test(model=model)
def test_sklearn_extra_tree_message_pack_md5_hash(self):
"""Test extra tree training and UDF application with message pack protocol and the machine learn model
uploaded to the UDF md5 hash based storage system"""
model = ExtraTreesRegressor(n_estimators=100,
max_depth=7,
max_features="log2",
min_samples_split=2,
min_samples_leaf=1,
verbose=0)
model_path = MachineLearningTestCase.train_sklearn_model(model=model)
request_model = RequestStorageModel(
uri=model_path,
title="This is a test model",
description="This is the test description.")
response = self.app.post('/storage', json=request_model.dict())
print(response.content)
self.assertEqual(response.status_code, 200)
md5_hash = response.content.decode("ascii").strip().replace("\"", "")
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_sklearn_ml.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
red = create_datacube(name="red",
value=1,
dims=("t", "x", "y"),
shape=(2, 2, 2))
nir = create_datacube(name="nir",
value=1,
dims=("t", "x", "y"),
shape=(2, 2, 2))
ml = MachineLearnModelConfig(
framework="sklearn",
name="random_forest",
description=
"A sklearn model that adds two numbers in range of [1,1]",
md5_hash=md5_hash)
udf_data = UdfData(proj={"EPSG": 4326},
datacube_list=[red, nir],
ml_model_list=[ml])
pprint.pprint(udf_data.to_dict())
run_user_code(code=udf_code.source, data=udf_data)
result = udf_data.to_dict()
self.assertAlmostEqual(2.0, result['datacubes'][0]['data'][0][0][0], 2)
#result = self.send_msgpack_request(data=udf_data, code=udf_code)
#self.assertAlmostEqual(2.0, result['datacubes'][0]['data'][0][0][0], 2)
response = self.app.delete(f'/storage/{md5_hash}')
self.assertEqual(response.status_code, 200)
class MachineLearningModelStorageTestCase(unittest.TestCase):
create_storage_directory()
def setUp(self):
self.app = TestClient(app=app)
def test_ml_storage_post_get_delete(self):
path = "/tmp/test_file"
content = b"content"
# Create a dummy file
file = open(path, "wb")
file.write(content)
file.close()
request_model = RequestStorageModel(
uri=path,
title="This is a test model",
description="This is the test description.")
response = self.app.post('/storage', json=request_model.dict())
print(response.content)
self.assertEqual(response.status_code, 200)
md5_hash = response.content.decode("ascii")
self.assertEqual(md5_hash, md5(content).hexdigest())
response = self.app.get('/storage')
pprint(response.json())
self.assertEqual(response.status_code, 200)
response = self.app.delete(f'/storage/{md5_hash}')
print(response.content)
self.assertEqual(response.status_code, 200)
md5_hash = response.content.decode("ascii")
self.assertEqual(md5_hash, md5(content).hexdigest())
def test_ml_storage_post_get_delete_url(self):
url = "https://storage.googleapis.com/datentransfer/europe_countries.geojson"
request_model = RequestStorageModel(
uri=url,
title="This is a test model",
description="This is the test description.")
response = self.app.post('/storage', json=request_model.dict())
print(response.content)
self.assertEqual(response.status_code, 200)
response = self.app.get('/storage')
pprint(response.json())
self.assertEqual(response.status_code, 200)
md5_hash = None
model_list: List[dict] = response.json()
for model in model_list:
model = ResponseStorageModel(**model)
if model.source == url:
md5_hash = model.md5_hash
self.assertIsNotNone(md5_hash)
response = self.app.delete(f'/storage/{md5_hash}')
print(response.content)
self.assertEqual(response.status_code, 200)
def test_ml_storage_post_url_error(self):
url = "https://nopopopop.de/file.txt"
request_model = RequestStorageModel(
uri=url,
title="This is a test model",
description="This is the test description.")
response = self.app.post('/storage', json=request_model.dict())
# pprint(response.json())
self.assertEqual(response.status_code, 400)
class MachineLearningPytorchTestCase(unittest.TestCase):
create_storage_directory()
def setUp(self):
self.app = TestClient(app=app)
@staticmethod
def train_pytorch_model(model):
"""Train an arbitrary pytroch model with two features
Args:
model: The machine learn model to be used for training
Returns:
str:
The filename of the resulting
"""
model_path = '/tmp/simple_linear_nn_pytorch.pt'
criterion = nn.MSELoss()
a = numpy.random.randint(1, 4, 10)
a = a.reshape([5, 2])
input = Variable(torch.Tensor(a))
target = Variable(torch.Tensor(a))
for i in range(200):
output = model(input)
loss = criterion(output, target)
model.zero_grad()
loss.backward()
optimizer = optim.SGD(model.parameters(), lr=0.1)
optimizer.step()
torch.save(model, model_path)
def test_pytorch_linear_nn(self):
"""Test linear pytorch model training and UDF application"""
model = SimpleNetwork()
MachineLearningPytorchTestCase.train_pytorch_model(model=model)
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_pytorch_ml.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
temp = create_datacube(name="temp",
value=1,
dims=("x", "y"),
shape=(2, 2))
ml = MachineLearnModelConfig(
framework="pytorch",
name="linear_model",
description=
"A pytorch model that adds two numbers in range of [1,1]",
path="/tmp/simple_linear_nn_pytorch.pt")
udf_data = UdfData(proj={"EPSG": 4326},
datacube_list=[temp],
ml_model_list=[ml])
run_user_code(code=udf_code.source, data=udf_data)
pprint.pprint(udf_data.to_dict())
class DataCubeNdviTestCase(unittest.TestCase):
create_storage_directory()
def setUp(self):
self.app = TestClient(app=app)
def test_DataCube_ndvi(self):
"""Test the DataCube NDVI computation"""
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_ndvi.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
hc_red = create_datacube(name="red",
value=1,
dims=("t", "y", "x"),
shape=(3, 3, 3))
hc_nir = create_datacube(name="nir",
value=3,
dims=("t", "y", "x"),
shape=(3, 3, 3))
udf_data = UdfData(proj={"EPSG": 4326}, datacube_list=[hc_red, hc_nir])
run_user_code(code=udf_code.source, data=udf_data)
self.checkDataCubeNdvi(udf_data=udf_data)
def unused_test_DataCube_ndvi_message_pack(self):
"""Test the DataCube NDVI computation with the message pack protocol"""
# TODO: Reactivate this test
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_ndvi.py")
udf_code = UdfCodeModel(language="python",
source=open(file_name, "r").read())
hc_red = create_datacube(name="red",
value=1,
dims=("t", "y", "x"),
shape=(3, 3, 3))
hc_nir = create_datacube(name="nir",
value=3,
dims=("t", "y", "x"),
shape=(3, 3, 3))
udf_data = UdfData(proj={"EPSG": 4326}, datacube_list=[hc_red, hc_nir])
udf_request = UdfRequestModel(data=udf_data.to_dict(), code=udf_code)
udf_request = base64.b64encode(
msgpack.packb(udf_request.dict(), use_bin_type=True))
response = self.app.post(
'/udf_message_pack',
data=udf_request,
headers={"Content-Type": "application/base64"})
self.assertEqual(response.status_code, 200)
blob = base64.b64decode(response.content)
udf_data = msgpack.unpackb(blob, raw=False)
self.checkDataCubeNdvi(udf_data=udf_data)
def checkDataCubeNdvi(self, udf_data: UdfData):
"""Check the ndvi hyper cube data that was processed in the UDF server"""
hc_ndvi: DataCube = udf_data.datacube_list[0]
self.assertEqual(hc_ndvi.id, "NDVI")
self.assertEqual(hc_ndvi.array.name, "NDVI")
self.assertEqual(hc_ndvi.array.data.shape, (3, 3, 3))
self.assertEqual(hc_ndvi.array.data[0][0][0], 0.5)
self.assertEqual(hc_ndvi.array.data[2][2][2], 0.5)