def test_async_persistent_processing(self):
iface = GRaaSInterface(self.gconf)
process_chain = {
"version": "1",
"list": [
{"id": "g_region_1",
"module": "g.region",
"flags": "g"}]}
status, resource_id = iface.async_persistent_processing(location="LL",
mapset="new_user_mapset",
process_chain=process_chain)
print(status)
print(resource_id)
self.assertEqual(status, 200)
status, info = iface.resource_info(resource_id)
print(status)
print(info)
time.sleep(2)
status, info = iface.resource_info(resource_id)
print(status)
print(info)
def test_list_strds(self):
iface = GRaaSInterface(self.gconf)
status, layers = iface.list_strds(location="ECAD", mapset="PERMANENT")
pprint(layers)
self.assertEqual(status, 200)
self.assertEqual(len(layers), 2)
def test_mapset_info(self):
iface = GRaaSInterface(self.gconf)
status, info = iface.mapset_info(location="ECAD", mapset="PERMANENT")
pprint(info)
self.assertEqual(status, 200)
self.assertTrue("region" in info)
self.assertTrue("projection" in info)
def test_strds_info(self):
iface = GRaaSInterface(self.gconf)
status, info = iface.layer_info(layer_name="ECAD.PERMANENT.strds.precipitation_1950_2013_yearly_mm")
pprint(info)
self.assertEqual(status, 200)
self.assertTrue("temporal_type" in info)
self.assertTrue("aggregation_type" in info)
self.assertTrue("creation_time" in info)
self.assertTrue("creator" in info)
self.assertTrue("granularity" in info)
self.assertTrue("modification_time" in info)
self.assertTrue("number_of_maps" in info)
def test_mapset_creation_deletion(self):
config = self.gconf
config.USER = "******"
iface = GRaaSInterface(config)
status, resource_id = iface.create_mapset(location="LL", mapset="new_mapset")
print(status)
self.assertEqual(status, 200)
print(resource_id)
status, resource_id = iface.delete_mapset(location="LL", mapset="new_mapset")
print(status)
self.assertEqual(status, 200)
print(resource_id)
def get_process_list(args):
"""Analyse the process description and return the GRaaS process chain and the name of the processing result layer
which is a single raster layer
:param args: The process description
:return: (output_name, pc)
"""
# Get the input description and the process chain to attach this process
input_names, process_list = process_definitions.analyse_process_graph(args)
output_names = []
for input_name in input_names:
location, mapset, datatype, layer_name = GRaaSInterface.layer_def_to_components(
input_name)
output_name = "%s_%s" % (layer_name, PROCESS_NAME)
output_names.append(output_name)
if "python_file_url" in args:
python_file_url = args["python_file_url"]
else:
raise Exception("Python fle is missing in the process description")
pc = create_graas_process_chain_entry(input_name=input_name,
python_file_url=python_file_url,
output_name=output_name)
process_list.append(pc)
return output_names, process_list
def create_graas_process_chain_entry(input_name):
"""Create a GRaaS command of the process chain that computes the regional statistics based on a
strds and a polygon.
:param input_name: The name of the raster layer
:return: A GRaaS process chain description
"""
location, mapset, datatype, layer_name = GRaaSInterface.layer_def_to_components(
input_name)
input_name = layer_name
if mapset is not None:
input_name = layer_name + "@" + mapset
rn = randint(0, 1000000)
pc = []
exporter = {
"id":
"exporter_%i" % rn,
"module":
"exporter",
"outputs": [{
"export": {
"type": "raster",
"format": "GTiff"
},
"param": "map",
"value": input_name
}]
}
pc.append(exporter)
return pc
def get_process_list(args):
"""Analyse the process description and return the GRaaS process chain and the name of the processing result
strds that was filtered by start and end date
:param args: The process description
:return: (output_name, pc)
"""
# Get the input description and the process chain to attach this process
input_names, process_list = process_definitions.analyse_process_graph(args)
output_names = []
for input_name in input_names:
location, mapset, datatype, layer_name = GRaaSInterface.layer_def_to_components(
input_name)
output_name = "%s_%s" % (layer_name, PROCESS_NAME)
output_names.append(output_name)
start_time = None
end_time = None
if "from" in args:
start_time = args["from"]
if "to" in args:
end_time = args["to"]
pc = create_graas_process_chain_entry(input_name=input_name,
start_time=start_time,
end_time=end_time,
output_name=output_name)
process_list.append(pc)
return output_names, process_list
def create_graas_process_chain_entry(nir_time_series, red_time_series, output_time_series):
"""Create a GRaaS process description that uses t.rast.series to create the minimum
value of the time series.
:param nir_time_series: The NIR band time series name
:param red_time_series: The RED band time series name
:param output_time_series: The name of the output time series
:return: A list of GRaaS process chain descriptions
"""
location, mapset, datatype, layer_name = GRaaSInterface.layer_def_to_components(nir_time_series)
nir_time_series = layer_name
if mapset is not None:
nir_time_series = layer_name + "@" + mapset
location, mapset, datatype, layer_name = GRaaSInterface.layer_def_to_components(red_time_series)
red_time_series = layer_name
if mapset is not None:
red_time_series = layer_name + "@" + mapset
location, mapset, datatype, output_name = GRaaSInterface.layer_def_to_components(output_time_series)
rn = randint(0, 1000000)
pc = [
{"id": "t_rast_mapcalc_%i" % rn,
"module": "t.rast.mapcalc",
"inputs": [{"param": "expression",
"value": "%(result)s = float((%(nir)s - %(red)s)/"
"(%(nir)s + %(red)s))" % {"result": output_name,
"nir": nir_time_series,
"red": red_time_series}},
{"param": "inputs",
"value": "%(nir)s,%(red)s"%{"nir": nir_time_series,
"red": red_time_series}},
{"param": "basename",
"value": "ndvi"},
{"param": "output",
"value": output_name}]},
{"id": "t_rast_color_%i" % rn,
"module": "t.rast.colors",
"inputs": [{"param": "input",
"value": output_name},
{"param": "color",
"value": "ndvi"}]}]
return pc
def create_graas_process_chain_entry(input_name, start_time, end_time,
output_name):
"""Create a GRaaS command of the process chain that uses t.rast.extract to create a subset of a strds
:param strds_name: The name of the strds
:param start_time:
:param end_time:
:return: A GRaaS process chain description
"""
location, mapset, datatype, layer_name = GRaaSInterface.layer_def_to_components(
input_name)
input_name = layer_name
if mapset is not None:
input_name = layer_name + "@" + mapset
base_name = "%s_extract" % layer_name
# Get info about the time series to extract its resolution settings and bbox
rn = randint(0, 1000000)
pc = {
"id":
"t_rast_extract_%i" % rn,
"module":
"t.rast.extract",
"inputs": [{
"param": "input",
"value": input_name
}, {
"param": "where",
"value": "start_time >= '%(start)s' "
"AND end_time <= '%(end)s'" % {
"start": start_time,
"end": end_time
}
}, {
"param": "output",
"value": output_name
}, {
"param": "expression",
"value": "1.0 * %s" % input_name
}, {
"param": "basename",
"value": base_name
}, {
"param": "suffix",
"value": "num"
}]
}
return pc
def get_process_list(args):
"""Analyse the process description and return the GRaaS process chain and the name of the processing result layer
which is a single raster layer
:param args: The process description arguments
:return: (output_name, pc)
"""
input_names, process_list = process_definitions.analyse_process_graph(args)
output_names = []
for input_name in input_names:
location, mapset, datatype, layer_name = GRaaSInterface.layer_def_to_components(
input_name)
output_name = "%s_%s" % (layer_name, PROCESS_NAME)
output_names.append(output_name)
pc = create_graas_process_chain_entry(input_name, output_name)
process_list.append(pc)
return output_names, process_list
def get_process_list(args):
"""Analyse the process description and return the GRaaS process chain and the name of the processing result
:param args: The process description arguments
:return: (output_time_series, pc)
"""
input_names, process_list = process_definitions.analyse_process_graph(args)
output_names = []
# Two input names are required
if len(input_names) != 2:
raise Exception("At least two input time series are required")
for i in range(0, len(input_names), 2):
input_tuple = (input_names[i], input_names[i + 1])
nir_time_series = None
red_time_series = None
for input_name in input_tuple:
if "nir" in args and args["nir"] in input_name:
nir_time_series = input_name
if "red" in args and args["red"] in input_name:
red_time_series = input_name
if nir_time_series is None or red_time_series is None:
raise Exception("Band information is missing from process description")
location, mapset, datatype, layer_name = GRaaSInterface.layer_def_to_components(nir_time_series)
output_name = "%s_%s" % (layer_name, PROCESS_NAME)
output_names.append(output_name)
pc = create_graas_process_chain_entry(nir_time_series, red_time_series, output_name)
process_list.extend(pc)
return output_names, process_list
def create_graas_process_chain_entry(input_name, python_file_url, output_name):
"""Create a GRaaS command of the process chain that uses g.region to create a valid computational region
for the provide input strds
:param strds_name: The name of the strds
:param python_file_url: The URL to the python file that defines the UDF
:param output_name: The name of the output raster layer
:return: A GRaaS process chain description
"""
location, mapset, datatype, layer_name = GRaaSInterface.layer_def_to_components(
input_name)
input_name = layer_name
if mapset is not None:
input_name = layer_name + "@" + mapset
pc = {
"id":
"t_rast_aggr_func",
"module":
"t.rast.aggr_func",
"inputs": [{
"import_descr": {
"source": python_file_url,
"type": "file"
},
"param": "pyfile",
"value": "$file::my_py_func"
}, {
"param": "input",
"value": input_name
}, {
"param": "output",
"value": output_name
}]
}
return pc
def create_graas_process_chain_entry(input_name, output_name):
"""Create a GRaaS process description that uses t.rast.series to create the minimum
value of the time series.
:param input_time_series: The input time series name
:param output_map: The name of the output map
:return: A GRaaS process chain description
"""
location, mapset, datatype, layer_name = GRaaSInterface.layer_def_to_components(
input_name)
input_name = layer_name
if mapset is not None:
input_name = layer_name + "@" + mapset
rn = randint(0, 1000000)
pc = {
"id":
"t_rast_series_%i" % rn,
"module":
"t.rast.series",
"inputs": [{
"param": "input",
"value": input_name
}, {
"param": "method",
"value": "minimum"
}, {
"param": "output",
"value": output_name
}],
"flags":
"t"
}
return pc
def __init__(self):
self.iface = GRaaSInterface()
class GRaaSDataProductId(DataProductId):
def __init__(self):
self.iface = GRaaSInterface()
@swagger.doc(GET_DATA_PRODUCT_ID_DOC)
def get(self, product_id):
# List strds maps from the GRASS location
location, mapset, datatype, layer = self.iface.layer_def_to_components(
product_id)
status_code, layer_data = self.iface.layer_info(layer_name=product_id)
if status_code != 200:
return make_response(
jsonify(
{
"description":
"An internal error occurred "
"while catching GRASS GIS layer information "
"for layer <%s>!\n Error: %s"
"" % (product_id, str(layer_data))
}, 400))
# Get the projection from the GRASS mapset
status_code, mapset_info = self.iface.mapset_info(location=location,
mapset=mapset)
if status_code != 200:
return make_response(
jsonify(
{
"description":
"An internal error occurred "
"while catching mapset info "
"for mapset <%s>!" % mapset
}, 400))
description = "Raster dataset"
if datatype.lower() == "strds":
description = "Space time raster dataset"
if datatype.lower() == "vector":
description = "Vector dataset"
source = "GRASS GIS location/mapset path: /%s/%s" % (location, mapset)
srs = mapset_info["projection"]
extent = SpatialExtent(left=float(layer_data["west"]),
right=float(layer_data["east"]),
top=float(layer_data["north"]),
bottom=float(layer_data["south"]),
srs=srs)
print(layer_data)
if datatype.lower() == "strds":
time = DateTime()
time["from"] = layer_data["start_time"]
time["to"] = layer_data["end_time"]
bands = BandDescription(band_id=product_id)
info = dict(product_id=product_id,
extent=extent,
source=source,
description=description,
time=time,
bands=bands,
temporal_type=layer_data["start_time"],
number_of_maps=layer_data["number_of_maps"],
min_min=layer_data["min_min"],
min_max=layer_data["min_max"],
max_min=layer_data["max_min"],
max_max=layer_data["max_max"],
ewres_max=layer_data["ewres_max"],
ewres_min=layer_data["ewres_min"],
nsres_max=layer_data["nsres_max"],
nsres_min=layer_data["nsres_min"],
map_time=layer_data["map_time"],
granularity=layer_data["granularity"],
aggregation_type=layer_data["aggregation_type"],
creation_time=layer_data["creation_time"],
modification_time=layer_data["modification_time"],
mapset=mapset,
location=location)
else:
info = dict(
product_id=product_id,
extent=extent,
source=source,
description=description,
mapset=mapset,
location=location,
title=layer_data["title"],
comments=layer_data["comments"],
datatype=layer_data["datatype"],
cells=layer_data["cells"],
cols=layer_data["cols"],
rows=layer_data["rows"],
ewres=layer_data["ewres"],
nsres=layer_data["nsres"],
)
return make_response(jsonify(info), 200)
def test_layer_exists_4(self):
iface = GRaaSInterface(self.gconf)
status = iface.check_layer_exists(layer_name="ECAD.PERMANENT.raster.precipitation_yearly_mm_0")
self.assertTrue(status)
class GRaaSJobsJobId(JobsJobId):
def __init__(self):
self.iface = GRaaSInterface()
self.db = GraphDB()
@swagger.doc(GET_JOBS_ID_DOC)
def get(self, job_id):
try:
status, response = self.iface.resource_info(job_id)
if status == 200:
process_graph = self.db[job_id]
info = dict(job_id=job_id,
user_id=response["user_id"],
status=response["status"],
process_graph=process_graph,
submitted=response["accept_datetime"],
last_update=response["datetime"],
consumed_credits=response["time_delta"],
job_info=response)
if "urls" in response and "resources" in response["urls"]:
info["resources"] = response["urls"]["resources"]
return make_response(jsonify(info), 200)
else:
process_graph = self.db[job_id]
info = dict(job_id=job_id,
status="error",
process_graph=process_graph,
job_info=response)
return make_response(jsonify(info), status)
except Exception as e:
return make_response(jsonify({"error": str(e)}), 500)
@swagger.doc(DELETE_JOBS_ID_DOC)
def delete(self, job_id):
try:
status, response = self.iface.resource_info(job_id)
if status == 200:
process_graph = self.db[job_id]
info = dict(job_id=job_id,
user_id="scheduled",
status="submitted",
process_graph=process_graph,
submitted=response["accept_datetime"],
last_update=response["datetime"],
consumed_credits=response["time_delta"],
job_info=response)
status, response = self.iface.delete_resource(job_id)
if status != 200:
process_graph = self.db[job_id]
info = dict(job_id=job_id,
status="error",
process_graph=process_graph,
job_info=response)
return make_response(jsonify(info), status)
else:
process_graph = self.db[job_id]
info = dict(job_id=job_id,
status="error",
process_graph=process_graph,
job_info=response)
return make_response(jsonify(info), status)
except Exception as e:
return make_response(jsonify({"error": str(e)}), 500)
def test_health_check(self):
iface = GRaaSInterface(self.gconf)
self.assertTrue(iface.check_health())
class GRaaSJobs(Jobs):
def __init__(self):
self.iface = GRaaSInterface()
self.db = GraphDB()
@swagger.doc(POST_JOBS_DOC)
def put(self):
"""Modify the existing database by running the job in a persistent mapset
:return:
"""
try:
# Empty the process location
graas_openeo_core_wrapper.PROCESS_LOCATION = {}
process_graph = request.get_json()
# Transform the process graph into a process chain and store the input location
# Check all locations in the process graph
result_name, process_list = analyse_process_graph(process_graph)
if len(graas_openeo_core_wrapper.PROCESS_LOCATION) == 0 or len(
graas_openeo_core_wrapper.PROCESS_LOCATION) > 1:
return make_response(
jsonify(
{
"description":
"Processes can only be defined for a single location!"
}, 400))
location = graas_openeo_core_wrapper.PROCESS_LOCATION.keys()
location = list(location)[0]
status_code, mapsets = self.iface.list_mapsets(location=location)
if status_code != 200:
return make_response(
jsonify(
{
"description":
"An internal error occurred "
"while catching mapsets!"
}, 400))
count = 0
name = "openeo_mapset"
new_mapset = "%s_%i" % (name, count)
while new_mapset in mapsets:
count += 1
new_mapset = "%s_%i" % (name, count)
process_chain = dict(list=process_list, version="1")
# pprint.pprint(process_chain)
status, response = self.iface.async_persistent_processing(
location=location,
mapset=new_mapset,
process_chain=process_chain)
# pprint.pprint(response)
# Save the process graph into the graph db
self.db[response["resource_id"]] = process_graph
if status == 200:
return make_response(
jsonify({
"job_id": response["resource_id"],
"job_info": response
}), status)
else:
return make_response(jsonify(response), status)
except Exception as e:
return make_response(jsonify({"error": str(e)}), 400)
@swagger.doc(POST_JOBS_DOC)
def post(self):
"""Run the job in an ephemeral mapset
:return:
"""
try:
# Empty the process location
graas_openeo_core_wrapper.PROCESS_LOCATION = {}
process_graph = request.get_json()
# Transform the process graph into a process chain and store the input location
# Check all locations in the process graph
result_name, process_list = analyse_process_graph(process_graph)
if len(graas_openeo_core_wrapper.PROCESS_LOCATION) == 0 or len(
graas_openeo_core_wrapper.PROCESS_LOCATION) > 1:
return make_response(
jsonify(
{
"description":
"Processes can only be defined for a single location!"
}, 400))
location = graas_openeo_core_wrapper.PROCESS_LOCATION.keys()
location = list(location)[0]
process_chain = dict(list=process_list, version="1")
# pprint.pprint(process_chain)
status, response = self.iface.async_ephemeral_processing_export(
location=location, process_chain=process_chain)
# pprint.pprint(response)
# Save the process graph into the graph db
self.db[response["resource_id"]] = process_graph
if status == 200:
return make_response(
jsonify({
"job_id": response["resource_id"],
"job_info": response
}), status)
else:
return make_response(jsonify(response), status)
except Exception as e:
return make_response(jsonify({"error": str(e)}), 400)
def test_list_mapsets(self):
iface = GRaaSInterface(self.gconf)
status, mapsets = iface.list_mapsets(location="ECAD")
pprint(mapsets)
self.assertEqual(status, 200)
class GRaaSData(Data):
def __init__(self):
self.iface = GRaaSInterface()
@swagger.doc(GET_DATA_DOC)
def get(self, ):
dataset_list = []
for location in Config.LOCATIONS:
status_code, mapsets = self.iface.list_mapsets(location=location)
if status_code != 200:
return make_response(
jsonify(
{
"description":
"An internal error occurred "
"while catching mapset "
"from location %s!" % location
}, 400))
for mapset in mapsets:
# List strds maps from the GRASS location
status_code, strds_data = self.iface.list_strds(
location=location, mapset=mapset)
if status_code != 200:
return make_response(
jsonify(
{
"description":
"An internal error occurred "
"while catching strds layers!"
}, 400))
for entry in strds_data:
strds_id = "%s.%s.strds.%s" % (location, mapset, entry)
ds = DataSetListEntry(
product_id=strds_id,
description="Space time raster dataset",
source="GRASS GIS location/mapset path: "
"/%s/%s" % (location, mapset))
dataset_list.append(ds)
# List raster maps from the GRASS location
status_code, raster_data = self.iface.list_raster(
location=location, mapset=mapset)
if status_code != 200:
return make_response(
jsonify(
{
"description":
"An internal error occurred "
"while catching strds layers!"
}, 400))
for entry in raster_data:
raster_id = "%s.%s.raster.%s" % (location, mapset, entry)
ds = DataSetListEntry(
product_id=raster_id,
description="Raster dataset",
source="GRASS GIS location/mapset path: "
"/%s/%s" % (location, mapset))
dataset_list.append(ds)
return make_response(jsonify(dataset_list), 200)
def test_layer_exists_2_error(self):
iface = GRaaSInterface(self.gconf)
status = iface.check_layer_exists(layer_name="ECAD.PERMANENT.strds.precipitation_1950_2013_yearly_mm_nope")
self.assertFalse(status)
def __init__(self):
self.iface = GRaaSInterface()
self.db = GraphDB()
def create_graas_process_chain_entry(input_name, regions):
"""Create a GRaaS command of the process chain that computes the regional statistics based on a
strds and a polygon.
The computational region will be set to the vector map, the previous region will be saved and after processing
restored. A mask will be set that uses the vector file as input. This mask will be removed in the end.
:param input_name: The name of the strds
:param regions: The URL to the vector file that defines the regions of interest
:return: A GRaaS process chain description
"""
location, mapset, datatype, layer_name = GRaaSInterface.layer_def_to_components(
input_name)
input_name = layer_name
if mapset is not None:
input_name = layer_name + "@" + mapset
rn = randint(0, 1000000)
pc = []
importer = {
"id":
"importer_%i" % rn,
"module":
"importer",
"inputs": [{
"import_descr": {
"source": regions,
"type": "vector"
},
"param": "map",
"value": "polygon"
}]
}
g_region_1 = {
"id": "g_region_%i" % rn,
"module": "g.region",
"inputs": [{
"param": "save",
"value": "previous_region"
}],
"flags": "g"
}
g_region_2 = {
"id": "g_region_%i" % rn,
"module": "g.region",
"inputs": [{
"param": "vector",
"value": "polygon"
}],
"flags": "g"
}
r_mask_1 = {
"id": "r_mask_%i" % rn,
"module": "r.mask",
"inputs": [{
"param": "vector",
"value": "polygon"
}]
}
t_rast_univar = {
"id": "t_rast_univar_%i" % rn,
"module": "t.rast.univar",
"inputs": [{
"param": "input",
"value": input_name
}]
}
r_mask_2 = {"id": "r_mask_%i" % rn, "module": "r.mask", "flags": "r"}
g_region_3 = {
"id": "g_region_%i" % rn,
"module": "g.region",
"inputs": [{
"param": "region",
"value": "previous_region"
}],
"flags": "g"
}
pc.append(importer)
pc.append(g_region_1)
pc.append(g_region_2)
pc.append(r_mask_1)
pc.append(t_rast_univar)
pc.append(r_mask_2)
pc.append(g_region_3)
return pc