def test_job_resumption_config_error(self):
"""Test if the job resumption fails if save_interim_results is set to
False in the actinia.cfg
"""
tpl = Template(json_dumps(process_chain_1))
rv = self.server.post(URL_PREFIX + self.endpoint,
headers=self.admin_auth_header,
data=tpl.render(map="elevation2@PERMANENT"),
content_type="application/json")
resp = self.waitAsyncStatusAssertHTTP(rv,
headers=self.admin_auth_header,
http_status=400,
status="error")
status_url = resp["urls"]["status"].split(URL_PREFIX)[-1]
rv2 = self.server.put(URL_PREFIX + status_url,
headers=self.admin_auth_header,
data=tpl.render(map="elevation@PERMANENT"),
content_type="application/json")
self.assertEqual(rv2.status_code, 404)
resp_data = json_loads(rv2.data)
self.assertEqual(resp_data['status'], 'error')
self.assertEqual(resp_data['message'],
'Interim results are not set in the configureation')
def check_for_json(self):
"""Check if the Payload is a JSON document
Return: bool:
True in case of success, False otherwise
"""
# First check for the data field and create JSON from it
if hasattr(request, "data") is True:
try:
self.request_data = json_loads(request.data)
except Exception as e:
self.create_error_response(
message="No JSON data in request: Exception: %s" % str(e))
return False
if request.is_json is False:
self.create_error_response(message="No JSON data in request")
return False
self.request_data = request.get_json()
return True
def test_saved_interim_results(self):
"""Test if the interim results are removed
"""
self.create_new_mapset(self.mapset, self.location)
tpl = Template(json_dumps(process_chain_1))
rv = self.server.post(URL_PREFIX + self.endpoint,
headers=self.admin_auth_header,
data=tpl.render(map1="elevation@PERMANENT",
map2="baum"),
content_type="application/json")
self.waitAsyncStatusAssertHTTP(rv,
headers=self.admin_auth_header,
http_status=200,
status="finished")
# check if interim results are saved
resp_data = json_loads(rv.data)
rv_user_id = resp_data["user_id"]
rv_resource_id = resp_data["resource_id"]
interim_dir = os.path.join(global_config.GRASS_RESOURCE_DIR,
rv_user_id, "interim", rv_resource_id)
self.assertTrue(
not os.path.isdir(interim_dir),
"Interim results are not stored in the expected folder")
def test_job_resumption_stdout(self):
"""Test job resumption with processing_async endpoint and stdout
"""
tpl = Template(json_dumps(process_chain_5_stdout))
rv = self.server.post(URL_PREFIX + self.endpoint,
headers=self.admin_auth_header,
data=tpl.render(map1="elevation2@PERMANENT",
map2="baum"),
content_type="application/json")
resp = self.waitAsyncStatusAssertHTTP(rv,
headers=self.admin_auth_header,
http_status=400,
status="error")
status_url = resp["urls"]["status"].split(URL_PREFIX)[-1]
# check if interim results are saved
resp_data = json_loads(rv.data)
rv_user_id = resp_data["user_id"]
rv_resource_id = resp_data["resource_id"]
interim_dir = os.path.join(global_config.GRASS_RESOURCE_DIR,
rv_user_id, "interim", rv_resource_id)
self.assertTrue(
os.path.isdir(interim_dir),
"Interim results are not stored in the expected folder")
# resumption of the job
rv2 = self.server.put(URL_PREFIX + status_url,
headers=self.admin_auth_header,
data=tpl.render(map1="elevation@PERMANENT",
map2="baum"),
content_type="application/json")
resp2 = self.waitAsyncStatusAssertHTTP(rv2,
headers=self.admin_auth_header,
http_status=200,
status="finished")
self.compare_stdout(resp2)
def test_strds_create_register_unregister_1(self):
new_mapset = "raster_test_mapset"
self.create_new_mapset(new_mapset, "ECAD")
# Create success
rv = self.server.post(
URL_PREFIX +
'/locations/ECAD/mapsets/%s/strds/test_strds_register' %
new_mapset,
headers=self.admin_auth_header,
data=json_dumps({
"temporaltype": "absolute",
"title": "A nice title",
"description": "A nice description"
}),
content_type="application/json")
pprint(json_loads(rv.data))
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
# Create the raster layer
rv = self.server.post(
URL_PREFIX +
'/locations/ECAD/mapsets/%s/raster_layers/test_layer_1' %
new_mapset,
headers=self.admin_auth_header,
data=json_dumps({"expression": "1"}),
content_type="application/json")
pprint(json_loads(rv.data))
rv = self.server.post(
URL_PREFIX +
'/locations/ECAD/mapsets/%s/raster_layers/test_layer_2' %
new_mapset,
headers=self.admin_auth_header,
data=json_dumps({"expression": "2"}),
content_type="application/json")
pprint(json_loads(rv.data))
rv = self.server.post(
URL_PREFIX +
'/locations/ECAD/mapsets/%s/raster_layers/test_layer_3' %
new_mapset,
headers=self.admin_auth_header,
data=json_dumps({"expression": "3"}),
content_type="application/json")
pprint(json_loads(rv.data))
raster_layers = [{
"name": "test_layer_1",
"start_time": "2000-01-01 00:00:00",
"end_time": "2000-01-02 00:00:00"
}, {
"name": "test_layer_2",
"start_time": "2000-01-02 00:00:00",
"end_time": "2000-01-03 00:00:00"
}, {
"name": "test_layer_3",
"start_time": "2000-01-03 00:00:00",
"end_time": "2000-01-04 00:00:00"
}]
rv = self.server.put(
URL_PREFIX +
"/locations/ECAD/mapsets/%s/strds/test_strds_register/raster_layers"
% new_mapset,
data=json_dumps(raster_layers),
content_type="application/json",
headers=self.admin_auth_header)
pprint(json_loads(rv.data))
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
# Check strds
rv = self.server.get(
URL_PREFIX +
"/locations/ECAD/mapsets/%s/strds/test_strds_register" %
new_mapset,
headers=self.admin_auth_header)
pprint(json_loads(rv.data))
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
min_min = json_loads(rv.data)["process_results"]["min_min"]
max_max = json_loads(rv.data)["process_results"]["max_max"]
num_maps = json_loads(rv.data)["process_results"]["number_of_maps"]
self.assertEqual(min_min, "1.0")
self.assertEqual(max_max, "3.0")
self.assertEqual(num_maps, "3")
# Unregister the raster layers
raster_layers = ["test_layer_1", "test_layer_2", "test_layer_3"]
rv = self.server.delete(
URL_PREFIX +
"/locations/ECAD/mapsets/%s/strds/test_strds_register/raster_layers"
% new_mapset,
data=json_dumps(raster_layers),
content_type="application/json",
headers=self.user_auth_header)
pprint(json_loads(rv.data))
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
# Check strds
rv = self.server.get(
URL_PREFIX +
"/locations/ECAD/mapsets/%s/strds/test_strds_register" %
new_mapset,
headers=self.user_auth_header)
pprint(json_loads(rv.data))
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
min_min = json_loads(rv.data)["process_results"]["min_min"]
max_max = json_loads(rv.data)["process_results"]["max_max"]
num_maps = json_loads(rv.data)["process_results"]["number_of_maps"]
self.assertEqual(min_min, "None")
self.assertEqual(max_max, "None")
self.assertEqual(num_maps, "0")
# Delete the strds
rv = self.server.delete(
URL_PREFIX +
'/locations/ECAD/mapsets/%s/strds/test_strds_register' %
new_mapset,
headers=self.user_auth_header)
pprint(json_loads(rv.data))
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
def test_superadmin_termination(self):
"""Test the termination of one running resources from superadmin user
"""
# Create a random test user id that are used for login as admin
user_id = "horst" + str(randint(0, 10000000))
user_group = self.user_group
password = "******"
# We need to create an HTML basic authorization header
auth_header = Headers()
auth = bytes('%s:%s' % (user_id, password), "utf-8")
auth_header.add('Authorization',
'Basic ' + base64.b64encode(auth).decode())
# Make sure the user database is empty
user = ActiniaUser(user_id)
if user.exists():
user.delete()
# Create a user in the database and reduce its credentials
self.user = ActiniaUser.create_user(
user_id,
user_group,
password,
user_role="superadmin",
accessible_datasets={
"nc_spm_08": ["PERMANENT", "user1", "landsat", "test_mapset"],
"ECAD": ["PERMANENT"]
},
process_num_limit=3,
process_time_limit=100)
# Start three processes that will be terminated
rv = self.server.post(URL_PREFIX + '/custom_process/sleep',
headers=auth_header,
data=json_dumps(["20"]),
content_type="application/json")
# Test guest termination error
rv = self.server.delete(URL_PREFIX + '/resources/%s' % user_id,
headers=self.guest_auth_header)
print(rv.data.decode())
self.assertEqual(rv.status_code, 401,
"HTML status code is wrong %i" % rv.status_code)
# self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype)
# Test user termination error
rv = self.server.delete(URL_PREFIX + '/resources/%s' % user_id,
headers=self.user_auth_header)
print(rv.data.decode())
self.assertEqual(rv.status_code, 401,
"HTML status code is wrong %i" % rv.status_code)
# self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype)
# Test admin termination error
rv = self.server.delete(URL_PREFIX + '/resources/%s' % user_id,
headers=self.admin_auth_header)
print(rv.data.decode())
self.assertEqual(rv.status_code, 401,
"HTML status code is wrong %i" % rv.status_code)
# self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype)
# Test superadmin termination success
rv = self.server.delete(URL_PREFIX + '/resources/%s' % user_id,
headers=self.root_auth_header)
print(rv.data.decode())
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
# Wait for termination
time.sleep(5)
rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id,
headers=auth_header)
print(rv.data.decode())
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
resource_list = json_loads(rv.data)["resource_list"]
self.assertTrue(len(resource_list) == 1)
# Count return stats
terminated = 0
for resource in resource_list:
terminated += int(resource["status"] == "terminated")
print(resource["status"])
self.assertTrue(terminated == 1)
def test_user_status_requests_1(self):
"""Resource list with 3 finished resources
"""
# Create a random test user id that are used for login as admin
user_id = "heinz" + str(randint(0, 10000000))
user_group = self.user_group
password = "******"
# We need to create an HTML basic authorization header
auth_header = Headers()
auth = bytes('%s:%s' % (user_id, password), "utf-8")
auth_header.add('Authorization',
'Basic ' + base64.b64encode(auth).decode())
# Make sure the user database is empty
user = ActiniaUser(user_id)
if user.exists():
user.delete()
# Create a user in the database and reduce its credentials
self.user = ActiniaUser.create_user(
user_id,
user_group,
password,
user_role="user",
accessible_datasets={
"nc_spm_08":
["PERMANENT", "user1", "landsat", "modis_lst", "test_mapset"],
"ECAD": ["PERMANENT"]
},
process_num_limit=3,
process_time_limit=2)
# Create three successfully run resources
rv = self.server.post(URL_PREFIX + '/custom_process/uname',
headers=auth_header,
data=json_dumps(["-a"]),
content_type="application/json")
self.waitAsyncStatusAssertHTTP(rv, headers=self.admin_auth_header)
rv = self.server.post(URL_PREFIX + '/custom_process/uname',
headers=auth_header,
data=json_dumps(["-a"]),
content_type="application/json")
self.waitAsyncStatusAssertHTTP(rv, headers=self.admin_auth_header)
rv = self.server.post(URL_PREFIX + '/custom_process/uname',
headers=auth_header,
data=json_dumps(["-a"]),
content_type="application/json")
self.waitAsyncStatusAssertHTTP(rv, headers=self.admin_auth_header)
rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id,
headers=auth_header)
# print(rv.data.decode())
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
resource_list = json_loads(rv.data)["resource_list"]
pprint(resource_list)
self.assertTrue(len(resource_list) == 3)
# Count return stats
finished = 0
for resource in resource_list:
finished += int(resource["status"] == "finished")
# print(resource["status"])
self.assertTrue(finished == 3)
# Check the different resource list parameters
rv = self.server.get(URL_PREFIX + '/resources/%s?num=1' % user_id,
headers=auth_header)
self.assertTrue(len(json_loads(rv.data)["resource_list"]) == 1)
rv = self.server.get(URL_PREFIX + '/resources/%s?num=2' % user_id,
headers=auth_header)
self.assertTrue(len(json_loads(rv.data)["resource_list"]) == 2)
rv = self.server.get(URL_PREFIX +
'/resources/%s?num=2&type=finished' % user_id,
headers=auth_header)
self.assertTrue(len(json_loads(rv.data)["resource_list"]) == 2)
rv = self.server.get(URL_PREFIX +
'/resources/%s?num=2&type=all' % user_id,
headers=auth_header)
self.assertTrue(len(json_loads(rv.data)["resource_list"]) == 2)
rv = self.server.get(URL_PREFIX + '/resources/%s?type=all' % user_id,
headers=auth_header)
self.assertTrue(len(json_loads(rv.data)["resource_list"]) == 3)
rv = self.server.get(URL_PREFIX +
'/resources/%s?type=finished' % user_id,
headers=auth_header)
self.assertTrue(len(json_loads(rv.data)["resource_list"]) == 3)
rv = self.server.get(URL_PREFIX +
'/resources/%s?type=terminated' % user_id,
headers=auth_header)
self.assertTrue(len(json_loads(rv.data)["resource_list"]) == 0)
rv = self.server.get(URL_PREFIX +
'/resources/%s?type=unknown' % user_id,
headers=auth_header)
self.assertTrue(len(json_loads(rv.data)["resource_list"]) == 0)
# Check permission access using the default users
rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id,
headers=self.guest_auth_header)
# print(rv.data.decode())
self.assertEqual(rv.status_code, 401,
"HTML status code is wrong %i" % rv.status_code)
# self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype)
rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id,
headers=self.user_auth_header)
# print(rv.data.decode())
self.assertEqual(rv.status_code, 401,
"HTML status code is wrong %i" % rv.status_code)
# self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype)
rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id,
headers=self.admin_auth_header)
# print(rv.data.decode())
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
# self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype)
rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id,
headers=self.root_auth_header)
# print(rv.data.decode())
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
def test_user_status_requests_3(self):
"""Empty resource list test
"""
# Create a random test user id that are used for login as admin
user_id = "heinz" + str(randint(0, 10000000))
user_group = "test"
password = "******"
# We need to create an HTML basic authorization header
auth_header = Headers()
auth = bytes('%s:%s' % (user_id, password), "utf-8")
auth_header.add('Authorization',
'Basic ' + base64.b64encode(auth).decode())
# Make sure the user database is empty
user = ActiniaUser(user_id)
if user.exists():
user.delete()
# Create a user in the database and reduce its credentials
self.user = ActiniaUser.create_user(user_id, user_group, password)
rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id,
headers=auth_header)
print(rv.data.decode())
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
resource_list = json_loads(rv.data)["resource_list"]
self.assertTrue(len(resource_list) == 0)
# Check permission access using the default users
rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id,
headers=self.guest_auth_header)
print(rv.data.decode())
self.assertEqual(rv.status_code, 401,
"HTML status code is wrong %i" % rv.status_code)
# self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype)
rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id,
headers=self.user_auth_header)
print(rv.data.decode())
self.assertEqual(rv.status_code, 401,
"HTML status code is wrong %i" % rv.status_code)
# self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype)
rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id,
headers=self.admin_auth_header)
print(rv.data.decode())
self.assertEqual(rv.status_code, 401,
"HTML status code is wrong %i" % rv.status_code)
# self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype)
rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id,
headers=self.root_auth_header)
print(rv.data.decode())
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
def test_strds_create_remove(self):
new_mapset = "strds_test"
self.create_new_mapset(mapset_name=new_mapset, location_name="ECAD")
# Create success
rv = self.server.post(
URL_PREFIX +
'/locations/ECAD/mapsets/%s/strds/test_strds' % new_mapset,
headers=self.admin_auth_header,
data=json_dumps({
"temporaltype": "absolute",
"title": "A nice title",
"description": "A nice description"
}),
content_type="application/json")
print(rv.data)
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
# Create failure since the strds already exists
rv = self.server.post(
URL_PREFIX +
'/locations/ECAD/mapsets/%s/strds/test_strds' % new_mapset,
headers=self.admin_auth_header,
data=json_dumps({
"temporaltype": "absolute",
"title": "A nice title",
"description": "A nice description"
}),
content_type="application/json")
# print(rv.data)
self.assertEqual(rv.status_code, 400,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
# Read/check information of the new strds
rv = self.server.get(
URL_PREFIX +
'/locations/ECAD/mapsets/%s/strds/test_strds' % new_mapset,
headers=self.user_auth_header)
print(rv.data)
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
start_time = json_loads(rv.data)["process_results"]["start_time"]
self.assertEquals(start_time, "'None'")
# Delete the strds
rv = self.server.delete(
URL_PREFIX +
'/locations/ECAD/mapsets/%s/strds/test_strds' % new_mapset,
headers=self.admin_auth_header)
print(rv.data)
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
# Try to delete the strds again to produce an error
rv = self.server.delete(
URL_PREFIX +
'/locations/ECAD/mapsets/%s/strds/test_strds' % new_mapset,
headers=self.admin_auth_header)
print(rv.data)
self.assertEqual(rv.status_code, 400,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
rv = self.server.get(
URL_PREFIX +
'/locations/ECAD/mapsets/%s/strds/test_strds' % new_mapset,
headers=self.user_auth_header)
print(rv.data)
self.assertEqual(rv.status_code, 400,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
def get_sentinel_urls(self, product_ids, bands=["B04", "B08"]):
"""Receive the download urls and time stamps for a list of Sentinel2 product ids from AWS service
1. Transform the Sentinel ID into the path of the productInfo.json url that is required to get the tile urls
2. Parse the productInfo.json file and extract the tile urls
3. Create the download links for each tile based ond each band
4. Include the tileInfo.json, xml and preview url's
Args:
product_ids: A list of sentinel product ids
bands: A list of band names
Returns:
dict
A dictionary that contains the time stamp, the tile names, the map names in GRASS,
the aws cloud storage urls, the tile GeoJSON footprint and the XML metadata file path
download url's.
Example:
[{'product_id': 'S2A_MSIL1C_20170202T090201_N0204_R007_T36TVT_20170202T090155',
'tiles':[{'B04': {'file_name' : 'S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V20151207T003302_20151207T003302_tile_14_band_B04.jp2',
'map_name' : 'S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V20151207T003302_20151207T003302_tile_14_band_B04',
'public_url': 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/V/XE/2015/12/7/0/B04.jp2'},
'B08': {'file_name' : 'S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V20151207T003302_20151207T003302_tile_14_band_B08.jp2',
'map_name' : 'S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V20151207T003302_20151207T003302_tile_14_band_B08',
'public_url': 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/V/XE/2015/12/7/0/B08.jp2'},
'info' : 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/V/XE/2015/12/7/0/tileInfo.json',
'metadata' : 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/V/XE/2015/12/7/0/metadata.xml',
'preview' : 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/V/XE/2015/12/7/0/preview.jpg',
'timestamp' : '2015-12-07T00:33:02.634Z',
'url' : 'http://sentinel-s2-l1c.s3-website.eu-central-1.amazonaws.com/#tiles/57/V/XE/2015/12/7/0/'}]}]
"""
# Old format
# products/2015/12/7/S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V20151207T003302_20151207T003302
# https://sentinel-s2-l1c.s3.amazonaws.com/products/2015/10/1/S2A_OPER_PRD_MSIL1C_PDMC_20161220T003744_R006_V20151001T071826_20151001T071826/productInfo.json
# New format
# products/2017/10/31/S2A_MSIL1C_20171031T000721_N0206_R016_T01WCP_20171031T015145
# 1 2 3 4 5 6 7 8
# 012345678901234567890123456789012345678901234567890123456789012345678901234567890
# S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V20151207T003302_20151207T003302
# 1 2 3 4 5 6 7 8
# 012345678901234567890123456789012345678901234567890123456789012345678901234567890
# S2A_MSIL1C_20171031T000721_N0206_R016_T01WCP_20171031T015145
try:
for band in bands:
if band not in self.sentinel_bands:
raise Exception("Unknown sentinel 2A band name <%s>"%band)
result = []
for product_id in product_ids:
product_id = product_id.replace(".SAFE", "")
if "S2A_OPER" in product_id:
year = product_id[63:67]
month = product_id[67:69]
day = product_id[69:71]
else:
year = product_id[45:49]
month = product_id[49:51]
day = product_id[51:53]
if month != "10":
month = month.replace("0", "")
if day not in ["10", "20", "30"]:
day = day.replace("0", "")
# Get the product info JSON file
json_url = "%(base)s/products/%(year)s/%(month)s/%(day)s/%(id)s/" \
"productInfo.json"%{"base":self.aws_sentinel_base_url,
"year":year,
"month":month,
"day":day,
"id":product_id}
response = urlopen(json_url)
product_info = response.read()
# print(product_info)
try:
info = json_loads(product_info)
except:
raise Exception("Unable to read the productInfo.json file from URL: %s. Error: %s"%(json_url,
product_info))
if info:
scene_entry = {}
scene_entry["product_id"] = product_id
scene_entry["tiles"] = []
tile_num = 0
for tile in info["tiles"]:
tile_num += 1
tile_info = {}
tile_info["timestamp"] = info["timestamp"]
# http://sentinel-s2-l1c.s3.amazonaws.com/tiles/58/V/CK/2015/12/7/0/metadata.xml
# http://sentinel-s2-l1c.s3.amazonaws.com/tiles/58/V/CK/2015/12/7/0/tileInfo.json
# http://sentinel-s2-l1c.s3.amazonaws.com/tiles/58/V/CK/2015/12/7/0/preview.jpg
metadata_url = self.aws_sentinel_base_url + "/" + tile["path"] + "/metadata.xml"
info_url = self.aws_sentinel_base_url + "/" + tile["path"] + "/tileInfo.json"
preview_url = self.aws_sentinel_base_url + "/" + tile["path"] + "/preview.jpg"
tile_info["url"] = self.aws_sentinel_base_eu_central_url + "/#" + tile["path"] + "/"
tile_info["metadata"] = metadata_url
tile_info["info"] = info_url
tile_info["preview"] = preview_url
public_url = self.aws_sentinel_base_url + "/" + tile["path"]
for band in bands:
tile_name = "%s_tile_%i_band_%s.jp2"%(product_id, tile_num, band)
map_name = "%s_tile_%i_band_%s"%(product_id, tile_num, band)
tile_info[band] = {}
tile_info[band]["file_name"] = tile_name
tile_info[band]["map_name"] = map_name
tile_info[band]["public_url"] = "%s/%s.jp2"%(public_url, band)
scene_entry["tiles"].append(tile_info)
result.append(scene_entry)
return result
except:
raise
def test_3_existing_mapset_lock(self):
"""
Try to run two request on a locked mapset
:return:
"""
self.check_remove_test_mapset()
# Create new mapset
rv = self.server.post(URL_PREFIX +
'/locations/nc_spm_08/mapsets/test_mapset',
headers=self.admin_auth_header)
print(rv.data)
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
# Run the processing inside the new mapset
rv = self.server.post(
URL_PREFIX +
'/locations/nc_spm_08/mapsets/test_mapset/processing_async',
headers=self.admin_auth_header,
data=json_dumps(process_chain_long),
content_type="application/json")
print(rv.data)
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
resp = json_loads(rv.data)
rv_user_id = resp["user_id"]
rv_resource_id = resp["resource_id"]
time.sleep(0.5)
# Try to lock again and again
# Run the processing inside the new mapset
# Second runner
rv_lock_1 = self.server.post(
URL_PREFIX +
'/locations/nc_spm_08/mapsets/test_mapset/processing_async',
headers=self.admin_auth_header,
data=json_dumps(process_chain_short),
content_type="application/json")
print(rv_lock_1.data)
self.assertEqual(rv_lock_1.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv_lock_1.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
# Third runner
rv_lock_2 = self.server.post(
URL_PREFIX +
'/locations/nc_spm_08/mapsets/test_mapset/processing_async',
headers=self.admin_auth_header,
data=json_dumps(process_chain_long),
content_type="application/json")
print(rv_lock_2.data)
self.assertEqual(rv_lock_2.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv_lock_2.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
# Check the first runner
while True:
rv = self.server.get(URL_PREFIX + "/resources/%s/%s" %
(rv_user_id, rv_resource_id),
headers=self.admin_auth_header)
print(rv.data)
resp = json_loads(rv.data)
if resp["status"] == "finished" or resp["status"] == "error":
break
time.sleep(0.2)
self.assertEqual(resp["status"], "finished")
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
# Check the second runner
resp = json_loads(rv_lock_1.data)
rv_user_id = resp["user_id"]
rv_resource_id = resp["resource_id"]
while True:
rv = self.server.get(URL_PREFIX + "/resources/%s/%s" %
(rv_user_id, rv_resource_id),
headers=self.admin_auth_header)
print(rv.data)
resp = json_loads(rv.data)
if resp["status"] == "finished" or resp["status"] == "error":
break
time.sleep(0.2)
self.assertEqual(resp["status"], "error")
self.assertEqual(rv.status_code, 400,
"HTML status code is wrong %i" % rv.status_code)
# Check the third runner
resp = json_loads(rv_lock_2.data)
rv_user_id = resp["user_id"]
rv_resource_id = resp["resource_id"]
# Check the first runner
while True:
rv = self.server.get(URL_PREFIX + "/resources/%s/%s" %
(rv_user_id, rv_resource_id),
headers=self.admin_auth_header)
print(rv.data)
resp = json_loads(rv.data)
if resp["status"] == "finished" or resp["status"] == "error":
break
time.sleep(0.2)
self.assertEqual(resp["status"], "error")
self.assertEqual(rv.status_code, 400,
"HTML status code is wrong %i" % rv.status_code)
# Remove the mapset
rv = self.server.delete(URL_PREFIX +
'/locations/nc_spm_08/mapsets/test_mapset',
headers=self.admin_auth_header)
print(rv.data)
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
self.assertEqual(rv.mimetype, "application/json",
"Wrong mimetype %s" % rv.mimetype)
def test_job_2_times_resumption_exporter(self):
"""Test job 2 times resumption with processing_async_export endpoint and
exporter
"""
tpl = Template(json_dumps(process_chain_4_exporter))
rv = self.server.post(URL_PREFIX + self.endpoint,
headers=self.admin_auth_header,
data=tpl.render(map1="elevation2@PERMANENT",
map2="baum"),
content_type="application/json")
resp = self.waitAsyncStatusAssertHTTP(rv,
headers=self.admin_auth_header,
http_status=400,
status="error")
status_url = resp["urls"]["status"].split(URL_PREFIX)[-1]
# first job resumption
rv2 = self.server.put(URL_PREFIX + status_url,
headers=self.admin_auth_header,
data=tpl.render(map1="elevation@PERMANENT",
map2="baum1"),
content_type="application/json")
resp2 = self.waitAsyncStatusAssertHTTP(rv2,
headers=self.admin_auth_header,
http_status=400,
status="error")
status_url = resp2["urls"]["status"].split(URL_PREFIX)[-1]
# check if interim results are saved
resp_data = json_loads(rv2.data)
rv_user_id = resp_data["user_id"]
rv_resource_id = resp_data["resource_id"]
interim_dir = os.path.join(global_config.GRASS_RESOURCE_DIR,
rv_user_id, "interim", rv_resource_id)
self.assertTrue(
os.path.isdir(interim_dir),
"Interim results are not stored in the expected folder")
# second job resumption
rv3 = self.server.put(URL_PREFIX + status_url,
headers=self.admin_auth_header,
data=tpl.render(map1="elevation@PERMANENT",
map2="baum"),
content_type="application/json")
resp3 = self.waitAsyncStatusAssertHTTP(rv3,
headers=self.admin_auth_header,
http_status=200,
status="finished")
# Get the exported results
urls = resp3["urls"]["resources"]
for url in urls:
print(url)
rv = self.server.get(url, headers=self.admin_auth_header)
self.assertEqual(rv.status_code, 200,
"HTML status code is wrong %i" % rv.status_code)
if url.endswith('.tif'):
self.assertEqual(rv.mimetype, "image/tiff",
"Wrong mimetype %s" % rv.mimetype)
elif url.endswith('.zip'):
self.assertEqual(rv.mimetype, "application/zip",
"Wrong mimetype %s" % rv.mimetype)
self.__class__.resource_user_id = rv_user_id
self.__class__.resource_resource_id = rv_resource_id
def is_json(data):
try:
json_loads(data)
except JSONDecodeError:
return False
return True
def get_sentinel_urls(self, product_ids, bands=None):
"""Receive the download urls and time stamps for a list of Sentinel2
product ids from AWS service
1. Transform the Sentinel ID into the path of the productInfo.json url
that is required to get the tile urls
2. Parse the productInfo.json file and extract the tile urls
3. Create the download links for each tile based on each band
4. Include the tileInfo.json, xml and preview url's
Args:
product_ids: A list of sentinel product ids
bands: A list of band names
Returns:
dict
A dictionary that contains the time stamp, the tile names, the map
names in GRASS, the aws cloud storage urls, the tile GeoJSON footprint
and the XML metadata file path download url's.
Example:
[{'product_id': 'S2A_MSIL1C_20170202T090201_N0204_R007_T36TVT_'
'20170202T090155',
'tiles':[{
'B04': {
'file_name' : 'S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_'
'V20151207T003302_20151207T003302_tile_14_band_'
'B04.jp2',
'map_name' : 'S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_'
'V20151207T003302_20151207T003302_tile_14_band_B04',
'public_url': 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/'
'V/XE/2015/12/7/0/B04.jp2'},
'B08': {
'file_name' : 'S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V2015'
'1207T003302_20151207T003302_tile_14_band_B08.jp2',
'map_name' : 'S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V2015'
'1207T003302_20151207T003302_tile_14_band_B08',
'public_url': 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/'
'V/XE/2015/12/7/0/B08.jp2'},
'info' : 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/V/'
'XE/2015/12/7/0/tileInfo.json',
'metadata' : 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/V/'
'XE/2015/12/7/0/metadata.xml',
'preview' : 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/V/'
'XE/2015/12/7/0/preview.jpg',
'timestamp' : '2015-12-07T00:33:02.634Z',
'url' : 'http://sentinel-s2-l1c.s3-website.eu-central-1.'
'amazonaws.com/#tiles/57/V/XE/2015/12/7/0/'}]}]
"""
# Old format
# products/2015/12/7/S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V20151207T003302_20151207T003302
# https://sentinel-s2-l1c.s3.amazonaws.com/products/2015/10/1/S2A_OPER_PRD_MSIL1C_PDMC_20161220T003744_R006_V20151001T071826_20151001T071826/productInfo.json
# New format
# products/2017/10/31/S2A_MSIL1C_20171031T000721_N0206_R016_T01WCP_20171031T015145
# 1 2 3 4 5 6 7 8
# 012345678901234567890123456789012345678901234567890123456789012345678901234567890
# S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V20151207T003302_20151207T003302
# 1 2 3 4 5 6 7 8
# 012345678901234567890123456789012345678901234567890123456789012345678901234567890
# S2A_MSIL1C_20171031T000721_N0206_R016_T01WCP_20171031T015145
# Assign default bands
if bands is None:
bands = ["B04", "B08"]
try:
for band in bands:
if band not in self.sentinel_bands:
raise Exception("Unknown Sentinel-2 band name <%s>" % band)
result = []
for product_id in product_ids:
product_id = product_id.replace(".SAFE", "")
year, month, day = get_sentinel_date(product_id)
# Get the product info JSON file
json_url = "%(base)s/products/%(year)s/%(month)s/%(day)s/%(id)s/" \
"productInfo.json" % {"base": self.aws_sentinel_base_url,
"year": year,
"month": month,
"day": day,
"id": product_id}
response = urlopen(json_url)
product_info = response.read()
try:
info = json_loads(product_info)
except Exception:
raise Exception(
"Unable to read the productInfo.json file from URL: "
"%s. Error: %s" % (json_url, product_info))
if info:
scene_entry = self._parse_scene_info(
bands, product_id, info)
result.append(scene_entry)
return result
except Exception:
raise
