def test_job_resumption_config_error(self): """Test if the job resumption fails if save_interim_results is set to False in the actinia.cfg """ tpl = Template(json_dumps(process_chain_1)) rv = self.server.post(URL_PREFIX + self.endpoint, headers=self.admin_auth_header, data=tpl.render(map="elevation2@PERMANENT"), content_type="application/json") resp = self.waitAsyncStatusAssertHTTP(rv, headers=self.admin_auth_header, http_status=400, status="error") status_url = resp["urls"]["status"].split(URL_PREFIX)[-1] rv2 = self.server.put(URL_PREFIX + status_url, headers=self.admin_auth_header, data=tpl.render(map="elevation@PERMANENT"), content_type="application/json") self.assertEqual(rv2.status_code, 404) resp_data = json_loads(rv2.data) self.assertEqual(resp_data['status'], 'error') self.assertEqual(resp_data['message'], 'Interim results are not set in the configureation')
def check_for_json(self): """Check if the Payload is a JSON document Return: bool: True in case of success, False otherwise """ # First check for the data field and create JSON from it if hasattr(request, "data") is True: try: self.request_data = json_loads(request.data) except Exception as e: self.create_error_response( message="No JSON data in request: Exception: %s" % str(e)) return False if request.is_json is False: self.create_error_response(message="No JSON data in request") return False self.request_data = request.get_json() return True
def test_saved_interim_results(self): """Test if the interim results are removed """ self.create_new_mapset(self.mapset, self.location) tpl = Template(json_dumps(process_chain_1)) rv = self.server.post(URL_PREFIX + self.endpoint, headers=self.admin_auth_header, data=tpl.render(map1="elevation@PERMANENT", map2="baum"), content_type="application/json") self.waitAsyncStatusAssertHTTP(rv, headers=self.admin_auth_header, http_status=200, status="finished") # check if interim results are saved resp_data = json_loads(rv.data) rv_user_id = resp_data["user_id"] rv_resource_id = resp_data["resource_id"] interim_dir = os.path.join(global_config.GRASS_RESOURCE_DIR, rv_user_id, "interim", rv_resource_id) self.assertTrue( not os.path.isdir(interim_dir), "Interim results are not stored in the expected folder")
def test_job_resumption_stdout(self): """Test job resumption with processing_async endpoint and stdout """ tpl = Template(json_dumps(process_chain_5_stdout)) rv = self.server.post(URL_PREFIX + self.endpoint, headers=self.admin_auth_header, data=tpl.render(map1="elevation2@PERMANENT", map2="baum"), content_type="application/json") resp = self.waitAsyncStatusAssertHTTP(rv, headers=self.admin_auth_header, http_status=400, status="error") status_url = resp["urls"]["status"].split(URL_PREFIX)[-1] # check if interim results are saved resp_data = json_loads(rv.data) rv_user_id = resp_data["user_id"] rv_resource_id = resp_data["resource_id"] interim_dir = os.path.join(global_config.GRASS_RESOURCE_DIR, rv_user_id, "interim", rv_resource_id) self.assertTrue( os.path.isdir(interim_dir), "Interim results are not stored in the expected folder") # resumption of the job rv2 = self.server.put(URL_PREFIX + status_url, headers=self.admin_auth_header, data=tpl.render(map1="elevation@PERMANENT", map2="baum"), content_type="application/json") resp2 = self.waitAsyncStatusAssertHTTP(rv2, headers=self.admin_auth_header, http_status=200, status="finished") self.compare_stdout(resp2)
def test_strds_create_register_unregister_1(self): new_mapset = "raster_test_mapset" self.create_new_mapset(new_mapset, "ECAD") # Create success rv = self.server.post( URL_PREFIX + '/locations/ECAD/mapsets/%s/strds/test_strds_register' % new_mapset, headers=self.admin_auth_header, data=json_dumps({ "temporaltype": "absolute", "title": "A nice title", "description": "A nice description" }), content_type="application/json") pprint(json_loads(rv.data)) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) # Create the raster layer rv = self.server.post( URL_PREFIX + '/locations/ECAD/mapsets/%s/raster_layers/test_layer_1' % new_mapset, headers=self.admin_auth_header, data=json_dumps({"expression": "1"}), content_type="application/json") pprint(json_loads(rv.data)) rv = self.server.post( URL_PREFIX + '/locations/ECAD/mapsets/%s/raster_layers/test_layer_2' % new_mapset, headers=self.admin_auth_header, data=json_dumps({"expression": "2"}), content_type="application/json") pprint(json_loads(rv.data)) rv = self.server.post( URL_PREFIX + '/locations/ECAD/mapsets/%s/raster_layers/test_layer_3' % new_mapset, headers=self.admin_auth_header, data=json_dumps({"expression": "3"}), content_type="application/json") pprint(json_loads(rv.data)) raster_layers = [{ "name": "test_layer_1", "start_time": "2000-01-01 00:00:00", "end_time": "2000-01-02 00:00:00" }, { "name": "test_layer_2", "start_time": "2000-01-02 00:00:00", "end_time": "2000-01-03 00:00:00" }, { "name": "test_layer_3", "start_time": "2000-01-03 00:00:00", "end_time": "2000-01-04 00:00:00" }] rv = self.server.put( URL_PREFIX + "/locations/ECAD/mapsets/%s/strds/test_strds_register/raster_layers" % new_mapset, data=json_dumps(raster_layers), content_type="application/json", headers=self.admin_auth_header) pprint(json_loads(rv.data)) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) # Check strds rv = self.server.get( URL_PREFIX + "/locations/ECAD/mapsets/%s/strds/test_strds_register" % new_mapset, headers=self.admin_auth_header) pprint(json_loads(rv.data)) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) min_min = json_loads(rv.data)["process_results"]["min_min"] max_max = json_loads(rv.data)["process_results"]["max_max"] num_maps = json_loads(rv.data)["process_results"]["number_of_maps"] self.assertEqual(min_min, "1.0") self.assertEqual(max_max, "3.0") self.assertEqual(num_maps, "3") # Unregister the raster layers raster_layers = ["test_layer_1", "test_layer_2", "test_layer_3"] rv = self.server.delete( URL_PREFIX + "/locations/ECAD/mapsets/%s/strds/test_strds_register/raster_layers" % new_mapset, data=json_dumps(raster_layers), content_type="application/json", headers=self.user_auth_header) pprint(json_loads(rv.data)) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) # Check strds rv = self.server.get( URL_PREFIX + "/locations/ECAD/mapsets/%s/strds/test_strds_register" % new_mapset, headers=self.user_auth_header) pprint(json_loads(rv.data)) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) min_min = json_loads(rv.data)["process_results"]["min_min"] max_max = json_loads(rv.data)["process_results"]["max_max"] num_maps = json_loads(rv.data)["process_results"]["number_of_maps"] self.assertEqual(min_min, "None") self.assertEqual(max_max, "None") self.assertEqual(num_maps, "0") # Delete the strds rv = self.server.delete( URL_PREFIX + '/locations/ECAD/mapsets/%s/strds/test_strds_register' % new_mapset, headers=self.user_auth_header) pprint(json_loads(rv.data)) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype)
def test_superadmin_termination(self): """Test the termination of one running resources from superadmin user """ # Create a random test user id that are used for login as admin user_id = "horst" + str(randint(0, 10000000)) user_group = self.user_group password = "******" # We need to create an HTML basic authorization header auth_header = Headers() auth = bytes('%s:%s' % (user_id, password), "utf-8") auth_header.add('Authorization', 'Basic ' + base64.b64encode(auth).decode()) # Make sure the user database is empty user = ActiniaUser(user_id) if user.exists(): user.delete() # Create a user in the database and reduce its credentials self.user = ActiniaUser.create_user( user_id, user_group, password, user_role="superadmin", accessible_datasets={ "nc_spm_08": ["PERMANENT", "user1", "landsat", "test_mapset"], "ECAD": ["PERMANENT"] }, process_num_limit=3, process_time_limit=100) # Start three processes that will be terminated rv = self.server.post(URL_PREFIX + '/custom_process/sleep', headers=auth_header, data=json_dumps(["20"]), content_type="application/json") # Test guest termination error rv = self.server.delete(URL_PREFIX + '/resources/%s' % user_id, headers=self.guest_auth_header) print(rv.data.decode()) self.assertEqual(rv.status_code, 401, "HTML status code is wrong %i" % rv.status_code) # self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) # Test user termination error rv = self.server.delete(URL_PREFIX + '/resources/%s' % user_id, headers=self.user_auth_header) print(rv.data.decode()) self.assertEqual(rv.status_code, 401, "HTML status code is wrong %i" % rv.status_code) # self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) # Test admin termination error rv = self.server.delete(URL_PREFIX + '/resources/%s' % user_id, headers=self.admin_auth_header) print(rv.data.decode()) self.assertEqual(rv.status_code, 401, "HTML status code is wrong %i" % rv.status_code) # self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) # Test superadmin termination success rv = self.server.delete(URL_PREFIX + '/resources/%s' % user_id, headers=self.root_auth_header) print(rv.data.decode()) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) # Wait for termination time.sleep(5) rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id, headers=auth_header) print(rv.data.decode()) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) resource_list = json_loads(rv.data)["resource_list"] self.assertTrue(len(resource_list) == 1) # Count return stats terminated = 0 for resource in resource_list: terminated += int(resource["status"] == "terminated") print(resource["status"]) self.assertTrue(terminated == 1)
def test_user_status_requests_1(self): """Resource list with 3 finished resources """ # Create a random test user id that are used for login as admin user_id = "heinz" + str(randint(0, 10000000)) user_group = self.user_group password = "******" # We need to create an HTML basic authorization header auth_header = Headers() auth = bytes('%s:%s' % (user_id, password), "utf-8") auth_header.add('Authorization', 'Basic ' + base64.b64encode(auth).decode()) # Make sure the user database is empty user = ActiniaUser(user_id) if user.exists(): user.delete() # Create a user in the database and reduce its credentials self.user = ActiniaUser.create_user( user_id, user_group, password, user_role="user", accessible_datasets={ "nc_spm_08": ["PERMANENT", "user1", "landsat", "modis_lst", "test_mapset"], "ECAD": ["PERMANENT"] }, process_num_limit=3, process_time_limit=2) # Create three successfully run resources rv = self.server.post(URL_PREFIX + '/custom_process/uname', headers=auth_header, data=json_dumps(["-a"]), content_type="application/json") self.waitAsyncStatusAssertHTTP(rv, headers=self.admin_auth_header) rv = self.server.post(URL_PREFIX + '/custom_process/uname', headers=auth_header, data=json_dumps(["-a"]), content_type="application/json") self.waitAsyncStatusAssertHTTP(rv, headers=self.admin_auth_header) rv = self.server.post(URL_PREFIX + '/custom_process/uname', headers=auth_header, data=json_dumps(["-a"]), content_type="application/json") self.waitAsyncStatusAssertHTTP(rv, headers=self.admin_auth_header) rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id, headers=auth_header) # print(rv.data.decode()) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) resource_list = json_loads(rv.data)["resource_list"] pprint(resource_list) self.assertTrue(len(resource_list) == 3) # Count return stats finished = 0 for resource in resource_list: finished += int(resource["status"] == "finished") # print(resource["status"]) self.assertTrue(finished == 3) # Check the different resource list parameters rv = self.server.get(URL_PREFIX + '/resources/%s?num=1' % user_id, headers=auth_header) self.assertTrue(len(json_loads(rv.data)["resource_list"]) == 1) rv = self.server.get(URL_PREFIX + '/resources/%s?num=2' % user_id, headers=auth_header) self.assertTrue(len(json_loads(rv.data)["resource_list"]) == 2) rv = self.server.get(URL_PREFIX + '/resources/%s?num=2&type=finished' % user_id, headers=auth_header) self.assertTrue(len(json_loads(rv.data)["resource_list"]) == 2) rv = self.server.get(URL_PREFIX + '/resources/%s?num=2&type=all' % user_id, headers=auth_header) self.assertTrue(len(json_loads(rv.data)["resource_list"]) == 2) rv = self.server.get(URL_PREFIX + '/resources/%s?type=all' % user_id, headers=auth_header) self.assertTrue(len(json_loads(rv.data)["resource_list"]) == 3) rv = self.server.get(URL_PREFIX + '/resources/%s?type=finished' % user_id, headers=auth_header) self.assertTrue(len(json_loads(rv.data)["resource_list"]) == 3) rv = self.server.get(URL_PREFIX + '/resources/%s?type=terminated' % user_id, headers=auth_header) self.assertTrue(len(json_loads(rv.data)["resource_list"]) == 0) rv = self.server.get(URL_PREFIX + '/resources/%s?type=unknown' % user_id, headers=auth_header) self.assertTrue(len(json_loads(rv.data)["resource_list"]) == 0) # Check permission access using the default users rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id, headers=self.guest_auth_header) # print(rv.data.decode()) self.assertEqual(rv.status_code, 401, "HTML status code is wrong %i" % rv.status_code) # self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id, headers=self.user_auth_header) # print(rv.data.decode()) self.assertEqual(rv.status_code, 401, "HTML status code is wrong %i" % rv.status_code) # self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id, headers=self.admin_auth_header) # print(rv.data.decode()) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) # self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id, headers=self.root_auth_header) # print(rv.data.decode()) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code)
def test_user_status_requests_3(self): """Empty resource list test """ # Create a random test user id that are used for login as admin user_id = "heinz" + str(randint(0, 10000000)) user_group = "test" password = "******" # We need to create an HTML basic authorization header auth_header = Headers() auth = bytes('%s:%s' % (user_id, password), "utf-8") auth_header.add('Authorization', 'Basic ' + base64.b64encode(auth).decode()) # Make sure the user database is empty user = ActiniaUser(user_id) if user.exists(): user.delete() # Create a user in the database and reduce its credentials self.user = ActiniaUser.create_user(user_id, user_group, password) rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id, headers=auth_header) print(rv.data.decode()) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) resource_list = json_loads(rv.data)["resource_list"] self.assertTrue(len(resource_list) == 0) # Check permission access using the default users rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id, headers=self.guest_auth_header) print(rv.data.decode()) self.assertEqual(rv.status_code, 401, "HTML status code is wrong %i" % rv.status_code) # self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id, headers=self.user_auth_header) print(rv.data.decode()) self.assertEqual(rv.status_code, 401, "HTML status code is wrong %i" % rv.status_code) # self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id, headers=self.admin_auth_header) print(rv.data.decode()) self.assertEqual(rv.status_code, 401, "HTML status code is wrong %i" % rv.status_code) # self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) rv = self.server.get(URL_PREFIX + '/resources/%s' % user_id, headers=self.root_auth_header) print(rv.data.decode()) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype)
def test_strds_create_remove(self): new_mapset = "strds_test" self.create_new_mapset(mapset_name=new_mapset, location_name="ECAD") # Create success rv = self.server.post( URL_PREFIX + '/locations/ECAD/mapsets/%s/strds/test_strds' % new_mapset, headers=self.admin_auth_header, data=json_dumps({ "temporaltype": "absolute", "title": "A nice title", "description": "A nice description" }), content_type="application/json") print(rv.data) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) # Create failure since the strds already exists rv = self.server.post( URL_PREFIX + '/locations/ECAD/mapsets/%s/strds/test_strds' % new_mapset, headers=self.admin_auth_header, data=json_dumps({ "temporaltype": "absolute", "title": "A nice title", "description": "A nice description" }), content_type="application/json") # print(rv.data) self.assertEqual(rv.status_code, 400, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) # Read/check information of the new strds rv = self.server.get( URL_PREFIX + '/locations/ECAD/mapsets/%s/strds/test_strds' % new_mapset, headers=self.user_auth_header) print(rv.data) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) start_time = json_loads(rv.data)["process_results"]["start_time"] self.assertEquals(start_time, "'None'") # Delete the strds rv = self.server.delete( URL_PREFIX + '/locations/ECAD/mapsets/%s/strds/test_strds' % new_mapset, headers=self.admin_auth_header) print(rv.data) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) # Try to delete the strds again to produce an error rv = self.server.delete( URL_PREFIX + '/locations/ECAD/mapsets/%s/strds/test_strds' % new_mapset, headers=self.admin_auth_header) print(rv.data) self.assertEqual(rv.status_code, 400, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) rv = self.server.get( URL_PREFIX + '/locations/ECAD/mapsets/%s/strds/test_strds' % new_mapset, headers=self.user_auth_header) print(rv.data) self.assertEqual(rv.status_code, 400, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype)
def get_sentinel_urls(self, product_ids, bands=["B04", "B08"]): """Receive the download urls and time stamps for a list of Sentinel2 product ids from AWS service 1. Transform the Sentinel ID into the path of the productInfo.json url that is required to get the tile urls 2. Parse the productInfo.json file and extract the tile urls 3. Create the download links for each tile based ond each band 4. Include the tileInfo.json, xml and preview url's Args: product_ids: A list of sentinel product ids bands: A list of band names Returns: dict A dictionary that contains the time stamp, the tile names, the map names in GRASS, the aws cloud storage urls, the tile GeoJSON footprint and the XML metadata file path download url's. Example: [{'product_id': 'S2A_MSIL1C_20170202T090201_N0204_R007_T36TVT_20170202T090155', 'tiles':[{'B04': {'file_name' : 'S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V20151207T003302_20151207T003302_tile_14_band_B04.jp2', 'map_name' : 'S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V20151207T003302_20151207T003302_tile_14_band_B04', 'public_url': 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/V/XE/2015/12/7/0/B04.jp2'}, 'B08': {'file_name' : 'S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V20151207T003302_20151207T003302_tile_14_band_B08.jp2', 'map_name' : 'S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V20151207T003302_20151207T003302_tile_14_band_B08', 'public_url': 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/V/XE/2015/12/7/0/B08.jp2'}, 'info' : 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/V/XE/2015/12/7/0/tileInfo.json', 'metadata' : 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/V/XE/2015/12/7/0/metadata.xml', 'preview' : 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/V/XE/2015/12/7/0/preview.jpg', 'timestamp' : '2015-12-07T00:33:02.634Z', 'url' : 'http://sentinel-s2-l1c.s3-website.eu-central-1.amazonaws.com/#tiles/57/V/XE/2015/12/7/0/'}]}] """ # Old format # products/2015/12/7/S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V20151207T003302_20151207T003302 # https://sentinel-s2-l1c.s3.amazonaws.com/products/2015/10/1/S2A_OPER_PRD_MSIL1C_PDMC_20161220T003744_R006_V20151001T071826_20151001T071826/productInfo.json # New format # products/2017/10/31/S2A_MSIL1C_20171031T000721_N0206_R016_T01WCP_20171031T015145 # 1 2 3 4 5 6 7 8 # 012345678901234567890123456789012345678901234567890123456789012345678901234567890 # S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V20151207T003302_20151207T003302 # 1 2 3 4 5 6 7 8 # 012345678901234567890123456789012345678901234567890123456789012345678901234567890 # S2A_MSIL1C_20171031T000721_N0206_R016_T01WCP_20171031T015145 try: for band in bands: if band not in self.sentinel_bands: raise Exception("Unknown sentinel 2A band name <%s>"%band) result = [] for product_id in product_ids: product_id = product_id.replace(".SAFE", "") if "S2A_OPER" in product_id: year = product_id[63:67] month = product_id[67:69] day = product_id[69:71] else: year = product_id[45:49] month = product_id[49:51] day = product_id[51:53] if month != "10": month = month.replace("0", "") if day not in ["10", "20", "30"]: day = day.replace("0", "") # Get the product info JSON file json_url = "%(base)s/products/%(year)s/%(month)s/%(day)s/%(id)s/" \ "productInfo.json"%{"base":self.aws_sentinel_base_url, "year":year, "month":month, "day":day, "id":product_id} response = urlopen(json_url) product_info = response.read() # print(product_info) try: info = json_loads(product_info) except: raise Exception("Unable to read the productInfo.json file from URL: %s. Error: %s"%(json_url, product_info)) if info: scene_entry = {} scene_entry["product_id"] = product_id scene_entry["tiles"] = [] tile_num = 0 for tile in info["tiles"]: tile_num += 1 tile_info = {} tile_info["timestamp"] = info["timestamp"] # http://sentinel-s2-l1c.s3.amazonaws.com/tiles/58/V/CK/2015/12/7/0/metadata.xml # http://sentinel-s2-l1c.s3.amazonaws.com/tiles/58/V/CK/2015/12/7/0/tileInfo.json # http://sentinel-s2-l1c.s3.amazonaws.com/tiles/58/V/CK/2015/12/7/0/preview.jpg metadata_url = self.aws_sentinel_base_url + "/" + tile["path"] + "/metadata.xml" info_url = self.aws_sentinel_base_url + "/" + tile["path"] + "/tileInfo.json" preview_url = self.aws_sentinel_base_url + "/" + tile["path"] + "/preview.jpg" tile_info["url"] = self.aws_sentinel_base_eu_central_url + "/#" + tile["path"] + "/" tile_info["metadata"] = metadata_url tile_info["info"] = info_url tile_info["preview"] = preview_url public_url = self.aws_sentinel_base_url + "/" + tile["path"] for band in bands: tile_name = "%s_tile_%i_band_%s.jp2"%(product_id, tile_num, band) map_name = "%s_tile_%i_band_%s"%(product_id, tile_num, band) tile_info[band] = {} tile_info[band]["file_name"] = tile_name tile_info[band]["map_name"] = map_name tile_info[band]["public_url"] = "%s/%s.jp2"%(public_url, band) scene_entry["tiles"].append(tile_info) result.append(scene_entry) return result except: raise
def test_3_existing_mapset_lock(self): """ Try to run two request on a locked mapset :return: """ self.check_remove_test_mapset() # Create new mapset rv = self.server.post(URL_PREFIX + '/locations/nc_spm_08/mapsets/test_mapset', headers=self.admin_auth_header) print(rv.data) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) # Run the processing inside the new mapset rv = self.server.post( URL_PREFIX + '/locations/nc_spm_08/mapsets/test_mapset/processing_async', headers=self.admin_auth_header, data=json_dumps(process_chain_long), content_type="application/json") print(rv.data) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) resp = json_loads(rv.data) rv_user_id = resp["user_id"] rv_resource_id = resp["resource_id"] time.sleep(0.5) # Try to lock again and again # Run the processing inside the new mapset # Second runner rv_lock_1 = self.server.post( URL_PREFIX + '/locations/nc_spm_08/mapsets/test_mapset/processing_async', headers=self.admin_auth_header, data=json_dumps(process_chain_short), content_type="application/json") print(rv_lock_1.data) self.assertEqual(rv_lock_1.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv_lock_1.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) # Third runner rv_lock_2 = self.server.post( URL_PREFIX + '/locations/nc_spm_08/mapsets/test_mapset/processing_async', headers=self.admin_auth_header, data=json_dumps(process_chain_long), content_type="application/json") print(rv_lock_2.data) self.assertEqual(rv_lock_2.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv_lock_2.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype) # Check the first runner while True: rv = self.server.get(URL_PREFIX + "/resources/%s/%s" % (rv_user_id, rv_resource_id), headers=self.admin_auth_header) print(rv.data) resp = json_loads(rv.data) if resp["status"] == "finished" or resp["status"] == "error": break time.sleep(0.2) self.assertEqual(resp["status"], "finished") self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) # Check the second runner resp = json_loads(rv_lock_1.data) rv_user_id = resp["user_id"] rv_resource_id = resp["resource_id"] while True: rv = self.server.get(URL_PREFIX + "/resources/%s/%s" % (rv_user_id, rv_resource_id), headers=self.admin_auth_header) print(rv.data) resp = json_loads(rv.data) if resp["status"] == "finished" or resp["status"] == "error": break time.sleep(0.2) self.assertEqual(resp["status"], "error") self.assertEqual(rv.status_code, 400, "HTML status code is wrong %i" % rv.status_code) # Check the third runner resp = json_loads(rv_lock_2.data) rv_user_id = resp["user_id"] rv_resource_id = resp["resource_id"] # Check the first runner while True: rv = self.server.get(URL_PREFIX + "/resources/%s/%s" % (rv_user_id, rv_resource_id), headers=self.admin_auth_header) print(rv.data) resp = json_loads(rv.data) if resp["status"] == "finished" or resp["status"] == "error": break time.sleep(0.2) self.assertEqual(resp["status"], "error") self.assertEqual(rv.status_code, 400, "HTML status code is wrong %i" % rv.status_code) # Remove the mapset rv = self.server.delete(URL_PREFIX + '/locations/nc_spm_08/mapsets/test_mapset', headers=self.admin_auth_header) print(rv.data) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) self.assertEqual(rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype)
def test_job_2_times_resumption_exporter(self): """Test job 2 times resumption with processing_async_export endpoint and exporter """ tpl = Template(json_dumps(process_chain_4_exporter)) rv = self.server.post(URL_PREFIX + self.endpoint, headers=self.admin_auth_header, data=tpl.render(map1="elevation2@PERMANENT", map2="baum"), content_type="application/json") resp = self.waitAsyncStatusAssertHTTP(rv, headers=self.admin_auth_header, http_status=400, status="error") status_url = resp["urls"]["status"].split(URL_PREFIX)[-1] # first job resumption rv2 = self.server.put(URL_PREFIX + status_url, headers=self.admin_auth_header, data=tpl.render(map1="elevation@PERMANENT", map2="baum1"), content_type="application/json") resp2 = self.waitAsyncStatusAssertHTTP(rv2, headers=self.admin_auth_header, http_status=400, status="error") status_url = resp2["urls"]["status"].split(URL_PREFIX)[-1] # check if interim results are saved resp_data = json_loads(rv2.data) rv_user_id = resp_data["user_id"] rv_resource_id = resp_data["resource_id"] interim_dir = os.path.join(global_config.GRASS_RESOURCE_DIR, rv_user_id, "interim", rv_resource_id) self.assertTrue( os.path.isdir(interim_dir), "Interim results are not stored in the expected folder") # second job resumption rv3 = self.server.put(URL_PREFIX + status_url, headers=self.admin_auth_header, data=tpl.render(map1="elevation@PERMANENT", map2="baum"), content_type="application/json") resp3 = self.waitAsyncStatusAssertHTTP(rv3, headers=self.admin_auth_header, http_status=200, status="finished") # Get the exported results urls = resp3["urls"]["resources"] for url in urls: print(url) rv = self.server.get(url, headers=self.admin_auth_header) self.assertEqual(rv.status_code, 200, "HTML status code is wrong %i" % rv.status_code) if url.endswith('.tif'): self.assertEqual(rv.mimetype, "image/tiff", "Wrong mimetype %s" % rv.mimetype) elif url.endswith('.zip'): self.assertEqual(rv.mimetype, "application/zip", "Wrong mimetype %s" % rv.mimetype) self.__class__.resource_user_id = rv_user_id self.__class__.resource_resource_id = rv_resource_id
def is_json(data): try: json_loads(data) except JSONDecodeError: return False return True
def get_sentinel_urls(self, product_ids, bands=None): """Receive the download urls and time stamps for a list of Sentinel2 product ids from AWS service 1. Transform the Sentinel ID into the path of the productInfo.json url that is required to get the tile urls 2. Parse the productInfo.json file and extract the tile urls 3. Create the download links for each tile based on each band 4. Include the tileInfo.json, xml and preview url's Args: product_ids: A list of sentinel product ids bands: A list of band names Returns: dict A dictionary that contains the time stamp, the tile names, the map names in GRASS, the aws cloud storage urls, the tile GeoJSON footprint and the XML metadata file path download url's. Example: [{'product_id': 'S2A_MSIL1C_20170202T090201_N0204_R007_T36TVT_' '20170202T090155', 'tiles':[{ 'B04': { 'file_name' : 'S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_' 'V20151207T003302_20151207T003302_tile_14_band_' 'B04.jp2', 'map_name' : 'S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_' 'V20151207T003302_20151207T003302_tile_14_band_B04', 'public_url': 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/' 'V/XE/2015/12/7/0/B04.jp2'}, 'B08': { 'file_name' : 'S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V2015' '1207T003302_20151207T003302_tile_14_band_B08.jp2', 'map_name' : 'S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V2015' '1207T003302_20151207T003302_tile_14_band_B08', 'public_url': 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/' 'V/XE/2015/12/7/0/B08.jp2'}, 'info' : 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/V/' 'XE/2015/12/7/0/tileInfo.json', 'metadata' : 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/V/' 'XE/2015/12/7/0/metadata.xml', 'preview' : 'http://sentinel-s2-l1c.s3.amazonaws.com/tiles/57/V/' 'XE/2015/12/7/0/preview.jpg', 'timestamp' : '2015-12-07T00:33:02.634Z', 'url' : 'http://sentinel-s2-l1c.s3-website.eu-central-1.' 'amazonaws.com/#tiles/57/V/XE/2015/12/7/0/'}]}] """ # Old format # products/2015/12/7/S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V20151207T003302_20151207T003302 # https://sentinel-s2-l1c.s3.amazonaws.com/products/2015/10/1/S2A_OPER_PRD_MSIL1C_PDMC_20161220T003744_R006_V20151001T071826_20151001T071826/productInfo.json # New format # products/2017/10/31/S2A_MSIL1C_20171031T000721_N0206_R016_T01WCP_20171031T015145 # 1 2 3 4 5 6 7 8 # 012345678901234567890123456789012345678901234567890123456789012345678901234567890 # S2A_OPER_PRD_MSIL1C_PDMC_20151207T031157_R102_V20151207T003302_20151207T003302 # 1 2 3 4 5 6 7 8 # 012345678901234567890123456789012345678901234567890123456789012345678901234567890 # S2A_MSIL1C_20171031T000721_N0206_R016_T01WCP_20171031T015145 # Assign default bands if bands is None: bands = ["B04", "B08"] try: for band in bands: if band not in self.sentinel_bands: raise Exception("Unknown Sentinel-2 band name <%s>" % band) result = [] for product_id in product_ids: product_id = product_id.replace(".SAFE", "") year, month, day = get_sentinel_date(product_id) # Get the product info JSON file json_url = "%(base)s/products/%(year)s/%(month)s/%(day)s/%(id)s/" \ "productInfo.json" % {"base": self.aws_sentinel_base_url, "year": year, "month": month, "day": day, "id": product_id} response = urlopen(json_url) product_info = response.read() try: info = json_loads(product_info) except Exception: raise Exception( "Unable to read the productInfo.json file from URL: " "%s. Error: %s" % (json_url, product_info)) if info: scene_entry = self._parse_scene_info( bands, product_id, info) result.append(scene_entry) return result except Exception: raise