def test_query_for_sentinel_scene_empty_query(self):
gsqi = GoogleSatelliteBigQueryInterface(global_config)
result = gsqi.get_sentinel_urls([
"S2A_MSIL1C_20170208T092131_N0204_R093_T35TLF_NOPENOPENOPE",
], ["B04", "B08"])
pprint(result)
self.assertTrue(not result)
def test_query_for_landsat_scene_empty_query(self):
gsqi = GoogleSatelliteBigQueryInterface(global_config)
result = gsqi.get_landsat_urls([
"LE72_NOPENOPENOPE",
], ["B1", "MTL"])
pprint(result)
self.assertTrue(not result)
def __init__(self, rdc):
"""
Setup the variables of this class
Args:
rdc (ResourceDataContainer): The data container that contains all required variables for processing
"""
EphemeralProcessingWithExport.__init__(self, rdc)
self.query_interface = GoogleSatelliteBigQueryInterface(self.config)
self.product_id = self.rdc.user_data
self.sentinel2_band_file_list = {}
self.gml_footprint = ""
self.user_download_cache_path = os.path.join(
self.config.DOWNLOAD_CACHE, self.user_id)
self.raster_result_list = [
] # The raster layer names which must be exported, stats computed
# and preview image created
self.module_results = [
] # A list of r.univar output classes for each vegetation index
self.response_model_class = SentinelNDVIResponseModel # The class that is used to create the response
self.required_bands = [
"B08", "B04"
] # The Sentinel 2A bands that are required for NDVI processing
self.query_result = None
def test_query_for_sentinel_scene_sinlge(self):
gsqi = GoogleSatelliteBigQueryInterface(global_config)
# A very small scene
result = gsqi.get_sentinel_urls([
"S2A_MSIL1C_20170212T104141_N0204_R008_T31TGJ_20170212T104138",
], ["B12", "B08"])
pprint(result)
self.assertTrue(len(result) == 1)
def test_query_for_landsat_scene_one_missing(self):
gsqi = GoogleSatelliteBigQueryInterface(global_config)
result = gsqi.get_landsat_urls([
"LC80440342016259LGN00", "LC80440342013106LGN01",
"LC80440342013154LGN00_WRONG"
], ["B1", "MTL"])
pprint(result)
self.assertTrue(len(result) == 2)
def test_query_for_sentinel_scene_two_missing(self):
gsqi = GoogleSatelliteBigQueryInterface(global_config)
result = gsqi.get_sentinel_urls([
"S2A_MSIL1C_20170301T225331_N0204_R115_T56DNG_20170301T225347",
"S2A_MSIL1C_20170221T093031_N0204_R136_T34UGV_20170221T093310",
"S2A_MSIL1C_NOPE", "S2A_MSIL1C_POPE"
], ["B04", "B08"])
pprint(result)
self.assertTrue(len(result) == 2)
def test_query_for_sentinel2_scene_time(self):
gsqi = GoogleSatelliteBigQueryInterface(global_config)
start = "2017-01-01T10:05:00"
end = "2017-01-01T11:00:00"
result = gsqi.query_sentinel2_archive(start_time=start, end_time=end)
pprint(result)
self.assertEqual(len(result), 154)
def test_query_for_landsat_scene_time(self):
gsqi = GoogleSatelliteBigQueryInterface(global_config)
start = "2017-01-01T10:45:00"
end = "2017-01-01T10:55:00"
result = gsqi.query_landsat_archive(start_time=start, end_time=end)
pprint(result)
self.assertTrue(len(result) >= 1)
def test_query_for_sentinel_scene_four(self):
gsqi = GoogleSatelliteBigQueryInterface(global_config)
result = gsqi.get_sentinel_urls([
"S2A_MSIL1C_20170202T090201_N0204_R007_T36TVT_20170202T090155",
"S2A_MSIL1C_20170221T093031_N0204_R136_T34UGV_20170221T093310",
"S2A_MSIL1C_20170211T060951_N0204_R134_T44WPT_20170211T060950",
"S2A_MSIL1C_20170218T143751_N0204_R096_T20PRT_20170218T143931"
], ["B04", "B08"])
pprint(result)
self.assertTrue(len(result) == 4)
def test_query_for_sentinel2_scene_time_coords(self):
gsqi = GoogleSatelliteBigQueryInterface(global_config)
start = "2016-11-01T00:00:00"
end = "2016-11-30T00:00:00"
result = gsqi.query_sentinel2_archive(start_time=start,
end_time=end,
lon=-45.0,
lat=-7.0)
pprint(result)
self.assertTrue(len(result) >= 1)
def _get(self, satellite):
args = self.query_parser.parse_args()
start_time = None
end_time = None
lon = None
lat = None
cloud_cover = None
scene_id = None
spacecraft_id = None
if "start_time" in args:
start_time = args["start_time"]
if "end_time" in args:
end_time = args["end_time"]
if "lon" in args:
lon = args["lon"]
if "lat" in args:
lat = args["lat"]
if "cloud_cover" in args:
cloud_cover = args["cloud_cover"]
if "scene_id" in args:
scene_id = args["scene_id"]
if "spacecraft_id" in args:
spacecraft_id = args["spacecraft_id"]
try:
iface = GoogleSatelliteBigQueryInterface(global_config)
if satellite == "landsat":
result = iface.query_landsat_archive(
start_time=start_time,
end_time=end_time,
lon=lon,
lat=lat,
cloud_cover=cloud_cover,
scene_id=scene_id,
spacecraft_id=spacecraft_id)
else:
result = iface.query_sentinel2_archive(start_time=start_time,
end_time=end_time,
lon=lon,
lat=lat,
cloud_cover=cloud_cover,
scene_id=scene_id)
return make_response(jsonify(result), 200)
except Exception as e:
result = {"status": "error", "message": str(e)}
return make_response(jsonify(result), 400)
def __init__(self, rdc):
"""Constructor
Args:
rdc (ResourceDataContainer): The data container that contains all required variables for processing
"""
PersistentProcessing.__init__(self, rdc)
# This works only if the mapset snot already exists
self.temp_mapset_name = self.mapset_name
self.query_interface = GoogleSatelliteBigQueryInterface(self.config)
self.product_ids = self.rdc.request_data["product_ids"]
self.strds_ids = self.rdc.request_data["strds"]
self.required_bands = self.rdc.request_data["bands"]
self.user_download_cache_path = os.path.join(
self.config.DOWNLOAD_CACHE, self.user_id)
self.query_result = None
class LandsatTimeSeriesCreator(PersistentProcessing):
"""Create a space time raster dataset from all provided scene_ids for each Sentinel2A band
in a new mapset.
The Sentiel2A scenes are downloaded , imported and pre-processed before they are registered
in the band specific space time datasets.
"""
def __init__(self, rdc):
"""Constructor
Args:
rdc (ResourceDataContainer): The data container that contains all required variables for processing
"""
PersistentProcessing.__init__(self, rdc)
# This works only if the mapset snot already exists
self.temp_mapset_name = self.mapset_name
self.query_interface = GoogleSatelliteBigQueryInterface(self.config)
self.scene_ids = self.rdc.request_data["scene_ids"]
self.strds_basename = self.rdc.request_data["strds"]
self.atcor_method = self.rdc.request_data["atcor_method"]
self.user_download_cache_path = os.path.join(
self.config.DOWNLOAD_CACHE, self.user_id)
self.query_result = None
self.required_bands = None
self.sensor_id = None
def _prepare_download(self):
"""Check the download cache if the file already exists, to avoid redundant downloads.
The downloaded files will be stored in a temporary directory. After the download of all files
completes, the downloaded files will be moved to the download cache. This avoids broken
files in case a download was interrupted or stopped by termination.
"""
# Create the download cache directory if it does not exists
if os.path.exists(self.config.DOWNLOAD_CACHE):
pass
else:
os.mkdir(self.config.DOWNLOAD_CACHE)
# Create the user specific download cache directory to put the downloaded files into it
if os.path.exists(self.user_download_cache_path):
pass
else:
os.mkdir(self.user_download_cache_path)
# Switch into the tempfile directory
os.chdir(self.temp_file_path)
# We have to set the home directory to create the grass location
os.putenv("HOME", "/tmp")
sensor_ids = {}
for scene_id in self.scene_ids:
# Check if the the list of scenes contains scenes from different satellites
self.sensor_id = extract_sensor_id_from_scene_id(scene_id=scene_id)
sensor_ids[self.sensor_id] = self.sensor_id
if len(sensor_ids) > 2:
raise AsyncProcessError(
"Different satellites are in the list of scenes to be imported. "
"Only scenes from a single satellite an be imported at once."
)
# All bands are imported, except the MTL file
self.required_bands = SCENE_BANDS[self.sensor_id][0:-1]
self._send_resource_update("Sending Google BigQuery request.")
try:
self.query_result = self.query_interface.get_landsat_urls(
self.scene_ids, self.required_bands)
except Exception as e:
raise AsyncProcessError("Error in querying Landsat product <%s> "
"in Google BigQuery Landsat database. "
"Error: %s" % (self.scene_ids, str(e)))
if not self.query_result:
raise AsyncProcessError("Unable to find Landsat product <%s> "
"in Google BigQuery Landsat database" %
self.scene_ids)
def _import_scenes(self):
"""Import all found Sentinel2 scenes with their bands and create the space time raster datasets
to register the maps in.
Raises:
AsyncProcessError: In case something went wrong
"""
all_commands = []
counter = 0
for scene_id in self.query_result:
process_lib = LandsatProcessing(
config=self.config,
scene_id=scene_id,
temp_file_path=self.temp_file_path,
download_cache=self.user_download_cache_path,
send_resource_update=self._send_resource_update,
message_logger=self.message_logger)
download_commands, self.sentinel2_band_file_list = process_lib.get_download_process_list(
)
# Download the Landsat scene if it is not in the download cache
if download_commands:
all_commands.extend(download_commands)
# Import and atmospheric correction
import_commands = process_lib.get_import_process_list()
all_commands.extend(import_commands)
atcor_method_commands = process_lib.get_i_landsat_toar_process_list(
atcor_method=self.atcor_method)
all_commands.extend(atcor_method_commands)
# Run the commands
self._update_num_of_steps(len(all_commands))
self._execute_process_list(process_list=all_commands)
# IMPORTANT:
# The registration must be performed in the temporary mapset with the same name as the target mapset,
# since the temporal database will contain the mapset name.
register_commands = {}
result_dict = {}
for i in range(len(self.required_bands)):
band = self.required_bands[i]
strds = self.strds_basename + "_%s" % band
result_dict[band] = []
create_strds = {
"module": "t.create",
"inputs": {
"output": strds,
"title": "Landsat time series for band %s" % band,
"description": "Landsat time series for band %s" % band,
"temporaltype": "absolute",
"type": "strds"
}
}
register_commands[str(counter)] = create_strds
counter += 1
# Create the input file
map_list_file_name = os.path.join(self.user_download_cache_path,
strds)
map_list_file = open(map_list_file_name, "w")
# Use only the product ids that were found in the big query
for scene_id in self.query_result:
# We need to create a time interval, otherwise the temporal algebra will not work :/
start_time = dtparser.parse(
self.query_result[scene_id]["timestamp"].split(".")[0])
end_time = start_time + timedelta(seconds=1)
index = SCENE_BANDS[self.sensor_id].index(band)
raster_suffix = RASTER_SUFFIXES[self.sensor_id][index]
map_name = "%s_%s%s" % (scene_id, self.atcor_method,
raster_suffix)
result_dict[band].append(
[map_name, str(start_time),
str(end_time)])
map_list_file.write("%s|%s|%s\n" %
(map_name, str(start_time), str(end_time)))
map_list_file.close()
register_maps = {
"module": "t.register",
"inputs": {
"input": strds,
"file": map_list_file_name,
"type": "raster"
}
}
register_commands[str(counter)] = register_maps
counter += 1
process_list = self._validate_process_chain(
process_chain=register_commands, skip_permission_check=True)
self._execute_process_list(process_list=process_list)
self.module_results = result_dict
def _execute(self):
# Setup the user credentials and logger
self._setup()
# Check and lock the target and temp mapsets
self._check_lock_target_mapset()
if self.target_mapset_exists is True:
raise AsyncProcessError(
"Sentinel time series can only be create in a new mapset. "
"Mapset <%s> already exists." % self.target_mapset_name)
# Init GRASS environment and create the temporary mapset with the same name as the target mapset
# This is required to register the raster maps in the temporary directory, but use them in
# persistent directory
# Create the temp database and link the
# required mapsets into it
self._create_temp_database(self.required_mapsets)
# Initialize the GRASS environment and switch into PERMANENT
# mapset, which is always linked
self._create_grass_environment(
grass_data_base=self.temp_grass_data_base, mapset_name="PERMANENT")
# Create the temporary mapset and switch into it
self._create_temporary_mapset(temp_mapset_name=self.target_mapset_name)
# Setup the download cache and query the BigQuery database of google for scene_ids
self._prepare_download()
# Check if all product ids were found
missing_scene_ids = []
for scene_id in self.scene_ids:
if scene_id not in self.query_result:
missing_scene_ids.append(scene_id)
# Abort if a single scene is missing
if len(missing_scene_ids) > 0:
raise AsyncProcessError("Unable to find product ids <%s> in the "
"Google BigQuery database" %
str(missing_scene_ids))
self._import_scenes()
# Copy local mapset to original location
self._copy_merge_tmp_mapset_to_target_mapset()
class EphemeralSentinelProcessing(EphemeralProcessingWithExport):
"""
"""
def __init__(self, rdc):
"""
Setup the variables of this class
Args:
rdc (ResourceDataContainer): The data container that contains all required variables for processing
"""
EphemeralProcessingWithExport.__init__(self, rdc)
self.query_interface = GoogleSatelliteBigQueryInterface(self.config)
self.product_id = self.rdc.user_data
self.sentinel2_band_file_list = {}
self.gml_footprint = ""
self.user_download_cache_path = os.path.join(
self.config.DOWNLOAD_CACHE, self.user_id)
self.raster_result_list = [
] # The raster layer names which must be exported, stats computed
# and preview image created
self.module_results = [
] # A list of r.univar output classes for each vegetation index
self.response_model_class = SentinelNDVIResponseModel # The class that is used to create the response
self.required_bands = [
"B08", "B04"
] # The Sentinel 2A bands that are required for NDVI processing
self.query_result = None
def _prepare_sentinel2_download(self):
"""Check the download cache if the file already exists, to avoid redundant downloads.
The downloaded files will be stored in a temporary directory. After the download of all files
completes, the downloaded files will be moved to the download cache. This avoids broken
files in case a download was interrupted or stopped by termination.
"""
# Create the download cache directory if it does not exists
if os.path.exists(self.config.DOWNLOAD_CACHE):
pass
else:
os.mkdir(self.config.DOWNLOAD_CACHE)
# Create the user specific download cache directory to put the downloaded files into it
if os.path.exists(self.user_download_cache_path):
pass
else:
os.mkdir(self.user_download_cache_path)
# Switch into the tempfile directory
os.chdir(self.temp_file_path)
# We have to set the home directory to create the grass location
os.putenv("HOME", "/tmp")
try:
self.query_result = self.query_interface.get_sentinel_urls([
self.product_id,
], self.required_bands)
except Exception as e:
raise AsyncProcessError(
"Error in querying Sentinel 2A product <%s> "
"in Google BigQuery Sentinel 2A database. "
"Error: %s" % (self.product_id, str(e)))
if not self.query_result:
raise AsyncProcessError("Unable to find Sentinel 2A product <%s> "
"in Google BigQuery Sentinel 2A database" %
self.product_id)
def _create_temp_database(self, mapsets=[]):
"""Create a temporary gis database and location with a PERMANENT mapset for processing
Raises:
This function raises AsyncProcessError in case of an error.
"""
if not self.sentinel2_band_file_list:
raise AsyncProcessError(
"Unable to create a temporary GIS database, no data is available"
)
try:
geofile = self.sentinel2_band_file_list[self.required_bands[0]][0]
self._send_resource_update(geofile)
# We have to set the home directory to create the grass location
os.putenv("HOME", "/tmp")
# Switch into the GRASS temporary database directory
os.chdir(self.temp_grass_data_base)
executable_params = list()
executable_params.append(self.config.GRASS_GIS_START_SCRIPT)
executable_params.append("-e")
executable_params.append("-c")
executable_params.append(geofile)
executable_params.append(
os.path.join(self.temp_grass_data_base, self.location_name))
self.message_logger.info(
"%s %s" %
(self.config.GRASS_GIS_START_SCRIPT, executable_params))
self._update_num_of_steps(1)
p = Process(exec_type="exec",
executable="python2",
executable_params=executable_params)
# Create the GRASS location, this will create the location and mapset paths
self._run_process(p)
except Exception as e:
raise AsyncProcessError(
"Unable to create a temporary GIS database and location at <%s>"
", Exception: %s" %
(os.path.join(self.temp_grass_data_base, self.location_name,
"PERMANENT"), str(e)))
def _run_r_univar_command(self, raster_name):
"""Compute the univariate statistics for a raster layer
and put the result as dict in the module_result dict
Args:
raster_name:
"""
result_file = tempfile.mktemp(suffix=".univar",
dir=self.temp_file_path)
univar_command = dict()
univar_command["1"] = {
"module": "r.univar",
"inputs": {
"map": raster_name
},
"outputs": {
"output": {
"name": result_file
}
},
"flags": "g"
}
process_list = self._validate_process_chain(
process_chain=univar_command, skip_permission_check=True)
self._execute_process_list(process_list=process_list)
result_list = open(result_file, "r").readlines()
results = {"name": raster_name}
for line in result_list:
if "=" in line:
key, value = line.split("=")
results[key] = float(value.strip())
self.module_results.append(UnivarResultModel(**results))
def _render_preview_image(self, raster_name):
"""Setup the render environment and create a g.region
process chain entry to setup the extent from the options.
Args:
options: The parser options that contain n, s, e and w entries for region settings
result_file: The resulting PNG file name
Returns:
A process chain entry of g.region
"""
result_file = tempfile.mktemp(suffix=".png", dir=self.temp_file_path)
os.putenv("GRASS_RENDER_IMMEDIATE", "png")
os.putenv("GRASS_RENDER_WIDTH", "1300")
os.putenv("GRASS_RENDER_HEIGHT", "1000")
os.putenv("GRASS_RENDER_TRANSPARENT", "TRUE")
os.putenv("GRASS_RENDER_TRUECOLOR", "TRUE")
os.putenv("GRASS_RENDER_FILE", result_file)
os.putenv("GRASS_RENDER_FILE_READ", "TRUE")
render_commands = {}
render_commands["1"] = {
"module": "d.rast",
"inputs": {
"map": raster_name
}
}
# "flags":"n"}
render_commands["2"] = {
"module": "d.legend",
"inputs": {
"raster": raster_name,
"at": "8,92,0,7"
},
"flags": "n"
}
render_commands["3"] = {
"module": "d.barscale",
"inputs": {
"style": "line",
"at": "20,4"
},
# "flags":"n" # No "n" flag in grass72
}
# Run the selected modules
process_list = self._validate_process_chain(
process_chain=render_commands, skip_permission_check=True)
self._execute_process_list(process_list)
# Attach the png preview image to the resource URL list
# Generate the resource URL's from the url base and the file name
# Copy the png file to the resource directory
# file_name = "%s_preview.png"%(raster_name)
# resource_url = self.resource_url_base.replace("__None__", file_name)
# self.storage_interface.
# self.resource_url_list.append(resource_url)
# export_path = os.path.join(self.resource_export_path, file_name)
# shutil.move(result_file, export_path)
# Store the temporary file in the resource storage
# and receive the resource URL
resource_url = self.storage_interface.store_resource(result_file)
self.resource_url_list.append(resource_url)
def _create_output_resources(self, raster_result_list):
"""Create the output resources from the raster layer that are the result of the processing
The following resources will be computed
- Univariate statistics as result dictionary for each raster layer
- A PNG preview image for each raster layer
- A gzipped GeoTiff file
"""
for raster_name in raster_result_list:
self._run_r_univar_command(raster_name)
# Render a preview image for this raster layer
self._render_preview_image(raster_name)
export_dict = {
"name": raster_name,
"export": {
"format": "GTiff",
"type": "raster"
}
}
# Add the raster layer to the export list
self.resource_export_list.append(export_dict)
self._update_num_of_steps(len(raster_result_list))
# Export all resources and generate the finish response
self._export_resources(use_raster_region=True)
def _execute(self):
"""Overwrite this function in subclasses
- Setup user credentials and working paths
- Create the resource directory
- Download and store the sentinel2 scene files
- Initialize and create the temporal database and location
- Analyse the process chains
- Run the modules
- Export the results
- Cleanup
"""
# Setup the user credentials and logger
self._setup()
# Create and check the resource directory
self.storage_interface.setup()
# Setup the download cache
self._prepare_sentinel2_download()
process_lib = Sentinel2Processing(
self.config, self.product_id, self.query_result,
self.required_bands, self.temp_file_path,
self.user_download_cache_path, self._send_resource_update,
self.message_logger)
download_commands, self.sentinel2_band_file_list = process_lib.get_sentinel2_download_process_list(
)
# Download the sentinel scene if not in the download cache
if download_commands:
self._update_num_of_steps(len(download_commands))
self._execute_process_list(process_list=download_commands)
# Setup GRASS
self._create_temporary_grass_environment(
source_mapset_name="PERMANENT")
# Import and prepare the sentinel scenes
import_commands = process_lib.get_sentinel2_import_process_list()
self._update_num_of_steps(len(import_commands))
self._execute_process_list(process_list=import_commands)
# Generate the ndvi command
nir = self.sentinel2_band_file_list["B08"][1]
red = self.sentinel2_band_file_list["B04"][1]
ndvi_commands = process_lib.get_ndvi_r_mapcalc_process_list(
red, nir, "ndvi")
self._update_num_of_steps(len(ndvi_commands))
self._execute_process_list(process_list=ndvi_commands)
self.raster_result_list.append("ndvi")
# Create the output resources: stats, preview and geotiff
self._create_output_resources(self.raster_result_list)
def _final_cleanup(self):
"""Overwrite this function in subclasses to perform the final cleanup
"""
# Clean up and remove the temporary gisdbase
self._cleanup()
# Remove resource directories
if "error" in self.run_state or "terminated" in self.run_state:
self.storage_interface.remove_resources()