def get_image_bounds(image_file: str) -> str:
"""Loads the boundaries from an image file
Arguments:
image_file: path to the image to load the bounds from
Return:
Returns the GEOJSON of the bounds if they could be loaded and converted (if necessary).
None is returned if the bounds are loaded or can't be converted
"""
# If the file has a geo shape we store it for clipping
bounds = image_get_geobounds(image_file)
epsg = get_epsg(image_file)
if bounds[0] != np.nan:
ring = ogr.Geometry(ogr.wkbLinearRing)
ring.AddPoint(bounds[2], bounds[1]) # Upper left
ring.AddPoint(bounds[3], bounds[1]) # Upper right
ring.AddPoint(bounds[3], bounds[0]) # lower right
ring.AddPoint(bounds[2], bounds[0]) # lower left
ring.AddPoint(bounds[2], bounds[1]) # Closing the polygon
poly = ogr.Geometry(ogr.wkbPolygon)
poly.AddGeometry(ring)
ref_sys = osr.SpatialReference()
if ref_sys.ImportFromEPSG(int(epsg)) == ogr.OGRERR_NONE:
poly.AssignSpatialReference(ref_sys)
return geometry_to_geojson(poly)
logging.warning("Failed to import EPSG %s for image file %s", str(epsg), image_file)
return None
def find_image_files(self, files):
"""Finds files that are needed for extracting plots from an orthomosaic
Args:
files(list): the list of file to look through and access
Returns:
Returns a dict of georeferenced image files (indexed by filename and containing an
object with the calculated image bounds as an ogr polygon and a list of the
bounds as a tuple)
The bounds are assumed to be rectilinear with the upper-left corner directly
pulled from the file and the lower-right corner calculated based upon the geometry
information stored in the file.
The polygon points start at the upper left corner and proceed clockwise around the
boundary. The returned polygon is closed: the first and last point are the same.
The bounds tuple contains the min and max Y point values, followed by the min and
max X point values.
"""
imagefiles = {}
for onefile in files:
ext = os.path.splitext(os.path.basename(onefile))[1].lstrip('.')
if not ext in self.known_non_image_ext:
if file_is_image_type(self.args.identify_binary, onefile,
onefile +
self.file_infodata_file_ending):
# If the file has a geo shape we store it for clipping
bounds = image_get_geobounds(onefile)
epsg = get_epsg(onefile)
if bounds[0] != np.nan:
ring = ogr.Geometry(ogr.wkbLinearRing)
ring.AddPoint(bounds[2], bounds[1]) # Upper left
ring.AddPoint(bounds[3], bounds[1]) # Upper right
ring.AddPoint(bounds[3], bounds[0]) # lower right
ring.AddPoint(bounds[2], bounds[0]) # lower left
ring.AddPoint(bounds[2],
bounds[1]) # Closing the polygon
poly = ogr.Geometry(ogr.wkbPolygon)
poly.AddGeometry(ring)
ref_sys = osr.SpatialReference()
if ref_sys.ImportFromEPSG(
int(epsg)) != ogr.OGRERR_NONE:
logging.error(
"Failed to import EPSG %s for image file %s",
str(epsg), onefile)
else:
poly.AssignSpatialReference(ref_sys)
imagefiles[onefile] = {'bounds': poly}
# Return what we've found
return imagefiles
def load_image_files(files):
"""Loads image file nboundaries
Args:
files(list): the list of file to look through and access
Returns:
Returns a dict of georeferenced image files (indexed by filename and containing an
object with the calculated image bounds as an ogr polygon and a list of the
bounds as a tuple)
The bounds are assumed to be rectilinear with the upper-left corner directly
pulled from the file and the lower-right corner calculated based upon the geometry
information stored in the file.
The polygon points start at the upper left corner and proceed clockwise around the
boundary. The returned polygon is closed: the first and last point are the same.
The bounds tuple contains the min and max Y point values, followed by the min and
max X point values.
"""
imagefiles = {}
for onefile in files:
# If the file has a geo shape we store it for clipping
bounds = image_get_geobounds(onefile)
epsg = get_epsg(onefile)
if bounds[0] != np.nan:
ring = ogr.Geometry(ogr.wkbLinearRing)
ring.AddPoint(bounds[2], bounds[1]) # Upper left
ring.AddPoint(bounds[3], bounds[1]) # Upper right
ring.AddPoint(bounds[3], bounds[0]) # lower right
ring.AddPoint(bounds[2], bounds[0]) # lower left
ring.AddPoint(bounds[2], bounds[1]) # Closing the polygon
poly = ogr.Geometry(ogr.wkbPolygon)
poly.AddGeometry(ring)
ref_sys = osr.SpatialReference()
if epsg:
if ref_sys.ImportFromEPSG(int(epsg)) != ogr.OGRERR_NONE:
raise RuntimeError("Failed to import EPSG " + str(epsg) +
" for image file " + onefile)
else:
poly.AssignSpatialReference(ref_sys)
else:
raise RuntimeError("File is missing an EPSG code: " + onefile)
imagefiles[onefile] = {'bounds': poly}
# Return what we've found
return imagefiles
def get_image_bounds_json(file_path: str,
default_epsg: int = None) -> Optional[str]:
"""Loads the boundaries of the image file and returns the GeoJSON
representing the bounds (including EPSG code)
Arguments:
file_path: path to the file from which to load the bounds
default_epsg: the default EPSG to assume if a file has a boundary but not a coordinate system
Return:
Returns the JSON representing the image boundary, or None if the
bounds could not be loaded
Notes:
If a file doesn't have a coordinate system and a default epsg is specified, the
return JSON will use the default_epsg.
If a file doesn't have a coordinate system and there isn't a default epsg specified, the boundary
of the image is not returned (None) and a warning is logged.
"""
# Get the bounds (if they exist)
bounds = image_get_geobounds(file_path)
if bounds[0] == np.nan:
return None
epsg = get_epsg(file_path)
if epsg is None:
if default_epsg:
epsg = default_epsg
else:
logging.warning(
"Files does not have a coordinate system defined and no default was specified: '%s'",
file_path)
return None
ring = ogr.Geometry(ogr.wkbLinearRing)
ring.AddPoint(bounds[2], bounds[1]) # Upper left
ring.AddPoint(bounds[3], bounds[1]) # Upper right
ring.AddPoint(bounds[3], bounds[0]) # lower right
ring.AddPoint(bounds[2], bounds[0]) # lower left
ring.AddPoint(bounds[2], bounds[1]) # Closing the polygon
poly = ogr.Geometry(ogr.wkbPolygon)
poly.AddGeometry(ring)
ref_sys = osr.SpatialReference()
if ref_sys.ImportFromEPSG(int(epsg)) == ogr.OGRERR_NONE:
poly.AssignSpatialReference(ref_sys)
return geometry_to_geojson(poly)
logging.error("Failed to import EPSG %s for image file %s", str(epsg),
file_path)
return None
def process_message(self, connector, host, secret_key, resource, parameters):
super(rgbEnhancementExtractor, self).process_message(connector, host, secret_key,
resource, parameters)
self.start_message(resource)
# Get left/right files and metadata
process_files = []
if not self.get_terraref_metadata is None:
process_files = find_terraref_files(resource)
else:
process_files = find_image_files(self.args.identify_binary, resource,
self.file_infodata_file_ending)
# Get the best username, password, and space
old_un, old_pw, old_space = (self.clowder_user, self.clowder_pass, self.clowderspace)
self.clowder_user, self.clowder_pass, self.clowderspace = self.get_clowder_context()
# Ensure that the clowder information is valid
if not confirm_clowder_info(host, secret_key, self.clowderspace, self.clowder_user,
self.clowder_pass):
self.log_error(resource, "Clowder configuration is invalid. Not processing " +\
"request")
self.clowder_user, self.clowder_pass, self.clowderspace = (old_un, old_pw, old_space)
self.end_message(resource)
return
# Change the base path of files to include the user by tweaking the sensor's value
sensor_old_base = None
if self.get_terraref_metadata is None:
_, new_base = self.get_username_with_base_path(host, secret_key, resource['id'],
self.sensors.base)
sensor_old_base = self.sensors.base
self.sensors.base = new_base
# Prepare for processing files
timestamp = timestamp_to_terraref(self.find_timestamp(resource['dataset_info']['name']))
target_dsid = resource['id']
uploaded_file_ids = []
ratios = []
try:
for one_file in process_files:
mask_source = one_file
# Make sure the source image is in the correct EPSG space
epsg = get_epsg(one_file)
if epsg != self.default_epsg:
self.log_info(resource, "Reprojecting from " + str(epsg) +
" to default " + str(self.default_epsg))
_, tmp_name = tempfile.mkstemp()
src = gdal.Open(one_file)
gdal.Warp(tmp_name, src, dstSRS='EPSG:'+str(self.default_epsg))
mask_source = tmp_name
# Get the bounds of the image to see if we can process it. Also get the mask filename
rgb_mask_tif, bounds = self.get_maskfilename_bounds(mask_source, timestamp)
if bounds is None:
self.log_skip(resource, "Skipping non-georeferenced image: " + \
os.path.basename(one_file))
if mask_source != one_file:
os.remove(mask_source)
continue
if not file_exists(rgb_mask_tif) or self.overwrite:
self.log_info(resource, "creating %s" % rgb_mask_tif)
mask_ratio, mask_rgb = gen_cc_enhanced(mask_source)
ratios.append(mask_ratio)
# Bands must be reordered to avoid swapping R and B
mask_rgb = cv2.cvtColor(mask_rgb, cv2.COLOR_BGR2RGB)
create_geotiff(mask_rgb, bounds, rgb_mask_tif, None, False, self.extractor_info,
self.get_terraref_metadata)
compress_geotiff(rgb_mask_tif)
# Remove any temporary file
if mask_source != one_file:
os.remove(mask_source)
self.created += 1
self.bytes += os.path.getsize(rgb_mask_tif)
found_in_dest = check_file_in_dataset(connector, host, secret_key, target_dsid,
rgb_mask_tif, remove=self.overwrite)
if not found_in_dest:
self.log_info(resource, "uploading %s" % rgb_mask_tif)
fileid = upload_to_dataset(connector, host, self.clowder_user, self.clowder_pass,
target_dsid, rgb_mask_tif)
uploaded_file_ids.append(host + ("" if host.endswith("/") else "/") +
"files/" + fileid)
# Tell Clowder this is completed so subsequent file updates don't daisy-chain
if not self.get_terraref_metadata is None:
ratios_len = len(ratios)
left_ratio = (ratios[0] if ratios_len > 0 else None)
right_ratio = (ratios[1] if ratios_len > 1 else None)
md = {
"files_created": uploaded_file_ids
}
if not left_ratio is None:
md["left_mask_ratio"] = left_ratio
if not self.leftonly and not right_ratio is None:
md["right_mask_ratio"] = right_ratio
extractor_md = build_metadata(host, self.extractor_info, target_dsid, md, 'dataset')
self.log_info(resource, "uploading extractor metadata to Lv1 dataset")
remove_metadata(connector, host, secret_key, resource['id'],
self.extractor_info['name'])
upload_metadata(connector, host, secret_key, resource['id'], extractor_md)
finally:
# Signal end of processing message and restore changed variables. Be sure to restore
# changed variables above with early returns
if not sensor_old_base is None:
self.sensors.base = sensor_old_base
self.clowder_user, self.clowder_pass, self.clowderspace = (old_un, old_pw, old_space)
self.end_message(resource)
def find_shape_image_files(self, files, triggering_file):
"""Finds files that are needed for extracting plots from an orthomosaic
Args:
files(list): the list of file to look through and access
triggering_file(str): optional parameter specifying the file that triggered the
extraction
Returns:
Returns a list containing the shapefile name, its optional associated DBF file,
and a dict of georeferenced image files (indexed by filename and containing an
object with the calculated image bounds as an ogr polygon and a list of the
bounds as a tuple)
The bounds are assumed to be rectilinear with the upper-left corner directly
pulled from the file and the lower-right corner calculated based upon the geometry
information stored in the file.
The polygon points start at the upper left corner and proceed clockwise around the
boundary. The returned polygon is closed: the first and last point are the same.
The bounds tuple contains the min and max Y point values, followed by the min and
max X point values.
"""
shapefile, shxfile, dbffile = None, None, None
imagefiles = {}
for onefile in files:
if onefile.endswith(".shp") and shapefile is None:
# We give priority to the shapefile that triggered the extraction over any other
# shapefiles that may exist
if triggering_file is None or triggering_file.endswith(
onefile):
shapefile = onefile
filename_test = os.path.splitext(shapefile)[0] + ".shx"
if os.path.isfile(filename_test):
shxfile = filename_test
filename_test = os.path.splitext(shapefile)[0] + ".dbf"
if os.path.isfile(filename_test):
dbffile = filename_test
else:
ext = os.path.splitext(
os.path.basename(onefile))[1].lstrip('.')
if not ext in self.known_non_image_ext:
if file_is_image_type(
self.args.identify_binary, onefile,
onefile + self.file_infodata_file_ending):
# If the file has a geo shape we store it for clipping
bounds = image_get_geobounds(onefile)
epsg = get_epsg(onefile)
if bounds[0] != nan:
ring = ogr.Geometry(ogr.wkbLinearRing)
ring.AddPoint(bounds[2], bounds[1]) # Upper left
ring.AddPoint(bounds[3], bounds[1]) # Upper right
ring.AddPoint(bounds[3], bounds[0]) # lower right
ring.AddPoint(bounds[2], bounds[0]) # lower left
ring.AddPoint(bounds[2],
bounds[1]) # Closing the polygon
poly = ogr.Geometry(ogr.wkbPolygon)
poly.AddGeometry(ring)
ref_sys = osr.SpatialReference()
if ref_sys.ImportFromEPSG(
int(epsg)) != ogr.OGRERR_NONE:
logging.error(
"Failed to import EPSG %s for image file %s",
str(epsg), onefile)
else:
poly.AssignSpatialReference(ref_sys)
# pylint: disable=line-too-long
imagefiles[onefile] = {'bounds': poly}
# pylint: enable=line-too-long
# Return what we've found
return (shapefile, shxfile, dbffile, imagefiles)
