def __init__(self):
# same as LibInfo variables
self.app_name = name
self.app_version = __version__
self.app_author = "G.Masetti (CCOM/JHC)"
self.app_author_email = "*****@*****.**"
self.app_license = "LGPL v3"
self.app_license_url = "https://www.hydroffice.org/license/"
self.app_path = os.path.abspath(os.path.dirname(__file__))
self.app_url = "https://www.hydroffice.org/abc/"
self.app_manual_url = "https://www.hydroffice.org/manuals/abc/index.html"
self.app_support_email = "*****@*****.**"
self.app_latest_url = "https://www.hydroffice.org/latest/abc.txt"
# additional AppInfo-specific variables
self.app_media_path = os.path.join(self.app_path, "media")
self.app_main_window_object_name = "MainWindow"
self.app_license_path = os.path.join(self.app_media_path, "LICENSE")
self.app_icon_path = os.path.join(self.app_media_path, "app_icon.png")
self.app_tabs_icon_size = 36
self.app_toolbars_icon_size = 24
GdalAux.check_gdal_data()
GdalAux.check_proj4_data()
def get_transformed_point(cls,
long_in,
lat_in,
epsg_inputs=4326,
epsg_calc=3857):
"""" Perform coordinate pair transformation
Args:
long_in: Input longitude
lat_in: Input latitude
epsg_inputs: Input EPSG CRS
epsg_calc: Output EPSG CRS
Returns:
list: X, Y
"""
GdalAux.check_gdal_data()
osr_inputs = osr.SpatialReference()
osr_inputs.ImportFromEPSG(epsg_inputs)
osr_calc = osr.SpatialReference()
if epsg_calc == 3857:
osr_calc.ImportFromWkt(wkt_3857)
else:
osr_calc.ImportFromEPSG(epsg_calc)
coord_transform = osr.CoordinateTransformation(osr_inputs, osr_calc)
# print(type(coord_transform))
point = ogr.Geometry(ogr.wkbPoint)
point.AddPoint(long_in, lat_in)
point.Transform(coord_transform)
# print(point)
return point.GetX(), point.GetY()
def __init__(self):
""" Initialization """
GdalAux.check_gdal_data()
GdalAux.check_proj4_data()
self.geo = Geod(ellps='WGS84')
def write_bluenotes(cls, feature_list, path, list_of_list=True):
if not os.path.exists(os.path.dirname(path)):
raise RuntimeError("the passed path does not exist: %s" % path)
path = Helper.truncate_too_long(path)
if not isinstance(feature_list, list):
raise RuntimeError(
"the passed parameter as feature_list is not a list: %s" %
type(feature_list))
if os.path.splitext(path)[-1] == '.shp':
path = path[:-4]
GdalAux()
# create the data source
try:
ds = GdalAux.create_ogr_data_source(
ogr_format=GdalAux.ogr_formats['ESRI Shapefile'],
output_path=path)
lyr = cls._create_ogr_point_lyr_and_fields(ds)
except RuntimeError as e:
logger.error("%s" % e)
return
if list_of_list:
if len(feature_list[0]) != len(feature_list[1]):
raise RuntimeError("invalid input for list of list")
tmp_list = feature_list
feature_list = list()
for i, x in enumerate(tmp_list[0]):
if len(tmp_list) >= 4:
feature_list.append(
[x, tmp_list[1][i], tmp_list[2][i], tmp_list[3][i]])
else:
feature_list.append([x, tmp_list[1][i], tmp_list[2][i]])
for feature in feature_list:
ft = ogr.Feature(lyr.GetLayerDefn())
ft.SetField('note', feature[2])
if len(feature) >= 4:
ft.SetField('info', feature[3])
pt = ogr.Geometry(ogr.wkbPoint25D)
pt.SetPoint(0, feature[0], feature[1])
try:
ft.SetGeometry(pt)
except Exception as e:
RuntimeError("%s > pt: %s, %s" % (e, feature[0], feature[1]))
if lyr.CreateFeature(ft) != 0:
raise RuntimeError("Unable to create feature")
ft.Destroy()
return True
def __init__(self,
s57,
grids,
version="2018",
scale=10000,
with_laser=True,
is_target_detection=False,
progress_bar=None):
if version not in ["2015", "2016", "2017", "2018"]:
raise RuntimeError("unsupported specs version: %s" % version)
super().__init__(s57=s57, grids=grids)
self.type = valsou_algos["VALSOU_CHECK_v7"]
self.version = version
self.scale = scale
self.with_laser = with_laser
self.is_target_detection = is_target_detection
self.all_features = self.s57.rec10s
self.progress = progress_bar
self.dist_2mm = None
self.valsou_features = list()
self.valsou_2mm_features = list()
self.valsou_1node_features = list()
self.valsou_geo = None
self.valsou_utm = None
self.valsou_closest = None
self.valsou_closest_2 = None
self.valsou_visited = None
self.out_of_bbox = False
self.deconflicted = False
self.geo2loc = None
# dtm
self.bathy_nodata = None
self.bathy_values = None
self.bathy_is_double = False
self.bathy_hrs = None
self.bathy_transform = None
self.bathy_rows = None
self.bathy_cols = None
GdalAux.check_gdal_data()
def write_soundings(cls, feature_list, path):
"""Feature list as list of long, lat, depth"""
if not os.path.exists(os.path.dirname(path)):
raise RuntimeError("the passed path does not exist: %s" % path)
path = Helper.truncate_too_long(path)
if not isinstance(feature_list, list):
raise RuntimeError(
"the passed parameter as feature_list is not a list: %s" %
type(feature_list))
if os.path.splitext(path)[-1] == '.shp':
path = path[:-4]
GdalAux()
# create the data source
try:
ds = GdalAux.create_ogr_data_source(
ogr_format=GdalAux.ogr_formats['ESRI Shapefile'],
output_path=path)
lyr = cls._create_ogr_point_lyr_and_fields(ds)
except RuntimeError as e:
logger.error("%s" % e)
return
for feature in feature_list:
ft = ogr.Feature(lyr.GetLayerDefn())
ft.SetField('info', "%.1f" % feature[2])
pt = ogr.Geometry(ogr.wkbPoint25D)
pt.SetPoint(0, feature[0], feature[1], feature[2])
try:
ft.SetGeometry(pt)
except Exception as e:
RuntimeError("%s > pt: %s, %s, %s" %
(e, feature[0], feature[1], feature[2]))
if lyr.CreateFeature(ft) != 0:
raise RuntimeError("Unable to create feature")
ft.Destroy()
return True
def test_create_ogr_data_source(self):
for ogr_format in GdalAux.ogr_formats.keys():
output_file = self.tp.output_data_folder().joinpath(
"ex_gdal_aux%s" % GdalAux.ogr_exts[ogr_format])
if output_file.exists():
output_file.unlink()
output_ds = GdalAux.create_ogr_data_source(
ogr_format=GdalAux.ogr_formats[ogr_format],
output_path=str(output_file))
lyr = output_ds.CreateLayer("test", None, ogr.wkbPoint)
self.assertIsNotNone(lyr)
output_ds = None
def _georef_fliers(self):
"""Helper function that looks at the flagged array and store the node != 0 as feature fliers"""
# selected holes
fliers_x = list()
fliers_y = list()
fliers_z = list()
fliers_ck = list()
nz_y, nz_x = self.flag_grid.nonzero()
for i, x in enumerate(nz_x):
y = nz_y[i]
fliers_x.append(x)
fliers_y.append(y)
fliers_z.append(self.bathy_values[y, x])
fliers_ck.append(self.flag_grid[y, x])
GdalAux.check_gdal_data()
# logger.debug("crs: %s" % self.bathy_crs)
try:
osr_csar = osr.SpatialReference()
osr_csar.ImportFromWkt(self.bathy_hrs)
osr_geo = osr.SpatialReference()
osr_geo.ImportFromEPSG(4326) # geographic WGS84
loc2geo = osr.CoordinateTransformation(osr_csar, osr_geo)
except Exception as e:
raise IOError("unable to create a valid coords transform: %s" % e)
if len(fliers_x) == 0:
logger.info(
"No fliers detected in current slice, total fliers: %s" %
len(self.flagged_fliers))
return
tile = self.grids.tiles[0]
flagged_xs = list()
flagged_ys = list()
flagged_zs = list()
flagged_cks = list()
for i, x in enumerate(fliers_x):
e = tile.convert_easting(int(x))
n = tile.convert_northing(int(fliers_y[i]))
z = fliers_z[i]
c = fliers_ck[i]
logger.debug("#%d: %.0f, %.0f -> %.2f %.2f %.2f : %d" %
(i, x, fliers_y[i], e, n, z, c))
flagged_xs.append(e)
flagged_ys.append(n)
flagged_zs.append(z)
flagged_cks.append(c)
logger.info("Initial lists length: %s, %s, %s, %s" %
(len(self.flagged_xs), len(self.flagged_ys),
len(self.flagged_zs), len(self.flagged_cks)))
# convert flagged nodes to geographic coords
try:
xs = np.array(flagged_xs)
ys = np.array(flagged_ys)
cks = np.array(flagged_cks)
# logger.debug("xs: %s" % xs)
# logger.debug("ys: %s" % ys)
# convert to geographic
if gdal.__version__[0] == '3':
lonlat = np.array(
loc2geo.TransformPoints(np.vstack((xs, ys)).transpose()),
np.float64)
lonlat.T[[0, 1]] = lonlat.T[[1, 0]]
# print(lonlat)
else:
lonlat = np.array(
loc2geo.TransformPoints(np.vstack((xs, ys)).transpose()),
np.float64)
# add checks
lonlat[:, 2] = cks
# store as list of list
self.flagged_fliers += lonlat.tolist()
self.flagged_xs += flagged_xs
self.flagged_ys += flagged_ys
self.flagged_zs += flagged_zs
self.flagged_cks += flagged_cks
logger.info("Detected %s possible fliers" %
len(self.flagged_fliers))
logger.info("Resulting lists lengths: %s, %s, %s, %s" %
(len(self.flagged_xs), len(self.flagged_ys),
len(self.flagged_zs), len(self.flagged_cks)))
logger.debug(f"Flagged fliers: {self.flagged_fliers}")
except Exception as e:
raise RuntimeError(
"Unable to perform conversion of the flagged fliers to geographic: %s"
% e)
def generate_output(self, output_folder, output_name):
logger.debug("do EXIF: %s" % self.do_exif)
# create ascii file
self.output_ascii = os.path.join(output_folder, "%s.ascii" % output_name)
ascii_fod = open(self.output_ascii, 'w')
ascii_fod.write('Latitude;Longitude;Observed time;Colour;Nature of surface - qualifying terms;'
'Nature of surface;Remarks;Source date;Source indication;Images;'
'CMECS Substrate Name;CMECS Substrate Code;'
'CMECS Co-occurring Element 1 Name;CMECS Co-occurring Element 1 Code;'
'CMECS Co-occurring Element 2 Name;CMECS Co-occurring Element 2 Code\n')
# create output folder
self.cmecs_output_folder = os.path.join(output_folder, output_name)
if not os.path.exists(self.cmecs_output_folder):
os.mkdir(self.cmecs_output_folder)
# create 'Images' output folder
self.images_output_folder = os.path.join(self.cmecs_output_folder, "Images")
if not os.path.exists(self.images_output_folder):
os.mkdir(self.images_output_folder)
# create shapefile
self.output_shp = os.path.join(self.cmecs_output_folder, output_name)
GdalAux()
try:
ds = GdalAux.create_ogr_data_source(ogr_format=GdalAux.ogr_formats['ESRI Shapefile'],
output_path=self.output_shp)
lyr = self._create_ogr_point_lyr_and_fields(ds)
except RuntimeError as e:
logger.error("%s" % e)
return False
# populate
for idx, feature in enumerate(self.sbdare_features):
# create OGR feature
ft = ogr.Feature(lyr.GetLayerDefn())
# retrieve position for ASCII format
lat = Geodesy.dd2dms(feature.centroid.y)
lon = Geodesy.dd2dms(feature.centroid.x)
lat_str = "%02.0f-%02.0f-%05.2f%s" % (abs(lat[0]), lat[1], lat[2], ("N" if (lat[0] > 0) else "S"))
lon_str = "%03.0f-%02.0f-%05.2f%s" % (abs(lon[0]), lon[1], lon[2], ("E" if (lon[0] > 0) else "W"))
# print(lat_str, lon_str)
# retrieve position for shapefile format
pt = ogr.Geometry(ogr.wkbPoint)
pt.SetPoint(0, feature.centroid.x, feature.centroid.y)
try:
ft.SetGeometry(pt)
except Exception as e:
RuntimeError("%s > #%d pt: %s, %s" % (e, idx, feature.centroid.x, feature.centroid.y))
info = self._retrieve_info(feature=feature, feature_idx=idx)
info = self._calc_cmecs(info=info, feature_idx=idx)
# for each SBDARE, write a row in the ASCII file
ascii_fod.write("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s\n"
% (lat_str, lon_str,
info.observed_time, info.colour, info.natqua, info.natsur,
info.remrks, info.sordat, info.sorind,
info.images, info.c_subn, info.c_subc,
info.c_cen1, info.c_cec1, info.c_cen2, info.c_cec2))
# actually write the feature in the shapefile
self._write_shape_attributes(ft, info)
if lyr.CreateFeature(ft) != 0:
raise RuntimeError("Unable to create feature")
ft.Destroy()
# finalize ASCII file
ascii_fod.close()
return self._finalize_generate_output(root_folder=output_folder, base_folder=output_name, remove_folder=False)
def crs_id(self) -> Optional[str]:
if self._crs is None:
return None
return GdalAux.crs_id(self._crs)
import logging
from pathlib import Path
from osgeo import ogr
from hyo2.abc.lib.testing_paths import TestingPaths
from hyo2.abc.lib.gdal_aux import GdalAux
from hyo2.abc.lib.logging import set_logging
logger = logging.getLogger(__name__)
set_logging(ns_list=["hyo2.abc"])
tp = TestingPaths(root_folder=Path(__file__).parent.parent.parent.resolve())
gdal_version = GdalAux.current_gdal_version()
logger.debug("GDAL version: %s" % gdal_version)
for ogr_format in GdalAux.ogr_formats.keys():
output_file = tp.output_data_folder().joinpath(
"ex_gdal_aux%s" % GdalAux.ogr_exts[ogr_format])
if output_file.exists():
output_file.unlink()
logger.debug("file: %s" % output_file)
output_ds = GdalAux.create_ogr_data_source(
ogr_format=GdalAux.ogr_formats[ogr_format],
output_path=str(output_file))
lyr = output_ds.CreateLayer("test", None, ogr.wkbPoint)
output_ds = None
from PySide2 import QtCore, QtWidgets from hyo2.abc.lib.gdal_aux import GdalAux GdalAux.check_proj4_data() from mpl_toolkits.basemap import Basemap, __version__ import matplotlib.pyplot as plt print(__version__) map = Basemap(projection='ortho', lat_0=0, lon_0=0) map.drawmapboundary(fill_color='aqua') map.fillcontinents(color='coral', lake_color='aqua') map.drawcoastlines() lons = [0, 10, -20, -20] lats = [0, -10, 40, -20] x, y = map(lons, lats) map.scatter(x, y, marker='D', color='m') plt.show()
def test_gdal_version(self):
GdalAux.current_gdal_version()
def test_crs_id(self):
wkt = """
GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]
"""
self.assertEqual(GdalAux.crs_id(wkt=wkt), "4326")
def test_gdal_data(self):
GdalAux.check_gdal_data()
def write_tin(cls, feature_list_a, feature_list_b, path, list_of_list=True):
if not os.path.exists(os.path.dirname(path)):
raise RuntimeError("the passed path does not exist: %s" % path)
path = Helper.truncate_too_long(path)
if not isinstance(feature_list_a, list):
raise RuntimeError("the passed parameter as feature_list_a is not a list: %s" % type(feature_list_a))
if not isinstance(feature_list_b, list):
raise RuntimeError("the passed parameter as feature_list_b is not a list: %s" % type(feature_list_b))
if os.path.splitext(path)[-1] == '.kml':
path = path[:-4]
GdalAux()
# create the data source
try:
ds = GdalAux.create_ogr_data_source(ogr_format=GdalAux.ogr_formats['KML'],
output_path=path)
lyr = cls._create_ogr_line_lyr_and_fields(ds)
except RuntimeError as e:
logger.error("%s" % e)
return
if list_of_list:
if len(feature_list_a[0]) != len(feature_list_a[1]):
raise RuntimeError("invalid input for list of list")
if len(feature_list_b[0]) != len(feature_list_b[1]):
raise RuntimeError("invalid input for list of list")
if len(feature_list_a) != len(feature_list_b):
raise RuntimeError("invalid input for list of list")
tmp_list_a = feature_list_a
feature_list_a = list()
for i, x in enumerate(tmp_list_a[0]):
feature_list_a.append([x, tmp_list_a[1][i]])
tmp_list_b = feature_list_b
feature_list_b = list()
for i, x in enumerate(tmp_list_b[0]):
feature_list_b.append([x, tmp_list_b[1][i]])
for i, point in enumerate(feature_list_a):
ft = ogr.Feature(lyr.GetLayerDefn())
ft.SetField('note', "tin edge")
ln = ogr.Geometry(ogr.wkbLineString)
ln.AddPoint(point[0], point[1])
ln.AddPoint(feature_list_b[i][0], feature_list_b[i][1])
try:
ft.SetGeometry(ln)
except Exception as e:
RuntimeError("%s > ln: %s, %s / %s, %s"
% (e, point[0], point[1], feature_list_b[i][0], feature_list_b[i][1]))
if lyr.CreateFeature(ft) != 0:
raise RuntimeError("Unable to create feature")
ft.Destroy()
return True
def export_profiles_metadata(self, project_name, output_folder, ogr_format=GdalAux.ogr_formats['ESRI Shapefile'],
filter_fields=None):
self.filter_fields = filter_fields
if self.filter_fields is None:
self.filter_fields = ExportDbFields()
GdalAux()
output = os.path.join(self.export_folder(output_folder=output_folder), project_name)
# create the data source
try:
ds = GdalAux.create_ogr_data_source(ogr_format=ogr_format, output_path=output)
lyr = self._create_ogr_lyr_and_fields(ds)
except RuntimeError as e:
logger.error("%s" % e)
return
rows = self.db.list_profiles()
if rows is None:
raise RuntimeError("Unable to retrieve profiles. Empty database?")
if len(rows) == 0:
raise RuntimeError("Unable to retrieve profiles. Empty database?")
for row in rows:
ft = ogr.Feature(lyr.GetLayerDefn())
if self.filter_fields.fields['pk']:
ft.SetField('pk', int(row[0]))
if self.filter_fields.fields['datetime']:
ft.SetField('datetime', row[1].isoformat())
if self.filter_fields.fields['sensor']:
ft.SetField('sensor', Dicts.first_match(Dicts.sensor_types, row[3]))
if self.filter_fields.fields['probe']:
ft.SetField('probe', Dicts.first_match(Dicts.probe_types, row[4]))
if self.filter_fields.fields['path']:
ft.SetField('path', row[5])
if self.filter_fields.fields['agency']:
if row[6]:
ft.SetField('agency', row[6])
if self.filter_fields.fields['survey']:
if row[7]:
ft.SetField('survey', row[7])
if self.filter_fields.fields['vessel']:
if row[8]:
ft.SetField('vessel', row[8])
if self.filter_fields.fields['sn']:
if row[9]:
ft.SetField('sn', row[9])
if self.filter_fields.fields['proc_time']:
ft.SetField('proc_time', row[10].isoformat())
if self.filter_fields.fields['proc_info']:
ft.SetField('proc_info', row[11])
if self.filter_fields.fields['surveyline']:
if row[12]:
ft.SetField('surveyline', row[12])
if self.filter_fields.fields['comments']:
if row[13]:
ft.SetField('comments', row[13])
if self.filter_fields.fields['press_uom']:
if row[14]:
ft.SetField('press_uom', row[14])
if self.filter_fields.fields['depth_uom']:
if row[15]:
ft.SetField('depth_uom', row[15])
if self.filter_fields.fields['ss_uom']:
if row[16]:
ft.SetField('ss_uom', row[16])
if self.filter_fields.fields['temp_uom']:
if row[17]:
ft.SetField('temp_uom', row[17])
if self.filter_fields.fields['cond_uom']:
if row[18]:
ft.SetField('cond_uom', row[18])
if self.filter_fields.fields['sal_uom']:
if row[19]:
ft.SetField('sal_uom', row[19])
if self.filter_fields.fields['ss_at_mind']:
if row[20]:
ft.SetField('ss_at_mind', row[20])
if self.filter_fields.fields['min_depth']:
if row[21]:
ft.SetField('min_depth', row[21])
if self.filter_fields.fields['max_depth']:
if row[22]:
ft.SetField('max_depth', row[22])
if self.filter_fields.fields['max_raw_d']:
if row[23]:
ft.SetField('max_raw_d', row[23])
pt = ogr.Geometry(ogr.wkbPoint)
lat = row[2].y
lon = row[2].x
if lon > 180.0: # Go back to negative longitude
lon -= 360.0
pt.SetPoint_2D(0, lon, lat)
if self.filter_fields.fields['POINT_X']:
ft.SetField('POINT_X', lon)
if self.filter_fields.fields['POINT_Y']:
ft.SetField('POINT_Y', lat)
try:
ft.SetGeometry(pt)
except Exception as e:
RuntimeError("%s > pt: %s, %s" % (e, lon, lat))
if lyr.CreateFeature(ft) != 0:
raise RuntimeError("Unable to create feature")
ft.Destroy()
ds = None
return True
