def pyproj_crs_to_osgeo(proj_crs: Union[CRS, int]):
"""
Convert from the pyproj CRS object to osgeo SpatialReference
See https://pyproj4.github.io/pyproj/stable/crs_compatibility.html
Parameters
----------
proj_crs
pyproj CRS or an integer epsg code
Returns
-------
SpatialReference
converted SpatialReference
"""
if isinstance(proj_crs, int):
proj_crs = CRS.from_epsg(proj_crs)
osr_crs = SpatialReference()
if osgeo.version_info.major < 3:
osr_crs.ImportFromWkt(proj_crs.to_wkt(WktVersion.WKT1_GDAL))
else:
osr_crs.ImportFromWkt(proj_crs.to_wkt())
return osr_crs
def read_raster(path, band_number=1):
"""
Create a raster dataset from a single raster file
Parameters
----------
path: string
Path to the raster file. Can be either local or s3/gs.
band_number: int
The band number to use
Returns
-------
RasterDataset
"""
path = fileutils.get_path(path)
ds = gdal.Open(path, GA_ReadOnly)
xsize = ds.RasterXSize
ysize = ds.RasterYSize
proj = SpatialReference()
proj.ImportFromWkt(ds.GetProjection())
geo_transform = ds.GetGeoTransform()
return RasterDataset(ds, xsize, ysize, geo_transform, proj)
def _nor_roms(xp=3991,
yp=2230,
dx=800,
ylon=70,
name='NK800',
metric_unit=False):
if metric_unit:
unit_str = 'UNIT["metre",1,AUTHORITY["EPSG","9001"]]'
dx_unit = dx
else:
unit_str = f'UNIT["Cells",{dx}]]'
dx_unit = 1
wkt = f"""PROJCS["{name}",
GEOGCS["ETRS89",
DATUM["European_Terrestrial_Reference_System_1989",
SPHEROID["GRS 1980",6378137,298.257222101,
AUTHORITY["EPSG","7019"]],
AUTHORITY["EPSG","6258"]],
PRIMEM["Greenwich",0,
AUTHORITY["EPSG","8901"]],
UNIT["degree",0.01745329251994328,
AUTHORITY["EPSG","9122"]],
AUTHORITY["EPSG","4258"]],
PROJECTION["Polar_Stereographic"],
PARAMETER["latitude_of_origin",60],
PARAMETER["central_meridian",{ylon}],
PARAMETER["scale_factor",1],
PARAMETER["false_easting",{xp * dx_unit}],
PARAMETER["false_northing",{yp * dx_unit}],
{unit_str}"""
sr = SpatialReference()
sr.ImportFromWkt(wkt)
return sr
def __init__(self, src):
"""
initialize shapefile reader
"""
if isinstance(src, unicode):
src = src.encode('ascii', 'ignore')
src = self.find_source(src)
self.shpSrc = src
self.sr = shapefile.Reader(src)
self.recs = []
self.shapes = {}
self.load_records()
self.proj = None
# Check if there's a spatial reference
prj_src = src[:-4] + '.prj'
if exists(prj_src):
prj_text = open(prj_src).read()
srs = SpatialReference()
if srs.ImportFromWkt(prj_text):
raise ValueError("Error importing PRJ information from: %s" %
prj_file)
if srs.IsProjected():
try:
self.proj = pyproj.Proj(srs.ExportToProj4())
except: # e.g. ERROR 6: No translation for Lambert_Conformal_Conic to PROJ.4 format is known.
srs.MorphFromESRI()
self.proj = pyproj.Proj(srs.ExportToProj4())
def load_data(self):
filename = self.path.split(os.sep)[-1]
# Check for VELMA filename matches
for pattern in [self.VELMA_FILENAME_BASIC_RE, self.VELMA_FILENAME_NO_LAYER_RE,
self.VELMA_FILENAME_SUBDAY_RE, self.VELMA_FILENAME_SUBDAY_NO_LAYER_RE]:
match = pattern.match(filename)
if match:
self._velma_pattern = pattern
self.data_name = match.group(1)
self._loop = match.group(2)
self._has_layer = pattern in [self.VELMA_FILENAME_BASIC_RE, self.VELMA_FILENAME_SUBDAY_RE]
if self._has_layer:
self._layer = match.group(3)
self._is_subday = pattern in [self.VELMA_FILENAME_SUBDAY_RE, self.VELMA_FILENAME_SUBDAY_NO_LAYER_RE]
self._is_velma = True
break
projection = None
prj_file = self.path.replace('.asc', '.prj')
if os.path.exists(prj_file):
with open(prj_file, 'r') as f:
ref = SpatialReference()
ref.ImportFromWkt(f.read())
projection = Proj(ref.ExportToProj4())
# Capture georeference
with rasterio.open(self.path) as src:
self.affine = src.transform
self.shape = src.shape
self.resolution = src.res[0] # Assumed to be square
self.extent = Extent(*list(src.bounds), projection=projection)
self._nodata_value = src.nodata
self.time_info = TemporalInfo()
if not self._is_velma:
self.data_name = self.path.split(os.sep)[-1].split('.')[0]
return
timestamps = []
for f in os.listdir(os.path.dirname(os.path.abspath(self.path))):
filename = f.split(os.sep)[-1]
match = self._velma_pattern.match(filename)
# Skip if match is None and if match name or loop is wrong
if match and match.group(1) == self.data_name and match.group(2) == self._loop:
# Skip if match layer is wrong, if applicable
if self._has_layer and match.group(3) != self._layer:
continue
years = int(match.group(4 if self._has_layer else 3))
days = int(match.group(5 if self._has_layer else 4))
hours = int(match.group(6 if self._has_layer else 5)) if self._is_subday else 0
minutes = int(match.group(7 if self._has_layer else 6)) if self._is_subday else 0
insort(timestamps, datetime.datetime(years, 1, 1, hours, minutes) + datetime.timedelta(days - 1))
self.time_info.timestamps = timestamps
def crs_from_gridmapping(grid_mapping):
"""Create projection from grid_mapping variable"""
if 'crs_wkt' not in grid_mapping.attrs:
raise NotImplementedError('At present, a "crs_wkt" attr is required')
wkt = grid_mapping.attrs['crs_wkt']
sr = SpatialReference()
sr.ImportFromWkt(wkt)
return sr
def __new__(cls, ds, band_number=1):
"""
Create an in-memory representation for a single band in
a raster. (0,0) in pixel coordinates represents the
upper left corner of the raster which corresponds to
(min_lon, max_lat). Inherits from ndarray, so you can
use it like a numpy array.
Parameters
----------
ds: gdal.Dataset
band_number: int
The band number to use
Attributes
----------
data: np.ndarray[xsize, ysize]
The raster data
"""
if not isinstance(ds, gdal.Dataset):
path = fileutils.get_path(ds)
ds = gdal.Open(path, GA_ReadOnly)
band = ds.GetRasterBand(band_number)
if band is None:
msg = "Unable to load band %d " % band_number
msg += "in raster %s" % ds.GetDescription()
raise ValueError(msg)
gdal_type = band.DataType
dtype = np.dtype(GDAL2NP_CONVERSION[gdal_type])
self = np.asarray(band.ReadAsArray().astype(dtype)).view(cls)
self.gdal_type = gdal_type
proj = SpatialReference()
proj.ImportFromWkt(ds.GetProjection())
geo_transform = ds.GetGeoTransform()
# Initialize the base class with coordinate information.
RasterBase.__init__(self, ds.RasterXSize, ds.RasterYSize,
geo_transform, proj)
self.nan = band.GetNoDataValue()
#self = np.ma.masked_equal(self, band.GetNoDataValue(), copy=False)
ctable = band.GetColorTable()
if ctable is not None:
self.colors = np.array(
[ctable.GetColorEntry(i) for i in range(256)], dtype=np.uint8)
else:
self.colors = None
ds = None
return self
def crs_from_wkt(wkt):
"""Create SpatialReference from Well Known Text (WKT)
:param wkt:
WKT representation of the spatial reference frame
:type wkt: str
:returns:
SpatialReference object
:rtype: SpatialReference
"""
proj = SpatialReference()
proj.ImportFromWkt(wkt)
return proj
def testClipReproject(self):
# Build the test file.
imageFile = self._createTestFile()
# Build the envelope.
ulx = 367080
uly = 4209230
lrx = 509200
lry = 4095100
srs = SpatialReference()
srs.ImportFromEPSG(32612)
env = Envelope()
env.addPoint(ulx, uly, 0, srs)
env.addPoint(lrx, lry, 0, srs)
# Reprojection parameter
targetSRS = SpatialReference()
targetSRS.ImportFromEPSG(4326)
# Clip, reproject and resample.
imageFile.clipReproject(env, targetSRS,)
# Check the results.
dataset = gdal.Open(imageFile.fileName(), gdalconst.GA_ReadOnly)
if not dataset:
raise RuntimeError('Unable to read ' + imageFile.fileName() + '.')
xform = dataset.GetGeoTransform()
xScale = xform[1]
yScale = xform[5]
width = dataset.RasterXSize
height = dataset.RasterYSize
clippedUlx = xform[0]
clippedUly = xform[3]
clippedLrx = clippedUlx + width * xScale
clippedLry = clippedUly + height * yScale
self.assertAlmostEqual(clippedUlx, -112.49369402670872, places=12)
self.assertAlmostEqual(clippedUly, 38.03073206024332, places=11)
self.assertAlmostEqual(clippedLrx, -110.89516946364738, places=12)
self.assertAlmostEqual(clippedLry, 36.99265291293727, places=11)
outSRS = SpatialReference()
outSRS.ImportFromWkt(dataset.GetProjection())
self.assertTrue(outSRS.IsSame(targetSRS))
# Delete the test file.
os.remove(imageFile.fileName())
def __init__(self,
pathToFile,
spatialReference=None,
subdataset=None,
logger=None):
# Initialize the base class.
super(GeospatialImageFile, self).__init__(pathToFile, subdataset)
self.logger = logger
# The passed SRS overrides any internal SRS.
if spatialReference and spatialReference.Validate() == 0:
self._dataset.SetSpatialRef(spatialReference)
# Does the image file have a valid SRS?
if self._dataset.GetSpatialRef() and \
self._dataset.GetSpatialRef().Validate() == 0:
return
# Can the image file's projection be used as an SRS?
wkt = self.getDataset().GetProjection()
if wkt:
projSRS = SpatialReference()
projSRS.ImportFromWkt(wkt)
if projSRS.Validate() == 0:
self._dataset.SetSpatialRef(projSRS)
# # If there is still no SRS in the image, try to use the one passed.
# if not self._dataset.GetSpatialRef() or \
# self._dataset.GetSpatialRef().Validate() != 0:
#
# if spatialReference and spatialReference.Validate() == 0:
# self._dataset.SetSpatialRef(spatialReference)
# After all that, is there a valid SRS in the image?
if not self._dataset.GetSpatialRef() or \
self._dataset.GetSpatialRef().Validate() != 0:
raise RuntimeError('Spatial reference for ' + pathToFile,
' is invalid.')
def filter_by_id(self, ids):
"""Return a vector layer with only those shapes with
id in ids.
Parameters
----------
ids: iterable
The ids to filter on"""
assert hasattr(ids, "__iter__"), "ids must be iterable"
if not isinstance(ids, pd.Index):
ids = self._make_ids(ids)
geoms = [self[i].Clone() for i in ids]
proj = SpatialReference()
proj.ImportFromWkt(self.proj.ExportToWkt())
[g.AssignSpatialReference(proj) for g in geoms]
return VectorLayer(geoms, index=ids)
def __init__(self, src):
"""
initialize shapefile reader
"""
if isinstance(src, unicode):
src = src.encode('ascii', 'ignore')
src = self.find_source(src)
self.shpSrc = src
self.sr = shapefile.Reader(src)
self.recs = []
self.intersect_tol = .3
self.max_area_for_circle = .002
self.high_exp_factor = 1.75
self.shapes = {}
self.geoms = {}
self.load_records()
self.proj = None
# Check if there's a spatial reference
prj_src = src[:-4] + '.prj'
if exists(prj_src):
prj_text = open(prj_src).read()
srs = SpatialReference()
wkt_ret = srs.ImportFromWkt(prj_text)
# print 'prj_text={0}'.format(prj_text)
#print "srs={0}".format(srs)
if wkt_ret:
raise ValueError("Error importing PRJ information from: %s" %
prj_file)
if srs.IsProjected():
export_srs = srs.ExportToProj4()
# print 'srs.IsProjected'
#print "Groomp"
# self.proj=pyproj.Proj(proj='utm',zone=10,ellps='WGS84')
self.proj = pyproj.Proj(export_srs)
else:
self.proj = None
# print 'self.proj = None'
#export_srs=srs.ExportToProj4()
#self.proj=pyproj.Proj(init='epsg:26915')
#self.proj = pyproj.Proj(export_srs)
else:
print 'choo'
def crs_local(lon, lat):
"""Create local metric coordinate system based on ETRS89 and transverse
mercator.
:param lon:
Longitude of the central location
:param lat:
Latitude of the central location
:returns:
SpatialReference object
:rtype: SpatialReference
"""
wkt = f"""
PROJCS["Local ETRS89",
GEOGCS["ETRS89",
DATUM["European_Terrestrial_Reference_System_1989",
SPHEROID["GRS 1980",6378137,298.257222101,
AUTHORITY["EPSG","7019"]],
AUTHORITY["EPSG","6258"]],
PRIMEM["Greenwich",0,
AUTHORITY["EPSG","8901"]],
UNIT["degree",0.01745329251994328,
AUTHORITY["EPSG","9122"]],
AUTHORITY["EPSG","4258"]],
UNIT["metre",1,
AUTHORITY["EPSG","9001"]],
PROJECTION["Transverse_Mercator"],
PARAMETER["latitude_of_origin", {lat}],
PARAMETER["central_meridian", {lon}],
PARAMETER["scale_factor",1.0],
PARAMETER["false_easting",0],
PARAMETER["false_northing",0],
AXIS["Easting",EAST],
AXIS["Northing",NORTH]]
"""
sr = SpatialReference()
sr.ImportFromWkt(wkt)
return sr
def testReproject(self):
# Build the test file.
imageFile = self._createTestFile()
# Reproject.
targetSRS = SpatialReference()
targetSRS.ImportFromEPSG(4326)
imageFile.clipReproject(outputSRS=targetSRS)
# Check the SRS.
dataset = gdal.Open(imageFile.fileName(), gdalconst.GA_ReadOnly)
if not dataset:
raise RuntimeError('Unable to read ' + imageFile.fileName() + '.')
outSRS = SpatialReference()
outSRS.ImportFromWkt(dataset.GetProjection())
self.assertTrue(outSRS.IsSame(targetSRS))
# Delete the test file.
os.remove(imageFile.fileName())
def get_spatial_ref_from_wkt(wkt_or_crs_name):
'''
Function to return SpatialReference object for supplied WKT
@param wkt: Well-known text or CRS name for SpatialReference, including "EPSG:XXXX"
@return spatial_ref: SpatialReference from WKT
'''
spatial_ref = SpatialReference()
# Try to resolve WKT
result = spatial_ref.ImportFromWkt(wkt_or_crs_name)
if not result:
return spatial_ref
# Try to resolve CRS name - either mapped or original
result = spatial_ref.SetWellKnownGeogCS(
CRS_NAME_MAPPING.get(wkt_or_crs_name) or wkt_or_crs_name)
if not result:
return spatial_ref
# Try common formulations for UTM zones
#TODO: Fix this so it works in the Northern hemisphere
modified_crs_name = re.sub('\s+', '', wkt_or_crs_name.strip().upper())
utm_match = (re.match('(\w+)/MGAZONE(\d+)', modified_crs_name)
or re.match('(\w+)/(\d+)S', modified_crs_name)
or re.match('(EPSG:283)(\d{2})', modified_crs_name))
if utm_match:
modified_crs_name = utm_match.group(1)
utm_zone = int(utm_match.group(2))
result = spatial_ref.SetWellKnownGeogCS(
CRS_NAME_MAPPING.get(modified_crs_name) or modified_crs_name)
if not result:
spatial_ref.SetUTM(
utm_zone, False
) # Put this here to avoid potential side effects in downstream code
return spatial_ref
assert not result, 'Invalid WKT or CRS name'
def crs_to_osgeo(input_crs: Union[CRS, str, int]):
"""
Take in a CRS in several formats and returns an osr SpatialReference for use with GDAL/OGR
Supports pyproj CRS object, crs in Proj4 format, crs in Wkt format, epsg code as integer/string
Parameters
----------
input_crs
input crs in one of the accepted forms
Returns
-------
SpatialReference
osr SpatialReference for the provided CRS
"""
if isinstance(input_crs, CRS):
crs = pyproj_crs_to_osgeo(input_crs)
else:
crs = SpatialReference()
try: # in case someone passes a str that is an epsg
epsg = int(input_crs)
err = crs.ImportFromEPSG(epsg)
if err:
raise ValueError(
'Error trying to ImportFromEPSG: {}'.format(epsg))
except ValueError: # a wkt or proj4 is provided
err = crs.ImportFromWkt(input_crs)
if err:
err = crs.ImportFromProj4(input_crs)
if err:
raise ValueError(
'{} is neither a valid Wkt or Proj4 string'.format(
input_crs))
return crs
class CRS(object):
def __init__(self,
init=None,
epsg=None,
proj4=None,
gdal=None,
pyproj=None,
wkt=None,
esri=None,
mapping=None,
**kwargs):
self._proj4 = None
self._gdal = None
if epsg is not None:
self._gdal = SpatialReference()
error = self._gdal.ImportFromEPSG(int(epsg))
if error != 0:
raise ValueError(
"unable to interpret EPSG code '{}'".format(epsg))
elif isinstance(init, CRS):
self._proj4 = init._proj4
self._gdal = init._gdal
elif isinstance(init, Proj):
self._proj4 = init.srs
elif isinstance(pyproj, Proj):
self._proj4 = pyproj.srs
elif isinstance(init, SpatialReference):
self._gdal = init
elif isinstance(gdal, SpatialReference):
self._gdal = gdal
elif isinstance(init, dict):
self._proj4 = mapping_to_proj4(init)
elif isinstance(mapping, dict):
self._proj4 = mapping_to_proj4(mapping)
elif isinstance(init, string_types):
projection_string = str(init)
self._gdal = SpatialReference()
error = -1
if error != 0:
try:
error = self._gdal.ImportFromProj4(projection_string)
except:
error = -1
if error != 0:
try:
error = self._gdal.ImportFromWkt(projection_string)
except:
error = -1
if error != 0:
try:
error = self._gdal.ImportFromESRI(projection_string)
except:
error = -1
if error != 0:
try:
error = self._gdal.ImportFromUrl(projection_string)
except:
error = -1
if error != 0:
try:
error = self._gdal.ImportFromXML(projection_string)
except:
error = -1
if error != 0:
try:
error = self._gdal.ImportFromEPSG(projection_string)
except:
error = -1
if error != 0:
try:
error = self._gdal.ImportFromEPSGA(projection_string)
except:
error = -1
if error != 0:
try:
error = self._gdal.ImportFromERM(projection_string)
except:
error = -1
if error != 0:
try:
error = self._gdal.ImportFromMICoordSys(projection_string)
except:
error = -1
if error != 0:
try:
error = self._gdal.ImportFromOzi(projection_string)
except:
error = -1
if error != 0:
try:
error = self._gdal.ImportFromPCI(projection_string)
except:
error = -1
if error != 0:
try:
error = self._gdal.ImportFromUSGS(projection_string)
except:
error = -1
if error != 0:
pyproj = Proj(projection_string)
self._proj4 = pyproj.srs
elif proj4 is not None:
self._proj4 = proj4
self._gdal = SpatialReference()
error = self._gdal.ImportFromProj4(proj4)
if error != 0:
raise ValueError(
"unable to interpret PROJ4 string '{}'".format(proj4))
elif wkt is not None:
self._gdal = SpatialReference()
error = self._gdal.ImportFromWkt(wkt)
if error != 0:
raise ValueError(
"unable to interpret WKT string '{}'".format(wkt))
elif esri is not None:
self._gdal = SpatialReference()
error = self._gdal.ImportFromESRI(esri)
if error != 0:
raise ValueError(
"unable to interpret ESRI string '{}'".format(esri))
@staticmethod
def from_epsg(epsg):
return CRS(epsg=epsg)
@staticmethod
def from_proj4(proj4):
return CRS(proj4=proj4)
@staticmethod
def from_wkt(wkt):
return CRS(wkt=wkt)
@staticmethod
def from_esri(esri):
return CRS(esri=esri)
@staticmethod
def from_pyproj(pyproj):
return CRS(pyproj=pyproj)
@staticmethod
def from_gdal(gdal):
return CRS(gdal=gdal)
@property
def proj4(self):
if self._proj4 is not None:
pass
elif self._gdal is not None:
self._proj4 = self._gdal.ExportToProj4()
else:
raise ValueError("unable to produce PROJ4 string")
return self._proj4
@property
def pyproj(self):
return Proj(self.proj4)
@property
def mapping(self):
return proj4_to_mapping(self.proj4)
@property
def gdal(self):
if self._gdal is not None:
pass
elif self._proj4 is not None:
self._gdal = SpatialReference()
self._gdal.ImportFromProj4(self._proj4)
return self._gdal
@property
def wkt(self):
return self.gdal.ExportToWkt()
@property
def pretty_wkt(self):
return self.gdal.ExportToPrettyWkt()
@property
def usgs(self):
return self.gdal.ExportToUSGS()
@property
def xml(self):
return self.gdal.ExportToXML()
@property
def pci(self):
return self.gdal.ExportToPCI()
@property
def mi(self):
return self.gdal.ExportToMICoordSys()
@property
def is_latlon(self):
return self.pyproj.is_latlong()
@property
def is_geocentric(self):
return self.pyproj.is_geocent()
@property
def _cstype(self):
return 'GEOGCS' if self.is_latlon else 'PROJCS'
@property
def authority(self):
return self.gdal.GetAuthorityName(self._cstype)
@property
def code(self):
return self.gdal.GetAuthorityCode(self._cstype)
def __repr__(self):
return "CRS('{}')".format(self.proj4)
def get_spatial_ref_from_wkt(wkt_or_crs_name):
'''
Function to return SpatialReference object for supplied WKT
@param wkt: Well-known text or CRS name for SpatialReference, including "EPSG:XXXX"
@return spatial_ref: SpatialReference from WKT
'''
if not wkt_or_crs_name:
return None
spatial_ref = SpatialReference()
result = spatial_ref.SetFromUserInput(wkt_or_crs_name)
if not result:
logger.debug(
'CRS determined using SpatialReference.SetFromUserInput({})'.
format(wkt_or_crs_name))
return spatial_ref
# Try to resolve WKT
result = spatial_ref.ImportFromWkt(wkt_or_crs_name)
if not result:
logger.debug(
'CRS determined using SpatialReference.ImportFromWkt({})'.format(
wkt_or_crs_name))
return spatial_ref
# Try to resolve CRS name - either mapped or original
modified_crs_name = CRS_NAME_MAPPING.get(
wkt_or_crs_name) or wkt_or_crs_name
result = spatial_ref.SetWellKnownGeogCS(modified_crs_name)
if not result:
logger.debug(
'CRS determined using SpatialReference.SetWellKnownGeogCS({})'.
format(modified_crs_name))
return spatial_ref
match = re.match('EPSG:(\d+)$', wkt_or_crs_name, re.IGNORECASE)
if match:
epsg_code = int(match.group(1))
result = spatial_ref.ImportFromEPSG(epsg_code)
if not result:
logger.debug(
'CRS determined using SpatialReference.ImportFromEPSG({})'.
format(epsg_code))
return spatial_ref
# Try common formulations for UTM zones
#TODO: Fix this so it works in the Northern hemisphere
modified_crs_name = re.sub('\s+', '', wkt_or_crs_name.strip().upper())
utm_match = (re.match('(\w+)/MGAZONE(\d+)$', modified_crs_name)
or re.match('(\w+)/(\d+)S$', modified_crs_name)
or re.match('(EPSG:283)(\d{2})$', modified_crs_name)
or re.match('(MGA)(\d{2}$)', modified_crs_name))
if utm_match:
modified_crs_name = utm_match.group(1)
modified_crs_name = CRS_NAME_MAPPING.get(
modified_crs_name) or modified_crs_name
utm_zone = int(utm_match.group(2))
result = spatial_ref.SetWellKnownGeogCS(modified_crs_name)
if not result:
spatial_ref.SetUTM(
utm_zone, False
) # Put this here to avoid potential side effects in downstream code
logger.debug(
'UTM CRS determined using SpatialReference.SetWellKnownGeogCS({}) (zone{})'
.format(modified_crs_name, utm_zone))
return spatial_ref
assert not result, 'Invalid WKT or CRS name: "{}"'.format(wkt_or_crs_name)
def set_netcdf_metadata_attributes(
self, to_crs='EPSG:4326', do_stats=False):
'''
Function to set all NetCDF metadata attributes using self.METADATA_MAPPING to map from NetCDF ACDD global attribute name to metadata path (e.g. xpath)
Parameter:
to_crs: EPSG or WKT for spatial metadata
do_stats: Boolean flag indicating whether minmax stats should be determined (slow)
'''
assert self.METADATA_MAPPING, 'No metadata mapping defined'
assert self._netcdf_dataset, 'NetCDF output dataset not defined.'
# assert self._metadata_dict, 'No metadata acquired'
# Set geospatial attributes
try:
grid_mapping = [variable.grid_mapping for variable in self._netcdf_dataset.variables.values(
) if hasattr(variable, 'grid_mapping')][0]
except:
logger.error(
'Unable to determine grid_mapping for spatial reference')
raise
crs = self._netcdf_dataset.variables[grid_mapping]
spatial_ref = crs.spatial_ref
geoTransform = [float(string)
for string in crs.GeoTransform.strip().split(' ')]
xpixels, ypixels = (
dimension.size for dimension in self._netcdf_dataset.dimensions.values())
dimension_names = (
dimension.name for dimension in self._netcdf_dataset.dimensions.values())
# Create nested list of bounding box corner coordinates
bbox_corners = [[geoTransform[0] + (x_pixel_offset * geoTransform[1]) + (y_pixel_offset * geoTransform[2]),
geoTransform[3] + (x_pixel_offset * geoTransform[4]) + (y_pixel_offset * geoTransform[5])]
for x_pixel_offset in [0, xpixels]
for y_pixel_offset in [0, ypixels]]
if to_crs: # Coordinate transformation required
from_spatial_ref = SpatialReference()
from_spatial_ref.ImportFromWkt(spatial_ref)
to_spatial_ref = SpatialReference()
# Check for EPSG then Well Known Text
epsg_match = re.match('^EPSG:(\d+)$', to_crs)
if epsg_match:
to_spatial_ref.ImportFromEPSG(int(epsg_match.group(1)))
else: # Assume valid WKT definition
to_spatial_ref.ImportFromWkt(to_crs)
coord_trans = CoordinateTransformation(
from_spatial_ref, to_spatial_ref)
extents = np.array(
[coord[0:2] for coord in coord_trans.TransformPoints(bbox_corners)])
spatial_ref = to_spatial_ref.ExportToWkt()
centre_pixel_coords = [coord[0:2] for coord in coord_trans.TransformPoints(
[[geoTransform[0] + (x_pixel_offset * geoTransform[1]) + (y_pixel_offset * geoTransform[2]),
geoTransform[3] + (x_pixel_offset * geoTransform[4]) + (y_pixel_offset * geoTransform[5])]
for x_pixel_offset in [xpixels // 2, xpixels // 2 + 1]
for y_pixel_offset in [ypixels // 2, ypixels // 2 + 1]]
)
]
# Use Pythagoras to compute centre pixel size in new coordinates
# (never mind the angles)
yres = pow(pow(centre_pixel_coords[1][0] - centre_pixel_coords[0][0], 2) + pow(
centre_pixel_coords[1][1] - centre_pixel_coords[0][1], 2), 0.5)
xres = pow(pow(centre_pixel_coords[2][0] - centre_pixel_coords[0][0], 2) + pow(
centre_pixel_coords[2][1] - centre_pixel_coords[0][1], 2), 0.5)
# TODO: Make this more robust - could pull single unit from WKT
if to_spatial_ref.IsGeographic():
xunits, yunits = ('degrees_east', 'degrees_north')
elif to_spatial_ref.IsProjected():
xunits, yunits = ('m', 'm')
else:
xunits, yunits = ('unknown', 'unknown')
else: # Use native coordinates
extents = np.array(bbox_corners)
xres = round(geoTransform[1], Geophys2NetCDF.DECIMAL_PLACES)
yres = round(geoTransform[5], Geophys2NetCDF.DECIMAL_PLACES)
xunits, yunits = (self._netcdf_dataset.variables[
dimension_name].units for dimension_name in dimension_names)
xmin = np.min(extents[:, 0])
ymin = np.min(extents[:, 1])
xmax = np.max(extents[:, 0])
ymax = np.max(extents[:, 1])
attribute_dict = dict(zip(['geospatial_lon_min', 'geospatial_lat_min', 'geospatial_lon_max', 'geospatial_lat_max'],
[xmin, ymin, xmax, ymax]
)
)
attribute_dict['geospatial_lon_resolution'] = xres
attribute_dict['geospatial_lat_resolution'] = yres
attribute_dict['geospatial_lon_units'] = xunits
attribute_dict['geospatial_lat_units'] = yunits
try:
convex_hull = [coordinate[0:2] for coordinate in coord_trans.TransformPoints(
netcdf2convex_hull(self.netcdf_dataset, 2000000000))] # Process dataset in pieces <= 2GB in size
except:
logger.info('Unable to compute convex hull. Using rectangular bounding box instead.')
convex_hull = [coordinate[0:2] for coordinate in coord_trans.TransformPoints(bbox_corners + [bbox_corners[0]])]
attribute_dict['geospatial_bounds'] = 'POLYGON((' + ', '.join([' '.join(
['%.4f' % ordinate for ordinate in coordinates]) for coordinates in convex_hull]) + '))'
attribute_dict['geospatial_bounds_crs'] = spatial_ref
for key, value in attribute_dict.items():
setattr(self._netcdf_dataset, key, value)
# Set attributes defined in self.METADATA_MAPPING
# Scan list in reverse to give priority to earlier entries
#TODO: Improve this coding - it's a bit crap
keys_read = []
for key, metadata_path in self.METADATA_MAPPING:
# Skip any keys already read
if key in keys_read:
continue
value = self.get_metadata(metadata_path)
if value is not None:
logger.debug('Setting %s to %s', key, value)
# TODO: Check whether hierarchical metadata required
setattr(self._netcdf_dataset, key, value)
keys_read.append(key)
else:
logger.warning(
'WARNING: Metadata path %s not found', metadata_path)
unread_keys = sorted(
list(set([item[0] for item in self.METADATA_MAPPING]) - set(keys_read)))
if unread_keys:
logger.warning(
'WARNING: No value found for metadata attribute(s) %s' % ', '.join(unread_keys))
# Ensure only one DOI is stored - could be multiple, comma-separated
# entries
if hasattr(self._netcdf_dataset, 'doi'):
url_list = [url.strip()
for url in self._netcdf_dataset.doi.split(',')]
doi_list = [url for url in url_list if url.startswith(
'http://dx.doi.org/')]
if len(url_list) > 1: # If more than one URL in list
try: # Give preference to proper DOI URL
url = doi_list[0] # Use first (preferably only) DOI URL
except:
url = url_list[0] # Just use first URL if no DOI found
url = url.replace('&', '&')
self._netcdf_dataset.doi = url
# Set metadata_link to NCI metadata URL
self._netcdf_dataset.metadata_link = 'https://pid.nci.org.au/dataset/%s' % self.uuid
self._netcdf_dataset.Conventions = 'CF-1.6, ACDD-1.3'
if do_stats:
datastats = DataStats(netcdf_dataset=self.netcdf_dataset,
netcdf_path=None, max_bytes=2000000000) # 2GB pieces
datastats.data_variable.actual_range = np.array(
[datastats.value('min'), datastats.value('max')], dtype='float32')
# Remove old fields - remove this later
if hasattr(self._netcdf_dataset, 'id'):
del self._netcdf_dataset.id
if hasattr(self._netcdf_dataset, 'ga_uuid'):
del self._netcdf_dataset.ga_uuid
if hasattr(self._netcdf_dataset, 'keywords_vocabulary'):
del self._netcdf_dataset.keywords_vocabulary
