def get_gids_for_intertable(self):
# returns gids of messages to use to create interpolation tables
val = grib_get_double_array(self._gid_main_res, 'values')
val2 = None
if self._gid_ext_res:
val2 = grib_get_double_array(self._gid_ext_res, 'values')
return self._gid_main_res, val, self._gid_ext_res, val2
def _regularise_shape(grib_message):
"""
Calculate the regularised shape of the reduced message and push
dummy regularised values into the message to force the gribapi
to update the message grid type from reduced to regular.
"""
# Make sure to read any missing values as NaN.
gribapi.grib_set_double(grib_message, "missingValue", np.nan)
# Get full longitude values, these describe the longitude value of
# *every* point in the grid, they are not 1d monotonic coordinates.
lons = gribapi.grib_get_double_array(grib_message, "longitudes")
# Compute the new longitude coordinate for the regular grid.
new_nx = max(gribapi.grib_get_long_array(grib_message, "pl"))
new_x_step = (max(lons) - min(lons)) / (new_nx - 1)
if gribapi.grib_get_long(grib_message, "iScansNegatively"):
new_x_step *= -1
gribapi.grib_set_long(grib_message, "Nx", int(new_nx))
gribapi.grib_set_double(grib_message, "iDirectionIncrementInDegrees",
float(new_x_step))
# Spoof gribapi with false regularised values.
nj = gribapi.grib_get_long(grib_message, 'Nj')
temp = np.zeros((nj * new_nx,), dtype=np.float)
gribapi.grib_set_double_array(grib_message, 'values', temp)
gribapi.grib_set_long(grib_message, "jPointsAreConsecutive", 0)
gribapi.grib_set_long(grib_message, "PLPresent", 0)
def __getattr__(self, key):
"""Return a grib key, or one of our extra keys."""
# is it in the grib message?
try:
# we just get <type 'float'> as the type of the "values" array...special case here...
if key in ["values", "pv", "latitudes", "longitudes"]:
res = gribapi.grib_get_double_array(self.grib_message, key)
elif key in ('typeOfFirstFixedSurface', 'typeOfSecondFixedSurface'):
res = np.int32(gribapi.grib_get_long(self.grib_message, key))
else:
key_type = gribapi.grib_get_native_type(self.grib_message, key)
if key_type == int:
res = np.int32(gribapi.grib_get_long(self.grib_message, key))
elif key_type == float:
# Because some computer keys are floats, like
# longitudeOfFirstGridPointInDegrees, a float32 is not always enough...
res = np.float64(gribapi.grib_get_double(self.grib_message, key))
elif key_type == str:
res = gribapi.grib_get_string(self.grib_message, key)
else:
raise ValueError("Unknown type for %s : %s" % (key, str(key_type)))
except gribapi.GribInternalError:
res = None
#...or is it in our list of extras?
if res is None:
if key in self.extra_keys:
res = self.extra_keys[key]
else:
#must raise an exception for the hasattr() mechanism to work
raise AttributeError("Cannot find GRIB key %s" % key)
return res
def _regularise_shape(grib_message):
"""
Calculate the regularised shape of the reduced message and push
dummy regularised values into the message to force the gribapi
to update the message grid type from reduced to regular.
"""
# Make sure to read any missing values as NaN.
gribapi.grib_set_double(grib_message, "missingValue", np.nan)
# Get full longitude values, these describe the longitude value of
# *every* point in the grid, they are not 1d monotonic coordinates.
lons = gribapi.grib_get_double_array(grib_message, "longitudes")
# Compute the new longitude coordinate for the regular grid.
new_nx = max(gribapi.grib_get_long_array(grib_message, "pl"))
new_x_step = (max(lons) - min(lons)) / (new_nx - 1)
if gribapi.grib_get_long(grib_message, "iScansNegatively"):
new_x_step *= -1
gribapi.grib_set_long(grib_message, "Nx", int(new_nx))
gribapi.grib_set_double(grib_message, "iDirectionIncrementInDegrees",
float(new_x_step))
# Spoof gribapi with false regularised values.
nj = gribapi.grib_get_long(grib_message, 'Nj')
temp = np.zeros((nj * new_nx,), dtype=np.float)
gribapi.grib_set_double_array(grib_message, 'values', temp)
gribapi.grib_set_long(grib_message, "jPointsAreConsecutive", 0)
gribapi.grib_set_long(grib_message, "PLPresent", 0)
def __getattr__(self, key):
"""Return a grib key, or one of our extra keys."""
# is it in the grib message?
try:
# we just get <type 'float'> as the type of the "values" array...special case here...
if key in ["values", "pv", "latitudes", "longitudes"]:
res = gribapi.grib_get_double_array(self.grib_message, key)
elif key in ('typeOfFirstFixedSurface', 'typeOfSecondFixedSurface'):
res = np.int32(gribapi.grib_get_long(self.grib_message, key))
else:
key_type = gribapi.grib_get_native_type(self.grib_message, key)
if key_type == int:
res = np.int32(gribapi.grib_get_long(self.grib_message, key))
elif key_type == float:
# Because some computer keys are floats, like
# longitudeOfFirstGridPointInDegrees, a float32 is not always enough...
res = np.float64(gribapi.grib_get_double(self.grib_message, key))
elif key_type == str:
res = gribapi.grib_get_string(self.grib_message, key)
else:
raise ValueError("Unknown type for %s : %s" % (key, str(key_type)))
except gribapi.GribInternalError:
res = None
#...or is it in our list of extras?
if res is None:
if key in self.extra_keys:
res = self.extra_keys[key]
else:
#must raise an exception for the hasattr() mechanism to work
raise AttributeError("Cannot find GRIB key %s" % key)
return res
def _message_values(grib_message, shape):
gribapi.grib_set_double(grib_message, 'missingValue', np.nan)
data = gribapi.grib_get_double_array(grib_message, 'values')
data = data.reshape(shape)
# Handle missing values in a sensible way.
mask = np.isnan(data)
if mask.any():
data = ma.array(data, mask=mask, fill_value=np.nan)
return data
def _message_values(grib_message, shape):
gribapi.grib_set_double(grib_message, 'missingValue', np.nan)
data = gribapi.grib_get_double_array(grib_message, 'values')
data = data.reshape(shape)
# Handle missing values in a sensible way.
mask = np.isnan(data)
if mask.any():
data = ma.array(data, mask=mask, fill_value=np.nan)
return data
def select_messages(self, **kwargs):
self._selected_grbs = self._get_gids(**kwargs)
if len(self._selected_grbs) > 0:
self._gid_main_res = self._selected_grbs[0]
grid = GribGridDetails(self._selected_grbs[0])
# some cumulated messages come with the message at step=0 as instant, to permit aggregation
# cumulated rainfall rates could have the step zero instant message as kg/m^2, instead of kg/(m^2*s)
if len(self._selected_grbs) > 1:
unit = grib_get(self._selected_grbs[1], 'units')
type_of_step = grib_get(self._selected_grbs[1], 'stepType')
else:
type_of_step = grib_get(self._selected_grbs[0], 'stepType')
unit = grib_get(self._selected_grbs[0], 'units')
type_of_level = grib_get(self._selected_grbs[0], 'levelType')
missing_value = grib_get(self._selected_grbs[0], 'missingValue')
all_values = {}
all_values_second_res = {}
grid2 = None
input_step = self._step_grib
for g in self._selected_grbs:
start_step = grib_get(g, 'startStep')
end_step = grib_get(g, 'endStep')
points_meridian = grib_get(g, 'Nj')
if '{}-{}'.format(start_step,
end_step) == self._change_step_at:
# second time resolution
input_step = self._step_grib2
key = Step(start_step, end_step, points_meridian, input_step)
if points_meridian != grid.num_points_along_meridian and grid.get_2nd_resolution(
) is None:
# found second resolution messages
grid2 = GribGridDetails(g)
self._gid_ext_res = g
values = grib_get_double_array(g, 'values')
if not grid2:
all_values[key] = values
elif points_meridian != grid.num_points_along_meridian:
all_values_second_res[key] = values
if grid2:
key_2nd_spatial_res = min(all_values_second_res.keys())
grid.set_2nd_resolution(grid2, key_2nd_spatial_res)
return Messages(all_values, missing_value, unit, type_of_level,
type_of_step, grid, all_values_second_res)
# no messages found
else:
raise ValueError('No messages in grib file')
def randomise_grib(filename):
"""Randomise the data in the given GRIB file."""
in_file = open(filename, "rb")
out_file = open("out.grib", "wb")
while True:
grib_message = gribapi.grib_new_from_file(in_file)
if not grib_message:
break
values = gribapi.grib_get_double_array(grib_message, "values")
values = numpy.random.rand(len(values))
gribapi.grib_set_double_array(grib_message, "values", values)
gribapi.grib_write(grib_message, out_file)
os.remove(filename)
os.rename("out.grib", filename)
def randomise_grib(filename):
"""Randomise the data in the given GRIB file."""
in_file = open(filename, "rb")
out_file = open("out.grib", "wb")
while True:
grib_message = gribapi.grib_new_from_file(in_file)
if not grib_message:
break
values = gribapi.grib_get_double_array(grib_message, "values")
values = numpy.random.rand(len(values))
gribapi.grib_set_double_array(grib_message, "values", values)
gribapi.grib_write(grib_message, out_file)
os.remove(filename)
os.rename("out.grib", filename)
def _regularise(grib_message):
"""
Transform a reduced grid to a regular grid using interpolation.
Uses 1d linear interpolation at constant latitude to make the grid
regular. If the longitude dimension is circular then this is taken
into account by the interpolation. If the longitude dimension is not
circular then extrapolation is allowed to make sure all end regular
grid points get a value. In practice this extrapolation is likely to
be minimal.
"""
# Make sure to read any missing values as NaN.
gribapi.grib_set_double(grib_message, "missingValue", np.nan)
# Get full longitude values, these describe the longitude value of
# *every* point in the grid, they are not 1d monotonic coordinates.
lons = gribapi.grib_get_double_array(grib_message, "longitudes")
# Compute the new longitude coordinate for the regular grid.
new_nx = max(gribapi.grib_get_long_array(grib_message, "pl"))
new_x_step = (max(lons) - min(lons)) / (new_nx - 1)
if gribapi.grib_get_long(grib_message, "iScansNegatively"):
new_x_step *= -1
new_lons = np.arange(new_nx) * new_x_step + lons[0]
# Get full latitude and data values, these describe the latitude and
# data values of *every* point in the grid, they are not 1d monotonic
# coordinates.
lats = gribapi.grib_get_double_array(grib_message, "latitudes")
values = gribapi.grib_get_double_array(grib_message, "values")
# Retrieve the distinct latitudes from the GRIB message. GRIBAPI docs
# don't specify if these points are guaranteed to be oriented correctly so
# the safe option is to sort them into ascending (south-to-north) order
# and then reverse the order if necessary.
new_lats = gribapi.grib_get_double_array(grib_message, "distinctLatitudes")
new_lats.sort()
if not gribapi.grib_get_long(grib_message, "jScansPositively"):
new_lats = new_lats[::-1]
ny = new_lats.shape[0]
# Use 1d linear interpolation along latitude circles to regularise the
# reduced data.
cyclic = _longitude_is_cyclic(new_lons)
new_values = np.empty([ny, new_nx], dtype=values.dtype)
for ilat, lat in enumerate(new_lats):
idx = np.where(lats == lat)
llons = lons[idx]
vvalues = values[idx]
if cyclic:
# For cyclic data we insert dummy points at each end to ensure
# we can interpolate to all output longitudes using pure
# interpolation.
cgap = (360 - llons[-1] - llons[0])
llons = np.concatenate(
(llons[0:1] - cgap, llons, llons[-1:] + cgap))
vvalues = np.concatenate(
(vvalues[-1:], vvalues, vvalues[0:1]))
fixed_latitude_interpolator = scipy.interpolate.interp1d(
llons, vvalues)
else:
# Allow extrapolation for non-cyclic data sets to ensure we can
# interpolate to all output longitudes.
fixed_latitude_interpolator = Linear1dExtrapolator(
scipy.interpolate.interp1d(llons, vvalues))
new_values[ilat] = fixed_latitude_interpolator(new_lons)
new_values = new_values.flatten()
# Set flags for the regularised data.
if np.isnan(new_values).any():
# Account for any missing data.
gribapi.grib_set_double(grib_message, "missingValue", np.inf)
gribapi.grib_set(grib_message, "bitmapPresent", 1)
new_values = np.where(np.isnan(new_values), np.inf, new_values)
gribapi.grib_set_long(grib_message, "Nx", int(new_nx))
gribapi.grib_set_double(grib_message,
"iDirectionIncrementInDegrees", float(new_x_step))
gribapi.grib_set_double_array(grib_message, "values", new_values)
gribapi.grib_set_long(grib_message, "jPointsAreConsecutive", 0)
gribapi.grib_set_long(grib_message, "PLPresent", 0)
def _compute_extra_keys(self):
"""Compute our extra keys."""
global unknown_string
self.extra_keys = {}
forecastTime = self.startStep
# regular or rotated grid?
try:
longitudeOfSouthernPoleInDegrees = \
self.longitudeOfSouthernPoleInDegrees
latitudeOfSouthernPoleInDegrees = \
self.latitudeOfSouthernPoleInDegrees
except AttributeError:
longitudeOfSouthernPoleInDegrees = 0.0
latitudeOfSouthernPoleInDegrees = 90.0
centre = gribapi.grib_get_string(self.grib_message, "centre")
# default values
self.extra_keys = {'_referenceDateTime': -1.0,
'_phenomenonDateTime': -1.0,
'_periodStartDateTime': -1.0,
'_periodEndDateTime': -1.0,
'_levelTypeName': unknown_string,
'_levelTypeUnits': unknown_string,
'_firstLevelTypeName': unknown_string,
'_firstLevelTypeUnits': unknown_string,
'_firstLevel': -1.0,
'_secondLevelTypeName': unknown_string,
'_secondLevel': -1.0,
'_originatingCentre': unknown_string,
'_forecastTime': None,
'_forecastTimeUnit': unknown_string,
'_coord_system': None,
'_x_circular': False,
'_x_coord_name': unknown_string,
'_y_coord_name': unknown_string,
# These are here to avoid repetition in the rules
# files, and reduce the very long line lengths.
'_x_points': None,
'_y_points': None,
'_cf_data': None}
# cf phenomenon translation
# Get centre code (N.B. self.centre has default type = string)
centre_number = gribapi.grib_get_long(self.grib_message, "centre")
# Look for a known grib1-to-cf translation (or None).
cf_data = gptx.grib1_phenom_to_cf_info(
table2_version=self.table2Version,
centre_number=centre_number,
param_number=self.indicatorOfParameter)
self.extra_keys['_cf_data'] = cf_data
# reference date
self.extra_keys['_referenceDateTime'] = \
datetime.datetime(int(self.year), int(self.month), int(self.day),
int(self.hour), int(self.minute))
# forecast time with workarounds
self.extra_keys['_forecastTime'] = forecastTime
# verification date
processingDone = self._get_processing_done()
# time processed?
if processingDone.startswith("time"):
validityDate = str(self.validityDate)
validityTime = "{:04}".format(int(self.validityTime))
endYear = int(validityDate[:4])
endMonth = int(validityDate[4:6])
endDay = int(validityDate[6:8])
endHour = int(validityTime[:2])
endMinute = int(validityTime[2:4])
# fixed forecastTime in hours
self.extra_keys['_periodStartDateTime'] = \
(self.extra_keys['_referenceDateTime'] +
datetime.timedelta(hours=int(forecastTime)))
self.extra_keys['_periodEndDateTime'] = \
datetime.datetime(endYear, endMonth, endDay, endHour,
endMinute)
else:
self.extra_keys['_phenomenonDateTime'] = \
self._get_verification_date()
# originating centre
# TODO #574 Expand to include sub-centre
self.extra_keys['_originatingCentre'] = CENTRE_TITLES.get(
centre, "unknown centre %s" % centre)
# forecast time unit as a cm string
# TODO #575 Do we want PP or GRIB style forecast delta?
self.extra_keys['_forecastTimeUnit'] = self._timeunit_string()
# shape of the earth
soe_code = self.shapeOfTheEarth
# As this class is now *only* for GRIB1, 'shapeOfTheEarth' is not a
# value read from the actual file : It is really a GRIB2 param, and
# the value is merely what eccodes (gribapi) gives as the default.
# This was always = 6, until eccodes 0.19, when it changed to 0.
# See https://jira.ecmwf.int/browse/ECC-811
# The two represent different sized spherical earths.
if soe_code not in (6, 0):
raise ValueError('Unexpected shapeOfTheEarth value =', soe_code)
soe_code = 6
# *FOR NOW* maintain the old behaviour (radius=6371229) in all cases,
# for backwards compatibility.
# However, this does not match the 'radiusOfTheEarth' default from the
# gribapi so is probably incorrect (see above issue ECC-811).
# So we may change this in future.
if soe_code == 0:
# New supposedly-correct default value, matches 'radiusOfTheEarth'.
geoid = coord_systems.GeogCS(semi_major_axis=6367470)
elif soe_code == 6:
# Old value, does *not* match the 'radiusOfTheEarth' parameter.
geoid = coord_systems.GeogCS(6371229)
gridType = gribapi.grib_get_string(self.grib_message, "gridType")
if gridType in ["regular_ll", "regular_gg", "reduced_ll",
"reduced_gg"]:
self.extra_keys['_x_coord_name'] = "longitude"
self.extra_keys['_y_coord_name'] = "latitude"
self.extra_keys['_coord_system'] = geoid
elif gridType == 'rotated_ll':
# TODO: Confirm the translation from angleOfRotation to
# north_pole_lon (usually 0 for both)
self.extra_keys['_x_coord_name'] = "grid_longitude"
self.extra_keys['_y_coord_name'] = "grid_latitude"
southPoleLon = longitudeOfSouthernPoleInDegrees
southPoleLat = latitudeOfSouthernPoleInDegrees
self.extra_keys['_coord_system'] = \
coord_systems.RotatedGeogCS(
-southPoleLat,
math.fmod(southPoleLon + 180.0, 360.0),
self.angleOfRotation, geoid)
elif gridType == 'polar_stereographic':
self.extra_keys['_x_coord_name'] = "projection_x_coordinate"
self.extra_keys['_y_coord_name'] = "projection_y_coordinate"
if self.projectionCentreFlag == 0:
pole_lat = 90
elif self.projectionCentreFlag == 1:
pole_lat = -90
else:
raise TranslationError("Unhandled projectionCentreFlag")
# Note: I think the grib api defaults LaDInDegrees to 60 for grib1.
self.extra_keys['_coord_system'] = \
coord_systems.Stereographic(
pole_lat, self.orientationOfTheGridInDegrees, 0, 0,
self.LaDInDegrees, ellipsoid=geoid)
elif gridType == 'lambert':
self.extra_keys['_x_coord_name'] = "projection_x_coordinate"
self.extra_keys['_y_coord_name'] = "projection_y_coordinate"
flag_name = "projectionCenterFlag"
if getattr(self, flag_name) == 0:
pole_lat = 90
elif getattr(self, flag_name) == 1:
pole_lat = -90
else:
raise TranslationError("Unhandled projectionCentreFlag")
LambertConformal = coord_systems.LambertConformal
self.extra_keys['_coord_system'] = LambertConformal(
self.LaDInDegrees, self.LoVInDegrees, 0, 0,
secant_latitudes=(self.Latin1InDegrees, self.Latin2InDegrees),
ellipsoid=geoid)
else:
raise TranslationError("unhandled grid type: {}".format(gridType))
if gridType in ["regular_ll", "rotated_ll"]:
self._regular_longitude_common()
j_step = self.jDirectionIncrementInDegrees
if not self.jScansPositively:
j_step = -j_step
self._y_points = (np.arange(self.Nj, dtype=np.float64) * j_step +
self.latitudeOfFirstGridPointInDegrees)
elif gridType in ['regular_gg']:
# longitude coordinate is straight-forward
self._regular_longitude_common()
# get the distinct latitudes, and make sure they are sorted
# (south-to-north) and then put them in the right direction
# depending on the scan direction
latitude_points = gribapi.grib_get_double_array(
self.grib_message, 'distinctLatitudes').astype(np.float64)
latitude_points.sort()
if not self.jScansPositively:
# we require latitudes north-to-south
self._y_points = latitude_points[::-1]
else:
self._y_points = latitude_points
elif gridType in ["polar_stereographic", "lambert"]:
# convert the starting latlon into meters
cartopy_crs = self.extra_keys['_coord_system'].as_cartopy_crs()
x1, y1 = cartopy_crs.transform_point(
self.longitudeOfFirstGridPointInDegrees,
self.latitudeOfFirstGridPointInDegrees,
ccrs.Geodetic())
if not np.all(np.isfinite([x1, y1])):
raise TranslationError("Could not determine the first latitude"
" and/or longitude grid point.")
self._x_points = x1 + self.DxInMetres * np.arange(self.Nx,
dtype=np.float64)
self._y_points = y1 + self.DyInMetres * np.arange(self.Ny,
dtype=np.float64)
elif gridType in ["reduced_ll", "reduced_gg"]:
self._x_points = self.longitudes
self._y_points = self.latitudes
else:
raise TranslationError("unhandled grid type")
def _compute_extra_keys(self):
"""Compute our extra keys."""
global unknown_string
self.extra_keys = {}
# work out stuff based on these values from the message
edition = self.edition
# time-processed forcast time is from reference time to start of period
if edition == 2:
forecastTime = self.forecastTime
uft = np.uint32(forecastTime)
BILL = 2**30
# Workaround grib api's assumption that forecast time is positive.
# Handles correctly encoded -ve forecast times up to one -1 billion.
if hindcast_workaround:
if 2 * BILL < uft < 3 * BILL:
msg = "Re-interpreting negative forecastTime from " \
+ str(forecastTime)
forecastTime = -(uft - 2 * BILL)
msg += " to " + str(forecastTime)
warnings.warn(msg)
else:
forecastTime = self.startStep
#regular or rotated grid?
try:
longitudeOfSouthernPoleInDegrees = self.longitudeOfSouthernPoleInDegrees
latitudeOfSouthernPoleInDegrees = self.latitudeOfSouthernPoleInDegrees
except AttributeError:
longitudeOfSouthernPoleInDegrees = 0.0
latitudeOfSouthernPoleInDegrees = 90.0
centre = gribapi.grib_get_string(self.grib_message, "centre")
#default values
self.extra_keys = {'_referenceDateTime':-1.0, '_phenomenonDateTime':-1.0,
'_periodStartDateTime':-1.0, '_periodEndDateTime':-1.0,
'_levelTypeName':unknown_string,
'_levelTypeUnits':unknown_string, '_firstLevelTypeName':unknown_string,
'_firstLevelTypeUnits':unknown_string, '_firstLevel':-1.0,
'_secondLevelTypeName':unknown_string, '_secondLevel':-1.0,
'_originatingCentre':unknown_string,
'_forecastTime':None, '_forecastTimeUnit':unknown_string,
'_coord_system':None, '_x_circular':False,
'_x_coord_name':unknown_string, '_y_coord_name':unknown_string,
# These are here to avoid repetition in the rules files,
# and reduce the very long line lengths.
'_x_points':None, '_y_points':None,
'_cf_data':None}
# cf phenomenon translation
if edition == 1:
# Get centre code (N.B. self.centre has default type = string)
centre_number = gribapi.grib_get_long(self.grib_message, "centre")
# Look for a known grib1-to-cf translation (or None).
cf_data = gptx.grib1_phenom_to_cf_info(
table2_version=self.table2Version,
centre_number=centre_number,
param_number=self.indicatorOfParameter)
self.extra_keys['_cf_data'] = cf_data
elif edition == 2:
# Don't attempt to interpret params if 'master tables version' is
# 255, as local params may then have same codes as standard ones.
if self.tablesVersion != 255:
# Look for a known grib2-to-cf translation (or None).
cf_data = gptx.grib2_phenom_to_cf_info(
param_discipline=self.discipline,
param_category=self.parameterCategory,
param_number=self.parameterNumber)
self.extra_keys['_cf_data'] = cf_data
#reference date
self.extra_keys['_referenceDateTime'] = \
datetime.datetime(int(self.year), int(self.month), int(self.day),
int(self.hour), int(self.minute))
# forecast time with workarounds
self.extra_keys['_forecastTime'] = forecastTime
#verification date
processingDone = self._get_processing_done()
#time processed?
if processingDone.startswith("time"):
if self.edition == 1:
validityDate = str(self.validityDate)
validityTime = "{:04}".format(int(self.validityTime))
endYear = int(validityDate[:4])
endMonth = int(validityDate[4:6])
endDay = int(validityDate[6:8])
endHour = int(validityTime[:2])
endMinute = int(validityTime[2:4])
elif self.edition == 2:
endYear = self.yearOfEndOfOverallTimeInterval
endMonth = self.monthOfEndOfOverallTimeInterval
endDay = self.dayOfEndOfOverallTimeInterval
endHour = self.hourOfEndOfOverallTimeInterval
endMinute = self.minuteOfEndOfOverallTimeInterval
# fixed forecastTime in hours
self.extra_keys['_periodStartDateTime'] = \
(self.extra_keys['_referenceDateTime'] +
datetime.timedelta(hours=int(forecastTime)))
self.extra_keys['_periodEndDateTime'] = \
datetime.datetime(endYear, endMonth, endDay, endHour, endMinute)
else:
self.extra_keys['_phenomenonDateTime'] = self._get_verification_date()
#originating centre
#TODO #574 Expand to include sub-centre
self.extra_keys['_originatingCentre'] = CENTRE_TITLES.get(
centre, "unknown centre %s" % centre)
#forecast time unit as a cm string
#TODO #575 Do we want PP or GRIB style forecast delta?
self.extra_keys['_forecastTimeUnit'] = self._timeunit_string()
#shape of the earth
#pre-defined sphere
if self.shapeOfTheEarth == 0:
geoid = coord_systems.GeogCS(semi_major_axis=6367470)
#custom sphere
elif self.shapeOfTheEarth == 1:
geoid = coord_systems.GeogCS(
self.scaledValueOfRadiusOfSphericalEarth *
10 ** -self.scaleFactorOfRadiusOfSphericalEarth)
#IAU65 oblate sphere
elif self.shapeOfTheEarth == 2:
geoid = coord_systems.GeogCS(6378160, inverse_flattening=297.0)
#custom oblate spheroid (km)
elif self.shapeOfTheEarth == 3:
geoid = coord_systems.GeogCS(
semi_major_axis=self.scaledValueOfEarthMajorAxis *
10 ** -self.scaleFactorOfEarthMajorAxis * 1000.,
semi_minor_axis=self.scaledValueOfEarthMinorAxis *
10 ** -self.scaleFactorOfEarthMinorAxis * 1000.)
#IAG-GRS80 oblate spheroid
elif self.shapeOfTheEarth == 4:
geoid = coord_systems.GeogCS(6378137, None, 298.257222101)
#WGS84
elif self.shapeOfTheEarth == 5:
geoid = \
coord_systems.GeogCS(6378137, inverse_flattening=298.257223563)
#pre-defined sphere
elif self.shapeOfTheEarth == 6:
geoid = coord_systems.GeogCS(6371229)
#custom oblate spheroid (m)
elif self.shapeOfTheEarth == 7:
geoid = coord_systems.GeogCS(
semi_major_axis=self.scaledValueOfEarthMajorAxis *
10 ** -self.scaleFactorOfEarthMajorAxis,
semi_minor_axis=self.scaledValueOfEarthMinorAxis *
10 ** -self.scaleFactorOfEarthMinorAxis)
elif self.shapeOfTheEarth == 8:
raise ValueError("unhandled shape of earth : grib earth shape = 8")
else:
raise ValueError("undefined shape of earth")
gridType = gribapi.grib_get_string(self.grib_message, "gridType")
if gridType in ["regular_ll", "regular_gg", "reduced_ll", "reduced_gg"]:
self.extra_keys['_x_coord_name'] = "longitude"
self.extra_keys['_y_coord_name'] = "latitude"
self.extra_keys['_coord_system'] = geoid
elif gridType == 'rotated_ll':
# TODO: Confirm the translation from angleOfRotation to
# north_pole_lon (usually 0 for both)
self.extra_keys['_x_coord_name'] = "grid_longitude"
self.extra_keys['_y_coord_name'] = "grid_latitude"
southPoleLon = longitudeOfSouthernPoleInDegrees
southPoleLat = latitudeOfSouthernPoleInDegrees
self.extra_keys['_coord_system'] = \
iris.coord_systems.RotatedGeogCS(
-southPoleLat,
math.fmod(southPoleLon + 180.0, 360.0),
self.angleOfRotation, geoid)
elif gridType == 'polar_stereographic':
self.extra_keys['_x_coord_name'] = "projection_x_coordinate"
self.extra_keys['_y_coord_name'] = "projection_y_coordinate"
if self.projectionCentreFlag == 0:
pole_lat = 90
elif self.projectionCentreFlag == 1:
pole_lat = -90
else:
raise TranslationError("Unhandled projectionCentreFlag")
# Note: I think the grib api defaults LaDInDegrees to 60 for grib1.
self.extra_keys['_coord_system'] = \
iris.coord_systems.Stereographic(
pole_lat, self.orientationOfTheGridInDegrees, 0, 0,
self.LaDInDegrees, ellipsoid=geoid)
elif gridType == 'lambert':
self.extra_keys['_x_coord_name'] = "projection_x_coordinate"
self.extra_keys['_y_coord_name'] = "projection_y_coordinate"
if self.edition == 1:
flag_name = "projectionCenterFlag"
else:
flag_name = "projectionCentreFlag"
if getattr(self, flag_name) == 0:
pole_lat = 90
elif getattr(self, flag_name) == 1:
pole_lat = -90
else:
raise TranslationError("Unhandled projectionCentreFlag")
LambertConformal = iris.coord_systems.LambertConformal
self.extra_keys['_coord_system'] = LambertConformal(
self.LaDInDegrees, self.LoVInDegrees, 0, 0,
secant_latitudes=(self.Latin1InDegrees, self.Latin2InDegrees),
ellipsoid=geoid)
else:
raise TranslationError("unhandled grid type: {}".format(gridType))
if gridType in ["regular_ll", "rotated_ll"]:
self._regular_longitude_common()
j_step = self.jDirectionIncrementInDegrees
if not self.jScansPositively:
j_step = -j_step
self._y_points = (np.arange(self.Nj, dtype=np.float64) * j_step +
self.latitudeOfFirstGridPointInDegrees)
elif gridType in ['regular_gg']:
# longitude coordinate is straight-forward
self._regular_longitude_common()
# get the distinct latitudes, and make sure they are sorted
# (south-to-north) and then put them in the right direction
# depending on the scan direction
latitude_points = gribapi.grib_get_double_array(
self.grib_message, 'distinctLatitudes').astype(np.float64)
latitude_points.sort()
if not self.jScansPositively:
# we require latitudes north-to-south
self._y_points = latitude_points[::-1]
else:
self._y_points = latitude_points
elif gridType in ["polar_stereographic", "lambert"]:
# convert the starting latlon into meters
cartopy_crs = self.extra_keys['_coord_system'].as_cartopy_crs()
x1, y1 = cartopy_crs.transform_point(
self.longitudeOfFirstGridPointInDegrees,
self.latitudeOfFirstGridPointInDegrees,
ccrs.Geodetic())
if not np.all(np.isfinite([x1, y1])):
raise TranslationError("Could not determine the first latitude"
" and/or longitude grid point.")
self._x_points = x1 + self.DxInMetres * np.arange(self.Nx,
dtype=np.float64)
self._y_points = y1 + self.DyInMetres * np.arange(self.Ny,
dtype=np.float64)
elif gridType in ["reduced_ll", "reduced_gg"]:
self._x_points = self.longitudes
self._y_points = self.latitudes
else:
raise TranslationError("unhandled grid type")
def _message_values(grib_message, shape):
gribapi.grib_set_double(grib_message, 'missingValue', np.nan)
data = gribapi.grib_get_double_array(grib_message, 'values')
data = data.reshape(shape)
return data
def _message_values(grib_message, shape):
gribapi.grib_set_double(grib_message, 'missingValue', np.nan)
data = gribapi.grib_get_double_array(grib_message, 'values')
data = data.reshape(shape)
return data
def _compute_extra_keys(self):
"""Compute our extra keys."""
global unknown_string
self.extra_keys = {}
forecastTime = self.startStep
# regular or rotated grid?
try:
longitudeOfSouthernPoleInDegrees = \
self.longitudeOfSouthernPoleInDegrees
latitudeOfSouthernPoleInDegrees = \
self.latitudeOfSouthernPoleInDegrees
except AttributeError:
longitudeOfSouthernPoleInDegrees = 0.0
latitudeOfSouthernPoleInDegrees = 90.0
centre = gribapi.grib_get_string(self.grib_message, "centre")
# default values
self.extra_keys = {
'_referenceDateTime': -1.0,
'_phenomenonDateTime': -1.0,
'_periodStartDateTime': -1.0,
'_periodEndDateTime': -1.0,
'_levelTypeName': unknown_string,
'_levelTypeUnits': unknown_string,
'_firstLevelTypeName': unknown_string,
'_firstLevelTypeUnits': unknown_string,
'_firstLevel': -1.0,
'_secondLevelTypeName': unknown_string,
'_secondLevel': -1.0,
'_originatingCentre': unknown_string,
'_forecastTime': None,
'_forecastTimeUnit': unknown_string,
'_coord_system': None,
'_x_circular': False,
'_x_coord_name': unknown_string,
'_y_coord_name': unknown_string,
# These are here to avoid repetition in the rules
# files, and reduce the very long line lengths.
'_x_points': None,
'_y_points': None,
'_cf_data': None
}
# cf phenomenon translation
# Get centre code (N.B. self.centre has default type = string)
centre_number = gribapi.grib_get_long(self.grib_message, "centre")
# Look for a known grib1-to-cf translation (or None).
cf_data = gptx.grib1_phenom_to_cf_info(
table2_version=self.table2Version,
centre_number=centre_number,
param_number=self.indicatorOfParameter)
self.extra_keys['_cf_data'] = cf_data
# reference date
self.extra_keys['_referenceDateTime'] = \
datetime.datetime(int(self.year), int(self.month), int(self.day),
int(self.hour), int(self.minute))
# forecast time with workarounds
self.extra_keys['_forecastTime'] = forecastTime
# verification date
processingDone = self._get_processing_done()
# time processed?
if processingDone.startswith("time"):
validityDate = str(self.validityDate)
validityTime = "{:04}".format(int(self.validityTime))
endYear = int(validityDate[:4])
endMonth = int(validityDate[4:6])
endDay = int(validityDate[6:8])
endHour = int(validityTime[:2])
endMinute = int(validityTime[2:4])
# fixed forecastTime in hours
self.extra_keys['_periodStartDateTime'] = \
(self.extra_keys['_referenceDateTime'] +
datetime.timedelta(hours=int(forecastTime)))
self.extra_keys['_periodEndDateTime'] = \
datetime.datetime(endYear, endMonth, endDay, endHour,
endMinute)
else:
self.extra_keys['_phenomenonDateTime'] = \
self._get_verification_date()
# originating centre
# TODO #574 Expand to include sub-centre
self.extra_keys['_originatingCentre'] = CENTRE_TITLES.get(
centre, "unknown centre %s" % centre)
# forecast time unit as a cm string
# TODO #575 Do we want PP or GRIB style forecast delta?
self.extra_keys['_forecastTimeUnit'] = self._timeunit_string()
# shape of the earth
# pre-defined sphere
if self.shapeOfTheEarth == 0:
geoid = coord_systems.GeogCS(semi_major_axis=6367470)
# custom sphere
elif self.shapeOfTheEarth == 1:
geoid = coord_systems.GeogCS(
self.scaledValueOfRadiusOfSphericalEarth *
10**-self.scaleFactorOfRadiusOfSphericalEarth)
# IAU65 oblate sphere
elif self.shapeOfTheEarth == 2:
geoid = coord_systems.GeogCS(6378160, inverse_flattening=297.0)
# custom oblate spheroid (km)
elif self.shapeOfTheEarth == 3:
geoid = coord_systems.GeogCS(
semi_major_axis=self.scaledValueOfEarthMajorAxis *
10**-self.scaleFactorOfEarthMajorAxis * 1000.,
semi_minor_axis=self.scaledValueOfEarthMinorAxis *
10**-self.scaleFactorOfEarthMinorAxis * 1000.)
# IAG-GRS80 oblate spheroid
elif self.shapeOfTheEarth == 4:
geoid = coord_systems.GeogCS(6378137, None, 298.257222101)
# WGS84
elif self.shapeOfTheEarth == 5:
geoid = \
coord_systems.GeogCS(6378137, inverse_flattening=298.257223563)
# pre-defined sphere
elif self.shapeOfTheEarth == 6:
geoid = coord_systems.GeogCS(6371229)
# custom oblate spheroid (m)
elif self.shapeOfTheEarth == 7:
geoid = coord_systems.GeogCS(
semi_major_axis=self.scaledValueOfEarthMajorAxis *
10**-self.scaleFactorOfEarthMajorAxis,
semi_minor_axis=self.scaledValueOfEarthMinorAxis *
10**-self.scaleFactorOfEarthMinorAxis)
elif self.shapeOfTheEarth == 8:
raise ValueError("unhandled shape of earth : grib earth shape = 8")
else:
raise ValueError("undefined shape of earth")
gridType = gribapi.grib_get_string(self.grib_message, "gridType")
if gridType in [
"regular_ll", "regular_gg", "reduced_ll", "reduced_gg"
]:
self.extra_keys['_x_coord_name'] = "longitude"
self.extra_keys['_y_coord_name'] = "latitude"
self.extra_keys['_coord_system'] = geoid
elif gridType == 'rotated_ll':
# TODO: Confirm the translation from angleOfRotation to
# north_pole_lon (usually 0 for both)
self.extra_keys['_x_coord_name'] = "grid_longitude"
self.extra_keys['_y_coord_name'] = "grid_latitude"
southPoleLon = longitudeOfSouthernPoleInDegrees
southPoleLat = latitudeOfSouthernPoleInDegrees
self.extra_keys['_coord_system'] = \
coord_systems.RotatedGeogCS(
-southPoleLat,
math.fmod(southPoleLon + 180.0, 360.0),
self.angleOfRotation, geoid)
elif gridType == 'polar_stereographic':
self.extra_keys['_x_coord_name'] = "projection_x_coordinate"
self.extra_keys['_y_coord_name'] = "projection_y_coordinate"
if self.projectionCentreFlag == 0:
pole_lat = 90
elif self.projectionCentreFlag == 1:
pole_lat = -90
else:
raise TranslationError("Unhandled projectionCentreFlag")
# Note: I think the grib api defaults LaDInDegrees to 60 for grib1.
self.extra_keys['_coord_system'] = \
coord_systems.Stereographic(
pole_lat, self.orientationOfTheGridInDegrees, 0, 0,
self.LaDInDegrees, ellipsoid=geoid)
elif gridType == 'lambert':
self.extra_keys['_x_coord_name'] = "projection_x_coordinate"
self.extra_keys['_y_coord_name'] = "projection_y_coordinate"
flag_name = "projectionCenterFlag"
if getattr(self, flag_name) == 0:
pole_lat = 90
elif getattr(self, flag_name) == 1:
pole_lat = -90
else:
raise TranslationError("Unhandled projectionCentreFlag")
LambertConformal = coord_systems.LambertConformal
self.extra_keys['_coord_system'] = LambertConformal(
self.LaDInDegrees,
self.LoVInDegrees,
0,
0,
secant_latitudes=(self.Latin1InDegrees, self.Latin2InDegrees),
ellipsoid=geoid)
else:
raise TranslationError("unhandled grid type: {}".format(gridType))
if gridType in ["regular_ll", "rotated_ll"]:
self._regular_longitude_common()
j_step = self.jDirectionIncrementInDegrees
if not self.jScansPositively:
j_step = -j_step
self._y_points = (np.arange(self.Nj, dtype=np.float64) * j_step +
self.latitudeOfFirstGridPointInDegrees)
elif gridType in ['regular_gg']:
# longitude coordinate is straight-forward
self._regular_longitude_common()
# get the distinct latitudes, and make sure they are sorted
# (south-to-north) and then put them in the right direction
# depending on the scan direction
latitude_points = gribapi.grib_get_double_array(
self.grib_message, 'distinctLatitudes').astype(np.float64)
latitude_points.sort()
if not self.jScansPositively:
# we require latitudes north-to-south
self._y_points = latitude_points[::-1]
else:
self._y_points = latitude_points
elif gridType in ["polar_stereographic", "lambert"]:
# convert the starting latlon into meters
cartopy_crs = self.extra_keys['_coord_system'].as_cartopy_crs()
x1, y1 = cartopy_crs.transform_point(
self.longitudeOfFirstGridPointInDegrees,
self.latitudeOfFirstGridPointInDegrees, ccrs.Geodetic())
if not np.all(np.isfinite([x1, y1])):
raise TranslationError("Could not determine the first latitude"
" and/or longitude grid point.")
self._x_points = x1 + self.DxInMetres * np.arange(self.Nx,
dtype=np.float64)
self._y_points = y1 + self.DyInMetres * np.arange(self.Ny,
dtype=np.float64)
elif gridType in ["reduced_ll", "reduced_gg"]:
self._x_points = self.longitudes
self._y_points = self.latitudes
else:
raise TranslationError("unhandled grid type")
def _regularise(grib_message):
"""
Transform a reduced grid to a regular grid using interpolation.
Uses 1d linear interpolation at constant latitude to make the grid
regular. If the longitude dimension is circular then this is taken
into account by the interpolation. If the longitude dimension is not
circular then extrapolation is allowed to make sure all end regular
grid points get a value. In practice this extrapolation is likely to
be minimal.
"""
# Make sure to read any missing values as NaN.
gribapi.grib_set_double(grib_message, "missingValue", np.nan)
# Get full longitude values, these describe the longitude value of
# *every* point in the grid, they are not 1d monotonic coordinates.
lons = gribapi.grib_get_double_array(grib_message, "longitudes")
# Compute the new longitude coordinate for the regular grid.
new_nx = max(gribapi.grib_get_long_array(grib_message, "pl"))
new_x_step = (max(lons) - min(lons)) / (new_nx - 1)
if gribapi.grib_get_long(grib_message, "iScansNegatively"):
new_x_step *= -1
new_lons = np.arange(new_nx) * new_x_step + lons[0]
# Get full latitude and data values, these describe the latitude and
# data values of *every* point in the grid, they are not 1d monotonic
# coordinates.
lats = gribapi.grib_get_double_array(grib_message, "latitudes")
values = gribapi.grib_get_double_array(grib_message, "values")
# Retrieve the distinct latitudes from the GRIB message. GRIBAPI docs
# don't specify if these points are guaranteed to be oriented correctly so
# the safe option is to sort them into ascending (south-to-north) order
# and then reverse the order if necessary.
new_lats = gribapi.grib_get_double_array(grib_message, "distinctLatitudes")
new_lats.sort()
if not gribapi.grib_get_long(grib_message, "jScansPositively"):
new_lats = new_lats[::-1]
ny = new_lats.shape[0]
# Use 1d linear interpolation along latitude circles to regularise the
# reduced data.
cyclic = _longitude_is_cyclic(new_lons)
new_values = np.empty([ny, new_nx], dtype=values.dtype)
for ilat, lat in enumerate(new_lats):
idx = np.where(lats == lat)
llons = lons[idx]
vvalues = values[idx]
if cyclic:
# For cyclic data we insert dummy points at each end to ensure
# we can interpolate to all output longitudes using pure
# interpolation.
cgap = (360 - llons[-1] - llons[0])
llons = np.concatenate(
(llons[0:1] - cgap, llons, llons[-1:] + cgap))
vvalues = np.concatenate(
(vvalues[-1:], vvalues, vvalues[0:1]))
fixed_latitude_interpolator = scipy.interpolate.interp1d(
llons, vvalues)
else:
# Allow extrapolation for non-cyclic data sets to ensure we can
# interpolate to all output longitudes.
fixed_latitude_interpolator = Linear1dExtrapolator(
scipy.interpolate.interp1d(llons, vvalues))
new_values[ilat] = fixed_latitude_interpolator(new_lons)
new_values = new_values.flatten()
# Set flags for the regularised data.
if np.isnan(new_values).any():
# Account for any missing data.
gribapi.grib_set_double(grib_message, "missingValue", np.inf)
gribapi.grib_set(grib_message, "bitmapPresent", 1)
new_values = np.where(np.isnan(new_values), np.inf, new_values)
gribapi.grib_set_long(grib_message, "Nx", int(new_nx))
gribapi.grib_set_double(grib_message,
"iDirectionIncrementInDegrees", float(new_x_step))
gribapi.grib_set_double_array(grib_message, "values", new_values)
gribapi.grib_set_long(grib_message, "jPointsAreConsecutive", 0)
gribapi.grib_set_long(grib_message, "PLPresent", 0)
def _compute_latlongs(gid):
lats = gribapi.grib_get_double_array(gid, 'latitudes')
lons = gribapi.grib_get_double_array(gid, 'longitudes')
return lats, lons
