def grib_generator(filename, auto_regularise=True):
"""
Returns a generator of :class:`~iris.fileformats.grib.GribWrapper`
fields from the given filename.
Args:
* filename (string):
Name of the file to generate fields from.
Kwargs:
* auto_regularise (*True* | *False*):
If *True*, any field defined on a reduced grid will be interpolated
to an equivalent regular grid. If *False*, any field defined on a
reduced grid will be loaded on the raw reduced grid with no shape
information. The default behaviour is to interpolate fields on a
reduced grid to an equivalent regular grid.
"""
with open(filename, 'rb') as grib_fh:
while True:
grib_message = gribapi.grib_new_from_file(grib_fh)
if grib_message is None:
break
grib_wrapper = GribWrapper(grib_message, grib_fh, auto_regularise)
yield grib_wrapper
# finished with the grib message - claimed by the ecmwf c library.
gribapi.grib_release(grib_message)
def scan_grib(self, gribs, kwargs):
v_selected = kwargs['shortName']
v_pert = kwargs.get('perturbationNumber', -1)
grib_append = gribs.append
if not utils.is_container(v_selected):
v_selected = [v_selected]
if self._grbindx:
for v in v_selected:
grib_index_select(self._grbindx, 'shortName', str(v))
if v_pert != -1:
grib_index_select(self._grbindx, 'perturbationNumber',
int(v_pert))
while 1:
gid = grib_new_from_index(self._grbindx)
if gid is None:
break
if GRIBReader._find(gid, **kwargs):
grib_append(gid)
else:
# release unused grib
grib_release(gid)
elif self._file_handler:
while 1:
gid = grib_new_from_file(self._file_handler)
if gid is None:
break
if GRIBReader._find(gid, **kwargs):
grib_append(gid)
else:
# release unused grib
grib_release(gid)
def test_gdt40_computed(self):
fname = tests.get_data_path(('GRIB', 'gaussian', 'regular_gg.grib2'))
with open(fname, 'rb') as grib_fh:
self.grib_id = gribapi.grib_new_from_file(grib_fh)
section = _Section(self.grib_id, None, [])
latitudes = section.get_computed_key('latitudes')
self.assertTrue(88.55 < latitudes[0] < 88.59)
def test_bounded_level(self):
cube = iris.load_cube(
tests.get_data_path(("GRIB", "uk_t", "uk_t.grib2")))
# Changing pressure to altitude due to grib api bug:
# https://github.com/SciTools/iris/pull/715#discussion_r5901538
cube.remove_coord("pressure")
cube.add_aux_coord(
iris.coords.AuxCoord(
1030.0,
long_name="altitude",
units="m",
bounds=np.array([111.0, 1949.0]),
))
with self.temp_filename(".grib2") as testfile:
iris.save(cube, testfile)
with open(testfile, "rb") as saved_file:
g = gribapi.grib_new_from_file(saved_file)
self.assertEqual(
gribapi.grib_get_double(g,
"scaledValueOfFirstFixedSurface"),
111.0,
)
self.assertEqual(
gribapi.grib_get_double(g,
"scaledValueOfSecondFixedSurface"),
1949.0,
)
def test_gdt40_computed(self):
fname = tests.get_data_path(('GRIB', 'gaussian', 'regular_gg.grib2'))
with open(fname, 'rb') as grib_fh:
self.grib_id = gribapi.grib_new_from_file(grib_fh)
section = Section(self.grib_id, None, [])
latitudes = section.get_computed_key('latitudes')
self.assertTrue(88.55 < latitudes[0] < 88.59)
def grib_generator(filename, auto_regularise=True):
"""
Returns a generator of :class:`~iris.fileformats.grib.GribWrapper`
fields from the given filename.
Args:
* filename (string):
Name of the file to generate fields from.
Kwargs:
* auto_regularise (*True* | *False*):
If *True*, any field defined on a reduced grid will be interpolated
to an equivalent regular grid. If *False*, any field defined on a
reduced grid will be loaded on the raw reduced grid with no shape
information. The default behaviour is to interpolate fields on a
reduced grid to an equivalent regular grid.
"""
with open(filename, 'rb') as grib_fh:
while True:
grib_message = gribapi.grib_new_from_file(grib_fh)
if grib_message is None:
break
grib_wrapper = GribWrapper(grib_message, grib_fh, auto_regularise)
yield grib_wrapper
# finished with the grib message - claimed by the ecmwf c library.
gribapi.grib_release(grib_message)
def setUp(self):
filename = tests.get_data_path(('GRIB', 'umukv', 'ukv_chan9.grib2'))
with open(filename, 'rb') as grib_fh:
self.grib_id = gribapi.grib_new_from_file(grib_fh)
self.keys = ['satelliteSeries', 'satelliteNumber', 'instrumentType',
'scaleFactorOfCentralWaveNumber',
'scaledValueOfCentralWaveNumber']
def setUp(self):
filename = tests.get_data_path(('GRIB', 'umukv', 'ukv_chan9.grib2'))
with open(filename, 'rb') as grib_fh:
self.grib_id = gribapi.grib_new_from_file(grib_fh)
self.keys = ['satelliteSeries', 'satelliteNumber', 'instrumentType',
'scaleFactorOfCentralWaveNumber',
'scaledValueOfCentralWaveNumber']
def from_file_offset(filename, offset):
with open(filename, 'rb') as f:
f.seek(offset)
message_id = gribapi.grib_new_from_file(f)
if message_id is None:
fmt = 'Invalid GRIB message: {} @ {}'
raise RuntimeError(fmt.format(filename, offset))
return _RawGribMessage(message_id)
def __getitem__(self, keys):
with open(self.path, 'rb') as grib_fh:
grib_fh.seek(self.offset)
grib_message = gribapi.grib_new_from_file(grib_fh)
data = _message_values(grib_message, self.shape)
gribapi.grib_release(grib_message)
return data.__getitem__(keys)
def from_file_offset(filename, offset):
with open(filename, 'rb') as f:
f.seek(offset)
message_id = gribapi.grib_new_from_file(f)
if message_id is None:
fmt = 'Invalid GRIB message: {} @ {}'
raise RuntimeError(fmt.format(filename, offset))
return _RawGribMessage(message_id)
def __getitem__(self, keys):
with open(self.path, 'rb') as grib_fh:
grib_fh.seek(self.offset)
grib_message = gribapi.grib_new_from_file(grib_fh)
data = _message_values(grib_message, self.shape)
gribapi.grib_release(grib_message)
return data.__getitem__(keys)
def __getitem__(self, keys):
with open(self.path, "rb") as grib_fh:
grib_fh.seek(self.offset)
grib_message = gribapi.grib_new_from_file(grib_fh)
if self.regularise and _is_quasi_regular_grib(grib_message):
_regularise(grib_message)
data = _message_values(grib_message, self.shape)
gribapi.grib_release(grib_message)
return data.__getitem__(keys)
def merge_prev_months(month, fin, fouts, writer):
while True:
gid = gribapi.grib_new_from_file(fin)
if (not gid): break
date = int(gribapi.grib_get(gid, "dataDate"))
mon = (date % 10000) / 100
if (mon == month):
code = make_grib_tuple(gribapi.grib_get(gid, "param"))
if (code in accum_codes): continue
fix_Pa_pressure_levels(gid)
writer(gid, fouts)
gribapi.grib_release(gid)
def __init__(self,
grib_file=None,
clone=None,
sample=None,
gribindex=None):
"""
Open a message and inform the GRIB file that it's been incremented.
If ``grib_file`` is not supplied, the message is cloned from
``GribMessage`` ``clone``. If neither is supplied, the ``GribMessage``
is cloned from ``sample``. If ``index`` is suppliea as a GribIndex, the
message is taken from the index.
"""
#: Unique GRIB ID, for GRIB API interface
self.gid = None
#: File containing message
self.grib_file = None
#: GribIndex referencing message
self.grib_index = None
# Strangely, if I test any of the input variables in an if-clause I
# the GRIB API no longer increments the file, so I've enclosed the gid
# assignments in try blocks. I wish there were a better way of doing
# this.
try:
self.gid = gribapi.grib_new_from_file(grib_file.file_handle)
self.grib_file = grib_file
self.grib_file.message += 1
self.grib_file.open_messages.append(self)
except AttributeError:
pass
try:
self.gid = gribapi.grib_clone(clone.gid)
except AttributeError:
pass
try:
self.gid = gribapi.grib_new_from_samples(sample)
except AssertionError:
pass
try:
self.gid = gribapi.grib_new_from_index(gribindex.iid)
if not self.gid:
raise IndexNotSelectedError("All keys must have selected "
"values before receiving message "
"from index.")
self.grib_index = gribindex
gribindex.open_messages.append(self)
except AttributeError:
pass
if not self.gid:
raise RuntimeError("Either grib_file, clone, sample or gribindex "
"must be provided.")
def __getitem__(self, keys):
# Avoid fetching file data just to return an 'empty' result.
# Needed because of how dask.array.from_array behaves since Dask v2.0.
result = _array_slice_ifempty(keys, self.shape, self.dtype)
if result is None:
with open(self.path, 'rb') as grib_fh:
grib_fh.seek(self.offset)
grib_message = gribapi.grib_new_from_file(grib_fh)
data = _message_values(grib_message, self.shape)
gribapi.grib_release(grib_message)
result = data.__getitem__(keys)
return result
def test_bounded_level(self):
cube = iris.load_cube(tests.get_data_path(("GRIB", "uk_t", "uk_t.grib2")))
# Changing pressure to altitude due to grib api bug:
# https://github.com/SciTools/iris/pull/715#discussion_r5901538
cube.remove_coord("pressure")
cube.add_aux_coord(
iris.coords.AuxCoord(1030.0, long_name="altitude", units="m", bounds=np.array([111.0, 1949.0]))
)
with self.temp_filename(".grib2") as testfile:
iris.save(cube, testfile)
with open(testfile, "rb") as saved_file:
g = gribapi.grib_new_from_file(saved_file)
self.assertEqual(gribapi.grib_get_double(g, "scaledValueOfFirstFixedSurface"), 111.0)
self.assertEqual(gribapi.grib_get_double(g, "scaledValueOfSecondFixedSurface"), 1949.0)
def get_grib_metadata(filename, shortname, level=None):
"""Restituisce l'elenco dei grib che fanno match con shortname e level passati"""
grib_get_or_none = lambda gid, key: gribapi.grib_get(gid, key) if gribapi.grib_is_defined(gid, key) else None
with open(filename) as fp:
# Itero sui messaggi GRIB
while True:
gid = gribapi.grib_new_from_file(fp)
# when None, there are no more messages
if gid is None:
break
# grib message should have these metadata
if grib_get_or_none(gid, "cfVarName") != shortname:
continue
if level is not None and grib_get_or_none(gid, "level") != level:
continue
global units
units=grib_get_or_none(gid, "units")
# custom scaling options
global scaling_offset
global scaling_factor
if units == 'K':
scaling_offset=-273.15
units = '°C'
# converting total cloud cover for cosmo in okta
if units == '%' and grib_get_or_none(gid, "cfVarName") == 'tcc':
scaling_factor = 0.08
units = 'okta'
# converting total cloud cover for ifs-ecmwf in okta
if units == '(0 - 1)' and grib_get_or_none(gid, "cfVarName") == 'tcc':
scaling_factor = 8.0
units = 'okta'
# converting precipitations in mm
if units == 'm' and grib_get_or_none(gid, "cfVarName") != 'hzerocl':
scaling_factor = 0.001
units = 'mm'
if units == 'pa':
scaling_factor = 0.01
units = 'hPa'
yield gid, grib_get_or_none(gid, "endStep")
def __init__(self, grib_file=None, clone=None, sample=None, gribindex=None):
"""
Open a message and inform the GRIB file that it's been incremented.
If ``grib_file`` is not supplied, the message is cloned from
``GribMessage`` ``clone``. If neither is supplied, the ``GribMessage``
is cloned from ``sample``. If ``index`` is suppliea as a GribIndex, the
message is taken from the index.
"""
#: Unique GRIB ID, for GRIB API interface
self.gid = None
#: File containing message
self.grib_file = None
#: GribIndex referencing message
self.grib_index = None
# Strangely, if I test any of the input variables in an if-clause I
# the GRIB API no longer increments the file, so I've enclosed the gid
# assignments in try blocks. I wish there were a better way of doing
# this.
try:
self.gid = gribapi.grib_new_from_file(grib_file.file_handle)
self.grib_file = grib_file
self.grib_file.message += 1
self.grib_file.open_messages.append(self)
except AttributeError:
pass
try:
self.gid = gribapi.grib_clone(clone.gid)
except AttributeError:
pass
try:
self.gid = gribapi.grib_new_from_samples(sample)
except AssertionError:
pass
try:
self.gid = gribapi.grib_new_from_index(gribindex.iid)
if not self.gid:
raise IndexNotSelectedError("All keys must have selected "
"values before receiving message "
"from index.")
self.grib_index = gribindex
gribindex.open_messages.append(self)
except AttributeError:
pass
if not self.gid:
raise RuntimeError("Either grib_file, clone, sample or gribindex "
"must be provided.")
def randomise_grib(filename):
"""Randomise the data in the given GRIB file."""
in_file = open(filename, "rb")
out_file = open("out.grib", "wb")
while True:
grib_message = gribapi.grib_new_from_file(in_file)
if not grib_message:
break
values = gribapi.grib_get_double_array(grib_message, "values")
values = numpy.random.rand(len(values))
gribapi.grib_set_double_array(grib_message, "values", values)
gribapi.grib_write(grib_message, out_file)
os.remove(filename)
os.rename("out.grib", filename)
def randomise_grib(filename):
"""Randomise the data in the given GRIB file."""
in_file = open(filename, "rb")
out_file = open("out.grib", "wb")
while True:
grib_message = gribapi.grib_new_from_file(in_file)
if not grib_message:
break
values = gribapi.grib_get_double_array(grib_message, "values")
values = numpy.random.rand(len(values))
gribapi.grib_set_double_array(grib_message, "values", values)
gribapi.grib_write(grib_message, out_file)
os.remove(filename)
os.rename("out.grib", filename)
def test_bounded_level(self):
cube = iris.load_cube(
tests.get_data_path(("GRIB", "uk_t", "uk_t.grib2")))
with self.temp_filename(".grib2") as testfile:
iris.save(cube, testfile)
with open(testfile, "rb") as saved_file:
g = gribapi.grib_new_from_file(saved_file)
self.assertEqual(
gribapi.grib_get_double(g,
"scaledValueOfFirstFixedSurface"),
0.0,
)
self.assertEqual(
gribapi.grib_get_double(g,
"scaledValueOfSecondFixedSurface"),
2147483647.0,
)
def ls_grib(src_path, keys):
keys = ['name', 'level', 'typeOfLevel'] + keys
messages = defaultdict(list)
with open(src_path, 'rb') as fin:
while True:
gid = gribapi.grib_new_from_file(fin)
if gid is None:
break
for k in keys:
try:
val = gribapi.grib_get(gid, k)
messages[k].append(val)
except Exception as e:
logging.error('Failed to get key (%s). Either key is not available for the message or it is an '
'array type.' % k)
return messages, keys
def messages_from_filename(filename, regularise=True):
"""
Return a generator of :class:`_GribMessage` instances; one for
each message in the supplied GRIB file.
Args:
* filename (string):
Name of the file to generate fields from.
"""
with open(filename, 'rb') as grib_fh:
while True:
offset = grib_fh.tell()
grib_id = gribapi.grib_new_from_file(grib_fh)
if grib_id is None:
break
raw_message = _RawGribMessage(grib_id)
recreate_raw = _MessageLocation(filename, offset)
yield _GribMessage(raw_message, recreate_raw, regularise)
def messages_from_filename(filename):
"""
Return a generator of :class:`_GribMessage` instances; one for
each message in the supplied GRIB file.
Args:
* filename (string):
Name of the file to generate fields from.
"""
with open(filename, 'rb') as grib_fh:
while True:
offset = grib_fh.tell()
grib_id = gribapi.grib_new_from_file(grib_fh)
if grib_id is None:
break
raw_message = _RawGribMessage(grib_id)
recreate_raw = _MessageLocation(filename, offset)
yield _GribMessage(raw_message, recreate_raw)
def grib_generator(filename, auto_regularise=True):
"""
Returns a generator of :class:`~iris.fileformats.grib.GribWrapper`
fields from the given filename.
.. deprecated:: 1.10
The function:
:meth:`iris.fileformats.grib.message.GribMessage.messages_from_filename`
provides alternative means of obtainig GRIB messages from a file.
Args:
* filename (string):
Name of the file to generate fields from.
Kwargs:
* auto_regularise (*True* | *False*):
If *True*, any field defined on a reduced grid will be interpolated
to an equivalent regular grid. If *False*, any field defined on a
reduced grid will be loaded on the raw reduced grid with no shape
information. The default behaviour is to interpolate fields on a
reduced grid to an equivalent regular grid.
"""
warn_deprecated('Deprecated at version 1.10')
with open(filename, 'rb') as grib_fh:
while True:
grib_message = gribapi.grib_new_from_file(grib_fh)
if grib_message is None:
break
grib_wrapper = GribWrapper(grib_message, grib_fh, auto_regularise)
yield grib_wrapper
# finished with the grib message - claimed by the ecmwf c library.
gribapi.grib_release(grib_message)
def grib_generator(filename, auto_regularise=True):
"""
Returns a generator of :class:`~iris_grib.GribWrapper`
fields from the given filename.
.. deprecated:: 1.10
The function:
:meth:`iris_grib.message.GribMessage.messages_from_filename`
provides alternative means of obtainig GRIB messages from a file.
Args:
* filename (string):
Name of the file to generate fields from.
Kwargs:
* auto_regularise (*True* | *False*):
If *True*, any field defined on a reduced grid will be interpolated
to an equivalent regular grid. If *False*, any field defined on a
reduced grid will be loaded on the raw reduced grid with no shape
information. The default behaviour is to interpolate fields on a
reduced grid to an equivalent regular grid.
"""
warn_deprecated('Deprecated at version 1.10')
with open(filename, 'rb') as grib_fh:
while True:
grib_message = gribapi.grib_new_from_file(grib_fh)
if grib_message is None:
break
grib_wrapper = GribWrapper(grib_message, grib_fh, auto_regularise)
yield grib_wrapper
# finished with the grib message - claimed by the ecmwf c library.
gribapi.grib_release(grib_message)
def messages_from_filename(filename):
"""
Return a generator of :class:`GribMessage` instances; one for
each message in the supplied GRIB file.
Args:
* filename (string):
Name of the file to generate fields from.
"""
grib_fh = open(filename, 'rb')
# create an _OpenFileRef to manage the closure of the file handle
file_ref = _OpenFileRef(grib_fh)
while True:
offset = grib_fh.tell()
grib_id = gribapi.grib_new_from_file(grib_fh)
if grib_id is None:
break
raw_message = _RawGribMessage(grib_id)
recreate_raw = _MessageLocation(filename, offset)
yield GribMessage(raw_message, recreate_raw, file_ref=file_ref)
def messages_from_filename(filename):
"""
Return a generator of :class:`GribMessage` instances; one for
each message in the supplied GRIB file.
Args:
* filename (string):
Name of the file to generate fields from.
"""
grib_fh = open(filename, 'rb')
# create an _OpenFileRef to manage the closure of the file handle
file_ref = _OpenFileRef(grib_fh)
while True:
offset = grib_fh.tell()
grib_id = gribapi.grib_new_from_file(grib_fh)
if grib_id is None:
break
raw_message = _RawGribMessage(grib_id)
recreate_raw = _MessageLocation(filename, offset)
yield GribMessage(raw_message, recreate_raw, file_ref=file_ref)
def __init__(self, grib_file=None, clone=None, sample=None, gribindex=None):
"""
Open a message and inform the GRIB file that it's been incremented.
If ``grib_file`` is not supplied, the message is cloned from
``GribMessage`` ``clone``. If neither is supplied, the ``GribMessage``
is cloned from ``sample``. If ``index`` is supplied as a GribIndex, the
message is taken from the index.
"""
#: Unique GRIB ID, for GRIB API interface
self.gid = None
#: File containing message
self.grib_file = None
#: GribIndex referencing message
self.grib_index = None
if grib_file is not None:
self.gid = gribapi.grib_new_from_file(grib_file.file_handle)
if self.gid is None:
raise IOError("Grib file %s is exhausted" % grib_file.name)
self.grib_file = grib_file
self.grib_file.message += 1
self.grib_file.open_messages.append(self)
elif clone is not None:
self.gid = gribapi.grib_clone(clone.gid)
elif sample is not None:
self.gid = gribapi.grib_new_from_samples(sample)
elif gribindex is not None:
self.gid = gribapi.grib_new_from_index(gribindex.iid)
if not self.gid:
raise IndexNotSelectedError("All keys must have selected "
"values before receiving message "
"from index.")
self.grib_index = gribindex
gribindex.open_messages.append(self)
else:
raise RuntimeError("Either grib_file, clone, sample or gribindex "
"must be provided.")
def read_lsm(res_num, input_path_oifs, output_path_oifs, exp_name_oifs,
num_fields):
'''
This function reads the oifs input file in grib format and save it into a
list of numpy arrays.
'''
print(' Opening Grib inpute file: %s ' % (input_path_oifs, ))
input_file_oifs = input_path_oifs + 'ICMGG' + exp_name_oifs + 'INIT'
gid = [None] * num_fields
gribfield = [None] * num_fields
with open(input_file_oifs, 'r+') as f:
keys = ['N', 'shortName']
for i in range(num_fields):
gid[i] = gribapi.grib_new_from_file(f)
if gid[i] is None:
break
for key in keys:
if not gribapi.grib_is_defined(gid[i], key):
raise ValueError("Key '%s' was not defined" % key)
print('%s=%s' % (key, gribapi.grib_get(gid[i], key)))
shortName = gribapi.grib_get(gid[i], 'shortName')
if shortName == 'lsm':
lsm_id = i
if shortName == 'slt':
slt_id = i
if shortName == 'cl':
cl_id = i
nres = gribapi.grib_get(gid[i], 'N')
gribfield[i] = gribapi.grib_get_values(gid[i])
return (gribfield, lsm_id, slt_id, cl_id, gid)
def plotData():
#
# G R I B A P I
#
f = open(INPUT)
gid = gribapi.grib_new_from_file(f)
values = gribapi.grib_get_values(gid)
# print values.dtype
# print values.shape
Ni = gribapi.grib_get(gid, 'Ni')
Nj = gribapi.grib_get(gid, 'Nj')
print 'GRIB_API: %d (%dx%d=%d) values found in %s' % (len(values), Nj, Ni,
Nj * Ni, INPUT)
latitudeOfFirstGridPointInDegrees = gribapi.grib_get(
gid, 'latitudeOfFirstGridPointInDegrees')
longitudeOfFirstGridPointInDegrees = gribapi.grib_get(
gid, 'longitudeOfFirstGridPointInDegrees')
latitudeOfLastGridPointInDegrees = gribapi.grib_get(
gid, 'latitudeOfLastGridPointInDegrees')
longitudeOfLastGridPointInDegrees = gribapi.grib_get(
gid, 'longitudeOfLastGridPointInDegrees')
jDirectionIncrementInDegrees = gribapi.grib_get(
gid, 'jDirectionIncrementInDegrees')
iDirectionIncrementInDegrees = gribapi.grib_get(
gid, 'iDirectionIncrementInDegrees')
# for key in ('max','min','average'):
# print ' %s=%.10e' % (key,gribapi.grib_get(gid,key))
gribapi.grib_release(gid)
f.close()
#
# M A G I C S
#
Magics.init()
Magics.setc("OUTPUT_FORMAT", "ps")
Magics.setc("OUTPUT_NAME", "py_arrays_from_grib")
Magics.setr("INPUT_FIELD_INITIAL_LATITUDE",
latitudeOfFirstGridPointInDegrees)
Magics.setr("INPUT_FIELD_INITIAL_LONGITUDE",
longitudeOfFirstGridPointInDegrees)
Magics.setr("INPUT_FIELD_LATITUDE_STEP", -jDirectionIncrementInDegrees)
Magics.setr("INPUT_FIELD_LONGITUDE_STEP", iDirectionIncrementInDegrees)
values2 = numpy.array(values - 273.15) # convert degree K to C
# print values2.dtype
# print values2.shape
val = values2.reshape(Nj, Ni)
# print val.dtype
# print val.shape
Magics.set2r("INPUT_FIELD", val)
# Magics.setc ("contour_grid_value_plot", "on")
# Magics.setc ("contour_grid_value_plot_type", "value")
# Magics.seti ("contour_grid_value_lat_frequency", 8)
# Magics.seti ("contour_grid_value_lon_frequency", 8)
# Magics.setr ("contour_grid_value_height", 0.3)
# Magics.setc ("contour", "off")
Magics.cont()
Magics.coast()
Magics.finalize()
import json
#Setting of the output file name
output = magics.output(output_formats=['png'], output_name='mars-array')
#Setting the projection attributes
europe = magics.mmap(subpage_lower_left_longitude=-20.,
subpage_upper_right_longitude=20.00,
subpage_upper_right_latitude=30.,
subpage_map_projection="cylindrical",
subpage_lower_left_latitude=70.)
file = open('z500.grb')
#Getting the first message from the file
field = grib.grib_new_from_file(file)
nj = grib.grib_get(field, "Nj")
ni = grib.grib_get(field, "Ni")
metadata = {
"paramId": grib.grib_get(field, "paramId"),
"units": grib.grib_get(field, "units"),
"typeOfLevel": grib.grib_get(field, "typeOfLevel"),
"marsType": grib.grib_get(field, "marsType"),
"marsClass": grib.grib_get(field, "marsClass"),
"marsStream": grib.grib_get_string(field, "marsStream"),
"level": grib.grib_get(field, "level")
}
firstlat = grib.grib_get(field, "latitudeOfFirstGridPointInDegrees")
# SET FILENAMES
filePrefix = 'nam.t'
fileSuffix = 'z.afwaca00.tm00.grib2'
# fileSuffix = 'z.afwaca00.grb2.tm00'
filenames = []
filePaths = []
for i in range(nfiles):
filenames.append(filePrefix + '{:02d}'.format(i * 6) + fileSuffix)
filePaths.append(os.path.join(inDir, filenames[i]))
#####
# OPEN FIRST GRIB FILE, GET MESSAGE HANDLERS AND CLOSE
f = open(filePaths[0], 'r')
gribapi.grib_multi_support_on()
mcount = gribapi.grib_count_in_file(f) # number of messages in file
gids = [gribapi.grib_new_from_file(f) for i in range(mcount)]
f.close()
#####
# GET NAME AND LEVEL OF EACH MESSAGE
varNames = []
levels = []
for i in range(mcount):
gid = gids[i]
varNames.append(gribapi.grib_get(gid, 'shortName'))
levels.append(gribapi.grib_get(gid, 'level'))
#####
# GET REQUIRED GIDS (AT REQUIRED LEVELS)
gidPRMSL = varNames.index("prmsl") + 1 # Pressure reduced to mean sea level
# Height
def read_next(self, headers_only=False):
self.record = gribapi.grib_new_from_file(self.file_object,
headers_only=headers_only)
return self.record is not None
def writeRec9():
PRES = 1013.0 # Sea level presure (replicate Sara's file)
# total accumulated rainfall from past hour (replicate Sara's file)
RAIN = 0.0
SC = 0 # snow cover
RADSW = 0.0 # SW radiation at surface (replicate Sara's file)
RADLW = 0.0 # LW radiation at top (replicate Sara's file)
VAPMR = 0.0 # Vagour mixing ratio (replicate Sara's file)
for t in range(nfiles):
print("Processing file " + filenames[t])
dateTime = parse(date) + dt.timedelta(hours=t * 6)
MYR = dateTime.year # Year of data block
MMO = dateTime.month # Month of data block
MDAY = dateTime.day # Day of data block
MHR = dateTime.hour # Hour of data block
# GRIB file processing:
f = open(filePaths[t], 'r') # Open GRIB file
gribapi.grib_multi_support_on() # Turn on multi-message support
mcount = gribapi.grib_count_in_file(f) # number of messages in file
[gribapi.grib_new_from_file(f)
for i in range(mcount)] # Get handles for all messages
f.close() # Close GRIB file
# Initialse 3D arrays for holding required fields:
HGTgrd = np.zeros(shape=(Nj, Ni, NZ))
TMPgrd = np.zeros(shape=(Nj, Ni, NZ))
Ugrd = np.zeros(shape=(Nj, Ni, NZ))
Vgrd = np.zeros(shape=(Nj, Ni, NZ))
Wgrd = np.zeros(shape=(Nj, Ni, NZ))
RHgrd = np.zeros(shape=(Nj, Ni, NZ))
# Loop through included levels and store the values in the appropriate k index of the 3D arrays
for k in range(NZ):
HGTvals = gribapi.grib_get_values(int(gidHGT[k]))
HGTgrd[:, :, k] = np.reshape(HGTvals, (Nj, Ni), 'C')
TMPvals = gribapi.grib_get_values(int(gidTMP[k]))
TMPgrd[:, :, k] = np.reshape(TMPvals, (Nj, Ni), 'C')
Uvals = gribapi.grib_get_values(int(gidU[k]))
Ugrd[:, :, k] = np.reshape(Uvals, (Nj, Ni), 'C')
Vvals = gribapi.grib_get_values(int(gidV[k]))
Vgrd[:, :, k] = np.reshape(Vvals, (Nj, Ni), 'C')
Wvals = gribapi.grib_get_values(int(gidW[k]))
Wgrd[:, :, k] = np.reshape(Wvals, (Nj, Ni), 'C')
RHvals = gribapi.grib_get_values(int(gidRH[k]))
RHgrd[:, :, k] = np.reshape(RHvals, (Nj, Ni), 'C')
WSgrd = np.sqrt(Ugrd**2 + Vgrd**2) # Calculate wins speed (pythagoras)
# Calculate wind direction:
# radians, between [-pi,pi], positive anticlockwise from positive x-axis
WDgrd = np.arctan2(Vgrd, Ugrd)
# degrees, between [-180,180], positive anticlockwise from positive x-axis
WDgrd *= 180 / np.pi
# degrees, between [0,360], positive anticlockwise from negative x-axis (Since we specify the direction the wind is blowing FROM, not TO)
WDgrd += 180
# degrees, between [-360,0], positive clockwise from negative x-axis (Since wind direction is positive clockwise)
WDgrd = -WDgrd
# degrees, between [-270,90], positive clockwise from positive y-axis (Since wind direction is from North)
WDgrd += 90
# degrees, between [0,360], positive clockwise from positive y-axis (DONE!)
WDgrd = np.mod(WDgrd, 360)
if t > 0:
dateTime = parse(date) + dt.timedelta(hours=(t * 6) - 3)
print('interpolating')
print(dateTime)
MYR = dateTime.year # Year of data block
MMO = dateTime.month # Month of data block
MDAY = dateTime.day # Day of data block
MHR = dateTime.hour # Hour of data block
HGTgrd = np.mean(np.array([HGTgrd + HGTgrd_ini]), axis=0)
TMPgrd = np.mean(np.array([TMPgrd + TMPgrd_ini]), axis=0)
Wgrd = np.mean(np.array([Wgrd + Wgrd_ini]), axis=0)
RHgrd = np.mean(np.array([RHgrd + RHgrd_ini]), axis=0)
WDgrd = np.mean(np.array([WDgrd + WDgrd_ini]), axis=0)
WSgrd = np.mean(np.array([WSgrd + WSgrd_ini]), axis=0)
inds = HGTgrd.argsort(axis=2)
HGTgrd = take_along_axis(HGTgrd, inds, axis=2)
TMPgrd = take_along_axis(TMPgrd, inds, axis=2)
Wgrd = take_along_axis(Wgrd, inds, axis=2)
RHgrd = take_along_axis(RHgrd, inds, axis=2)
WSgrd = take_along_axis(WSgrd, inds, axis=2)
WDgrd = take_along_axis(WDgrd, inds, axis=2)
# Loop over grid cells:
for j in range(NY):
JX = j + 1 # J-index of grid cell
for i in range(NX):
IX = i + 1 # i-index of grid cell
fout.write(
('{:4d}' + '{:02d}' * 3 + '{:3d}' * 2 +
'{:7.1f}{:5.2f}{:2d}' + '{:8.1f}' * 2 + '\n').format(
MYR, MMO, MDAY, MHR, IX, JX, PRES, RAIN, SC,
RADSW, RADLW))
for k in range(NZ):
PRES2 = levsIncl[k] # Pressure (mb)
# Elevation (m above sea level)
Z = int(HGTgrd[iLatMinGRIB + j, iLonMinGRIB + i, k])
# Temperature (Kelvin)
TEMPK = TMPgrd[iLatMinGRIB + j, iLonMinGRIB + i, k]
# Wind direction (degrees)
WD = int(WDgrd[iLatMinGRIB + j, iLonMinGRIB + i, k])
# Wind speed (m/s)
WS = WSgrd[iLatMinGRIB + j, iLonMinGRIB + i, k]
# Vertical velocity (m/s)
W = Wgrd[iLatMinGRIB + j, iLonMinGRIB + i, k]
# Relative humidity (%)
RH = int(RHgrd[iLatMinGRIB + j, iLonMinGRIB + i, k])
fout.write((
'{:4d}{:6d}{:6.1f}{:4d}{:5.1f}{:6.2f}{:3d}{:5.2f}\n'
).format(PRES2, Z, TEMPK, WD, WS, W, RH, VAPMR))
# Release all messages:
for i in range(mcount):
gribapi.grib_release(i + 1)
# GRIB file processing:
dateTime = parse(date) + dt.timedelta(hours=t * 6)
print('processing')
print(dateTime)
MYR = dateTime.year # Year of data block
MMO = dateTime.month # Month of data block
MDAY = dateTime.day # Day of data block
MHR = dateTime.hour # Hour of data block
f = open(filePaths[t], 'r') # Open GRIB file
gribapi.grib_multi_support_on() # Turn on multi-message support
mcount = gribapi.grib_count_in_file(f) # number of messages in file
[gribapi.grib_new_from_file(f)
for i in range(mcount)] # Get handles for all messages
f.close() # Close GRIB file
# Initialse 3D arrays for holding required fields:
HGTgrd = np.zeros(shape=(Nj, Ni, NZ))
TMPgrd = np.zeros(shape=(Nj, Ni, NZ))
Ugrd = np.zeros(shape=(Nj, Ni, NZ))
Vgrd = np.zeros(shape=(Nj, Ni, NZ))
Wgrd = np.zeros(shape=(Nj, Ni, NZ))
RHgrd = np.zeros(shape=(Nj, Ni, NZ))
# Loop through included levels and store the values in the appropriate k index of the 3D arrays
for k in range(NZ):
HGTvals = gribapi.grib_get_values(int(gidHGT[k]))
HGTgrd[:, :, k] = np.reshape(HGTvals, (Nj, Ni), 'C')
TMPvals = gribapi.grib_get_values(int(gidTMP[k]))
TMPgrd[:, :, k] = np.reshape(TMPvals, (Nj, Ni), 'C')
Uvals = gribapi.grib_get_values(int(gidU[k]))
Ugrd[:, :, k] = np.reshape(Uvals, (Nj, Ni), 'C')
Vvals = gribapi.grib_get_values(int(gidV[k]))
Vgrd[:, :, k] = np.reshape(Vvals, (Nj, Ni), 'C')
Wvals = gribapi.grib_get_values(int(gidW[k]))
Wgrd[:, :, k] = np.reshape(Wvals, (Nj, Ni), 'C')
RHvals = gribapi.grib_get_values(int(gidRH[k]))
RHgrd[:, :, k] = np.reshape(RHvals, (Nj, Ni), 'C')
HGTgrd_ini = HGTgrd
TMPgrd_ini = TMPgrd
Wgrd_ini = Wgrd
RHgrd_ini = RHgrd
WSgrd = np.sqrt(Ugrd**2 + Vgrd**2) # Calculate wins speed (pythagoras)
# Calculate wind direction:
# radians, between [-pi,pi], positive anticlockwise from positive x-axis
WDgrd = np.arctan2(Vgrd, Ugrd)
# degrees, between [-180,180], positive anticlockwise from positive x-axis
WDgrd *= 180 / np.pi
# degrees, between [0,360], positive anticlockwise from negative x-axis (Since we specify the direction the wind is blowing FROM, not TO)
WDgrd += 180
# degrees, between [-360,0], positive clockwise from negative x-axis (Since wind direction is positive clockwise)
WDgrd = -WDgrd
# degrees, between [-270,90], positive clockwise from positive y-axis (Since wind direction is from North)
WDgrd += 90
# degrees, between [0,360], positive clockwise from positive y-axis (DONE!)
WDgrd = np.mod(WDgrd, 360)
WDgrd_ini = WDgrd
WSgrd_ini = WSgrd
inds = HGTgrd.argsort(axis=2)
HGTgrd = take_along_axis(HGTgrd, inds, axis=2)
TMPgrd = take_along_axis(TMPgrd, inds, axis=2)
Wgrd = take_along_axis(Wgrd, inds, axis=2)
RHgrd = take_along_axis(RHgrd, inds, axis=2)
WSgrd = take_along_axis(WSgrd, inds, axis=2)
WDgrd = take_along_axis(WDgrd, inds, axis=2)
# Loop over grid cells:
for j in range(NY):
JX = j + 1 # J-index of grid cell
for i in range(NX):
IX = i + 1 # i-index of grid cell
fout.write(
('{:4d}' + '{:02d}' * 3 + '{:3d}' * 2 +
'{:7.1f}{:5.2f}{:2d}' + '{:8.1f}' * 2 + '\n').format(
MYR, MMO, MDAY, MHR, IX, JX, PRES, RAIN, SC, RADSW,
RADLW))
for k in range(NZ):
PRES2 = levsIncl[k] # Pressure (mb)
# Elevation (m above sea level)
Z = int(HGTgrd[iLatMinGRIB + j, iLonMinGRIB + i, k])
# Temperature (Kelvin)
TEMPK = TMPgrd[iLatMinGRIB + j, iLonMinGRIB + i, k]
# Wind direction (degrees)
WD = int(WDgrd[iLatMinGRIB + j, iLonMinGRIB + i, k])
# Wind speed (m/s)
WS = WSgrd[iLatMinGRIB + j, iLonMinGRIB + i, k]
# Vertical velocity (m/s)
W = Wgrd[iLatMinGRIB + j, iLonMinGRIB + i, k]
# Relative humidity (%)
RH = int(RHgrd[iLatMinGRIB + j, iLonMinGRIB + i, k])
fout.write(
('{:4d}{:6d}{:6.1f}{:4d}{:5.1f}{:6.2f}{:3d}{:5.2f}\n'
).format(PRES2, Z, TEMPK, WD, WS, W, RH, VAPMR))
# Release all messages:
for i in range(mcount):
gribapi.grib_release(i + 1)
def save(source, target, append=False, sample_file=_sample_file):
"""
Takes a dataset (source) and writes its contents
as grib 1 to file-like target. Grib 1 is used (instead
of grib 2) because some older forecast visualization
software can't read grib 2.
This is a heavily modified but none-the-less derivative of
the grib saving functions from the iris package.
Parameters
----------
source : Dataset
A netcdf-like file holding the dataset we want to write
as grib. This must contain time, longitude and latitude
coordinates in order to infer the grib grid and time params
target : string path or file-like
Where the contents should be written. If target is a string
the file is created or appended to.
append : boolean
When creating a new file from string you can optionally
append to the file.
"""
if not _has_gribapi:
raise ImportError("gripapi is required to write grib files.")
if isinstance(target, basestring):
grib_file = open(target, "ab" if append else "wb")
elif hasattr(target, "write"):
if hasattr(target, "mode") and "b" not in target.mode:
raise ValueError("Target not binary")
grib_file = target
else:
raise ValueError("Can only save grib to filename or writable")
if not 'latitude' in source.variables or not 'longitude' in source.variables:
raise ValueError("Did not find either latitude or longitude.")
if source['latitude'].ndim != 1 or source['longitude'].ndim != 1:
raise ValueError("Latitude and Longitude should be regular.")
if not 'time' in source.variables:
raise ValueError("Expected time coordinate")
# sort the lats and lons
source = source.indexed(latitude=np.argsort(source['latitude'].values))
lons = source['longitude'].values
if np.any(np.abs(np.diff(lons)) > 180.):
# the latitudes must cross the dateline since we only allow 180
# degree wide bounding boxes, and there is more than a 180 degree
# difference between longitudes. Instead we try converting to
# 0 to 360 degree longitudes before sorting.
lons = np.mod(lons, 360)
if np.any(np.abs(np.diff(lons)) > 180.):
# TODO: I'm sure theres a way to deal with arbitrary longitude
# specifications for global data ... but its not a high priority
# so that will wait for later.
raise ValueError("Longitudes span more than 180 degrees and the dateline?")
source['longitude'].values[:] = lons
source = source.indexed(longitude=np.argsort(lons))
# iterate over variables, unless they are considered
# auxiliary variables (ie, variables used by slocum
# but not in grib files).
auxilary_variables = ['wind_speed', 'wind_from_direction']
for single_var in (v for k, v in source.noncoordinates.iteritems()
if not k in auxilary_variables):
# then iterate over time slices
iter_time = (single_var.indexed(**{'time': [i]})
for i in range(single_var.coordinates['time'].size))
for obj in iter_time:
# Save this slice to the grib file
gribapi.grib_gribex_mode_off()
if sample_file is not None and os.path.exists(sample_file):
with open(sample_file, 'r') as f:
grib_message = gribapi.grib_new_from_file(f)
logger.info("Created grib message from file %s" % sample_file)
else:
logger.info("Creating grib message from gribapi sample: GRIB1")
grib_message = gribapi.grib_new_from_samples("GRIB1")
set_time(obj, grib_message)
set_product(obj, grib_message)
set_grid(obj, grib_message)
set_data(obj, grib_message)
gribapi.grib_write(grib_message, grib_file)
gribapi.grib_release(grib_message)
# if target was a string then we have to close the file we
# created, otherwise leave that up to the user.
if isinstance(target, basestring):
grib_file.close()
def setUp(self):
filename = tests.get_data_path(('GRIB', 'uk_t', 'uk_t.grib2'))
with open(filename, 'rb') as grib_fh:
self.grib_id = gribapi.grib_new_from_file(grib_fh)
def get_phenomena(self):
phen_list = []
phenomenon =\
{
"id" : "",
"attribute_count" : "",
"attributes" :[]
}
phen_keys = [
"paramId",
"cfNameECMF",
"cfName",
"cfVarName",
"units",
"nameECMF",
"name"
]
phen_attr =\
{
"name" : "",
"value": ""
}
try:
fd = open(self.file_path)
while 1:
gid = gapi.grib_new_from_file(fd)
if gid is None: break
phen_attr_list = []
attr_count = 0
for key in phen_keys:
if not gapi.grib_is_defined(gid, key):
continue
value = str(gapi.grib_get(gid, key))
if len(key) < util.MAX_ATTR_LENGTH \
and len(value) < util.MAX_ATTR_LENGTH \
and util.is_valid_phen_attr(value):
phen_attr["name"] = str(key.strip())
phen_attr["value"] = str(unicode(value).strip())
if phen_attr not in phen_attr_list:
phen_attr_list.append(phen_attr.copy())
attr_count = attr_count + 1
if len(phen_attr_list) > 0:
new_phenomenon = phenomenon.copy()
new_phenomenon["attributes"] = phen_attr_list
new_phenomenon["attribute_count"] = attr_count
if new_phenomenon not in phen_list:
phen_list.append(new_phenomenon)
gapi.grib_release(gid)
fd.close()
return phen_list
except Exception:
return None
def setUp(self):
filename = tests.get_data_path(('GRIB', 'uk_t', 'uk_t.grib2'))
with open(filename, 'rb') as grib_fh:
self.grib_id = gribapi.grib_new_from_file(grib_fh)
def get_metadata_level3(self):
phen_list = []
phenomenon =\
{
"id" : "",
"attribute_count" : "",
"attributes" :[]
}
phen_attr =\
{
"name" : "",
"value": ""
}
lat_f_l = []
lon_f_l = []
lat_l_l = []
lon_l_l = []
date_d_l = []
date_t_l = []
phen_keys = [
"paramId",
"cfNameECMF",
"cfName",
"cfVarName",
"units",
"nameECMF",
"name",
"Ni",
"Nj",
"latitudeOfFirstGridPointInDegrees",
"longitudeOfFirstGridPointInDegrees",
"latitudeOfLastGridPointInDegrees",
"longitudeOfLastGridPointInDegrees",
"dataDate",
"dataTime"
]
try:
fd = open(self.file_path)
while 1:
gid = gapi.grib_new_from_file(fd)
if gid is None: break
phen_attr_list = []
attr_count = 0
for key in phen_keys:
if not gapi.grib_is_defined(gid, key):
continue
value = str(gapi.grib_get(gid, key))
#So the file contains many records but all report the
#same spatial and temporal information. Only complete distinct records
#will be stored i.e the one that contain the full list of parameter
#and are unique. If evety record has got different spatial and temporal
#then th eindex must change because currently there is only on geo_shape_field.
if key == "latitudeOfFirstGridPointInDegrees":
lat_f_l.append(value)
elif key == "longitudeOfFirstGridPointInDegrees":
lon_f_l.append(value)
elif key == "latitudeOfLastGridPointInDegrees":
lat_l_l.append(value)
elif key =="longitudeOfLastGridPointInDegrees":
lon_l_l.append(value)
elif key == "dataDate":
date_d_l.append(value)
elif key == "dataTime":
date_t_l.append(value)
else:
if len(key) < util.MAX_ATTR_LENGTH \
and len(value) < util.MAX_ATTR_LENGTH \
and util.is_valid_phen_attr(value):
phen_attr["name"] = str(key.strip())
phen_attr["value"] = str(unicode(value).strip())
if phen_attr not in phen_attr_list:
phen_attr_list.append(phen_attr.copy())
attr_count = attr_count + 1
if len(phen_attr_list) > 0:
new_phenomenon = phenomenon.copy()
new_phenomenon["attributes"] = phen_attr_list
new_phenomenon["attribute_count"] = attr_count
if new_phenomenon not in phen_list:
phen_list.append(new_phenomenon)
gapi.grib_release(gid)
fd.close()
if len(lat_f_l) > 0 \
and len(lon_f_l) > 0 \
and len(lat_l_l) > 0 \
and len(lon_l_l) > 0 \
and len(date_d_l) > 0 \
and len(date_t_l):
geospatial_dict = {}
geospatial_dict["type"] = "envelope"
temporal_dict = {}
lat_f = min(lat_f_l)
lon_f = min(lon_f_l)
lat_l = max(lat_l_l)
lon_l = max(lon_l_l)
date_d = min(date_d_l)
date_t = min(date_t_l)
if float(lon_l) > 180:
lon_l = (float(lon_l) -180) - 180
geospatial_dict["coordinates"] = [[round(float(lon_f), 3), round(float(lat_f), 3)], [round(float(lon_l), 3), round(float(lat_l), 3) ]]
temporal_dict["start_time"] = date_d
temporal_dict["end_time"] = date_t
return (phen_list, geospatial_dict, temporal_dict)
else:
return (phen_list,)
except Exception as ex:
return None
def merge_cur_months(month, fin1, fin2, fouts, writer):
gidinst, gidcum = -1, -1
timinst, timcum = -1, -1
procinst, proccum = True, True
instmode = False
while True:
if (instmode):
if (procinst):
if (gidinst != -1): gribapi.grib_release(gidinst)
gidinst = gribapi.grib_new_from_file(fin1)
if (not gidinst):
break
time = int(gribapi.grib_get(gidinst, "dataTime"))
if (timinst == -1): timinst = time
if (time != timinst):
timinst = time
procinst = False
instmode = False
continue
else:
procinst = True
code = make_grib_tuple(gribapi.grib_get(gidinst, "param"))
if (code in accum_codes):
continue
date = int(gribapi.grib_get(gidinst, "dataDate"))
mon = (date % 10**4) / 10**2
if (mon == month):
fix_Pa_pressure_levels(gidinst)
writer(gidinst, fouts)
else:
if (proccum):
if (gidinst != -1): gribapi.grib_release(gidcum)
gidcum = gribapi.grib_new_from_file(fin2)
if (not gidcum):
procinst = False
instmode = True
continue
time = int(gribapi.grib_get(gidcum, "dataTime"))
if (timcum == -1): timcum = time
if (time != timcum):
timcum = time
proccum = False
instmode = True
continue
else:
proccum = True
code = make_grib_tuple(gribapi.grib_get(gidcum, "param"))
if (code not in accum_codes): continue
if (timeshift > 0):
date = int(gribapi.grib_get(gidcum, "dataDate"))
mon = (date % 10**4) / 10**2
newdate = date
newtime = time - 100 * timeshift
if (newtime < 0):
curdate = datetime.date(date / 10**4, mon, date % 10**2)
prevdate = curdate - datetime.timedelta(days=1)
mon = prevdate.month
newdate = prevdate.year * 10**4 + mon * 10**2 + prevdate.day
newtime = 2400 + newtime
gribapi.grib_set(gidcum, "dataDate", newdate)
gribapi.grib_set(gidcum, "dataTime", newtime)
if (mon == month):
fix_Pa_pressure_levels(gidcum)
writer(gidcum, fouts)
