def analyze(self, paths, filename_only=False):
inpath = paths[0]
name_attrs = self.parse_filename(inpath)
properties = Struct()
core = properties.core = Struct()
core.product_name = os.path.splitext(os.path.basename(inpath))[0]
core.creation_date = datetime.strptime(name_attrs['creation_date'],
"%Y%m%dT%H%M%S")
if name_attrs['validity_start'] == "00000000T000000":
core.validity_start = datetime.min
else:
core.validity_start = datetime.strptime(
name_attrs['validity_start'], "%Y%m%dT%H%M%S")
if name_attrs['validity_stop'] == "99999999T999999":
core.validity_stop = datetime.max
else:
core.validity_stop = datetime.strptime(name_attrs['validity_stop'],
"%Y%m%dT%H%M%S")
s5p = properties.s5p = Struct()
s5p.file_class = name_attrs['file_class']
s5p.file_type = name_attrs['file_type']
return properties
def analyze(self, paths, filename_only=False):
inpath = paths[0]
name_attrs = self.parse_filename(inpath)
properties = Struct()
core = properties.core = Struct()
core.product_name = os.path.splitext(os.path.basename(inpath))[0]
core.creation_date = datetime.strptime(name_attrs['creation_date'],
"%Y%m%dT%H%M%S")
core.validity_start = datetime.strptime(name_attrs['validity_start'],
"%Y%m%dT%H%M%S")
core.validity_stop = datetime.strptime(name_attrs['validity_stop'],
"%Y%m%dT%H%M%S")
if not filename_only:
core.footprint = get_footprint(inpath)
s5p = properties.s5p = Struct()
s5p.file_class = name_attrs['file_class']
s5p.file_type = name_attrs['file_type']
s5p.orbit = int(name_attrs['orbit'])
s5p.collection = int(name_attrs['collection'])
s5p.processor_version = int(name_attrs['processor_version'])
return properties
def _unpack_namespace_properties(self, namespace, description, values):
unpacked_properties = Struct()
schema = self._namespace_schema(namespace)
for property, value in zip(description, values):
if value is not None or not schema.is_optional(property):
unpacked_properties[property] = value
return unpacked_properties
def _unpack_product_properties(self, description, values):
unpacked_properties, start = Struct(), 0
for ns_name, ns_description in description:
end = start + len(ns_description)
# A value of None for the uuid field of namespaces other than the core namespace indicates the namespace is
# not defined for the product under consideration. In this case, the entire namespace will be skipped.
# Otherwise, only the uuid field is skipped, as it is an implementation detail (foreign key) and it is not
# part of the namespace itself.
#
if ns_name != "core":
assert (ns_description[0] == "uuid")
if values[start] is None:
# Skip the entire namespace.
start = end
continue
# Skip the uuid field.
start += 1
ns_description = ns_description[1:]
unpacked_ns_properties = self._unpack_namespace_properties(
ns_name, ns_description, values[start:end])
self._validate_namespace_properties(ns_name,
unpacked_ns_properties)
unpacked_properties[ns_name] = unpacked_ns_properties
start = end
return unpacked_properties
def analyze(self, paths, filename_only=False):
inpath = paths[0]
name_attrs = self.parse_filename(inpath)
properties = Struct()
core = properties.core = Struct()
core.product_name = os.path.splitext(os.path.basename(inpath))[0]
core.validity_start = datetime.strptime(name_attrs['validity_start'],
"%Y%m%d")
core.validity_stop = core.validity_start + timedelta(days=1)
core.creation_date = core.validity_start
s5p = properties.s5p = Struct()
s5p.file_class = "OPER"
s5p.file_type = "AUX_NISE__"
return properties
def _unpack_namespace_properties(self, namespace, description, values):
unpacked_properties = Struct()
schema = self._namespace_schema(namespace)
for identifier, value in zip(description, values):
if value is not None or not schema.is_optional(identifier):
if issubclass(schema[identifier], JSON):
value = json.loads(value)
unpacked_properties[identifier] = value
return unpacked_properties
def _unpack_namespace_properties(self, namespace, description, values):
unpacked_properties = Struct()
schema = self._namespace_schema(namespace)
for identifier, value in zip(description, values):
# We may get unicode from the (psycopg2) connection
# if, possibly by a third party, the UNICODE adapter is loaded.
# Muninn assumes strs
if is_python2_unicode(value):
value = value.encode(self._connection.encoding)
if value is not None or not schema.is_optional(identifier):
unpacked_properties[identifier] = value
return unpacked_properties
def get_core_properties(product_type, ecmwfmars, levtype_options=None):
date = datetime.datetime.strptime(ecmwfmars.date.replace('-', ''),
"%Y%m%d")
time = ecmwfmars.time.replace(':', '')
if len(time) >= 2:
if len(time) >= 4:
date += datetime.timedelta(hours=int(time[0:2]),
minutes=int(time[2:4]))
else:
date += datetime.timedelta(hours=int(time[0:2]))
core = Struct()
core.uuid = Archive.generate_uuid()
core.active = True
core.product_type = product_type
core.product_name = "%s_%s_%s_%s_%s_%s" % (
product_type, ecmwfmars.marsclass, ecmwfmars.stream, ecmwfmars.expver,
ecmwfmars.type, date.strftime("%Y%m%dT%H%M%S"))
if 'step' in ecmwfmars:
core.product_name += "_%03d" % (ecmwfmars.step, )
core.physical_name = "%s.grib" % (core.product_name, )
core.validity_start = date
if 'step' in ecmwfmars:
core.validity_start += datetime.timedelta(hours=ecmwfmars.step)
core.validity_stop = core.validity_start
# the creation date is set to the base time of the model
core.creation_date = date
if levtype_options:
core.remote_url = get_remote_url(core.physical_name, ecmwfmars,
levtype_options)
return core
def extract_grib_metadata(gribfile):
"""
this will return a tuple containing:
- ecmwfmars properties struct
- levtype_options struct (see set_remote_url())
"""
import coda
@contextlib.contextmanager
def coda_open(filename):
coda_handle = coda.open(filename)
try:
yield coda_handle
finally:
coda.close(coda_handle)
ecmwfmars = Struct()
levtype_options = {} # TODO: add extraction of levtype_options
with coda_open(gribfile) as coda_handle:
cursor = coda.Cursor()
coda.cursor_set_product(cursor, coda_handle)
num_messages = coda.cursor_get_num_elements(cursor)
coda.cursor_goto_first_array_element(cursor)
for i in range(num_messages):
index = coda.cursor_get_available_union_field_index(cursor)
coda.cursor_goto_record_field_by_index(cursor, index)
step = 0
if index == 0:
# grib1
centuryOfReferenceTimeOfData = coda.fetch(
cursor, "centuryOfReferenceTimeOfData")
yearOfCentury = coda.fetch(cursor, "yearOfCentury")
month = coda.fetch(cursor, "month")
day = coda.fetch(cursor, "day")
date = "%02d%02d-%02d-%02d" % (centuryOfReferenceTimeOfData -
1, yearOfCentury, month, day)
hour = coda.fetch(cursor, "hour")
minute = coda.fetch(cursor, "minute")
time = "%02d:%02d:00" % (hour, minute)
unitOfTimeRange = coda.fetch(cursor, "unitOfTimeRange")
if unitOfTimeRange != 0:
P1 = coda.fetch(cursor, "P1")
if unitOfTimeRange == 1:
step = P1
elif unitOfTimeRange == 2:
step = 24 * P1
elif unitOfTimeRange == 10:
step = 3 * P1
elif unitOfTimeRange == 11:
step = 6 * P1
elif unitOfTimeRange == 13:
step = 12 * P1
else:
raise Error("unsupported unitOfTimeRange: %d" %
(unitOfTimeRange, ))
local = coda.fetch(cursor, "local")
try:
local = local[1:9].tobytes()
except AttributeError:
# workaround for older numpy versions
local = local[1:9].tostring()
marsclass, marstype, stream, expver = struct.unpack(
'>BBH4s', local)
else:
# grib2
year = coda.fetch(cursor, "year")
month = coda.fetch(cursor, "month")
day = coda.fetch(cursor, "day")
date = "%04d-%02d-%02d" % (year, month, day)
hour = coda.fetch(cursor, "hour")
minute = coda.fetch(cursor, "minute")
second = coda.fetch(cursor, "second")
time = "%02d:%02d:%02d" % (hour, minute, second)
significanceOfReferenceTime = coda.fetch(
cursor, "significanceOfReferenceTime")
local = coda.fetch(cursor, "local[0]")
try:
local = local[2:12].tobytes()
except AttributeError:
# workaround for older numpy versions
local = local[2:12].tostring()
marsclass, marstype, stream, expver = struct.unpack(
'>HHH4s', local)
coda.cursor_goto_record_field_by_name(cursor, "data")
num_data = coda.cursor_get_num_elements(cursor)
coda.cursor_goto_first_array_element(cursor)
prev_step = None
for j in range(num_data):
forecastTime = coda.fetch(cursor, "forecastTime")
if forecastTime != 0:
indicatorOfUnitOfTimeRange = coda.fetch(
cursor, "indicatorOfUnitOfTimeRange")
if indicatorOfUnitOfTimeRange == 0:
# minutes
step = 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 1:
# hours
step = 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 2:
# days
step = 24 * 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 10:
# 3 hours
step = 3 * 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 11:
# 6 hours
step = 6 * 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 12:
# 12 hours
step = 12 * 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 13:
# seconds
step = forecastTime
step = int(step / 3600.) # convert seconds to hours
if prev_step is None:
prev_step = step
elif step != prev_step:
raise Error(
"not all data has the same 'step' time (%d) (%d)"
% (step, prev_step))
if j < num_data - 1:
coda.cursor_goto_next_array_element(cursor)
coda.cursor_goto_parent(cursor)
coda.cursor_goto_parent(cursor)
if marsclass not in MARSCLASSES:
raise Error("unsupported MARS class (%d)" % (marsclass, ))
marsclass = MARSCLASSES[marsclass]
if marstype not in MARSTYPES:
raise Error("unsupported MARS type (%d)" % (marstype, ))
marstype = MARSTYPES[marstype]
if stream not in MARSSTREAMS:
raise Error("unsupported MARS stream (%d)" % (stream, ))
stream = MARSSTREAMS[stream]
if 'date' in ecmwfmars:
if date != ecmwfmars.date:
raise Error("not all data is for the same date (%s) (%s)" %
(date, ecmwfmars.date))
if time != ecmwfmars.time:
raise Error("not all data is for the same time (%s) (%s)" %
(time, ecmwfmars.time))
if step != 0:
if 'step' in ecmwfmars:
if step != ecmwfmars.step:
raise Error(
"not all data has the same 'step' time (%d) (%d)"
% (step, ecmwfmars.step))
else:
raise Error("not all data has the same 'step' time")
else:
if 'step' in ecmwfmars and ecmwfmars.step != 0:
raise Error("not all data has the same 'step' time")
if marsclass != ecmwfmars.marsclass:
raise Error(
"not all data has the same MARS class (%s) (%s)" %
(marsclass, ecmwfmars.marsclass))
if marstype != ecmwfmars.type:
raise Error(
"not all data has the same MARS type (%s) (%s)" %
(marstype, ecmwfmars.type))
if stream != ecmwfmars.stream:
raise Error(
"not all data has the same MARS stream (%s) (%s)" %
(stream, ecmwfmars.stream))
if expver != ecmwfmars.expver:
raise Error(
"not all data has the same MARS experiment version (%s) (%s)"
% (expver, ecmwfmars.expver))
else:
ecmwfmars.date = date
ecmwfmars.time = time
if step != 0:
ecmwfmars.step = step
ecmwfmars.marsclass = marsclass
ecmwfmars.type = marstype
ecmwfmars.stream = stream
ecmwfmars.expver = expver
if (marsclass, stream, expver, marstype) in PUBLIC_DATASETS:
ecmwfmars.dataset = PUBLIC_DATASETS[(marsclass, stream,
expver, marstype)]
coda.cursor_goto_parent(cursor)
if i < num_messages - 1:
coda.cursor_goto_next_array_element(cursor)
return ecmwfmars, levtype_options