def print_data(cursor):
type_class = coda.cursor_get_type_class(cursor)
if type_class == coda.coda_array_class:
num_elements = coda.cursor_get_num_elements(cursor)
if num_elements > 0:
print "[",
coda.cursor_goto_first_array_element(cursor)
for i in xrange(num_elements):
print_data(cursor)
if i < num_elements - 1:
print ", ",
coda.cursor_goto_next_array_element(cursor)
print "]",
coda.cursor_goto_parent(cursor)
elif type_class == coda.coda_special_class:
special_type = coda.cursor_get_special_type(cursor)
if special_type == coda.coda_special_time:
data = coda.cursor_read_double(cursor)
time_string = coda.time_to_string(data)
print "%s" % time_string,
else:
print "*** Unexpected special type (%s) ***", coda.type_get_special_type_name(special_type),
else:
read_type = coda.cursor_get_read_type(cursor)
if read_type == coda.coda_native_type_int8 or \
read_type == coda.coda_native_type_int16 or \
read_type == coda.coda_native_type_int32:
data = coda.cursor_read_int32(cursor)
print "%ld" % data,
elif read_type == coda.coda_native_type_uint8 or \
read_type == coda.coda_native_type_uint16 or \
read_type == coda.coda_native_type_uint32:
data = coda.cursor_read_uint32(cursor)
print "%lu" % data,
elif read_type == coda.coda_native_type_int64:
data = coda.cursor_read_int64(cursor)
print "%ld" % data,
elif read_type == coda.coda_native_type_uint64:
data = coda.cursor_read_uint64(cursor)
print "%lu" % data,
elif read_type == coda.coda_native_type_float or \
read_type == coda.coda_native_type_double:
data = coda.cursor_read_double(cursor)
print "%g" % data,
elif read_type == coda.coda_native_type_char:
data = coda.cursor_read_char(cursor)
print "'%c'" % data,
elif read_type == coda.coda_native_type_string:
str = coda.cursor_read_string(cursor)
print "\"%s\"" % str,
else:
print "*** Unexpected read type (%s) ***", coda.type_get_native_type_name(read_type),
def print_record(cursor):
num_fields = coda.cursor_get_num_elements(cursor)
if num_fields > 0:
record_type = coda.cursor_get_type(cursor)
coda.cursor_goto_first_record_field(cursor)
for i in xrange(num_fields):
# We don't print fields that are hidden, like the first MPH field
# (with value 'PRODUCT=')
hidden = coda.type_get_record_field_hidden_status(record_type, i)
if hidden == 0:
field_name = coda.type_get_record_field_name(record_type, i)
print "%32s : " % field_name,
print_data(cursor)
print ""
if i < num_fields - 1:
coda.cursor_goto_next_record_field(cursor)
coda.cursor_goto_parent(cursor)
cursor = coda.Cursor()
coda.cursor_set_product(cursor, pf)
print " MPH :"
coda.cursor_goto_record_field_by_name(cursor, "mph")
print_record(cursor)
coda.cursor_goto_parent(cursor)
print " SPH :";
coda.cursor_goto_record_field_by_name(cursor, "sph")
print_record(cursor);
coda.cursor_goto_parent(cursor)
coda.cursor_goto_record_field_by_name(cursor, "dsd")
num_dsd = coda.cursor_get_num_elements(cursor)
if num_dsd > 0:
coda.cursor_goto_first_array_element(cursor)
for i in range(num_dsd):
print " DSD(%d) :" % i
print_record(cursor);
if i < num_dsd - 1:
coda.cursor_goto_next_array_element(cursor);
coda.cursor_goto_parent(cursor)
coda.cursor_goto_parent(cursor)
coda.close(pf);
def initialize(self):
Node.initialize(self)
self._count = coda.cursor_get_num_elements(self.cursor)
def process_states(pf):
global fpn,leak,badpx
state_name = [ "no measurement", "nadir", "limb", "occultation", "monitoring" ]
states_cursor = coda.Cursor()
mds_cursor = [coda.Cursor() for i in range(0,5)]
pixel_array = numpy.zeros(shape=(8,1024),dtype=numpy.float64)
# coda.Cursor mds_cursor[5] /* 1 dummy + 4x MDS : empty, nadir, limb, occultation, monitoring */
# Initialize STATES cursor
coda.cursor_set_product(states_cursor, pf)
coda.cursor_goto_record_field_by_name(states_cursor, "states")
num_states = coda.cursor_get_num_elements(states_cursor)
if (num_states == 0):
return
coda.cursor_goto_first_array_element(states_cursor)
# Initialize MDS cursors
# Skip #0 (dummy MDS cursor)
for i in range(1,5):
coda.cursor_set_product(mds_cursor[i], pf)
record_type = coda.cursor_get_type(mds_cursor[i])
field_index = coda.type_get_record_field_index_from_name(record_type, state_name[i])
available = coda.cursor_get_record_field_available_status(mds_cursor[i], field_index)
if (available):
coda.cursor_goto_record_field_by_name(mds_cursor[i], state_name[i])
coda.cursor_goto_first_array_element(mds_cursor[i])
# now walk the states
for stateNr in range(0,num_states):
# Read mds_type, num_clus, and num_dsr for this state
coda.cursor_goto_record_field_by_name(states_cursor, "mds_type")
mds_type = coda.cursor_read_uint8(states_cursor)
coda.cursor_goto_parent(states_cursor)
coda.cursor_goto_record_field_by_name(states_cursor, "num_clus")
num_clus = coda.cursor_read_uint16(states_cursor)
coda.cursor_goto_parent(states_cursor)
coda.cursor_goto_record_field_by_name(states_cursor, "num_dsr")
num_dsr = coda.cursor_read_uint16(states_cursor)
coda.cursor_goto_parent(states_cursor)
print "Processing state %d of %d" % (stateNr + 1, num_states)
print " mds_type .....: %d (%s)" % (mds_type, state_name[mds_type])
print " num_clus .....: %d" % (num_clus,)
print " num_dsr ......: %d" % (num_dsr,)
# traverse the MDSRs for this state */
for dsrNr in range(0,num_dsr):
# clus_config_cursor = coda.Cursor()
# clus_dat_cursor = coda.Cursor()
# Initialize pixel_array with zeros
for chan_num in range(0,8):
for pixelNr in range(0,1024):
pixel_array[chan_num][pixelNr] = 0.0
clus_config_cursor = copy.deepcopy(states_cursor)
coda.cursor_goto_record_field_by_name(clus_config_cursor, "clus_config")
coda.cursor_goto_first_array_element(clus_config_cursor)
clus_dat_cursor = copy.deepcopy(mds_cursor[mds_type])
coda.cursor_goto_record_field_by_name(clus_dat_cursor, "clus_dat")
coda.cursor_goto_first_array_element(clus_dat_cursor)
# traverse the clusters in this MDSR
for cluster in range(0,num_clus):
# Read clus_config
coda.cursor_goto_record_field_by_name(clus_config_cursor, "chan_num")
chan_num = coda.cursor_read_uint8(clus_config_cursor)
coda.cursor_goto_parent(clus_config_cursor)
chan_num -= 1
coda.cursor_goto_record_field_by_name(clus_config_cursor, "start_pix")
start_pix = coda.cursor_read_uint16(clus_config_cursor)
coda.cursor_goto_parent(clus_config_cursor)
coda.cursor_goto_record_field_by_name(clus_config_cursor, "clus_len")
clus_len = coda.cursor_read_uint16(clus_config_cursor)
coda.cursor_goto_parent(clus_config_cursor)
coda.cursor_goto_record_field_by_name(clus_config_cursor, "pet")
pet = coda.cursor_read_double(clus_config_cursor)
coda.cursor_goto_parent(clus_config_cursor)
coda.cursor_goto_record_field_by_name(clus_config_cursor, "intgr_time")
intgr_time = coda.cursor_read_double(clus_config_cursor)
coda.cursor_goto_parent(clus_config_cursor)
coda.cursor_goto_record_field_by_name(clus_config_cursor, "coadd_factor")
coadd_factor = coda.cursor_read_uint16(clus_config_cursor)
coda.cursor_goto_parent(clus_config_cursor)
coda.cursor_goto_record_field_by_name(clus_config_cursor, "num_readouts")
num_readouts = coda.cursor_read_uint16(clus_config_cursor)
coda.cursor_goto_parent(clus_config_cursor)
coda.cursor_goto_record_field_by_name(clus_config_cursor, "clus_data_type")
clus_data_type = coda.cursor_read_uint8(clus_config_cursor)
coda.cursor_goto_parent(clus_config_cursor)
# The integration time is not allways equal to intgr_time/16
# since the integration time is sometimes equal to 1/32 which
# can not be expressed with the intgr_time field. So always use
# 'coadd_factor * pet' to calculate the integration time.
it = coadd_factor * pet
# Read clus_dat
if (clus_data_type == 1):
coda.cursor_goto_record_field_by_name(clus_dat_cursor, "sig")
else:
coda.cursor_goto_record_field_by_name(clus_dat_cursor, "sigc")
coda.cursor_goto_first_array_element(clus_dat_cursor)
for readoutNr in range(0,num_readouts):
for pixelNr in range(start_pix,start_pix + clus_len):
# Read signal
coda.cursor_goto_record_field_by_name(clus_dat_cursor, "signal")
signal = coda.cursor_read_double(clus_dat_cursor)
pixel_array[chan_num][pixelNr] += signal
coda.cursor_goto_parent(clus_dat_cursor)
if ((readoutNr < num_readouts - 1) or (pixelNr < start_pix + clus_len - 1)):
coda.cursor_goto_next_array_element(clus_dat_cursor)
coda.cursor_goto_parent(clus_dat_cursor) # back to array
coda.cursor_goto_parent(clus_dat_cursor) # back to record
for pixelNr in range(start_pix,start_pix + clus_len):
# Take average
pixel_array[chan_num][pixelNr] /= num_readouts
# Perform correction for fixed pattern noise and leakage current
pixel_array[chan_num][pixelNr] = pixel_array[chan_num][pixelNr] / it - fpn[chan_num][pixelNr] / pet - leak[chan_num][pixelNr]
if (cluster < num_clus - 1):
coda.cursor_goto_next_array_element(clus_config_cursor)
coda.cursor_goto_next_array_element(clus_dat_cursor)
# Apply bad pixel map
for chan_num in range(0,8):
for pixelNr in range(0,1024):
if (badpx[chan_num][pixelNr]):
pixel_array[chan_num][pixelNr] = NO_VALUE
coda.cursor_goto_next_array_element(mds_cursor[mds_type])
if (stateNr < num_states - 1):
coda.cursor_goto_next_array_element(states_cursor)
import coda
pf = coda.open('/data/GOME2/GOME_xxx_1B/GOME_xxx_1B_M02_20100415122955Z_20100415123255Z_N_T_20100713095647Z')
cursor = coda.Cursor()
coda.cursor_set_product(cursor, pf)
coda.cursor_goto(cursor, '/MDR')
num_mdr = coda.cursor_get_num_elements(cursor)
if num_mdr > 0:
coda.cursor_goto_first_array_element(cursor)
for i in xrange(num_mdr):
index = coda.cursor_get_available_union_field_index(cursor)
if index == 0:
# Earthshine MDR
# Note that fetching the full MDR is rather slow, since it converts
# the full MDR to a hierarchicel set of Python structures
mdr = coda.fetch(cursor, 'Earthshine')
print mdr
# If you want e.g. just the wavelength and band data of band 1b, you could use:
# wavelength = coda.fetch(cursor, 'Earthshine', 'wavelength_1b')
# rad = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'rad')
# err = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'err_rad')
# which will be much faster
elif index == 1:
# Calibration MDR
pass
elif index == 2:
# Sun MDR
mdr = coda.fetch(cursor, 'Sun')
print mdr
elif index == 3:
# Moon MDR
import coda
pf = coda.open(
'/data/GOME2/GOME_xxx_1B/GOME_xxx_1B_M02_20100415122955Z_20100415123255Z_N_T_20100713095647Z'
)
cursor = coda.Cursor()
coda.cursor_set_product(cursor, pf)
coda.cursor_goto(cursor, '/MDR')
num_mdr = coda.cursor_get_num_elements(cursor)
if num_mdr > 0:
coda.cursor_goto_first_array_element(cursor)
for i in xrange(num_mdr):
index = coda.cursor_get_available_union_field_index(cursor)
if index == 0:
# Earthshine MDR
# Note that fetching the full MDR is rather slow, since it converts
# the full MDR to a hierarchicel set of Python structures
mdr = coda.fetch(cursor, 'Earthshine')
print mdr
# If you want e.g. just the wavelength and band data of band 1b, you could use:
# wavelength = coda.fetch(cursor, 'Earthshine', 'wavelength_1b')
# rad = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'rad')
# err = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'err_rad')
# which will be much faster
elif index == 1:
# Calibration MDR
pass
elif index == 2:
# Sun MDR
mdr = coda.fetch(cursor, 'Sun')
print mdr
def extract_grib_metadata(gribfile):
"""
this will return a tuple containing:
- ecmwfmars properties struct
- levtype_options struct (see set_remote_url())
"""
import coda
@contextlib.contextmanager
def coda_open(filename):
coda_handle = coda.open(filename)
try:
yield coda_handle
finally:
coda.close(coda_handle)
ecmwfmars = Struct()
levtype_options = {} # TODO: add extraction of levtype_options
with coda_open(gribfile) as coda_handle:
cursor = coda.Cursor()
coda.cursor_set_product(cursor, coda_handle)
num_messages = coda.cursor_get_num_elements(cursor)
coda.cursor_goto_first_array_element(cursor)
for i in range(num_messages):
index = coda.cursor_get_available_union_field_index(cursor)
coda.cursor_goto_record_field_by_index(cursor, index)
step = 0
if index == 0:
# grib1
centuryOfReferenceTimeOfData = coda.fetch(
cursor, "centuryOfReferenceTimeOfData")
yearOfCentury = coda.fetch(cursor, "yearOfCentury")
month = coda.fetch(cursor, "month")
day = coda.fetch(cursor, "day")
date = "%02d%02d-%02d-%02d" % (centuryOfReferenceTimeOfData -
1, yearOfCentury, month, day)
hour = coda.fetch(cursor, "hour")
minute = coda.fetch(cursor, "minute")
time = "%02d:%02d:00" % (hour, minute)
unitOfTimeRange = coda.fetch(cursor, "unitOfTimeRange")
if unitOfTimeRange != 0:
P1 = coda.fetch(cursor, "P1")
if unitOfTimeRange == 1:
step = P1
elif unitOfTimeRange == 2:
step = 24 * P1
elif unitOfTimeRange == 10:
step = 3 * P1
elif unitOfTimeRange == 11:
step = 6 * P1
elif unitOfTimeRange == 13:
step = 12 * P1
else:
raise Error("unsupported unitOfTimeRange: %d" %
(unitOfTimeRange, ))
local = coda.fetch(cursor, "local")
try:
local = local[1:9].tobytes()
except AttributeError:
# workaround for older numpy versions
local = local[1:9].tostring()
marsclass, marstype, stream, expver = struct.unpack(
'>BBH4s', local)
else:
# grib2
year = coda.fetch(cursor, "year")
month = coda.fetch(cursor, "month")
day = coda.fetch(cursor, "day")
date = "%04d-%02d-%02d" % (year, month, day)
hour = coda.fetch(cursor, "hour")
minute = coda.fetch(cursor, "minute")
second = coda.fetch(cursor, "second")
time = "%02d:%02d:%02d" % (hour, minute, second)
significanceOfReferenceTime = coda.fetch(
cursor, "significanceOfReferenceTime")
local = coda.fetch(cursor, "local[0]")
try:
local = local[2:12].tobytes()
except AttributeError:
# workaround for older numpy versions
local = local[2:12].tostring()
marsclass, marstype, stream, expver = struct.unpack(
'>HHH4s', local)
coda.cursor_goto_record_field_by_name(cursor, "data")
num_data = coda.cursor_get_num_elements(cursor)
coda.cursor_goto_first_array_element(cursor)
prev_step = None
for j in range(num_data):
forecastTime = coda.fetch(cursor, "forecastTime")
if forecastTime != 0:
indicatorOfUnitOfTimeRange = coda.fetch(
cursor, "indicatorOfUnitOfTimeRange")
if indicatorOfUnitOfTimeRange == 0:
# minutes
step = 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 1:
# hours
step = 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 2:
# days
step = 24 * 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 10:
# 3 hours
step = 3 * 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 11:
# 6 hours
step = 6 * 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 12:
# 12 hours
step = 12 * 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 13:
# seconds
step = forecastTime
step = int(step / 3600.) # convert seconds to hours
if prev_step is None:
prev_step = step
elif step != prev_step:
raise Error(
"not all data has the same 'step' time (%d) (%d)"
% (step, prev_step))
if j < num_data - 1:
coda.cursor_goto_next_array_element(cursor)
coda.cursor_goto_parent(cursor)
coda.cursor_goto_parent(cursor)
if marsclass not in MARSCLASSES:
raise Error("unsupported MARS class (%d)" % (marsclass, ))
marsclass = MARSCLASSES[marsclass]
if marstype not in MARSTYPES:
raise Error("unsupported MARS type (%d)" % (marstype, ))
marstype = MARSTYPES[marstype]
if stream not in MARSSTREAMS:
raise Error("unsupported MARS stream (%d)" % (stream, ))
stream = MARSSTREAMS[stream]
if 'date' in ecmwfmars:
if date != ecmwfmars.date:
raise Error("not all data is for the same date (%s) (%s)" %
(date, ecmwfmars.date))
if time != ecmwfmars.time:
raise Error("not all data is for the same time (%s) (%s)" %
(time, ecmwfmars.time))
if step != 0:
if 'step' in ecmwfmars:
if step != ecmwfmars.step:
raise Error(
"not all data has the same 'step' time (%d) (%d)"
% (step, ecmwfmars.step))
else:
raise Error("not all data has the same 'step' time")
else:
if 'step' in ecmwfmars and ecmwfmars.step != 0:
raise Error("not all data has the same 'step' time")
if marsclass != ecmwfmars.marsclass:
raise Error(
"not all data has the same MARS class (%s) (%s)" %
(marsclass, ecmwfmars.marsclass))
if marstype != ecmwfmars.type:
raise Error(
"not all data has the same MARS type (%s) (%s)" %
(marstype, ecmwfmars.type))
if stream != ecmwfmars.stream:
raise Error(
"not all data has the same MARS stream (%s) (%s)" %
(stream, ecmwfmars.stream))
if expver != ecmwfmars.expver:
raise Error(
"not all data has the same MARS experiment version (%s) (%s)"
% (expver, ecmwfmars.expver))
else:
ecmwfmars.date = date
ecmwfmars.time = time
if step != 0:
ecmwfmars.step = step
ecmwfmars.marsclass = marsclass
ecmwfmars.type = marstype
ecmwfmars.stream = stream
ecmwfmars.expver = expver
if (marsclass, stream, expver, marstype) in PUBLIC_DATASETS:
ecmwfmars.dataset = PUBLIC_DATASETS[(marsclass, stream,
expver, marstype)]
coda.cursor_goto_parent(cursor)
if i < num_messages - 1:
coda.cursor_goto_next_array_element(cursor)
return ecmwfmars, levtype_options
