def Initialize(self, node):
try:
valueString = model.GetDataAsString(node.type, coda.fetch(node.cursor))
except (coda.CodaError, coda.CodacError) as ex:
raise RenderError("[CODA] %s" % (str(ex),))
self.SetValue(valueString)
def _read(self, path, fields="all", return_header=False):
tmpdira = config.conf["main"]["tmpdir"]
tmpdirb = config.conf["main"]["tmpdirb"]
tmpdir = (tmpdira
if shutil.disk_usage(tmpdira).free > self.minspace
else tmpdirb)
with tempfile.NamedTemporaryFile(mode="wb", dir=tmpdir, delete=True) as tmpfile:
with gzip.open(str(path), "rb") as gzfile:
logging.debug("Decompressing {!s}".format(path))
gzcont = gzfile.read()
logging.debug("Writing decompressed file to {!s}".format(tmpfile.name))
tmpfile.write(gzcont)
del gzcont
# All the hard work is in coda
logging.debug("Reading {!s}".format(tmpfile.name))
cfp = coda.open(tmpfile.name)
c = coda.fetch(cfp)
logging.debug("Sorting info...")
n_scanlines = c.MPHR.TOTAL_MDR
start = datetime.datetime(*coda.time_double_to_parts_utc(c.MPHR.SENSING_START))
has_mdr = numpy.array([hasattr(m, 'MDR') for m in c.MDR],
dtype=numpy.bool)
bad = numpy.array([
(m.MDR.DEGRADED_PROC_MDR|m.MDR.DEGRADED_INST_MDR)
if hasattr(m, 'MDR') else True
for m in c.MDR],
dtype=numpy.bool)
dlt = numpy.concatenate(
[m.MDR.OnboardUTC[:, numpy.newaxis]
for m in c.MDR
if hasattr(m, 'MDR')], 1) - c.MPHR.SENSING_START
M = numpy.ma.zeros(
dtype=self._dtype,
shape=(n_scanlines, 30))
M["time"][has_mdr] = numpy.datetime64(start, "ms") + numpy.array(dlt*1e3, "m8[ms]").T
specall = self.__obtain_from_mdr(c, "GS1cSpect")
M["spectral_radiance"][has_mdr] = specall
locall = self.__obtain_from_mdr(c, "GGeoSondLoc")
M["lon"][has_mdr] = locall[:, :, :, 0]
M["lat"][has_mdr] = locall[:, :, :, 1]
satangall = self.__obtain_from_mdr(c, "GGeoSondAnglesMETOP")
M["satellite_zenith_angle"][has_mdr] = satangall[:, :, :, 0]
M["satellite_azimuth_angle"][has_mdr] = satangall[:, :, :, 1]
solangall = self.__obtain_from_mdr(c, "GGeoSondAnglesSUN")
M["solar_zenith_angle"][has_mdr] = solangall[:, :, :, 0]
M["solar_azimuth_angle"][has_mdr] = solangall[:, :, :, 1]
for fld in M.dtype.names:
M.mask[fld][~has_mdr, ...] = True
M.mask[fld][bad, ...] = True
m = c.MDR[0].MDR
wavenumber = (m.IDefSpectDWn1b * numpy.arange(m.IDefNsfirst1b, m.IDefNslast1b+0.1) * (1/ureg.metre))
if self.wavenumber is None:
self.wavenumber = wavenumber
elif abs(self.wavenumber - wavenumber).max() > (0.05 * 1/(ureg.centimetre)):
raise ValueError("Inconsistent wavenumbers")
return M
def Initialize(self, node):
for i in range(0, len(node)):
field = node[i]
if not (coda.get_option_filter_record_fields() and (field.hidden or not field.available)):
idx = self.InsertItem(i, str(i))
self.SetItem(idx, 1, field.real_name, model.GetNodeIcon(field, model.iconDictionary))
if field.available:
if field.type == model.TYPE_RECORD:
self.SetItem(idx, 2, "<record of %u field(s)>" % (len(field),))
elif field.type == model.TYPE_ARRAY:
if field.isRankZero():
self.SetItem(idx, 2, "<rank-0 array of %s>" % (model.GetTypeAsString(field.base),))
else:
dimensionString = ""
for dim in field.dimensions[:-1]:
dimensionString += "%u x " % (dim,)
dimensionString += "%u" % (field.dimensions[-1],)
self.SetItem(idx, 2, "[%s %s]" % (dimensionString, model.GetTypeAsString(field.base)))
else:
# a maximum size of _maxLineLength characters is specified, because
# GDK may crash if the length of a string item gets too long.
# also, if the line is really long the user will probably not want
# to view as a single line in a ListCtrl anyway.
try:
data = coda.fetch(field.cursor)
except (coda.CodaError, coda.CodacError) as ex:
raise RenderError("[CODA] %s" % (str(ex),))
self.SetItem(idx, 2, model.GetDataAsString(field.type, data, frame._maxLineLength))
else:
self.SetItem(idx, 2, "<unavailable>")
# _AutoSize() sets the width of a column to the maximum of the
# width of the longest item in the column and the width of the column
# header.
# I would like to _AutoSize() only column 0 and 1, and have column 2
# take up the rest of the available space (maybe constrained to a
# minimum width, i.e. the width of the smallest item in the column).
# This would be desirable, because sometimes there can be very long
# items, which result in very small scrollbars. However, there does not
# seem to be an easy way to achieve this. (Note that if an item is
# longer than the column width, it is automatically clipped and
# padded with '...' by the ListCtrl.)
#
# UPDATE: A specified maximum length (productbrowser.frame._maxLength) is
# now used to avoid very long items. Therefore, we can now _AutoSize()
# column 2 as well. However, _AutoSize()-ing only column 0 and 1, and
# having column 2 take up the rest of the available space would still
# be a better solution.
self._AutoSize((0, 1, 2))
def get_solar_zenith(self):
"""
获取太阳天顶角
:return:
"""
fp = coda.open(self.in_file)
angles = coda.fetch(fp, 'MDR', -1, 'MDR', 'GGeoSondAnglesSUN')
zenith = np.array([])
for i in angles:
z = i.reshape(-1)[0::2]
zenith = np.append(zenith, z)
return zenith
def Initialize(self, node):
assert node.type == model.TYPE_ARRAY, "PlotRenderer can only render node of type TYPE_ARRAY"
assert not isinstance(
node, model.ObjectArrayNode
), "PlotRenderer cannot render arrays of compound types"
# coda.fetch() will promote a rank-0 array to a 1x1 array,
# so a rank of 1 is the smallest possible rank.
try:
self.array = coda.fetch(node.cursor)
except (coda.CodaError, coda.CodacError) as ex:
raise RenderError("[CODA] %s" % (str(ex), ))
# array sanity check
ok = True
if node.isRankZero():
ok = (self.array.ndim == 1) and (self.array.shape[0] == 1)
else:
ok = (self.array.ndim == len(node.dimensions)) and (self.array.size
== len(node))
i = 0
while ok and i < len(node.dimensions):
ok = ok and (self.array.shape[i] == node.dimensions[i])
i += 1
if not ok:
raise RenderError(
"[PlotRenderer] Array read from product does not match description."
)
self.plot = PlotWindow(self, -1)
sizer = wx.BoxSizer(wx.VERTICAL)
sizer.Add(self.plot, 1, wx.EXPAND | wx.ALL, 5)
if self.array.ndim > 1:
self.slicer = NumpySlicer1D(self, -1, self.array, "select")
sizer.Add(self.slicer, 0, wx.EXPAND | wx.LEFT | wx.RIGHT, 5)
self.Bind(EVT_SLICE_CHANGED, self.OnSliceChanged)
self.SetSizer(sizer)
if self.array.ndim == 1:
self.plot.AddDataSet(numpy.arange(self.array.shape[0]), self.array)
else:
slice = self.slicer.GetSlice()
self.plot.AddDataSet(numpy.arange(slice.shape[0]), slice)
def Initialize(self, node):
try:
array = coda.fetch(node.cursor)
except (coda.CodaError, coda.CodacError) as ex:
raise RenderError("[CODA] %s" % (str(ex),))
# array sanity check
if not array.ndim == 1:
raise RenderError("[BytesRenderer] Array read from product does not match description.")
if array.size == 0:
raise RenderError("[BytesRenderer] Array read from product is empty.")
width, height = self.GetTextExtent("FF")
self.SetDefaultColSize(width + 10)
self.SetDefaultRowSize(height + 10)
self.SetDefaultCellAlignment(wx.ALIGN_CENTRE, wx.ALIGN_CENTRE)
self.SetTable(HexNumpyWrapper(array), True)
def get_sensor_azimuth(self):
"""
return sensor_azimuth
"""
if self.resolution == 40000:
satellite_type1 = ['METOP-A', 'METOP-B']
if self.satellite in satellite_type1:
try:
fp = coda.open(self.in_file)
angle = coda.fetch(fp, 'MDR', -1, 'Earthshine',
'GEO_EARTH', 'SAT_AZIMUTH')
coda.close(fp)
# angle = 30*3*32 = 1440, 取30*1*32 = 960, 取前959个
data_size = self.data_shape[0]
data_row = angle.shape[0]
data_col = angle[0].shape[1]
data_pre = np.full(self.data_shape, -999.)
for i in xrange(data_size):
row, col = np.unravel_index(i, (data_row, data_col))
# print row, col, angle[row][1][col]
data_pre[i] = angle[row][1][col]
# 过滤无效值
invalid_index = np.logical_or(data_pre < 0, data_pre > 360)
data_pre = data_pre.astype(np.float32)
data_pre[invalid_index] = np.nan
data = data_pre
except Exception as e:
print 'Open file error {}'.format(e)
return
else:
raise ValueError('Cant read this satellite`s data.: {}'.format(
self.satellite))
else:
raise ValueError(
'Cant read this data, please check its resolution: {}'.format(
self.in_file))
return data
def get_footprint(product):
try:
import coda
except ImportError:
return None
path = "/METADATA/EOP_METADATA/om_featureOfInterest/eop_multiExtentOf/gml_surfaceMembers/gml_exterior@gml_posList"
pf = coda.open(product)
try:
coord = coda.fetch(pf, path).split(' ')
except coda.CodacError:
return None
finally:
coda.close(pf)
if len(coord) % 2 != 0:
return None
return Polygon([
LinearRing([
Point(float(lon), float(lat))
for lat, lon in zip(coord[0::2], coord[1::2])
])
])
import os
# Change this to the location of your AEOLUS codadef file
os.putenv('CODA_DEFINITION', '/usr/local/share/coda/definitions')
# You can also remove this line and set the CODA_DEFINITION environment variable globally on your system
import coda
import numpy
# change this to the full path of your Aeolus L1B DBL file
filename = "AE_OPER_AUX_MET_12_20071107T090000_20071108T150000_0001.DBL"
product = coda.open(filename)
num_records = coda.fetch(product, '/sph/num_records_in_ds1')
num_layers = coda.fetch(product, '/sph/num_of_model_layers')
assert (num_records > 0 and num_layers > 0)
amd_pnom = numpy.empty([num_records, num_layers])
amd_znom = numpy.empty([num_records, num_layers])
amd_t = numpy.empty([num_records, num_layers])
amd_u = numpy.empty([num_records, num_layers])
cursor = coda.Cursor()
coda.cursor_set_product(cursor, product)
coda.cursor_goto(cursor, '/met_off_nadir[0]')
for i in range(num_records):
coda.cursor_goto(cursor, 'profile_data[0]')
for j in range(num_layers - 1):
filename = "/path/to/AE_OPER_ALD_U_N_1B_20151002T001857059_005787000_046339_0001.DBL"
product = coda.open(filename)
### Observation profiles ###
# Observation profiles provide a single profile per BRC.
# Reading all values will therefore return 'an array of arrays' (we will get an array for each '-1' in the coda.fetch()).
# The first array is the list of BRCs, and the second array the list of points in the profile.
# This array of arrays can be turned into a single 2D numpy array of shape [num_brc, num_vertical] by using 'vstack'.
# Mie observation wind profiles
print("Mie observation wind profiles")
latitude = coda.fetch(product, 'geolocation', -1, 'observation_geolocation/observation_mie_geolocation', -1, 'latitude_of_height_bin')
latitude = vstack(latitude)
longitude = coda.fetch(product, 'geolocation', -1, 'observation_geolocation/observation_mie_geolocation', -1, 'longitude_of_height_bin')
longitude = vstack(longitude)
altitude = coda.fetch(product, 'geolocation', -1, 'observation_geolocation/observation_mie_geolocation', -1, 'altitude_of_height_bin')
altitude = vstack(altitude)
wind_velocity = coda.fetch(product, 'wind_velocity', -1, 'observation_wind_profile/mie_altitude_bin_wind_info', -1, 'wind_velocity')
wind_velocity = vstack(wind_velocity)
print(wind_velocity.shape)
print(wind_velocity)
# Rayleigh observation wind profiles
os.putenv('CODA_DEFINITION', '/usr/local/share/coda/definitions')
# You can also remove this line and set the CODA_DEFINITION environment variable globally on your system
import coda
from numpy import hstack, vstack
# change this to the full path of your Aeolus L2A DBL file
filename = "/path/to/AE_OPER_ALD_U_N_2A_20071107T193753059_008688000_000578_0001.DBL"
product = coda.open(filename)
### Standard Correct Algorithm (SCA) at middle bins ###
if coda.get_field_available(product, 'sca_optical_properties'):
print("SCA optical properties at middle bins")
latitude = coda.fetch(product, 'sca_optical_properties', -1,
'geolocation_middle_bins', -1, 'latitude')
latitude = vstack(latitude)
longitude = coda.fetch(product, 'sca_optical_properties', -1,
'geolocation_middle_bins', -1, 'longitude')
longitude = vstack(longitude)
altitude = coda.fetch(product, 'sca_optical_properties', -1,
'geolocation_middle_bins', -1, 'altitude')
altitude = vstack(altitude)
extinction = coda.fetch(product, 'sca_optical_properties', -1,
'sca_optical_properties_mid_bins', -1,
'extinction')
extinction = vstack(extinction)
print(extinction.shape)
import os
# Change this to the location of your AEOLUS codadef file
os.putenv('CODA_DEFINITION', '/usr/local/share/coda/definitions')
# You can also remove this line and set the CODA_DEFINITION environment variable globally on your system
import coda
import numpy
# change this to the full path of your Aeolus L1B DBL file
filename = "AE_OPER_AUX_MET_12_20071107T090000_20071108T150000_0001.DBL"
product = coda.open(filename)
num_records = coda.fetch(product, '/sph/num_records_in_ds1')
num_layers = coda.fetch(product, '/sph/num_of_model_layers')
assert(num_records > 0 and num_layers > 0)
amd_pnom = numpy.empty([num_records, num_layers])
amd_znom = numpy.empty([num_records, num_layers])
amd_t = numpy.empty([num_records, num_layers])
amd_u = numpy.empty([num_records, num_layers])
cursor = coda.Cursor()
coda.cursor_set_product(cursor, product)
coda.cursor_goto(cursor, '/met_off_nadir[0]')
for i in range(num_records):
coda.cursor_goto(cursor, 'profile_data[0]')
for j in range(num_layers - 1):
def extract_grib_metadata(gribfile):
"""
this will return a tuple containing:
- ecmwfmars properties struct
- levtype_options struct (see set_remote_url())
"""
import coda
@contextlib.contextmanager
def coda_open(filename):
coda_handle = coda.open(filename)
try:
yield coda_handle
finally:
coda.close(coda_handle)
ecmwfmars = Struct()
levtype_options = {} # TODO: add extraction of levtype_options
with coda_open(gribfile) as coda_handle:
cursor = coda.Cursor()
coda.cursor_set_product(cursor, coda_handle)
num_messages = coda.cursor_get_num_elements(cursor)
coda.cursor_goto_first_array_element(cursor)
for i in range(num_messages):
index = coda.cursor_get_available_union_field_index(cursor)
coda.cursor_goto_record_field_by_index(cursor, index)
step = 0
if index == 0:
# grib1
centuryOfReferenceTimeOfData = coda.fetch(
cursor, "centuryOfReferenceTimeOfData")
yearOfCentury = coda.fetch(cursor, "yearOfCentury")
month = coda.fetch(cursor, "month")
day = coda.fetch(cursor, "day")
date = "%02d%02d-%02d-%02d" % (centuryOfReferenceTimeOfData -
1, yearOfCentury, month, day)
hour = coda.fetch(cursor, "hour")
minute = coda.fetch(cursor, "minute")
time = "%02d:%02d:00" % (hour, minute)
unitOfTimeRange = coda.fetch(cursor, "unitOfTimeRange")
if unitOfTimeRange != 0:
P1 = coda.fetch(cursor, "P1")
if unitOfTimeRange == 1:
step = P1
elif unitOfTimeRange == 2:
step = 24 * P1
elif unitOfTimeRange == 10:
step = 3 * P1
elif unitOfTimeRange == 11:
step = 6 * P1
elif unitOfTimeRange == 13:
step = 12 * P1
else:
raise Error("unsupported unitOfTimeRange: %d" %
(unitOfTimeRange, ))
local = coda.fetch(cursor, "local")
try:
local = local[1:9].tobytes()
except AttributeError:
# workaround for older numpy versions
local = local[1:9].tostring()
marsclass, marstype, stream, expver = struct.unpack(
'>BBH4s', local)
else:
# grib2
year = coda.fetch(cursor, "year")
month = coda.fetch(cursor, "month")
day = coda.fetch(cursor, "day")
date = "%04d-%02d-%02d" % (year, month, day)
hour = coda.fetch(cursor, "hour")
minute = coda.fetch(cursor, "minute")
second = coda.fetch(cursor, "second")
time = "%02d:%02d:%02d" % (hour, minute, second)
significanceOfReferenceTime = coda.fetch(
cursor, "significanceOfReferenceTime")
local = coda.fetch(cursor, "local[0]")
try:
local = local[2:12].tobytes()
except AttributeError:
# workaround for older numpy versions
local = local[2:12].tostring()
marsclass, marstype, stream, expver = struct.unpack(
'>HHH4s', local)
coda.cursor_goto_record_field_by_name(cursor, "data")
num_data = coda.cursor_get_num_elements(cursor)
coda.cursor_goto_first_array_element(cursor)
prev_step = None
for j in range(num_data):
forecastTime = coda.fetch(cursor, "forecastTime")
if forecastTime != 0:
indicatorOfUnitOfTimeRange = coda.fetch(
cursor, "indicatorOfUnitOfTimeRange")
if indicatorOfUnitOfTimeRange == 0:
# minutes
step = 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 1:
# hours
step = 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 2:
# days
step = 24 * 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 10:
# 3 hours
step = 3 * 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 11:
# 6 hours
step = 6 * 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 12:
# 12 hours
step = 12 * 60 * 60 * forecastTime
elif indicatorOfUnitOfTimeRange == 13:
# seconds
step = forecastTime
step = int(step / 3600.) # convert seconds to hours
if prev_step is None:
prev_step = step
elif step != prev_step:
raise Error(
"not all data has the same 'step' time (%d) (%d)"
% (step, prev_step))
if j < num_data - 1:
coda.cursor_goto_next_array_element(cursor)
coda.cursor_goto_parent(cursor)
coda.cursor_goto_parent(cursor)
if marsclass not in MARSCLASSES:
raise Error("unsupported MARS class (%d)" % (marsclass, ))
marsclass = MARSCLASSES[marsclass]
if marstype not in MARSTYPES:
raise Error("unsupported MARS type (%d)" % (marstype, ))
marstype = MARSTYPES[marstype]
if stream not in MARSSTREAMS:
raise Error("unsupported MARS stream (%d)" % (stream, ))
stream = MARSSTREAMS[stream]
if 'date' in ecmwfmars:
if date != ecmwfmars.date:
raise Error("not all data is for the same date (%s) (%s)" %
(date, ecmwfmars.date))
if time != ecmwfmars.time:
raise Error("not all data is for the same time (%s) (%s)" %
(time, ecmwfmars.time))
if step != 0:
if 'step' in ecmwfmars:
if step != ecmwfmars.step:
raise Error(
"not all data has the same 'step' time (%d) (%d)"
% (step, ecmwfmars.step))
else:
raise Error("not all data has the same 'step' time")
else:
if 'step' in ecmwfmars and ecmwfmars.step != 0:
raise Error("not all data has the same 'step' time")
if marsclass != ecmwfmars.marsclass:
raise Error(
"not all data has the same MARS class (%s) (%s)" %
(marsclass, ecmwfmars.marsclass))
if marstype != ecmwfmars.type:
raise Error(
"not all data has the same MARS type (%s) (%s)" %
(marstype, ecmwfmars.type))
if stream != ecmwfmars.stream:
raise Error(
"not all data has the same MARS stream (%s) (%s)" %
(stream, ecmwfmars.stream))
if expver != ecmwfmars.expver:
raise Error(
"not all data has the same MARS experiment version (%s) (%s)"
% (expver, ecmwfmars.expver))
else:
ecmwfmars.date = date
ecmwfmars.time = time
if step != 0:
ecmwfmars.step = step
ecmwfmars.marsclass = marsclass
ecmwfmars.type = marstype
ecmwfmars.stream = stream
ecmwfmars.expver = expver
if (marsclass, stream, expver, marstype) in PUBLIC_DATASETS:
ecmwfmars.dataset = PUBLIC_DATASETS[(marsclass, stream,
expver, marstype)]
coda.cursor_goto_parent(cursor)
if i < num_messages - 1:
coda.cursor_goto_next_array_element(cursor)
return ecmwfmars, levtype_options
os.putenv('CODA_DEFINITION', '/usr/local/share/coda/definitions')
# You can also remove this line and set the CODA_DEFINITION environment variable globally on your system
import coda
from numpy import hstack, vstack
# change this to the full path of your Aeolus L2A DBL file
filename = "/path/to/AE_OPER_ALD_U_N_2A_20071107T193753059_008688000_000578_0001.DBL"
product = coda.open(filename)
### Standard Correct Algorithm (SCA) at middle bins ###
if coda.get_field_available(product, 'sca_optical_properties'):
print("SCA optical properties at middle bins")
latitude = coda.fetch(product, 'sca_optical_properties', -1, 'geolocation_middle_bins', -1, 'latitude')
latitude = vstack(latitude)
longitude = coda.fetch(product, 'sca_optical_properties', -1, 'geolocation_middle_bins', -1, 'longitude')
longitude = vstack(longitude)
altitude = coda.fetch(product, 'sca_optical_properties', -1, 'geolocation_middle_bins', -1, 'altitude')
altitude = vstack(altitude)
extinction = coda.fetch(product, 'sca_optical_properties', -1, 'sca_optical_properties_mid_bins', -1, 'extinction')
extinction = vstack(extinction)
print(extinction.shape)
print(extinction)
backscatter = coda.fetch(product, 'sca_optical_properties', -1, 'sca_optical_properties_mid_bins', -1, 'backscatter')
backscatter = vstack(backscatter)
import coda
pf = coda.open('/data/GOME2/GOME_xxx_1B/GOME_xxx_1B_M02_20100415122955Z_20100415123255Z_N_T_20100713095647Z')
cursor = coda.Cursor()
coda.cursor_set_product(cursor, pf)
coda.cursor_goto(cursor, '/MDR')
num_mdr = coda.cursor_get_num_elements(cursor)
if num_mdr > 0:
coda.cursor_goto_first_array_element(cursor)
for i in xrange(num_mdr):
index = coda.cursor_get_available_union_field_index(cursor)
if index == 0:
# Earthshine MDR
# Note that fetching the full MDR is rather slow, since it converts
# the full MDR to a hierarchicel set of Python structures
mdr = coda.fetch(cursor, 'Earthshine')
print mdr
# If you want e.g. just the wavelength and band data of band 1b, you could use:
# wavelength = coda.fetch(cursor, 'Earthshine', 'wavelength_1b')
# rad = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'rad')
# err = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'err_rad')
# which will be much faster
elif index == 1:
# Calibration MDR
pass
elif index == 2:
# Sun MDR
mdr = coda.fetch(cursor, 'Sun')
print mdr
elif index == 3:
# Moon MDR
#print("Individual Mie HLOS wind points")
#latitude = coda.fetch(product, 'mie_geolocation', -1, 'windresult_geolocation/latitude_cog')
#longitude = coda.fetch(product, 'mie_geolocation', -1, 'windresult_geolocation/longitude_cog')
#altitude = coda.fetch(product, 'mie_geolocation', -1, 'windresult_geolocation/altitude_vcog')
#mie_wind_velocity = coda.fetch(product, 'mie_hloswind', -1, 'windresult/mie_wind_velocity')
#print(mie_wind_velocity.shape)
#print(mie_wind_velocity)
### Rayleigh observation wind profiles points ###
print("Individual Rayleight HLOS wind points")
latitude = coda.fetch(product, 'rayleigh_geolocation', -1,
'windresult_geolocation/latitude_cog')
longitude = coda.fetch(product, 'rayleigh_geolocation', -1,
'windresult_geolocation/longitude_cog')
altitude = coda.fetch(product, 'rayleigh_geolocation', -1,
'windresult_geolocation/altitude_vcog')
rayleigh_wind_velocity = coda.fetch(product, 'rayleigh_hloswind', -1,
'windresult/rayleigh_wind_velocity')
print(latitude.shape)
print(latitude)
print(longitude.shape)
print(longitude)
print(altitude.shape)
print(altitude)
print(rayleigh_wind_velocity.shape)
print(rayleigh_wind_velocity)
class CLASS_GOME_L1():
def __init__(self, BandLst):
self.k = 1.98644746103858e-9
# 字典类型物理量
self.Ref = {}
# 二维矩阵
self.Lons = []
self.Lats = []
self.Time = []
self.satAzimuth = []
self.satZenith = []
self.sunAzimuth = []
self.sunZenith = []
# 光谱信息
self.wavenumber = []
self.radiance = []
def Load(self, L1File):
print u'读取 LEO所有数据信息......'
if not os.path.isfile(L1File):
print 'Error: %s not found' % L1File
sys.exit(1)
try:
fp = coda.open(L1File)
except Exception, e:
print 'Open file error<%s> .' % (e)
return
try:
# EPS = EUMETSAT Polar System atmospheric products (GOME-2 and IASI)
# EPS = EUMETSAT极地大气系统产品（GOME-2和IASI）'
# 获取文件头信息
product_class = coda.get_product_class(fp)
product_type = coda.get_product_type(fp)
product_version = coda.get_product_version(fp)
product_format = coda.get_product_format(fp)
product_size = coda.get_product_file_size(fp)
print 'product_class ', product_class
print 'product_type', product_type
print 'product_version', product_version
print 'product_format', product_format
print 'product_size', product_size
record = beatl2.ingest(L1File)
WAVE_3 = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'WAVELENGTH_3')
WAVE_4 = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'WAVELENGTH_4')
LAMBDA_SMR = coda.fetch(fp, 'VIADR_SMR', -1, 'LAMBDA_SMR')
SMR = coda.fetch(fp, 'VIADR_SMR', -1, 'SMR')
print 'gome data is:'
print record
self.rec = record
SUN_Z = coda.fetch(
fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'SOLAR_ZENITH')
SUN_A = coda.fetch(
fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'SOLAR_AZIMUTH')
SAT_Z = coda.fetch(
fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'SAT_ZENITH')
SAT_A = coda.fetch(
fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'SAT_AZIMUTH')
print '太阳方位角度长度', SUN_Z.shape, SUN_Z[0].shape
# 数据观测总长度
dataLen = record.latitude.size
dataCol = SUN_Z[0].shape[1]
dataRow = SUN_Z.shape[0]
# 角度存放内存
self.satZenith = np.full((dataLen, 1), -999.)
self.satAzimuth = np.full((dataLen, 1), -999.)
self.sunZenith = np.full((dataLen, 1), -999.)
self.sunAzimuth = np.full((dataLen, 1), -999.)
# 开始赋值
for i in xrange(dataLen):
row, col = np.unravel_index(i, (dataRow, dataCol))
self.satAzimuth[i] = SAT_A[row][1][col]
self.satZenith[i] = SAT_Z[row][1][col]
self.sunAzimuth[i] = SUN_A[row][1][col]
self.sunZenith[i] = SUN_Z[row][1][col]
self.Lons = (record.longitude).reshape(dataLen, 1)
self.Lats = (record.latitude).reshape(dataLen, 1)
# 计算gome的辐亮度
self.radiance = record.spectral_radiance[:, 2048:]
for m in xrange(dataLen):
row, col = np.unravel_index(m, (dataRow, dataCol))
for i in xrange(2048):
if i < 1024:
self.radiance[m, i] = self.radiance[
m, i] * self.k / WAVE_3[row][i]
else:
self.radiance[m, i] = self.radiance[
m, i] * self.k / WAVE_4[row][i - 1024]
# 计算太阳辐亮度
self.vec_Solar_L = np.zeros((2048,)) # 太阳辐亮度
self.vec_Solar_WL = np.zeros((2048,)) # 太阳辐亮度对应的波长
for i in xrange(2048):
if i < 1024:
self.vec_Solar_L[i] = (
SMR[0][2][i] * self.k) / LAMBDA_SMR[0][2][i]
self.vec_Solar_WL[i] = LAMBDA_SMR[0][2][i]
else:
self.vec_Solar_L[i] = (
SMR[0][3][i - 1024] * self.k) / LAMBDA_SMR[0][3][i - 1024]
self.vec_Solar_WL[i] = LAMBDA_SMR[0][3][i - 1024]
print 'GOME数据观测长度 %d' % dataLen
print '太阳辐亮度波长最小最大值'
print np.min(self.vec_Solar_WL), self.vec_Solar_WL[0:3]
print np.max(self.vec_Solar_WL)
# 暂时取一个观测的光谱波数
self.wavenumber = record.wavelength[0, 2048:]
print 'GOME辐亮度波长最小最大值,取第一个观测点'
print np.min(self.wavenumber), self.wavenumber[0:3]
print np.max(self.wavenumber)
self.wavenumber = record.wavelength[9, 2048:]
print 'GOME辐亮度波长最小最大值，取第十个观测点'
print np.min(self.wavenumber), self.wavenumber[0:3]
print np.max(self.wavenumber)
v_ymd2seconds = np.vectorize(metop_ymd2seconds)
T1 = v_ymd2seconds(record.time)
self.Time = T1.reshape(dataLen, 1)
# print time.gmtime(self.Time[0, 0])
except Exception as e:
print str(e)
sys.exit(1)
finally:
coda.close(fp)
raise IOError(text)
# leaf albedo file generated from PROSPECT
# typical leaf level parameters generated based on metadata-analysis
# see spreadsheet in PROSPECT folder
l_lower = 580. # limits for plotting purposes
l_upper = 800.
l_fn = 'leaf_spectrum.txt'
leaf_alb_arr = np.genfromtxt(l_fn)
leaf_alb_arr = leaf_alb_arr[np.intersect1d(np.array(np.where(leaf_alb_arr[:,0]\
>=l_lower)),np.array(np.where(leaf_alb_arr[:,0]<=l_upper)))]
leaf_lam = leaf_alb_arr[:, 0]
leaf_alb_arr = leaf_alb_arr[:, 1] + leaf_alb_arr[:, 2]
# reading solar wavelength and irradiance in band 4
sol_lam = coda.fetch(ef, 'VIADR_SMR', 0, 'LAMBDA_SMR')[3]
n_sol_lam = len(sol_lam)
sol_irr = coda.fetch(ef, 'VIADR_SMR', 0, 'SMR')[3]
sol_irr = phot_Watt(sol_irr, sol_lam)
# reading total number of scans
n_mdr = coda.fetch(ef, 'MPHR', 'TOTAL_MDR')
# list for point lat, lon, radiance, irradiance, spectrum etc.
Lat = []
Lon = []
Spec = []
Rad = []
Irr = []
Alb_sur = []
Alb_toa = []
pf = coda.open(
'/data/GOME2/GOME_xxx_1B/GOME_xxx_1B_M02_20100415122955Z_20100415123255Z_N_T_20100713095647Z'
)
cursor = coda.Cursor()
coda.cursor_set_product(cursor, pf)
coda.cursor_goto(cursor, '/MDR')
num_mdr = coda.cursor_get_num_elements(cursor)
if num_mdr > 0:
coda.cursor_goto_first_array_element(cursor)
for i in xrange(num_mdr):
index = coda.cursor_get_available_union_field_index(cursor)
if index == 0:
# Earthshine MDR
# Note that fetching the full MDR is rather slow, since it converts
# the full MDR to a hierarchicel set of Python structures
mdr = coda.fetch(cursor, 'Earthshine')
print mdr
# If you want e.g. just the wavelength and band data of band 1b, you could use:
# wavelength = coda.fetch(cursor, 'Earthshine', 'wavelength_1b')
# rad = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'rad')
# err = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'err_rad')
# which will be much faster
elif index == 1:
# Calibration MDR
pass
elif index == 2:
# Sun MDR
mdr = coda.fetch(cursor, 'Sun')
print mdr
elif index == 3:
# Moon MDR
product = coda.open(filename)
# The L2B product stores all valid profile points (which are called 'results') as one big consecutive array
# in the datasets mie_hloswind and rayleigh_hloswind.
# The associated geolocation for those point results are stored in mie_geolocation and rayleigh_geolocation.
#
# The specification of which points belong together in which profile is kept in the mie_profile and rayleigh_profile
# datasets. These datasets contain indices into the mie_hloswind and rayleigh hloswind datasets to specify which point
# belongs at which level in a profile.
### Mie horizontal line of sight wind profile points ###
print("Individual Mie HLOS wind points")
latitude = coda.fetch(product, 'mie_geolocation', -1, 'windresult_geolocation/latitude_cog')
longitude = coda.fetch(product, 'mie_geolocation', -1, 'windresult_geolocation/longitude_cog')
altitude = coda.fetch(product, 'mie_geolocation', -1, 'windresult_geolocation/altitude_vcog')
mie_wind_velocity = coda.fetch(product, 'mie_hloswind', -1, 'windresult/mie_wind_velocity')
print(mie_wind_velocity.shape)
print(mie_wind_velocity)
### Rayleigh observation wind profiles points ###
print("Individual Rayleight HLOS wind points")
latitude = coda.fetch(product, 'rayleigh_geolocation', -1, 'windresult_geolocation/latitude_cog')
product = coda.open(filename)
### Observation profiles ###
# Observation profiles provide a single profile per BRC.
# Reading all values will therefore return 'an array of arrays' (we will get an array for each '-1' in the coda.fetch()).
# The first array is the list of BRCs, and the second array the list of points in the profile.
# This array of arrays can be turned into a single 2D numpy array of shape [num_brc, num_vertical] by using 'vstack'.
# Mie observation wind profiles
print("Mie observation wind profiles")
latitude = coda.fetch(product, 'geolocation', -1,
'observation_geolocation/observation_mie_geolocation',
-1, 'latitude_of_height_bin')
latitude = vstack(latitude)
longitude = coda.fetch(product, 'geolocation', -1,
'observation_geolocation/observation_mie_geolocation',
-1, 'longitude_of_height_bin')
longitude = vstack(longitude)
altitude = coda.fetch(product, 'geolocation', -1,
'observation_geolocation/observation_mie_geolocation',
-1, 'altitude_of_height_bin')
altitude = vstack(altitude)
wind_velocity = coda.fetch(
product, 'wind_velocity', -1,
def Initialize(self, node):
assert node.type == model.TYPE_ARRAY, "ArrayRenderer can only render node of type TYPE_ARRAY"
assert not isinstance(node, model.ObjectArrayNode), "ArrayRenderer cannot render arrays of compound types"
# coda.fetch() will promote a rank-0 array to a 1x1 array,
# so a rank of 1 is the smallest possible rank.
try:
self.array = coda.fetch(node.cursor)
except (coda.CodaError, coda.CodacError) as ex:
raise RenderError("[CODA] %s" % (str(ex),))
# array sanity check
ok = True
if node.isRankZero():
ok = (self.array.ndim == 1) and (self.array.shape[0] == 1)
else:
ok = (self.array.ndim == len(node.dimensions)) and (self.array.size == len(node))
i = 0
while ok and i < len(node.dimensions):
ok = ok and (self.array.shape[i] == node.dimensions[i])
i += 1
if not ok:
raise RenderError("[ArrayRenderer] Array read from product does not match description.")
self.base = node.base
self.slice = None
# coda.fetch() will promote a rank-0 array to a 1x1 array,
# so a rank of 1 is the smallest possible rank.
if self.array.ndim == 1:
self.array = self.array[:, numpy.newaxis]
self.valueText = wx.TextCtrl(self, -1, "")
valueSizer = wx.BoxSizer(wx.HORIZONTAL)
valueSizer.Add(wx.StaticText(self, -1, "Value:"), 0, wx.ALIGN_CENTER, 0)
valueSizer.Add((5, 5), 0, wx.EXPAND)
valueSizer.Add(self.valueText, 1, 0, 0)
self.sizer = wx.BoxSizer(wx.VERTICAL)
self.grid = wx.grid.Grid(self, -1, style=wx.BORDER_SUNKEN)
# TODO: use wxGrid::SetDefaultColSize() and row equiv. as an approx.
# of AutoSize().
if self.array.ndim > 2:
self.slicer = NumpySlicer2D(self, -1, self.array, ("row", "col"))
self.Bind(EVT_SLICE_CHANGED, self.OnSliceChanged)
self.UpdateGridData()
# _really_ slow, so only used for string data (which can require a large cell width).
if self.base == model.TYPE_STRING:
self.grid.AutoSize()
self.sizer.Add(valueSizer, 0, wx.EXPAND | wx.ALL, 5)
self.sizer.Add(wx.StaticLine(self, -1, style=wx.LI_HORIZONTAL), 0, wx.EXPAND | wx.LEFT | wx.RIGHT, 5)
self.sizer.Add(self.grid, 1, wx.EXPAND | wx.ALL, 5)
if self.array.ndim > 2:
self.sizer.Add(self.slicer, 0, wx.EXPAND | wx.LEFT | wx.RIGHT, 5)
self.SetSizer(self.sizer)
self.Bind(wx.grid.EVT_GRID_SELECT_CELL, self.OnSelectCell)
class CLASS_IASI_L1():
def __init__(self, BandLst):
# 字典类型物理量
self.Tbb = {}
self.Rad = {}
# 二维矩阵
self.Lons = []
self.Lats = []
self.Time = []
self.satAzimuth = []
self.satZenith = []
self.sunAzimuth = []
self.sunZenith = []
# 光谱信息
self.wavenumber = []
self.radiance = []
def Load(self, L1File):
print u'读取 LEO所有数据信息......'
if not os.path.isfile(L1File):
print 'Error: %s not found' % L1File
sys.exit(1)
try:
fp = coda.open(L1File)
except Exception, e:
print 'Open file error<%s> .' % (e)
return
try:
# EPS = EUMETSAT Polar System atmospheric products (GOME-2 and IASI)
# EPS = EUMETSAT极地大气系统产品（GOME-2和IASI）'
# 获取文件头信息
product_class = coda.get_product_class(fp)
product_type = coda.get_product_type(fp)
product_version = coda.get_product_version(fp)
product_format = coda.get_product_format(fp)
product_size = coda.get_product_file_size(fp)
print 'product_class ', product_class
print 'product_type', product_type
print 'product_version', product_version
print 'product_format', product_format
print 'product_size', product_size
record = beatl2.ingest(L1File)
print record
SAT_angle = coda.fetch(fp, 'MDR', -1, 'MDR', 'GGeoSondAnglesMETOP')
SUN_angle = coda.fetch(fp, 'MDR', -1, 'MDR', 'GGeoSondAnglesSUN')
all_sun_angle = []
all_sat_angle = []
for i in xrange(len(SAT_angle)):
tmp_sat = SAT_angle[i].reshape(-1)
tmp_sun = SUN_angle[i].reshape(-1)
if len(all_sat_angle) == 0:
all_sat_angle = tmp_sat
all_sun_angle = tmp_sun
else:
all_sat_angle = np.concatenate((all_sat_angle, tmp_sat), 0)
all_sun_angle = np.concatenate((all_sun_angle, tmp_sun), 0)
iasiLen = len(record.longitude)
self.satZenith = (all_sat_angle[0::2]).reshape(iasiLen, 1)
self.satAzimuth = (all_sat_angle[1::2]).reshape(iasiLen, 1)
self.sunZenith = (all_sun_angle[0::2]).reshape(iasiLen, 1)
self.sunAzimuth = (all_sun_angle[1::2]).reshape(iasiLen, 1)
self.Lons = (record.longitude).reshape(iasiLen, 1)
self.Lats = (record.latitude).reshape(iasiLen, 1)
self.radiance = record.spectral_radiance * 10**7
# 暂时取一个观测的光谱波数
self.wavenumber = record.wavenumber[0, :]
v_ymd2seconds = np.vectorize(metop_ymd2seconds)
T1 = v_ymd2seconds(record.time)
self.Time = T1.reshape(iasiLen, 1)
except Exception as e:
print str(e)
sys.exit(1)
finally:
coda.close(fp)
def get_spectral_response(self):
"""
return 光谱波数和响应值，1维，2维
"""
k = 1.98644746103858e-9
if self.resolution == 40000:
satellite_type1 = ['METOP-A', 'METOP-B']
if self.satellite in satellite_type1:
try:
fp = coda.open(self.in_file)
wave3 = coda.fetch(fp, 'MDR', -1, 'Earthshine',
'WAVELENGTH_3')
wave4 = coda.fetch(fp, 'MDR', -1, 'Earthshine',
'WAVELENGTH_4')
lambda_smr = coda.fetch(fp, 'VIADR_SMR', -1, 'LAMBDA_SMR')
smr = coda.fetch(fp, 'VIADR_SMR', -1, 'SMR')
sunz = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH',
'SOLAR_ZENITH')
coda.close(fp)
# gome 959x4096 取后2048个点的辐射值
wavelens = self.record.wavelength[0, 2048:]
response = self.record.spectral_radiance[:, 2048:]
data_size = self.data_shape[0]
data_row = sunz.shape[0]
data_col = sunz[0].shape[1]
# gome的辐亮度计算 wave3 wave4 是30*1024
for m in xrange(data_size):
row, col = np.unravel_index(m, (data_row, data_col))
for i in xrange(2048):
if i < 1024:
response[m,
i] = response[m,
i] * k / wave3[row][i]
else:
response[m, i] = response[
m, i] * k / wave4[row][i - 1024]
# 计算太阳辐亮度
sol_wavelens = np.zeros((2048, )) # 太阳辐亮度
sol_response = np.zeros((2048, )) # 太阳辐亮度对应的波长
for i in xrange(2048):
if i < 1024:
sol_response[i] = (smr[0][2][i] *
k) / lambda_smr[0][2][i]
sol_wavelens[i] = lambda_smr[0][2][i]
else:
sol_response[i] = (smr[0][3][i - 1024] *
k) / lambda_smr[0][3][i - 1024]
sol_wavelens[i] = lambda_smr[0][3][i - 1024]
idx = np.where(response < 0)
print len(idx[0])
if len(idx[0] > 0):
response[idx] = 0.
gome_wavelens, gome_response, solar_response = self.combine_gome_band34(
wavelens, response, sol_wavelens, sol_response)
s0, s1 = gome_response.shape
gome_response = gome_response.reshape(s0, 1, s1)
print gome_response.shape, solar_response.shape
except Exception as e:
print 'Open file error {}'.format(e)
return
self.solar_response = solar_response
return gome_wavelens, gome_response
def read_file(self,
filename,
vars_to_read=None,
return_as='dict',
loglevel=None):
"""method to read an ESA binary data file entirely
Parameters
----------
filename : str
absolute path to filename to read
vars_to_read : list
list of str with variable names to read; defaults to ['od550aer']
verbose : Bool
set to True to increase verbosity
Returns
--------
Either:
dictionary (default):
keys are 'time', 'latitude', 'longitude', 'altitude' and the variable names
'ec550aer', 'bs550aer', 'sr', 'lod' if the whole file is read
'time' is a 1d array, while the other dict values are a another dict with the
time as keys (the same ret['time']) and a numpy array as values. These values represent the profile.
Note 1: latitude and longitude are height dependent due to the tilt of the measurement.
Note 2: negative values indicate a NaN
2d ndarray of type float:
representing a 'point cloud' with all points
column 1: time in seconds since the Unix epoch with ms accuracy (same time for every height
in a profile)
column 2: latitude
column 3: longitude
column 4: altitude
column 5: extinction
column 6: backscatter
column 7: sr
column 8: lod
Note: negative values are put to np.nan already
The indexes are noted in pyaerocom.io.read_aeolus_l2a_data.ReadAeolusL2aData.<index_name>
e.g. the time index is named pyaerocom.io.read_aeolus_l2a_data.ReadAeolusL2aData._TIMEINDEX
have a look at the example to access the values
This is whats in one DBL file
codadump list /lustre/storeA/project/aerocom/aerocom1/ADM_CALIPSO_TEST/download/AE_OPER_ALD_U_N_2A_20070101T002249149_002772000_003606_0001.DBL
/mph/product
/mph/proc_stage
/mph/ref_doc
/mph/acquisition_station
/mph/proc_center
/mph/proc_time
/mph/software_ver
/mph/baseline
/mph/sensing_start
/mph/sensing_stop
/mph/phase
/mph/cycle
/mph/rel_orbit
/mph/abs_orbit
/mph/state_vector_time
/mph/delta_ut1
/mph/x_position
/mph/y_position
/mph/z_position
/mph/x_velocity
/mph/y_velocity
/mph/z_velocity
/mph/vector_source
/mph/utc_sbt_time
/mph/sat_binary_time
/mph/clock_step
/mph/leap_utc
/mph/leap_sign
/mph/leap_err
/mph/product_err
/mph/tot_size
/mph/sph_size
/mph/num_dsd
/mph/dsd_size
/mph/num_data_sets
/sph/sph_descriptor
/sph/intersect_start_lat
/sph/intersect_start_long
/sph/intersect_stop_lat
/sph/intersect_stop_long
/sph/sat_track
/sph/num_brc
/sph/num_meas_max_brc
/sph/num_bins_per_meas
/sph/num_prof_sca
/sph/num_prof_ica
/sph/num_prof_mca
/sph/num_group_tot
/dsd[?]/ds_name
/dsd[?]/ds_type
/dsd[?]/filename
/dsd[?]/ds_offset
/dsd[?]/ds_size
/dsd[?]/num_dsr
/dsd[?]/dsr_size
/dsd[?]/byte_order
/geolocation[?]/start_of_obs_time
/geolocation[?]/num_meas_eff
/geolocation[?]/measurement_geolocation[?]/centroid_time
/geolocation[?]/measurement_geolocation[?]/mie_geolocation_height_bin[25]/longitude_of_height_bin
/geolocation[?]/measurement_geolocation[?]/mie_geolocation_height_bin[25]/latitude_of_height_bin
/geolocation[?]/measurement_geolocation[?]/mie_geolocation_height_bin[25]/altitude_of_height_bin
/geolocation[?]/measurement_geolocation[?]/rayleigh_geolocation_height_bin[25]/longitude_of_height_bin
/geolocation[?]/measurement_geolocation[?]/rayleigh_geolocation_height_bin[25]/latitude_of_height_bin
/geolocation[?]/measurement_geolocation[?]/rayleigh_geolocation_height_bin[25]/altitude_of_height_bin
/geolocation[?]/measurement_geolocation[?]/longitude_of_dem_intersection
/geolocation[?]/measurement_geolocation[?]/latitude_of_dem_intersection
/geolocation[?]/measurement_geolocation[?]/altitude_of_dem_intersection
/geolocation[?]/geoid_separation
/meas_pcd[?]/start_of_obs_time
/meas_pcd[?]/l1b_input_screening/l1b_obs_screening
/meas_pcd[?]/l1b_input_screening/l1b_obs_screening_flags[40]
/meas_pcd[?]/l1b_input_screening/l1b_mie_meas_screening[?]/l1b_mie_meas_qc
/meas_pcd[?]/l1b_input_screening/l1b_mie_meas_screening[?]/l1b_mie_meas_qc_flags[8]
/meas_pcd[?]/l1b_input_screening/l1b_rayleigh_meas_screening[?]/l1b_rayleigh_meas_qc
/meas_pcd[?]/l1b_input_screening/l1b_rayleigh_meas_screening[?]/l1b_rayleigh_meas_qc_flags[8]
/meas_pcd[?]/l1b_cal_screening/cal_valid
/meas_pcd[?]/l2a_processing_qc/sca_applied
/meas_pcd[?]/l2a_processing_qc/ica_applied
/meas_pcd[?]/l2a_processing_qc/mca_applied
/meas_pcd[?]/l2a_processing_qc/feature_finder_indicators/layer_information[24]/bin_loaded
/meas_pcd[?]/l2a_processing_qc/feature_finder_indicators/layer_information[24]/seed[30]
/meas_pcd[?]/l2a_processing_qc/feature_finder_indicators/lowest_computable_bin[30]
/sca_pcd[?]/starttime
/sca_pcd[?]/firstmatchingbin
/sca_pcd[?]/qc_flag
/sca_pcd[?]/profile_pcd_bins[24]/extinction_variance
/sca_pcd[?]/profile_pcd_bins[24]/backscatter_variance
/sca_pcd[?]/profile_pcd_bins[24]/lod_variance
/sca_pcd[?]/profile_pcd_bins[24]/processing_qc_flag
/sca_pcd[?]/profile_pcd_mid_bins[23]/extinction_variance
/sca_pcd[?]/profile_pcd_mid_bins[23]/backscatter_variance
/sca_pcd[?]/profile_pcd_mid_bins[23]/lod_variance
/sca_pcd[?]/profile_pcd_mid_bins[23]/ber_variance
/sca_pcd[?]/profile_pcd_mid_bins[23]/processing_qc_flag
/ica_pcd[?]/starttime
/ica_pcd[?]/first_matching_bin
/ica_pcd[?]/qc_flag
/ica_pcd[?]/ica_processing_qc_flag_bin[24]
/mca_pcd[?]/starttime
/mca_pcd[?]/processing_qc_flag_bin[24]
/amd_pcd[?]/starttime
/amd_pcd[?]/l2b_amd_screening_qc
/amd_pcd[?]/l2b_amd_screening_qc_flags
/amd_pcd[?]/l2b_amd_collocations[?]/l2b_amd_collocation_qc
/amd_pcd[?]/l2b_amd_collocations[?]/l2b_amd_collocation_qc_flags
/group_pcd[?]/starttime
/group_pcd[?]/brc_start
/group_pcd[?]/measurement_start
/group_pcd[?]/brc_end
/group_pcd[?]/measurement_end
/group_pcd[?]/height_bin_index
/group_pcd[?]/upper_problem_flag
/group_pcd[?]/particle_extinction_variance
/group_pcd[?]/particle_backscatter_variance
/group_pcd[?]/particle_lod_variance
/group_pcd[?]/qc_flag
/group_pcd[?]/mid_particle_extinction_variance_top
/group_pcd[?]/mid_particle_backscatter_variance_top
/group_pcd[?]/mid_particle_lod_variance_top
/group_pcd[?]/mid_particle_ber_variance_top
/group_pcd[?]/mid_particle_extinction_variance_bot
/group_pcd[?]/mid_particle_backscatter_variance_bot
/group_pcd[?]/mid_particle_lod_variance_bot
/group_pcd[?]/mid_particle_ber_variance_bot
/sca_optical_properties[?]/starttime
/sca_optical_properties[?]/sca_optical_properties[24]/extinction
/sca_optical_properties[?]/sca_optical_properties[24]/backscatter
/sca_optical_properties[?]/sca_optical_properties[24]/lod
/sca_optical_properties[?]/sca_optical_properties[24]/sr
/sca_optical_properties[?]/geolocation_middle_bins[24]/longitude
/sca_optical_properties[?]/geolocation_middle_bins[24]/latitude
/sca_optical_properties[?]/geolocation_middle_bins[24]/altitude
/sca_optical_properties[?]/sca_optical_properties_mid_bins[23]/extinction
/sca_optical_properties[?]/sca_optical_properties_mid_bins[23]/backscatter
/sca_optical_properties[?]/sca_optical_properties_mid_bins[23]/lod
/sca_optical_properties[?]/sca_optical_properties_mid_bins[23]/ber
/ica_optical_properties[?]/starttime
/ica_optical_properties[?]/ica_optical_properties[24]/case
/ica_optical_properties[?]/ica_optical_properties[24]/extinction
/ica_optical_properties[?]/ica_optical_properties[24]/backscatter
/ica_optical_properties[?]/ica_optical_properties[24]/lod
/mca_optical_properties[?]/starttime
/mca_optical_properties[?]/mca_optical_properties[24]/climber
/mca_optical_properties[?]/mca_optical_properties[24]/extinction
/mca_optical_properties[?]/mca_optical_properties[24]/lod
/amd[?]/starttime
/amd[?]/amd_properties[24]/pressure_fp
/amd[?]/amd_properties[24]/temperature_fp
/amd[?]/amd_properties[24]/frequencyshift_fp
/amd[?]/amd_properties[24]/relativehumidity_fp
/amd[?]/amd_properties[24]/molecularlod_fp
/amd[?]/amd_properties[24]/molecularbackscatter_fp
/amd[?]/amd_properties[24]/pressure_fiz
/amd[?]/amd_properties[24]/temperature_fiz
/amd[?]/amd_properties[24]/frequencyshift_fiz
/amd[?]/amd_properties[24]/relativehumidity_fiz
/amd[?]/amd_properties[24]/molecularlod_fiz
/amd[?]/amd_properties[24]/molecularbackscatter_fiz
/group_optical_properties[?]/starttime
/group_optical_properties[?]/height_bin_index
/group_optical_properties[?]/group_optical_property/group_extinction
/group_optical_properties[?]/group_optical_property/group_backscatter
/group_optical_properties[?]/group_optical_property/group_lod
/group_optical_properties[?]/group_optical_property/group_sr
/group_optical_properties[?]/group_geolocation_middle_bins/start_longitude
/group_optical_properties[?]/group_geolocation_middle_bins/start_latitude
/group_optical_properties[?]/group_geolocation_middle_bins/start_altitude
/group_optical_properties[?]/group_geolocation_middle_bins/mid_longitude
/group_optical_properties[?]/group_geolocation_middle_bins/mid_latitude
/group_optical_properties[?]/group_geolocation_middle_bins/mid_altitude
/group_optical_properties[?]/group_geolocation_middle_bins/stop_longitude
/group_optical_properties[?]/group_geolocation_middle_bins/stop_latitude
/group_optical_properties[?]/group_geolocation_middle_bins/stop_altitude
/group_optical_properties[?]/group_optical_property_middle_bins/mid_extinction_top
/group_optical_properties[?]/group_optical_property_middle_bins/mid_backscatter_top
/group_optical_properties[?]/group_optical_property_middle_bins/mid_lod_top
/group_optical_properties[?]/group_optical_property_middle_bins/mid_ber_top
/group_optical_properties[?]/group_optical_property_middle_bins/mid_extinction_bot
/group_optical_properties[?]/group_optical_property_middle_bins/mid_backscatter_bot
/group_optical_properties[?]/group_optical_property_middle_bins/mid_lod_bot
/group_optical_properties[?]/group_optical_property_middle_bins/mid_ber_bot
/scene_classification[?]/starttime
/scene_classification[?]/height_bin_index
/scene_classification[?]/aladin_cloud_flag/clrh
/scene_classification[?]/aladin_cloud_flag/clsr
/scene_classification[?]/aladin_cloud_flag/downclber
/scene_classification[?]/aladin_cloud_flag/topclber
/scene_classification[?]/nwp_cloud_flag
/scene_classification[?]/l2a_group_class_reliability
The question mark indicates a variable size array
It is not entirely clear what we actually have to look at.
For simplicity the data of the group 'sca_optical_properties' is returned at this point
Example
-------
>>> import pyaerocom.io.read_aeolus_l2a_data
>>> obj = pyaerocom.io.read_aeolus_l2a_data.ReadAeolusL2aData(verbose=True)
>>> import os
>>> os.environ['CODA_DEFINITION']='/lustre/storeA/project/aerocom/aerocom1/ADM_CALIPSO_TEST/'
>>> filename = '/lustre/storeA/project/aerocom/aerocom1/ADM_CALIPSO_TEST/download/AE_OPER_ALD_U_N_2A_20070101T002249149_002772000_003606_0001.DBL'
>>> # read returning a ndarray
>>> filedata_numpy = obj.read_file(filename, vars_to_read=['ec550aer'], return_as='numpy')
>>> time_as_numpy_datetime64 = filedata_numpy[0,obj._TIMEINDEX].astype('datetime64[s]')
>>> print('time: {}'.format(time_as_numpy_datetime64))
>>> print('latitude: {}'.format(filedata_numpy[1,obj._LATINDEX]))
>>> # read returning a dictionary
>>> filedata = obj.read_file(filename, vars_to_read=['ec550aer'])
>>> print('time: {}'.format(filedata['time'][0].astype('datetime64[s]')))
>>> print('all latitudes of 1st time step: {}'.format(filedata['latitude'][filedata['time'][0]]))
"""
import time
import coda
# coda uses 2000-01-01T00:00:00 as epoch unfortunately.
# so calculate the difference in seconds to the Unix epoch
seconds_to_add = np.datetime64('2000-01-01T00:00:00') - np.datetime64(
'1970-01-01T00:00:00')
seconds_to_add = seconds_to_add.astype(np.float_)
# the same can be achieved using pandas, but we stick to numpy here
# base_time = pd.DatetimeIndex(['2000-01-01'])
# seconds_to_add = (base_time.view('int64') // pd.Timedelta(1, unit='s'))[0]
start = time.perf_counter()
file_data = {}
self.logger.info('reading file {}'.format(filename))
# read file
product = coda.open(filename)
if vars_to_read is None:
# read all variables
vars_to_read = list(self.DATA_COLNAMES.keys())
vars_to_read.extend(list(self.METADATA_COLNAMES.keys()))
# read data
# start with the time because it is only stored once
groups = self.TIME_PATH.split(self.GROUP_DELIMITER)
file_data[self._TIME_NAME] = coda.fetch(product, groups[0], -1,
groups[1])
# epoch is 1 January 2000 at ESA
# so add offset to move that to 1 January 1970
# and save it into a np.datetime64[ms] object
file_data[self._TIME_NAME] = \
((file_data[self._TIME_NAME] + seconds_to_add) * 1.E3).astype(np.int).astype('datetime64[ms]')
# read data in a simple dictionary
for var in vars_to_read:
groups = self._COLNAMES[var].split(self.GROUP_DELIMITER)
if len(groups) == 3:
file_data[var] = {}
for idx, key in enumerate(file_data[self._TIME_NAME]):
file_data[var][key] = coda.fetch(product, groups[0], idx,
groups[1], -1, groups[2])
elif len(groups) == 2:
file_data[var] = {}
for idx, key in enumerate(file_data[self._TIME_NAME]):
file_data[var][key] = coda.fetch(product, groups[0], -1,
groups[1])
else:
file_data[var] = {}
for idx, key in enumerate(file_data[self._TIME_NAME]):
file_data[var][key] = coda.fetch(product, groups[0])
if return_as == 'numpy':
# return as one multidimensional numpy array that can be put into self.data directly
# (column wise because the column numbers do not match)
index_pointer = 0
data = np.empty([self._ROWNO, self._COLNO], dtype=np.float_)
for idx, _time in enumerate(file_data['time'].astype(np.float_) /
1000.):
# file_data['time'].astype(np.float_) is milliseconds after the (Unix) epoch
# but we want to save the time as seconds since the epoch
for _index in range(
len(file_data['latitude'][file_data['time'][idx]])):
# this works because all variables have to have the same size
# (aka same number of height levels)
# This loop could be avoided using numpy index slicing
# do that in case we need more optimisations
data[index_pointer, self._TIMEINDEX] = _time
for var in vars_to_read:
data[index_pointer,
self.INDEX_DICT[var]] = file_data[var][
file_data['time'][idx]][_index]
# put negative values to np.nan if the variable is not a metadata variable
if data[index_pointer,
self.INDEX_DICT[var]] == self.NAN_DICT[var]:
data[index_pointer, self.INDEX_DICT[var]] = np.nan
index_pointer += 1
if index_pointer >= self._ROWNO:
# add another array chunk to self.data
data = np.append(data,
np.empty(
[self._CHUNKSIZE, self._COLNO],
dtype=np.float_),
axis=0)
self._ROWNO += self._CHUNKSIZE
# return only the needed elements...
file_data = data[0:index_pointer]
end_time = time.perf_counter()
elapsed_sec = end_time - start
temp = 'time for single file read [s]: {:.3f}'.format(elapsed_sec)
self.logger.info(temp)
# self.logger.info('{} points read'.format(index_pointer))
return file_data
class GOME_COMM():
def __init__(self):
self.k = 1.98644746103858e-9
# 中心 纬度 经度
self.centre_lat = np.zeros((30, 24))
self.centre_lon = np.zeros((30, 24))
self.centre_row = np.zeros((30, 24))
self.centre_col = np.zeros((30, 24))
# 角点 纬度 经度
self.corner_lat = np.zeros((30, 24, 4))
self.corner_lon = np.zeros((30, 24, 4))
self.corner_row = np.zeros((30, 24, 4))
self.corner_col = np.zeros((30, 24, 4))
self.sun_Z = np.zeros((30, 24)) # 太阳天顶角
self.sun_A = np.zeros((30, 24)) # 太阳方位角
self.sat_Z = np.zeros((30, 24)) # 卫星天顶角
self.sat_A = np.zeros((30, 24)) # 卫星方位角
self.band3 = np.zeros((30, 24, 1024)) # 辐射值
self.band4 = np.zeros((30, 24, 1024))
self.band3_ERR_RAD = np.zeros((30, 24, 1024))
self.band3_STOKES_FRACTION = np.zeros((30, 24, 1024))
self.band4_ERR_RAD = np.zeros((30, 24, 1024))
self.band4_STOKES_FRACTION = np.zeros((30, 24, 1024))
self.wave3 = np.zeros((30, 1024)) # 波长
self.wave4 = np.zeros((30, 1024))
# self.LAMBDA_SMR3 = np.zeros((1, 1024))
# self.LAMBDA_SMR4 = np.zeros((1, 1024))
# self.SMR3 = np.zeros((1, 1024))
# self.SMR4 = np.zeros((1, 1024))
# self.E_SMR3 = np.zeros((1, 1024))
# self.E_SMR4 = np.zeros((1, 1024))
# self.E_REL_SUN3 = np.zeros((1, 1024))
# self.E_REL_SUN4 = np.zeros((1, 1024))
self.E_SMR3 = np.zeros((1024))
self.E_SMR4 = np.zeros((1024))
self.E_REL_SUN3 = np.zeros((1024))
self.E_REL_SUN4 = np.zeros((1024))
self.arr_GOME_L = np.zeros((2048, 30, 24)) # 辐亮度
self.arr_GOME_WL = np.zeros((2048, 30, 24)) # 辐亮度对应的波长
self.vec_Solar_L = np.zeros((2, 1024)) # 太阳辐亮度
self.vec_Solar_WL = np.zeros((2, 1024)) # 太阳辐亮度对应的波长
# 如下变量是通过卷积计算的
self.vec_Radiance_Solar = np.zeros(15)
self.vec_Radiance = np.zeros((15, 30, 24))
self.arr_Ref = np.zeros((15, 30, 24))
def init_gome(self, in_proj_cfg, des_sensor):
'''
读取yaml格式配置文件
'''
if not os.path.isfile(in_proj_cfg):
print 'Not Found %s' % in_proj_cfg
sys.exit(-1)
with open(in_proj_cfg, 'r') as stream:
cfg = yaml.load(stream)
self.sat1 = cfg['INFO']['sat']
self.sensor1 = cfg['INFO']['sensor']
self.sensor2 = des_sensor
self.ymd = cfg['INFO']['ymd']
self.ifile = cfg['PATH']['ipath']
self.ofile = cfg['PATH']['opath']
self.cmd = cfg['PROJ']['cmd']
self.col = cfg['PROJ']['col']
self.row = cfg['PROJ']['row']
self.res = cfg['PROJ']['res']
def read_gome(self, infile):
# 打开gome文件
try:
fp = coda.open(infile)
except Exception, e:
print 'Open file error<%s> .' % (e)
return
# 获取文件头信息
product_class = coda.get_product_class(fp)
product_type = coda.get_product_type(fp)
product_version = coda.get_product_version(fp)
product_format = coda.get_product_format(fp)
product_size = coda.get_product_file_size(fp)
# EPS = EUMETSAT Polar System atmospheric products (GOME-2 and IASI)
# EUMETSAT极地大气系统产品（GOME-2和IASI）'
print 'product_class ', product_class
print 'product_type', product_type
print 'product_version', product_version
print 'product_format', product_format
print 'product_size', product_size
CENTRE = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'CENTRE')
CORNER = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'CORNER')
SUN_Z = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH',
'SOLAR_ZENITH')
SUN_A = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH',
'SOLAR_AZIMUTH')
SAT_Z = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH',
'SAT_ZENITH')
SAT_A = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH',
'SAT_AZIMUTH')
BAND_3 = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'BAND_3')
BAND_4 = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'BAND_4')
WAVE_3 = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'WAVELENGTH_3')
WAVE_4 = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'WAVELENGTH_4')
LAMBDA_SMR = coda.fetch(fp, 'VIADR_SMR', -1, 'LAMBDA_SMR')
SMR = coda.fetch(fp, 'VIADR_SMR', -1, 'SMR')
E_SMR = coda.fetch(fp, 'VIADR_SMR', -1, 'E_SMR')
E_REL_SUN = coda.fetch(fp, 'VIADR_SMR', -1, 'E_REL_SUN')
# fp.close()
# print 'CENTRE', CENTRE.shape, CENTRE[0].shape
# print 'CORNER', CORNER.shape, CORNER[0].shape
# print 'SUN_Z', SUN_Z.shape, SUN_Z[0].shape
# print 'SUN_A', SUN_A.shape, SUN_A[0].shape
# print 'SAT_Z', SAT_Z.shape, SAT_Z[0].shape
# print 'SAT_A', SAT_A.shape, SAT_A[0].shape
# print 'BAND_3', BAND_3.shape, BAND_3[0].shape
# print 'BAND_4', BAND_4.shape, BAND_4[0].shape
# print 'WAVE_3', WAVE_3.shape, WAVE_3[0].shape
# print 'WAVE_4', WAVE_4.shape, WAVE_4[0].shape
# print 'LAMBDA_SMR', LAMBDA_SMR.shape, LAMBDA_SMR[0].shape
# print 'SMR', SMR.shape, SMR[0].shape
# print 'E_SMR', E_SMR.shape, E_SMR[0].shape
# print 'E_REL_SUN', E_REL_SUN.shape, E_REL_SUN[0].shape
# print LAMBDA_SMR[0][0][1023]
# print BAND_3[0][23][1023].RAD
# print self.band3.shape
for i in range(30):
count = coda.get_size(fp, 'MDR', i, 'Earthshine', 'BAND_3')
for j in range(int(count[0] * 0.75)):
for m in range(1024):
self.band3[i][j][m] = BAND_3[i][j][m].RAD
self.band3_ERR_RAD[i][j][m] = BAND_3[i][j][m].ERR_RAD
self.band3_STOKES_FRACTION[i][j][m] = BAND_3[i][j][
m].STOKES_FRACTION
self.band4[i][j][m] = BAND_4[i][j][m].RAD
self.band4_ERR_RAD[i][j][m] = BAND_4[i][j][m].ERR_RAD
self.band4_STOKES_FRACTION[i][j][m] = BAND_4[i][j][
m].STOKES_FRACTION
for m in range(2048):
for i in range(30):
count = coda.get_size(fp, 'MDR', i, 'Earthshine', 'BAND_3')
for j in range(int(count[0] * 0.75)):
if m < 1024:
if BAND_3[i][j][m].RAD < 0:
BAND_3[i][j][m].RAD = 0
self.arr_GOME_L[m][i][j] = (BAND_3[i][j][m].RAD *
self.k) / WAVE_3[i][m]
self.arr_GOME_WL[m][i][j] = WAVE_3[i][m]
else:
if BAND_4[i][j][m - 1024].RAD < 0:
BAND_4[i][j][m - 1024].RAD = 0
self.arr_GOME_L[m][i][j] = (
BAND_4[i][j][m - 1024].RAD *
self.k) / WAVE_4[i][m - 1024]
self.arr_GOME_WL[m][i][j] = WAVE_4[i][m - 1024]
for i in range(2):
for j in range(1024):
if i == 0:
self.vec_Solar_L[i][j] = (
SMR[0][2][j] * self.k) / LAMBDA_SMR[0][2][j] # 太阳辐亮度
self.vec_Solar_WL[i][j] = LAMBDA_SMR[0][2][j]
self.E_SMR3[j] = E_SMR[0][2][j]
self.E_REL_SUN3[j] = E_REL_SUN[0][2][j]
elif i == 1:
self.vec_Solar_L[i][j] = (
SMR[0][3][j] * self.k) / LAMBDA_SMR[0][3][j] # 太阳辐亮度
self.vec_Solar_WL[i][j] = LAMBDA_SMR[0][3][j]
self.E_SMR4[j] = E_SMR[0][3][j]
self.E_REL_SUN4[j] = E_REL_SUN[0][3][j]
for i in range(30):
for j in range(24):
self.sun_A[i][j] = SUN_A[i][1][j]
self.sun_Z[i][j] = SUN_Z[i][1][j]
self.sat_A[i][j] = SAT_A[i][1][j]
self.sun_Z[i][j] = SAT_Z[i][1][j]
self.centre_lat[i][j] = CENTRE[i][j].latitude
self.centre_lon[i][j] = CENTRE[i][j].longitude
for i in range(30):
for j in range(24):
for m in range(4):
self.corner_lat[i][j][m] = CORNER[i][m][j].latitude
self.corner_lon[i][j][m] = CORNER[i][m][j].longitude
coda.close(fp)
def _read(self, path, fields="all", return_header=False):
tmpdira = config.conf["main"]["tmpdir"]
tmpdirb = config.conf["main"]["tmpdirb"]
tmpdir = (tmpdira
if shutil.disk_usage(tmpdira).free > self.minspace
else tmpdirb)
with tempfile.NamedTemporaryFile(mode="wb", dir=tmpdir, delete=True) as tmpfile:
with gzip.open(str(path), "rb") as gzfile:
logging.debug("Decompressing {!s}".format(path))
gzcont = gzfile.read()
logging.debug("Writing decompressed file to {!s}".format(tmpfile.name))
tmpfile.write(gzcont)
del gzcont
# All the hard work is in coda
logging.debug("Reading {!s}".format(tmpfile.name))
cfp = coda.open(tmpfile.name)
c = coda.fetch(cfp)
logging.debug("Sorting info...")
n_scanlines = c.MPHR.TOTAL_MDR
start = datetime.datetime(*coda.time_double_to_parts_utc(c.MPHR.SENSING_START))
has_mdr = numpy.array([hasattr(m, 'MDR') for m in c.MDR],
dtype=numpy.bool)
bad = numpy.array([
(m.MDR.DEGRADED_PROC_MDR|m.MDR.DEGRADED_INST_MDR)
if hasattr(m, 'MDR') else True
for m in c.MDR],
dtype=numpy.bool)
dlt = numpy.concatenate(
[m.MDR.OnboardUTC[:, numpy.newaxis]
for m in c.MDR
if hasattr(m, 'MDR')], 1) - c.MPHR.SENSING_START
M = numpy.ma.zeros(
dtype=self._dtype,
shape=(n_scanlines, 30))
M["time"][has_mdr] = numpy.datetime64(start, "ms") + numpy.array(dlt*1e3, "m8[ms]").T
specall = self.__obtain_from_mdr(c, "GS1cSpect").astype("f8")
# apply scale factors
first = c.MDR[0].MDR.IDefNsfirst1b
last = c.MDR[0].MDR.IDefNslast1b
for (slc_st, slc_fi, fact) in zip(
filter(None, c.GIADR_ScaleFactors.IDefScaleSondNsfirst),
c.GIADR_ScaleFactors.IDefScaleSondNslast,
c.GIADR_ScaleFactors.IDefScaleSondScaleFactor):
# Documented intervals are closed [a, b]; Python uses
# half-open [a, b).
specall[..., (slc_st-first):(slc_fi-first+1)] *= pow(10.0, -fact)
M["spectral_radiance"][has_mdr] = specall
locall = self.__obtain_from_mdr(c, "GGeoSondLoc")
M["lon"][has_mdr] = locall[:, :, :, 0]
M["lat"][has_mdr] = locall[:, :, :, 1]
satangall = self.__obtain_from_mdr(c, "GGeoSondAnglesMETOP")
M["satellite_zenith_angle"][has_mdr] = satangall[:, :, :, 0]
M["satellite_azimuth_angle"][has_mdr] = satangall[:, :, :, 1]
solangall = self.__obtain_from_mdr(c, "GGeoSondAnglesSUN")
M["solar_zenith_angle"][has_mdr] = solangall[:, :, :, 0]
M["solar_azimuth_angle"][has_mdr] = solangall[:, :, :, 1]
for fld in M.dtype.names:
M.mask[fld][~has_mdr, ...] = True
M.mask[fld][bad, ...] = True
m = c.MDR[0].MDR
wavenumber = (m.IDefSpectDWn1b * numpy.arange(m.IDefNsfirst1b, m.IDefNslast1b+0.1) * (1/ureg.metre))
if self.wavenumber is None:
self.wavenumber = wavenumber
elif abs(self.wavenumber - wavenumber).max() > (0.05 * 1/(ureg.centimetre)):
raise ValueError("Inconsistent wavenumbers")
return M
def get_solar_zenith(self):
"""
return solar_zenith
"""
if self.resolution == 24000:
satellite_type1 = ['METOP-A', 'METOP-B']
if self.satellite in satellite_type1:
try:
fp = coda.open(self.in_file)
angle = coda.fetch(fp, 'MDR', -1, 'MDR',
'GGeoSondAnglesSUN')
coda.close(fp)
all_angle = []
for i in xrange(len(angle)):
tmp_angle = angle[i].reshape(-1)
if len(all_angle) == 0:
all_angle = tmp_angle
else:
all_angle = np.concatenate((all_angle, tmp_angle))
s0 = self.data_shape[0]
# 开始间隔一个取一个值，取偶数位
data_pre = (all_angle[0::2]).reshape(s0, 1)
# 过滤无效值
invalid_index = np.logical_or(data_pre < 0, data_pre > 180)
data_pre = data_pre.astype(np.float32)
data_pre[invalid_index] = np.nan
data = data_pre
except Exception as e:
print 'Open file error {}'.format(e)
return
else:
raise ValueError('Cant read this satellite`s data.: {}'.format(
self.satellite))
elif self.resolution == 24001:
satellite_type1 = ['METOP-A', 'METOP-B']
if self.satellite in satellite_type1:
try:
ncr = Dataset(self.in_file, 'r', format='NETCDF3_CLASSIC')
data_pre = ncr.variables['solar_zenith_angle'][:]
ncr.close()
# 过滤无效值
invalid_index = np.logical_or(data_pre < 0, data_pre > 180)
data_pre = data_pre.astype(np.float32)
data_pre[invalid_index] = np.nan
data = data_pre.reshape(self.data_shape)
except Exception as e:
print 'Open file error {}'.format(e)
return
else:
raise ValueError('Cant read this satellite`s data.: {}'.format(
self.satellite))
else:
raise ValueError(
'Cant read this data, please check its resolution: {}'.format(
self.in_file))
return data
def main():
parser = argparse.ArgumentParser(
'Read star reference spectra from a folder,'
' calculate mean, std, and quantiles (0.05, 0.50, 0.95),'
' save into a file in numpy format')
parser.add_argument('--path', help="input folder")
parser.add_argument('--out', help="output file")
parser.add_argument(
'--use-measures',
action="store_true",
help="use measurement data instead of the reference star spectrum")
parser.add_argument('--plot-all',
action="store_true",
help="plot all spectra and also the result")
parser.add_argument('--plot', action="store_true", help="plot the result")
args = parser.parse_args()
spectra = []
for file in os.listdir(args.path):
if file[-3:] != '.N1':
continue
# - see https://github.com/stcorp/codadef-documentation for product_class, product_type, version
# - to list field names: coda.get_field_names(h,""): [
# 'mph', 'sph', 'dsd', 'tra_summary_quality', 'tra_occultation_data',
# 'tra_nom_wav_assignment', 'tra_ref_star_spectrum', 'tra_ref_atm_dens_profile',
# 'tra_transmission', 'tra_satu_and_sfa_data', 'tra_auxiliary_data',
# 'tra_geolocation'
# ]
# - meanings of the fields: http://envisat.esa.int/handbooks/gomos/CNTR2-2-3.html
# file, product_class, product_type, version
h = coda.open_as(os.path.join(args.path, file), 'ENVISAT_GOMOS',
'GOM_TRA_1P', 1)
v = coda.fetch(h, 'sph')
m = re.match(r'\d*(.*)', v[12].strip())
bayer = m[1] if m else v[12].strip()
v = coda.fetch(h, 'tra_ref_star_spectrum')
spe_ref_raw = np.array(v[0][1]).reshape(
(1, -1)) # in electrons/pixel/0.5s
if args.use_measures:
# the actual measurement data:
n = 10
v = coda.fetch(h, 'tra_transmission')
spe_ref_raw = np.stack([np.array(r[2])
for r in v], axis=0)[:n, :] * spe_ref_raw
v = coda.fetch(h, 'tra_nom_wav_assignment')
lam = np.array(v[0][0]) # in nm
v = coda.fetch(h, 'tra_occultation_data')
sens_len = v[0][10] # how many elements the curve has
sens_lam = np.array(v[0][11])[0:sens_len] # in nm
sens_val = np.array(v[0][12])[
0:sens_len] # in (photons/s/cm2/nm) / (electrons/pixel/0.5s)
coda.close(h)
# hack to alleviate a problem arising from interpolation and a sharp fall in sensitivity around 389nm
dl = np.array([-0.3, 0, 0.3])
sens_lam = np.concatenate(
(sens_lam[:12], sens_lam[12:13] + dl, sens_lam[13:]))
sens_val = np.concatenate(
(sens_val[:12], sens_val[11:14], sens_val[13:]))
interp = interp1d(sens_lam, sens_val, kind='linear')
spe_ref = spe_ref_raw * interp(lam).reshape((1, -1))
spectra.append(spe_ref)
if args.plot_all:
plt.plot(
lam.reshape((1, -1)).repeat(len(spe_ref), axis=0), spe_ref)
# plt.plot(lam, spe_ref_raw)
# plt.plot(sens_lam, sens_val * 0.8*np.max(spe_ref)/np.max(sens_val), 'x-')
plt.title(file)
plt.xlabel('Wavelenth [nm]')
plt.ylabel('Photon flux density [ph/s/cm2/nm]')
plt.show()
if len(spectra) == 0:
raise Exception('no spectra found in folder %s' % args.path)
spectra = np.concatenate(spectra, axis=0)
quantiles = np.quantile(spectra, (0.05, 0.50, 0.95), axis=0)
mean = np.mean(spectra, axis=0)
std = np.std(spectra, axis=0)
outfile = args.out.replace('{{bayer}}', bayer.lower().replace(' ', '_'))
if outfile[-4:] != '.npz':
outfile = outfile + '.npz'
np.savez(outfile,
bayer=bayer,
lam=lam,
mean=mean,
std=std,
q05=quantiles[0, :],
q50=quantiles[1, :],
q95=quantiles[2, :])
if args.plot or args.plot_all:
plt.plot(lam, quantiles[1, :], 'C0-')
plt.plot(lam, quantiles[0, :], 'C1:')
plt.plot(lam, quantiles[2, :], 'C1:')
plt.title('Median spectrum of star %s' % (bayer, ))
plt.xlabel('Wavelength [nm]')
plt.ylabel('Photon flux density [ph/s/cm2/nm]')
plt.show()
