def set_file_attr(self):
"""
seft self.file_att is dict
"""
data = dict()
if self.resolution == 40000:
satellite_type1 = ['METOP-A', 'METOP-B']
if self.satellite in satellite_type1:
try:
fp = coda.open(self.in_file)
product_class = coda.get_product_class(fp)
product_type = coda.get_product_type(fp)
product_version = coda.get_product_version(fp)
product_format = coda.get_product_format(fp)
product_size = coda.get_product_file_size(fp)
coda.close(fp)
except Exception as e:
print str(e)
return
data['class'] = product_class
data['size'] = product_size
data['type'] = product_type
data['version'] = product_version
data['format'] = product_format
self.file_attr = data
else:
raise ValueError('Cant read this satellite`s data.: {}'.format(
self.satellite))
def read_gome(self, infile):
# 打开gome文件
try:
fp = coda.open(infile)
except Exception, e:
print 'Open file error<%s> .' % (e)
return
def _read(self, path, fields="all", return_header=False):
tmpdira = config.conf["main"]["tmpdir"]
tmpdirb = config.conf["main"]["tmpdirb"]
tmpdir = (tmpdira
if shutil.disk_usage(tmpdira).free > self.minspace
else tmpdirb)
with tempfile.NamedTemporaryFile(mode="wb", dir=tmpdir, delete=True) as tmpfile:
with gzip.open(str(path), "rb") as gzfile:
logging.debug("Decompressing {!s}".format(path))
gzcont = gzfile.read()
logging.debug("Writing decompressed file to {!s}".format(tmpfile.name))
tmpfile.write(gzcont)
del gzcont
# All the hard work is in coda
logging.debug("Reading {!s}".format(tmpfile.name))
cfp = coda.open(tmpfile.name)
c = coda.fetch(cfp)
logging.debug("Sorting info...")
n_scanlines = c.MPHR.TOTAL_MDR
start = datetime.datetime(*coda.time_double_to_parts_utc(c.MPHR.SENSING_START))
has_mdr = numpy.array([hasattr(m, 'MDR') for m in c.MDR],
dtype=numpy.bool)
bad = numpy.array([
(m.MDR.DEGRADED_PROC_MDR|m.MDR.DEGRADED_INST_MDR)
if hasattr(m, 'MDR') else True
for m in c.MDR],
dtype=numpy.bool)
dlt = numpy.concatenate(
[m.MDR.OnboardUTC[:, numpy.newaxis]
for m in c.MDR
if hasattr(m, 'MDR')], 1) - c.MPHR.SENSING_START
M = numpy.ma.zeros(
dtype=self._dtype,
shape=(n_scanlines, 30))
M["time"][has_mdr] = numpy.datetime64(start, "ms") + numpy.array(dlt*1e3, "m8[ms]").T
specall = self.__obtain_from_mdr(c, "GS1cSpect")
M["spectral_radiance"][has_mdr] = specall
locall = self.__obtain_from_mdr(c, "GGeoSondLoc")
M["lon"][has_mdr] = locall[:, :, :, 0]
M["lat"][has_mdr] = locall[:, :, :, 1]
satangall = self.__obtain_from_mdr(c, "GGeoSondAnglesMETOP")
M["satellite_zenith_angle"][has_mdr] = satangall[:, :, :, 0]
M["satellite_azimuth_angle"][has_mdr] = satangall[:, :, :, 1]
solangall = self.__obtain_from_mdr(c, "GGeoSondAnglesSUN")
M["solar_zenith_angle"][has_mdr] = solangall[:, :, :, 0]
M["solar_azimuth_angle"][has_mdr] = solangall[:, :, :, 1]
for fld in M.dtype.names:
M.mask[fld][~has_mdr, ...] = True
M.mask[fld][bad, ...] = True
m = c.MDR[0].MDR
wavenumber = (m.IDefSpectDWn1b * numpy.arange(m.IDefNsfirst1b, m.IDefNslast1b+0.1) * (1/ureg.metre))
if self.wavenumber is None:
self.wavenumber = wavenumber
elif abs(self.wavenumber - wavenumber).max() > (0.05 * 1/(ureg.centimetre)):
raise ValueError("Inconsistent wavenumbers")
return M
def get_solar_zenith(self):
"""
获取太阳天顶角
:return:
"""
fp = coda.open(self.in_file)
angles = coda.fetch(fp, 'MDR', -1, 'MDR', 'GGeoSondAnglesSUN')
zenith = np.array([])
for i in angles:
z = i.reshape(-1)[0::2]
zenith = np.append(zenith, z)
return zenith
def Load(self, L1File):
print u'读取 LEO所有数据信息......'
if not os.path.isfile(L1File):
print 'Error: %s not found' % L1File
sys.exit(1)
try:
fp = coda.open(L1File)
except Exception, e:
print 'Open file error<%s> .' % (e)
return
def set_file_attr(self):
"""
seft self.file_att is dict
"""
data = dict()
if self.resolution == 24000:
satellite_type1 = ['METOP-A', 'METOP-B']
if self.satellite in satellite_type1:
try:
fp = coda.open(self.in_file)
product_class = coda.get_product_class(fp)
product_type = coda.get_product_type(fp)
product_version = coda.get_product_version(fp)
product_format = coda.get_product_format(fp)
product_size = coda.get_product_file_size(fp)
coda.close(fp)
except Exception as e:
print str(e)
return
data['class'] = product_class
data['size'] = product_size
data['type'] = product_type
data['version'] = product_version
data['format'] = product_format
self.file_attr = data
else:
raise ValueError('Cant read this satellite`s data.: {}'.format(
self.satellite))
elif self.resolution == 24001:
satellite_type1 = ['METOP-A', 'METOP-B']
if self.satellite in satellite_type1:
try:
# 'NCETCDF4'
ncr = Dataset(self.in_file, 'r', format='NETCDF3_CLASSIC')
data = ncr.ncattrs()
ncr.close()
except Exception as e:
print str(e)
return
self.file_attr = data
else:
raise ValueError('Cant read this satellite`s data.: {}'.format(
self.satellite))
else:
raise ValueError("Cant handle this resolution: ".format(
self.resolution))
def get_sensor_azimuth(self):
"""
return sensor_azimuth
"""
if self.resolution == 40000:
satellite_type1 = ['METOP-A', 'METOP-B']
if self.satellite in satellite_type1:
try:
fp = coda.open(self.in_file)
angle = coda.fetch(fp, 'MDR', -1, 'Earthshine',
'GEO_EARTH', 'SAT_AZIMUTH')
coda.close(fp)
# angle = 30*3*32 = 1440, 取30*1*32 = 960, 取前959个
data_size = self.data_shape[0]
data_row = angle.shape[0]
data_col = angle[0].shape[1]
data_pre = np.full(self.data_shape, -999.)
for i in xrange(data_size):
row, col = np.unravel_index(i, (data_row, data_col))
# print row, col, angle[row][1][col]
data_pre[i] = angle[row][1][col]
# 过滤无效值
invalid_index = np.logical_or(data_pre < 0, data_pre > 360)
data_pre = data_pre.astype(np.float32)
data_pre[invalid_index] = np.nan
data = data_pre
except Exception as e:
print 'Open file error {}'.format(e)
return
else:
raise ValueError('Cant read this satellite`s data.: {}'.format(
self.satellite))
else:
raise ValueError(
'Cant read this data, please check its resolution: {}'.format(
self.in_file))
return data
def get_footprint(product):
try:
import coda
except ImportError:
return None
path = "/METADATA/EOP_METADATA/om_featureOfInterest/eop_multiExtentOf/gml_surfaceMembers/gml_exterior@gml_posList"
pf = coda.open(product)
try:
coord = coda.fetch(pf, path).split(' ')
except coda.CodacError:
return None
finally:
coda.close(pf)
if len(coord) % 2 != 0:
return None
return Polygon([
LinearRing([
Point(float(lon), float(lat))
for lat, lon in zip(coord[0::2], coord[1::2])
])
])
def get_solar_zenith(self):
"""
return solar_zenith
"""
if self.resolution == 24000:
satellite_type1 = ['METOP-A', 'METOP-B']
if self.satellite in satellite_type1:
try:
fp = coda.open(self.in_file)
angle = coda.fetch(fp, 'MDR', -1, 'MDR',
'GGeoSondAnglesSUN')
coda.close(fp)
all_angle = []
for i in xrange(len(angle)):
tmp_angle = angle[i].reshape(-1)
if len(all_angle) == 0:
all_angle = tmp_angle
else:
all_angle = np.concatenate((all_angle, tmp_angle))
s0 = self.data_shape[0]
# 开始间隔一个取一个值，取偶数位
data_pre = (all_angle[0::2]).reshape(s0, 1)
# 过滤无效值
invalid_index = np.logical_or(data_pre < 0, data_pre > 180)
data_pre = data_pre.astype(np.float32)
data_pre[invalid_index] = np.nan
data = data_pre
except Exception as e:
print 'Open file error {}'.format(e)
return
else:
raise ValueError('Cant read this satellite`s data.: {}'.format(
self.satellite))
elif self.resolution == 24001:
satellite_type1 = ['METOP-A', 'METOP-B']
if self.satellite in satellite_type1:
try:
ncr = Dataset(self.in_file, 'r', format='NETCDF3_CLASSIC')
data_pre = ncr.variables['solar_zenith_angle'][:]
ncr.close()
# 过滤无效值
invalid_index = np.logical_or(data_pre < 0, data_pre > 180)
data_pre = data_pre.astype(np.float32)
data_pre[invalid_index] = np.nan
data = data_pre.reshape(self.data_shape)
except Exception as e:
print 'Open file error {}'.format(e)
return
else:
raise ValueError('Cant read this satellite`s data.: {}'.format(
self.satellite))
else:
raise ValueError(
'Cant read this data, please check its resolution: {}'.format(
self.in_file))
return data
# default libRadtran input file text to be added to
inp_file = open('default.inp', 'r')
inp_default = inp_file.read()
inp_file.close()
# path and EPS file to read
if len(sys.argv) != 2:
text = '1 /path/file argument required and %d given.' % len(sys.argv) - 1
raise IOError(text)
e_fn = sys.argv[1]
if not (os.path.exists(e_fn)):
text = 'The file %s does not exist.' % e_fn
raise IOError(text)
#e_fn = '/home/malapradej/Documents/PhD_UCL/Data/GOME-2/l1b/2007/01/GOME_xxx_1B_M02_20070125124939Z_20070125143057Z_R_O_20120209112017Z'
# opening the file
ef = coda.open(e_fn)
# creates a base name without extension used in rest of code
base_fn = os.path.splitext(e_fn)[0] # removes any extension from path
base_fn = os.path.basename(base_fn) # selects only filename
# path to DASF data to be saved
path_dasf = '/home/ucfajrm/DATA/DASF/Data/DASF'
if not (os.path.exists(path_dasf)):
text = 'The path %s does not exist.' % path_dasf
raise IOError(text)
# leaf albedo file generated from PROSPECT
# typical leaf level parameters generated based on metadata-analysis
# see spreadsheet in PROSPECT folder
l_lower = 580. # limits for plotting purposes
#!/usr/bin/env python
#
# mipl2geo.py
import sys
import coda
if __name__ == "__main__":
if len(sys.argv) < 2:
print >> sys.stderr, "Usage: %s <mipas level 2 file> ..." % sys.argv[0]
sys.exit(1)
for f in sys.argv[1:]:
pf = coda.open(f)
product_class = coda.get_product_class(pf)
product_type = coda.get_product_type(pf)
if not product_class.startswith(
"ENVISAT") or product_type != "MIP_NL__2P":
print >> sys.stderr, "Error: file %s is not a MIPAS Level 2 product (product class = %s, product type = %s)" % (
sys.argv[1], product_class, product_type)
sys.exit(1)
print "Processing : %s" % f
cursor = coda.Cursor()
coda.cursor_set_product(cursor, pf)
coda.cursor_goto_record_field_by_name(cursor, "scan_geolocation_ads")
def read(self, vars_to_retrieve=None, files=[], first_file=None,
last_file=None, file_pattern=None, list_coda_paths=False,
local_temp_dir=None):
"""Method that reads list of files as instance of :class:`UngriddedData`
Parameters
----------
vars_to_retrieve : :obj:`list` or similar, optional,
list containing variable IDs that are supposed to be read. If None,
all variables in :attr:`PROVIDES_VARIABLES` are loaded
files : :obj:`list`, optional
list of files to be read. If None, then the file list is used that
is returned on :func:`get_file_list`.
first_file : :obj:`int`, optional
index of first file in file list to read. If None, the very first
file in the list is used. Note: is ignored if input parameter
`file_pattern` is specified.
last_file : :obj:`int`, optional
index of last file in list to read. If None, the very last file
in the list is used. Note: is ignored if input parameter
`file_pattern` is specified.
file_pattern : str, optional
string pattern for file search (cf :func:`get_file_list`)
:param local_temp_dir:
Returns
-------
UngriddedData
data object
Example:
>>> import pyaerocom as pya
>>> obj = pya.io.read_aeolus_l2a_data.ReadL2Data()
>>> testfiles = []
>>> testfiles.append('/lustre/storeB/project/fou/kl/admaeolus/data.rev.2A02/download/2018-12/01/AE_OPER_ALD_U_N_2A_20181201T033526026_005423993_001590_0001.TGZ')
>>> data=obj.read(files=testfiles)
>>> data=obj.read(files=testfiles, vars_to_retrieve='ec355aer')
"""
import pathlib
import tarfile
import os
import coda
if local_temp_dir is None:
local_temp_dir = self.LOCAL_TMP_DIR
if vars_to_retrieve is None:
vars_to_retrieve = self.DEFAULT_VARS
elif isinstance(vars_to_retrieve, str):
vars_to_retrieve = [vars_to_retrieve]
if files is None:
if len(self.files) == 0:
self.get_file_list(pattern=file_pattern)
files = self.files
if file_pattern is None:
if first_file is None:
first_file = 0
if last_file is None:
last_file = len(files)
files = files[first_file:last_file]
self.read_failed = []
temp_files = {}
data_obj = UngriddedData(num_points=self._COLNO, chunksize=self._CHUNKSIZE)
meta_key = 0.0
idx = 0
# check if the supplied file is a supported archive file (tar in this case)
# and extract the files with supported suffixes to const._cachedir
non_archive_files = []
for idx, _file in enumerate(sorted(files)):
# temp = 'reading file: {}'.format(_file)
self.logger.info('file: {}'.format(_file))
suffix = pathlib.Path(_file).suffix
if suffix in self.SUPPORTED_ARCHIVE_SUFFIXES:
temp = 'opening archive file; using {} as temp dir.'.format(local_temp_dir)
self.logger.info(temp)
# untar archive files first
tarhandle = tarfile.open(_file)
files_in_tar = tarhandle.getnames()
for file_in_tar in files_in_tar:
if pathlib.Path(file_in_tar).suffix in self.SUPPORTED_SUFFIXES:
# extract file to tmp path
member = tarhandle.getmember(file_in_tar)
temp = 'extracting file {}...'.format(member.name)
self.logger.info(temp)
tarhandle.extract(member, path=local_temp_dir, set_attrs=False)
extract_file = os.path.join(local_temp_dir, member.name)
non_archive_files.append(extract_file)
temp_files[extract_file] = True
tarhandle.close()
else:
non_archive_files.append(_file)
for idx, _file in enumerate(sorted(non_archive_files)):
# list coda data paths in the 1st file in case the user asked for that
if idx == 0 and list_coda_paths:
pass
coda_handle = coda.open(_file)
root_field_names = coda.get_field_names(coda_handle)
for field in root_field_names:
print(field)
coda.close(coda_handle)
data_obj = None
return data_obj
file_data = self.read_file(_file, vars_to_retrieve=vars_to_retrieve,
loglevel=logging.INFO, return_as='numpy')
self.logger.info('{} points read'.format(file_data.shape[0]))
# the metadata dict is left empty for L2 data
# the location in the data set is time step dependant!
if idx == 0:
data_obj._data = file_data
else:
data_obj._data = np.append(data_obj._data, file_data, axis=0)
data_obj._idx = data_obj._data.shape[0] + 1
file_data = None
# remove file if it was temporary one
if _file in temp_files:
os.remove(_file)
# pass
# tmp_obj = UngriddedData()
# tmp_obj._data = file_data
# tmp_obj._idx = data_obj._data.shape[0] + 1
# data_obj.append(tmp_obj)
self.logger.info('size of data object: {}'.format(data_obj._idx - 1))
return data_obj
if i < num_fields - 1:
coda.cursor_goto_next_record_field(cursor)
coda.cursor_goto_parent(cursor)
if __name__ == "__main__":
if len(sys.argv) != 2:
print >> sys.stderr, "Usage: %s <envisat file>" % sys.argv[0]
sys.exit(1)
coda.set_option_perform_conversions(0)
pf = coda.open(sys.argv[1])
product_class = coda.get_product_class(pf)
if not product_class.startswith("ENVISAT"):
print >>sys.stderr, "Error: file %s is not an ENVISAT product file (product class = %s)" % (sys.argv[1], product_class)
sys.exit(1)
cursor = coda.Cursor()
coda.cursor_set_product(cursor, pf)
print " MPH :"
coda.cursor_goto_record_field_by_name(cursor, "mph")
print_record(cursor)
coda.cursor_goto_parent(cursor)
print " SPH :";
coda.cursor_goto_record_field_by_name(cursor, "sph")
import os
# Change this to the location of your AEOLUS codadef file
os.putenv('CODA_DEFINITION', '/usr/local/share/coda/definitions')
# You can also remove this line and set the CODA_DEFINITION environment variable globally on your system
import coda
from numpy import hstack, vstack
# change this to the full path of your Aeolus L1B DBL file
filename = "/path/to/AE_OPER_ALD_U_N_1B_20151002T001857059_005787000_046339_0001.DBL"
product = coda.open(filename)
### Observation profiles ###
# Observation profiles provide a single profile per BRC.
# Reading all values will therefore return 'an array of arrays' (we will get an array for each '-1' in the coda.fetch()).
# The first array is the list of BRCs, and the second array the list of points in the profile.
# This array of arrays can be turned into a single 2D numpy array of shape [num_brc, num_vertical] by using 'vstack'.
# Mie observation wind profiles
print("Mie observation wind profiles")
latitude = coda.fetch(product, 'geolocation', -1, 'observation_geolocation/observation_mie_geolocation', -1, 'latitude_of_height_bin')
latitude = vstack(latitude)
longitude = coda.fetch(product, 'geolocation', -1, 'observation_geolocation/observation_mie_geolocation', -1, 'longitude_of_height_bin')
longitude = vstack(longitude)
def read_file(self,
filename,
vars_to_read=None,
return_as='dict',
loglevel=None):
"""method to read an ESA binary data file entirely
Parameters
----------
filename : str
absolute path to filename to read
vars_to_read : list
list of str with variable names to read; defaults to ['od550aer']
verbose : Bool
set to True to increase verbosity
Returns
--------
Either:
dictionary (default):
keys are 'time', 'latitude', 'longitude', 'altitude' and the variable names
'ec550aer', 'bs550aer', 'sr', 'lod' if the whole file is read
'time' is a 1d array, while the other dict values are a another dict with the
time as keys (the same ret['time']) and a numpy array as values. These values represent the profile.
Note 1: latitude and longitude are height dependent due to the tilt of the measurement.
Note 2: negative values indicate a NaN
2d ndarray of type float:
representing a 'point cloud' with all points
column 1: time in seconds since the Unix epoch with ms accuracy (same time for every height
in a profile)
column 2: latitude
column 3: longitude
column 4: altitude
column 5: extinction
column 6: backscatter
column 7: sr
column 8: lod
Note: negative values are put to np.nan already
The indexes are noted in pyaerocom.io.read_aeolus_l2a_data.ReadAeolusL2aData.<index_name>
e.g. the time index is named pyaerocom.io.read_aeolus_l2a_data.ReadAeolusL2aData._TIMEINDEX
have a look at the example to access the values
This is whats in one DBL file
codadump list /lustre/storeA/project/aerocom/aerocom1/ADM_CALIPSO_TEST/download/AE_OPER_ALD_U_N_2A_20070101T002249149_002772000_003606_0001.DBL
/mph/product
/mph/proc_stage
/mph/ref_doc
/mph/acquisition_station
/mph/proc_center
/mph/proc_time
/mph/software_ver
/mph/baseline
/mph/sensing_start
/mph/sensing_stop
/mph/phase
/mph/cycle
/mph/rel_orbit
/mph/abs_orbit
/mph/state_vector_time
/mph/delta_ut1
/mph/x_position
/mph/y_position
/mph/z_position
/mph/x_velocity
/mph/y_velocity
/mph/z_velocity
/mph/vector_source
/mph/utc_sbt_time
/mph/sat_binary_time
/mph/clock_step
/mph/leap_utc
/mph/leap_sign
/mph/leap_err
/mph/product_err
/mph/tot_size
/mph/sph_size
/mph/num_dsd
/mph/dsd_size
/mph/num_data_sets
/sph/sph_descriptor
/sph/intersect_start_lat
/sph/intersect_start_long
/sph/intersect_stop_lat
/sph/intersect_stop_long
/sph/sat_track
/sph/num_brc
/sph/num_meas_max_brc
/sph/num_bins_per_meas
/sph/num_prof_sca
/sph/num_prof_ica
/sph/num_prof_mca
/sph/num_group_tot
/dsd[?]/ds_name
/dsd[?]/ds_type
/dsd[?]/filename
/dsd[?]/ds_offset
/dsd[?]/ds_size
/dsd[?]/num_dsr
/dsd[?]/dsr_size
/dsd[?]/byte_order
/geolocation[?]/start_of_obs_time
/geolocation[?]/num_meas_eff
/geolocation[?]/measurement_geolocation[?]/centroid_time
/geolocation[?]/measurement_geolocation[?]/mie_geolocation_height_bin[25]/longitude_of_height_bin
/geolocation[?]/measurement_geolocation[?]/mie_geolocation_height_bin[25]/latitude_of_height_bin
/geolocation[?]/measurement_geolocation[?]/mie_geolocation_height_bin[25]/altitude_of_height_bin
/geolocation[?]/measurement_geolocation[?]/rayleigh_geolocation_height_bin[25]/longitude_of_height_bin
/geolocation[?]/measurement_geolocation[?]/rayleigh_geolocation_height_bin[25]/latitude_of_height_bin
/geolocation[?]/measurement_geolocation[?]/rayleigh_geolocation_height_bin[25]/altitude_of_height_bin
/geolocation[?]/measurement_geolocation[?]/longitude_of_dem_intersection
/geolocation[?]/measurement_geolocation[?]/latitude_of_dem_intersection
/geolocation[?]/measurement_geolocation[?]/altitude_of_dem_intersection
/geolocation[?]/geoid_separation
/meas_pcd[?]/start_of_obs_time
/meas_pcd[?]/l1b_input_screening/l1b_obs_screening
/meas_pcd[?]/l1b_input_screening/l1b_obs_screening_flags[40]
/meas_pcd[?]/l1b_input_screening/l1b_mie_meas_screening[?]/l1b_mie_meas_qc
/meas_pcd[?]/l1b_input_screening/l1b_mie_meas_screening[?]/l1b_mie_meas_qc_flags[8]
/meas_pcd[?]/l1b_input_screening/l1b_rayleigh_meas_screening[?]/l1b_rayleigh_meas_qc
/meas_pcd[?]/l1b_input_screening/l1b_rayleigh_meas_screening[?]/l1b_rayleigh_meas_qc_flags[8]
/meas_pcd[?]/l1b_cal_screening/cal_valid
/meas_pcd[?]/l2a_processing_qc/sca_applied
/meas_pcd[?]/l2a_processing_qc/ica_applied
/meas_pcd[?]/l2a_processing_qc/mca_applied
/meas_pcd[?]/l2a_processing_qc/feature_finder_indicators/layer_information[24]/bin_loaded
/meas_pcd[?]/l2a_processing_qc/feature_finder_indicators/layer_information[24]/seed[30]
/meas_pcd[?]/l2a_processing_qc/feature_finder_indicators/lowest_computable_bin[30]
/sca_pcd[?]/starttime
/sca_pcd[?]/firstmatchingbin
/sca_pcd[?]/qc_flag
/sca_pcd[?]/profile_pcd_bins[24]/extinction_variance
/sca_pcd[?]/profile_pcd_bins[24]/backscatter_variance
/sca_pcd[?]/profile_pcd_bins[24]/lod_variance
/sca_pcd[?]/profile_pcd_bins[24]/processing_qc_flag
/sca_pcd[?]/profile_pcd_mid_bins[23]/extinction_variance
/sca_pcd[?]/profile_pcd_mid_bins[23]/backscatter_variance
/sca_pcd[?]/profile_pcd_mid_bins[23]/lod_variance
/sca_pcd[?]/profile_pcd_mid_bins[23]/ber_variance
/sca_pcd[?]/profile_pcd_mid_bins[23]/processing_qc_flag
/ica_pcd[?]/starttime
/ica_pcd[?]/first_matching_bin
/ica_pcd[?]/qc_flag
/ica_pcd[?]/ica_processing_qc_flag_bin[24]
/mca_pcd[?]/starttime
/mca_pcd[?]/processing_qc_flag_bin[24]
/amd_pcd[?]/starttime
/amd_pcd[?]/l2b_amd_screening_qc
/amd_pcd[?]/l2b_amd_screening_qc_flags
/amd_pcd[?]/l2b_amd_collocations[?]/l2b_amd_collocation_qc
/amd_pcd[?]/l2b_amd_collocations[?]/l2b_amd_collocation_qc_flags
/group_pcd[?]/starttime
/group_pcd[?]/brc_start
/group_pcd[?]/measurement_start
/group_pcd[?]/brc_end
/group_pcd[?]/measurement_end
/group_pcd[?]/height_bin_index
/group_pcd[?]/upper_problem_flag
/group_pcd[?]/particle_extinction_variance
/group_pcd[?]/particle_backscatter_variance
/group_pcd[?]/particle_lod_variance
/group_pcd[?]/qc_flag
/group_pcd[?]/mid_particle_extinction_variance_top
/group_pcd[?]/mid_particle_backscatter_variance_top
/group_pcd[?]/mid_particle_lod_variance_top
/group_pcd[?]/mid_particle_ber_variance_top
/group_pcd[?]/mid_particle_extinction_variance_bot
/group_pcd[?]/mid_particle_backscatter_variance_bot
/group_pcd[?]/mid_particle_lod_variance_bot
/group_pcd[?]/mid_particle_ber_variance_bot
/sca_optical_properties[?]/starttime
/sca_optical_properties[?]/sca_optical_properties[24]/extinction
/sca_optical_properties[?]/sca_optical_properties[24]/backscatter
/sca_optical_properties[?]/sca_optical_properties[24]/lod
/sca_optical_properties[?]/sca_optical_properties[24]/sr
/sca_optical_properties[?]/geolocation_middle_bins[24]/longitude
/sca_optical_properties[?]/geolocation_middle_bins[24]/latitude
/sca_optical_properties[?]/geolocation_middle_bins[24]/altitude
/sca_optical_properties[?]/sca_optical_properties_mid_bins[23]/extinction
/sca_optical_properties[?]/sca_optical_properties_mid_bins[23]/backscatter
/sca_optical_properties[?]/sca_optical_properties_mid_bins[23]/lod
/sca_optical_properties[?]/sca_optical_properties_mid_bins[23]/ber
/ica_optical_properties[?]/starttime
/ica_optical_properties[?]/ica_optical_properties[24]/case
/ica_optical_properties[?]/ica_optical_properties[24]/extinction
/ica_optical_properties[?]/ica_optical_properties[24]/backscatter
/ica_optical_properties[?]/ica_optical_properties[24]/lod
/mca_optical_properties[?]/starttime
/mca_optical_properties[?]/mca_optical_properties[24]/climber
/mca_optical_properties[?]/mca_optical_properties[24]/extinction
/mca_optical_properties[?]/mca_optical_properties[24]/lod
/amd[?]/starttime
/amd[?]/amd_properties[24]/pressure_fp
/amd[?]/amd_properties[24]/temperature_fp
/amd[?]/amd_properties[24]/frequencyshift_fp
/amd[?]/amd_properties[24]/relativehumidity_fp
/amd[?]/amd_properties[24]/molecularlod_fp
/amd[?]/amd_properties[24]/molecularbackscatter_fp
/amd[?]/amd_properties[24]/pressure_fiz
/amd[?]/amd_properties[24]/temperature_fiz
/amd[?]/amd_properties[24]/frequencyshift_fiz
/amd[?]/amd_properties[24]/relativehumidity_fiz
/amd[?]/amd_properties[24]/molecularlod_fiz
/amd[?]/amd_properties[24]/molecularbackscatter_fiz
/group_optical_properties[?]/starttime
/group_optical_properties[?]/height_bin_index
/group_optical_properties[?]/group_optical_property/group_extinction
/group_optical_properties[?]/group_optical_property/group_backscatter
/group_optical_properties[?]/group_optical_property/group_lod
/group_optical_properties[?]/group_optical_property/group_sr
/group_optical_properties[?]/group_geolocation_middle_bins/start_longitude
/group_optical_properties[?]/group_geolocation_middle_bins/start_latitude
/group_optical_properties[?]/group_geolocation_middle_bins/start_altitude
/group_optical_properties[?]/group_geolocation_middle_bins/mid_longitude
/group_optical_properties[?]/group_geolocation_middle_bins/mid_latitude
/group_optical_properties[?]/group_geolocation_middle_bins/mid_altitude
/group_optical_properties[?]/group_geolocation_middle_bins/stop_longitude
/group_optical_properties[?]/group_geolocation_middle_bins/stop_latitude
/group_optical_properties[?]/group_geolocation_middle_bins/stop_altitude
/group_optical_properties[?]/group_optical_property_middle_bins/mid_extinction_top
/group_optical_properties[?]/group_optical_property_middle_bins/mid_backscatter_top
/group_optical_properties[?]/group_optical_property_middle_bins/mid_lod_top
/group_optical_properties[?]/group_optical_property_middle_bins/mid_ber_top
/group_optical_properties[?]/group_optical_property_middle_bins/mid_extinction_bot
/group_optical_properties[?]/group_optical_property_middle_bins/mid_backscatter_bot
/group_optical_properties[?]/group_optical_property_middle_bins/mid_lod_bot
/group_optical_properties[?]/group_optical_property_middle_bins/mid_ber_bot
/scene_classification[?]/starttime
/scene_classification[?]/height_bin_index
/scene_classification[?]/aladin_cloud_flag/clrh
/scene_classification[?]/aladin_cloud_flag/clsr
/scene_classification[?]/aladin_cloud_flag/downclber
/scene_classification[?]/aladin_cloud_flag/topclber
/scene_classification[?]/nwp_cloud_flag
/scene_classification[?]/l2a_group_class_reliability
The question mark indicates a variable size array
It is not entirely clear what we actually have to look at.
For simplicity the data of the group 'sca_optical_properties' is returned at this point
Example
-------
>>> import pyaerocom.io.read_aeolus_l2a_data
>>> obj = pyaerocom.io.read_aeolus_l2a_data.ReadAeolusL2aData(verbose=True)
>>> import os
>>> os.environ['CODA_DEFINITION']='/lustre/storeA/project/aerocom/aerocom1/ADM_CALIPSO_TEST/'
>>> filename = '/lustre/storeA/project/aerocom/aerocom1/ADM_CALIPSO_TEST/download/AE_OPER_ALD_U_N_2A_20070101T002249149_002772000_003606_0001.DBL'
>>> # read returning a ndarray
>>> filedata_numpy = obj.read_file(filename, vars_to_read=['ec550aer'], return_as='numpy')
>>> time_as_numpy_datetime64 = filedata_numpy[0,obj._TIMEINDEX].astype('datetime64[s]')
>>> print('time: {}'.format(time_as_numpy_datetime64))
>>> print('latitude: {}'.format(filedata_numpy[1,obj._LATINDEX]))
>>> # read returning a dictionary
>>> filedata = obj.read_file(filename, vars_to_read=['ec550aer'])
>>> print('time: {}'.format(filedata['time'][0].astype('datetime64[s]')))
>>> print('all latitudes of 1st time step: {}'.format(filedata['latitude'][filedata['time'][0]]))
"""
import time
import coda
# coda uses 2000-01-01T00:00:00 as epoch unfortunately.
# so calculate the difference in seconds to the Unix epoch
seconds_to_add = np.datetime64('2000-01-01T00:00:00') - np.datetime64(
'1970-01-01T00:00:00')
seconds_to_add = seconds_to_add.astype(np.float_)
# the same can be achieved using pandas, but we stick to numpy here
# base_time = pd.DatetimeIndex(['2000-01-01'])
# seconds_to_add = (base_time.view('int64') // pd.Timedelta(1, unit='s'))[0]
start = time.perf_counter()
file_data = {}
self.logger.info('reading file {}'.format(filename))
# read file
product = coda.open(filename)
if vars_to_read is None:
# read all variables
vars_to_read = list(self.DATA_COLNAMES.keys())
vars_to_read.extend(list(self.METADATA_COLNAMES.keys()))
# read data
# start with the time because it is only stored once
groups = self.TIME_PATH.split(self.GROUP_DELIMITER)
file_data[self._TIME_NAME] = coda.fetch(product, groups[0], -1,
groups[1])
# epoch is 1 January 2000 at ESA
# so add offset to move that to 1 January 1970
# and save it into a np.datetime64[ms] object
file_data[self._TIME_NAME] = \
((file_data[self._TIME_NAME] + seconds_to_add) * 1.E3).astype(np.int).astype('datetime64[ms]')
# read data in a simple dictionary
for var in vars_to_read:
groups = self._COLNAMES[var].split(self.GROUP_DELIMITER)
if len(groups) == 3:
file_data[var] = {}
for idx, key in enumerate(file_data[self._TIME_NAME]):
file_data[var][key] = coda.fetch(product, groups[0], idx,
groups[1], -1, groups[2])
elif len(groups) == 2:
file_data[var] = {}
for idx, key in enumerate(file_data[self._TIME_NAME]):
file_data[var][key] = coda.fetch(product, groups[0], -1,
groups[1])
else:
file_data[var] = {}
for idx, key in enumerate(file_data[self._TIME_NAME]):
file_data[var][key] = coda.fetch(product, groups[0])
if return_as == 'numpy':
# return as one multidimensional numpy array that can be put into self.data directly
# (column wise because the column numbers do not match)
index_pointer = 0
data = np.empty([self._ROWNO, self._COLNO], dtype=np.float_)
for idx, _time in enumerate(file_data['time'].astype(np.float_) /
1000.):
# file_data['time'].astype(np.float_) is milliseconds after the (Unix) epoch
# but we want to save the time as seconds since the epoch
for _index in range(
len(file_data['latitude'][file_data['time'][idx]])):
# this works because all variables have to have the same size
# (aka same number of height levels)
# This loop could be avoided using numpy index slicing
# do that in case we need more optimisations
data[index_pointer, self._TIMEINDEX] = _time
for var in vars_to_read:
data[index_pointer,
self.INDEX_DICT[var]] = file_data[var][
file_data['time'][idx]][_index]
# put negative values to np.nan if the variable is not a metadata variable
if data[index_pointer,
self.INDEX_DICT[var]] == self.NAN_DICT[var]:
data[index_pointer, self.INDEX_DICT[var]] = np.nan
index_pointer += 1
if index_pointer >= self._ROWNO:
# add another array chunk to self.data
data = np.append(data,
np.empty(
[self._CHUNKSIZE, self._COLNO],
dtype=np.float_),
axis=0)
self._ROWNO += self._CHUNKSIZE
# return only the needed elements...
file_data = data[0:index_pointer]
end_time = time.perf_counter()
elapsed_sec = end_time - start
temp = 'time for single file read [s]: {:.3f}'.format(elapsed_sec)
self.logger.info(temp)
# self.logger.info('{} points read'.format(index_pointer))
return file_data
# If you repeatedly need to access data from the same file then this will be orders of magnitude faster.
#
import os
# Change this to the location of your AEOLUS codadef file
os.putenv('CODA_DEFINITION', '/usr/local/share/coda/definitions')
# You can also remove this line and set the CODA_DEFINITION environment variable globally on your system
import coda
import numpy
# change this to the full path of your Aeolus L1B DBL file
filename = "AE_OPER_AUX_MET_12_20071107T090000_20071108T150000_0001.DBL"
product = coda.open(filename)
num_records = coda.fetch(product, '/sph/num_records_in_ds1')
num_layers = coda.fetch(product, '/sph/num_of_model_layers')
assert (num_records > 0 and num_layers > 0)
amd_pnom = numpy.empty([num_records, num_layers])
amd_znom = numpy.empty([num_records, num_layers])
amd_t = numpy.empty([num_records, num_layers])
amd_u = numpy.empty([num_records, num_layers])
cursor = coda.Cursor()
coda.cursor_set_product(cursor, product)
coda.cursor_goto(cursor, '/met_off_nadir[0]')
for i in range(num_records):
import coda
pf = coda.open('/data/GOME2/GOME_xxx_1B/GOME_xxx_1B_M02_20100415122955Z_20100415123255Z_N_T_20100713095647Z')
cursor = coda.Cursor()
coda.cursor_set_product(cursor, pf)
coda.cursor_goto(cursor, '/MDR')
num_mdr = coda.cursor_get_num_elements(cursor)
if num_mdr > 0:
coda.cursor_goto_first_array_element(cursor)
for i in xrange(num_mdr):
index = coda.cursor_get_available_union_field_index(cursor)
if index == 0:
# Earthshine MDR
# Note that fetching the full MDR is rather slow, since it converts
# the full MDR to a hierarchicel set of Python structures
mdr = coda.fetch(cursor, 'Earthshine')
print mdr
# If you want e.g. just the wavelength and band data of band 1b, you could use:
# wavelength = coda.fetch(cursor, 'Earthshine', 'wavelength_1b')
# rad = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'rad')
# err = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'err_rad')
# which will be much faster
elif index == 1:
# Calibration MDR
pass
elif index == 2:
# Sun MDR
mdr = coda.fetch(cursor, 'Sun')
print mdr
elif index == 3:
# Moon MDR
#
# mipl2geo.py
import sys
import coda
if __name__ == "__main__":
if len(sys.argv) < 2:
print >>sys.stderr, "Usage: %s <mipas level 2 file> ..." % sys.argv[0]
sys.exit(1)
for f in sys.argv[1:]:
pf = coda.open(f)
product_class = coda.get_product_class(pf)
product_type = coda.get_product_type(pf)
if not product_class.startswith("ENVISAT") or product_type != "MIP_NL__2P":
print >>sys.stderr, "Error: file %s is not a MIPAS Level 2 product (product class = %s, product type = %s)" % (sys.argv[1], product_class, product_type)
sys.exit(1)
print "Processing : %s" % f
cursor = coda.Cursor()
coda.cursor_set_product(cursor, pf)
coda.cursor_goto_record_field_by_name(cursor, "scan_geolocation_ads")
num_dsr = coda.cursor_get_num_elements(cursor)
def get_spectral_response(self):
"""
return 光谱波数和响应值，1维，2维
"""
k = 1.98644746103858e-9
if self.resolution == 40000:
satellite_type1 = ['METOP-A', 'METOP-B']
if self.satellite in satellite_type1:
try:
fp = coda.open(self.in_file)
wave3 = coda.fetch(fp, 'MDR', -1, 'Earthshine',
'WAVELENGTH_3')
wave4 = coda.fetch(fp, 'MDR', -1, 'Earthshine',
'WAVELENGTH_4')
lambda_smr = coda.fetch(fp, 'VIADR_SMR', -1, 'LAMBDA_SMR')
smr = coda.fetch(fp, 'VIADR_SMR', -1, 'SMR')
sunz = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH',
'SOLAR_ZENITH')
coda.close(fp)
# gome 959x4096 取后2048个点的辐射值
wavelens = self.record.wavelength[0, 2048:]
response = self.record.spectral_radiance[:, 2048:]
data_size = self.data_shape[0]
data_row = sunz.shape[0]
data_col = sunz[0].shape[1]
# gome的辐亮度计算 wave3 wave4 是30*1024
for m in xrange(data_size):
row, col = np.unravel_index(m, (data_row, data_col))
for i in xrange(2048):
if i < 1024:
response[m,
i] = response[m,
i] * k / wave3[row][i]
else:
response[m, i] = response[
m, i] * k / wave4[row][i - 1024]
# 计算太阳辐亮度
sol_wavelens = np.zeros((2048, )) # 太阳辐亮度
sol_response = np.zeros((2048, )) # 太阳辐亮度对应的波长
for i in xrange(2048):
if i < 1024:
sol_response[i] = (smr[0][2][i] *
k) / lambda_smr[0][2][i]
sol_wavelens[i] = lambda_smr[0][2][i]
else:
sol_response[i] = (smr[0][3][i - 1024] *
k) / lambda_smr[0][3][i - 1024]
sol_wavelens[i] = lambda_smr[0][3][i - 1024]
idx = np.where(response < 0)
print len(idx[0])
if len(idx[0] > 0):
response[idx] = 0.
gome_wavelens, gome_response, solar_response = self.combine_gome_band34(
wavelens, response, sol_wavelens, sol_response)
s0, s1 = gome_response.shape
gome_response = gome_response.reshape(s0, 1, s1)
print gome_response.shape, solar_response.shape
except Exception as e:
print 'Open file error {}'.format(e)
return
self.solar_response = solar_response
return gome_wavelens, gome_response
def _read(self, path, fields="all", return_header=False):
tmpdira = config.conf["main"]["tmpdir"]
tmpdirb = config.conf["main"]["tmpdirb"]
tmpdir = (tmpdira
if shutil.disk_usage(tmpdira).free > self.minspace
else tmpdirb)
with tempfile.NamedTemporaryFile(mode="wb", dir=tmpdir, delete=True) as tmpfile:
with gzip.open(str(path), "rb") as gzfile:
logging.debug("Decompressing {!s}".format(path))
gzcont = gzfile.read()
logging.debug("Writing decompressed file to {!s}".format(tmpfile.name))
tmpfile.write(gzcont)
del gzcont
# All the hard work is in coda
logging.debug("Reading {!s}".format(tmpfile.name))
cfp = coda.open(tmpfile.name)
c = coda.fetch(cfp)
logging.debug("Sorting info...")
n_scanlines = c.MPHR.TOTAL_MDR
start = datetime.datetime(*coda.time_double_to_parts_utc(c.MPHR.SENSING_START))
has_mdr = numpy.array([hasattr(m, 'MDR') for m in c.MDR],
dtype=numpy.bool)
bad = numpy.array([
(m.MDR.DEGRADED_PROC_MDR|m.MDR.DEGRADED_INST_MDR)
if hasattr(m, 'MDR') else True
for m in c.MDR],
dtype=numpy.bool)
dlt = numpy.concatenate(
[m.MDR.OnboardUTC[:, numpy.newaxis]
for m in c.MDR
if hasattr(m, 'MDR')], 1) - c.MPHR.SENSING_START
M = numpy.ma.zeros(
dtype=self._dtype,
shape=(n_scanlines, 30))
M["time"][has_mdr] = numpy.datetime64(start, "ms") + numpy.array(dlt*1e3, "m8[ms]").T
specall = self.__obtain_from_mdr(c, "GS1cSpect").astype("f8")
# apply scale factors
first = c.MDR[0].MDR.IDefNsfirst1b
last = c.MDR[0].MDR.IDefNslast1b
for (slc_st, slc_fi, fact) in zip(
filter(None, c.GIADR_ScaleFactors.IDefScaleSondNsfirst),
c.GIADR_ScaleFactors.IDefScaleSondNslast,
c.GIADR_ScaleFactors.IDefScaleSondScaleFactor):
# Documented intervals are closed [a, b]; Python uses
# half-open [a, b).
specall[..., (slc_st-first):(slc_fi-first+1)] *= pow(10.0, -fact)
M["spectral_radiance"][has_mdr] = specall
locall = self.__obtain_from_mdr(c, "GGeoSondLoc")
M["lon"][has_mdr] = locall[:, :, :, 0]
M["lat"][has_mdr] = locall[:, :, :, 1]
satangall = self.__obtain_from_mdr(c, "GGeoSondAnglesMETOP")
M["satellite_zenith_angle"][has_mdr] = satangall[:, :, :, 0]
M["satellite_azimuth_angle"][has_mdr] = satangall[:, :, :, 1]
solangall = self.__obtain_from_mdr(c, "GGeoSondAnglesSUN")
M["solar_zenith_angle"][has_mdr] = solangall[:, :, :, 0]
M["solar_azimuth_angle"][has_mdr] = solangall[:, :, :, 1]
for fld in M.dtype.names:
M.mask[fld][~has_mdr, ...] = True
M.mask[fld][bad, ...] = True
m = c.MDR[0].MDR
wavenumber = (m.IDefSpectDWn1b * numpy.arange(m.IDefNsfirst1b, m.IDefNslast1b+0.1) * (1/ureg.metre))
if self.wavenumber is None:
self.wavenumber = wavenumber
elif abs(self.wavenumber - wavenumber).max() > (0.05 * 1/(ureg.centimetre)):
raise ValueError("Inconsistent wavenumbers")
return M
def read(self,
vars_to_retrieve=None,
files=[],
first_file=None,
last_file=None,
file_pattern=None,
list_coda_paths=False,
local_temp_dir=None,
return_as='numpy',
apply_quality_flag=0.0):
"""Method that reads list of files as instance of :class:`UngriddedData`
Parameters
----------
vars_to_retrieve : :obj:`list` or similar, optional,
list containing variable IDs that are supposed to be read. If None,
all variables in :attr:`PROVIDES_VARIABLES` are loaded
files : :obj:`list`, optional
list of files to be read. If None, then the file list is used that
is returned on :func:`get_file_list`.
first_file : :obj:`int`, optional
index of first file in file list to read. If None, the very first
file in the list is used. Note: is ignored if input parameter
`file_pattern` is specified.
last_file : :obj:`int`, optional
index of last file in list to read. If None, the very last file
in the list is used. Note: is ignored if input parameter
`file_pattern` is specified.
file_pattern : str, optional
string pattern for file search (cf :func:`get_file_list`)
:param local_temp_dir:
Returns
-------
UngriddedData
data object
Example:
>>> import pyaerocom as pya
>>> obj = pya.io.read_aeolus_l2a_data.ReadL2Data()
>>> testfiles = []
>>> testfiles.append('/lustre/storeB/project/fou/kl/admaeolus/data.rev.2A02/download/2018-12/01/AE_OPER_ALD_U_N_2A_20181201T033526026_005423993_001590_0001.TGZ')
>>> data=obj.read(files=testfiles)
>>> data=obj.read(files=testfiles, vars_to_retrieve='ec355aer')
"""
import pathlib
import tarfile
import os
import coda
if local_temp_dir is None:
local_temp_dir = self.LOCAL_TMP_DIR
if vars_to_retrieve is None:
vars_to_retrieve = self.DEFAULT_VARS
elif isinstance(vars_to_retrieve, str):
vars_to_retrieve = [vars_to_retrieve]
if files is None:
if len(self.files) == 0:
self.get_file_list(pattern=file_pattern)
files = self.files
if file_pattern is None:
if first_file is None:
first_file = 0
if last_file is None:
last_file = len(files)
files = files[first_file:last_file]
self.read_failed = []
temp_files = {}
data_obj = UngriddedData(num_points=self._CHUNKSIZE)
meta_key = 0.0
idx = 0
# check if the supplied file is a supported archive file (tar in this case)
# and extract the files with supported suffixes to const.CACHEDIR
non_archive_files = []
for idx, _file in enumerate(sorted(files)):
# temp = 'reading file: {}'.format(_file)
self.logger.info('file: {}'.format(_file))
suffix = pathlib.Path(_file).suffix
if suffix in self.SUPPORTED_ARCHIVE_SUFFIXES:
temp = 'opening archive file; using {} as temp dir.'.format(
local_temp_dir)
self.logger.info(temp)
# untar archive files first
tarhandle = tarfile.open(_file)
files_in_tar = tarhandle.getnames()
for file_in_tar in files_in_tar:
if pathlib.Path(
file_in_tar).suffix in self.SUPPORTED_SUFFIXES:
# extract file to tmp path
member = tarhandle.getmember(file_in_tar)
temp = 'extracting file {}...'.format(member.name)
self.logger.info(temp)
tarhandle.extract(member,
path=local_temp_dir,
set_attrs=False)
extract_file = os.path.join(local_temp_dir,
member.name)
non_archive_files.append(extract_file)
temp_files[extract_file] = True
tarhandle.close()
else:
non_archive_files.append(_file)
for idx, _file in enumerate(sorted(non_archive_files)):
# list coda data paths in the 1st file in case the user asked for that
if idx == 0 and list_coda_paths:
pass
coda_handle = coda.open(_file)
root_field_names = coda.get_field_names(coda_handle)
for field in root_field_names:
print(field)
coda.close(coda_handle)
data_obj = None
return data_obj
file_data = self.read_file(_file,
vars_to_retrieve=vars_to_retrieve,
loglevel=logging.INFO,
return_as=return_as)
if return_as == 'numpy':
self.logger.info('{} points read'.format(file_data.shape[0]))
# the metadata dict is left empty for L2 data
# the location in the data set is time step dependant!
if idx == 0:
data_obj._data = file_data
else:
data_obj._data = np.append(data_obj._data,
file_data,
axis=0)
data_obj._idx = data_obj._data.shape[0] + 1
file_data = None
# remove file if it was temporary one
if _file in temp_files:
os.remove(_file)
# pass
# tmp_obj = UngriddedData()
# tmp_obj._data = file_data
# tmp_obj._idx = data_obj._data.shape[0] + 1
# data_obj.append(tmp_obj)
self.logger.info(
'size of data object: {}'.format(data_obj._idx - 1))
elif return_as == 'dict':
if idx == 0:
data_obj._data = {}
shape_store = {}
index_store = {}
file_start_index_arr = [0]
# apply quality flags
if apply_quality_flag > 0.:
qflags = file_data[self._QANAME]
keep_indexes = np.where(qflags >= apply_quality_flag)
elements_to_add = keep_indexes.size
else:
keep_indexes = np.arange(0,
len(file_data[self._QANAME]))
elements_to_add = file_data[self._QANAME].shape[0]
for _key in file_data:
# print('key: {}'.format(_key))
shape_store[_key] = file_data[_key].shape
index_store[_key] = file_data[_key].shape[0]
input_shape = list(file_data[_key].shape)
input_shape[0] = self._ROWNO
data_obj._data[_key] = np.empty(input_shape,
dtype=np.float_)
if len(input_shape) == 1:
data_obj._data[_key][0:file_data[_key].
shape[0]] = file_data[_key]
elif len(input_shape) == 2:
data_obj._data[_key][0:file_data[_key].
shape[0], :] = file_data[_key]
elif len(input_shape) == 3:
data_obj._data[_key][
0:file_data[_key].
shape[0], :, :] = file_data[_key]
elif len(input_shape) == 4:
data_obj._data[_key][
0:file_data[_key].
shape[0], :, :, :] = file_data[_key]
else:
pass
# 2nd + file
else:
if apply_quality_flag > 0.:
qflags = file_data[self._QANAME]
keep_indexes = np.where(qflags >= apply_quality_flag)
elements_to_add = keep_indexes.size
file_start_index_arr.append(
file_data[self.TSSIZENAME].shape[0])
for _key in file_data:
if _key in self.STATICFIELDNAMES:
print('key: {}'.format(_key))
continue
# shape_store[_key] = file_data[_key].shape
elements_to_add = file_data[_key].shape[0]
# extend data_obj._data[_key] if necessary
if index_store[_key] + elements_to_add > data_obj._data[
_key].shape[0]:
current_shape = list(data_obj._data[_key].shape)
current_shape[
0] = current_shape[0] + self._CHUNKSIZE
tmp_data = np.empty(current_shape, dtype=np.float_)
if len(current_shape) == 1:
tmp_data[0:data_obj._data[_key].
shape[0]] = data_obj._data[_key]
elif len(current_shape) == 2:
tmp_data[0:data_obj._data[_key].
shape[0], :] = data_obj._data[_key]
elif len(current_shape) == 3:
tmp_data[0:data_obj._data[_key].
shape[0], :, :] = data_obj._data[_key]
elif len(current_shape) == 4:
tmp_data[
0:data_obj._data[_key].
shape[0], :, :, :] = data_obj._data[_key]
else:
pass
input_shape = list(file_data[_key].shape)
if len(input_shape) == 1:
data_obj._data[_key][
index_store[_key]:index_store[_key] +
file_data[_key].shape[0]] = file_data[_key]
elif len(input_shape) == 2:
data_obj._data[_key][
index_store[_key]:index_store[_key] +
file_data[_key].shape[0], :] = file_data[_key]
elif len(input_shape) == 3:
data_obj._data[_key][
index_store[_key]:index_store[_key] +
file_data[_key].
shape[0], :, :] = file_data[_key]
elif len(input_shape) == 4:
data_obj._data[_key][
index_store[_key]:index_store[_key] +
file_data[_key].
shape[0], :, :, :] = file_data[_key]
else:
pass
index_store[_key] += elements_to_add
file_data = None
# remove file if it was temporary one
if _file in temp_files:
os.remove(_file)
else:
pass
# now shorten the data dict to the necessary size
if return_as == 'dict':
for _key in data_obj._data:
data_obj._data[_key] = data_obj._data[_key][:index_store[_key]]
data_obj._data['file_indexes'] = file_start_index_arr
# apply the quality flags
if apply_quality_flag > 0.:
pass
return data_obj
import coda
pf = coda.open(
'/data/GOME2/GOME_xxx_1B/GOME_xxx_1B_M02_20100415122955Z_20100415123255Z_N_T_20100713095647Z'
)
cursor = coda.Cursor()
coda.cursor_set_product(cursor, pf)
coda.cursor_goto(cursor, '/MDR')
num_mdr = coda.cursor_get_num_elements(cursor)
if num_mdr > 0:
coda.cursor_goto_first_array_element(cursor)
for i in xrange(num_mdr):
index = coda.cursor_get_available_union_field_index(cursor)
if index == 0:
# Earthshine MDR
# Note that fetching the full MDR is rather slow, since it converts
# the full MDR to a hierarchicel set of Python structures
mdr = coda.fetch(cursor, 'Earthshine')
print mdr
# If you want e.g. just the wavelength and band data of band 1b, you could use:
# wavelength = coda.fetch(cursor, 'Earthshine', 'wavelength_1b')
# rad = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'rad')
# err = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'err_rad')
# which will be much faster
elif index == 1:
# Calibration MDR
pass
elif index == 2:
# Sun MDR
mdr = coda.fetch(cursor, 'Sun')
print mdr
def coda_open(filename):
coda_handle = coda.open(filename)
try:
yield coda_handle
finally:
coda.close(coda_handle)
