def set_file_attr(self): """ seft self.file_att is dict """ data = dict() if self.resolution == 40000: satellite_type1 = ['METOP-A', 'METOP-B'] if self.satellite in satellite_type1: try: fp = coda.open(self.in_file) product_class = coda.get_product_class(fp) product_type = coda.get_product_type(fp) product_version = coda.get_product_version(fp) product_format = coda.get_product_format(fp) product_size = coda.get_product_file_size(fp) coda.close(fp) except Exception as e: print str(e) return data['class'] = product_class data['size'] = product_size data['type'] = product_type data['version'] = product_version data['format'] = product_format self.file_attr = data else: raise ValueError('Cant read this satellite`s data.: {}'.format( self.satellite))
def read_gome(self, infile): # 打开gome文件 try: fp = coda.open(infile) except Exception, e: print 'Open file error<%s> .' % (e) return
def _read(self, path, fields="all", return_header=False): tmpdira = config.conf["main"]["tmpdir"] tmpdirb = config.conf["main"]["tmpdirb"] tmpdir = (tmpdira if shutil.disk_usage(tmpdira).free > self.minspace else tmpdirb) with tempfile.NamedTemporaryFile(mode="wb", dir=tmpdir, delete=True) as tmpfile: with gzip.open(str(path), "rb") as gzfile: logging.debug("Decompressing {!s}".format(path)) gzcont = gzfile.read() logging.debug("Writing decompressed file to {!s}".format(tmpfile.name)) tmpfile.write(gzcont) del gzcont # All the hard work is in coda logging.debug("Reading {!s}".format(tmpfile.name)) cfp = coda.open(tmpfile.name) c = coda.fetch(cfp) logging.debug("Sorting info...") n_scanlines = c.MPHR.TOTAL_MDR start = datetime.datetime(*coda.time_double_to_parts_utc(c.MPHR.SENSING_START)) has_mdr = numpy.array([hasattr(m, 'MDR') for m in c.MDR], dtype=numpy.bool) bad = numpy.array([ (m.MDR.DEGRADED_PROC_MDR|m.MDR.DEGRADED_INST_MDR) if hasattr(m, 'MDR') else True for m in c.MDR], dtype=numpy.bool) dlt = numpy.concatenate( [m.MDR.OnboardUTC[:, numpy.newaxis] for m in c.MDR if hasattr(m, 'MDR')], 1) - c.MPHR.SENSING_START M = numpy.ma.zeros( dtype=self._dtype, shape=(n_scanlines, 30)) M["time"][has_mdr] = numpy.datetime64(start, "ms") + numpy.array(dlt*1e3, "m8[ms]").T specall = self.__obtain_from_mdr(c, "GS1cSpect") M["spectral_radiance"][has_mdr] = specall locall = self.__obtain_from_mdr(c, "GGeoSondLoc") M["lon"][has_mdr] = locall[:, :, :, 0] M["lat"][has_mdr] = locall[:, :, :, 1] satangall = self.__obtain_from_mdr(c, "GGeoSondAnglesMETOP") M["satellite_zenith_angle"][has_mdr] = satangall[:, :, :, 0] M["satellite_azimuth_angle"][has_mdr] = satangall[:, :, :, 1] solangall = self.__obtain_from_mdr(c, "GGeoSondAnglesSUN") M["solar_zenith_angle"][has_mdr] = solangall[:, :, :, 0] M["solar_azimuth_angle"][has_mdr] = solangall[:, :, :, 1] for fld in M.dtype.names: M.mask[fld][~has_mdr, ...] = True M.mask[fld][bad, ...] = True m = c.MDR[0].MDR wavenumber = (m.IDefSpectDWn1b * numpy.arange(m.IDefNsfirst1b, m.IDefNslast1b+0.1) * (1/ureg.metre)) if self.wavenumber is None: self.wavenumber = wavenumber elif abs(self.wavenumber - wavenumber).max() > (0.05 * 1/(ureg.centimetre)): raise ValueError("Inconsistent wavenumbers") return M
def get_solar_zenith(self): """ 获取太阳天顶角 :return: """ fp = coda.open(self.in_file) angles = coda.fetch(fp, 'MDR', -1, 'MDR', 'GGeoSondAnglesSUN') zenith = np.array([]) for i in angles: z = i.reshape(-1)[0::2] zenith = np.append(zenith, z) return zenith
def Load(self, L1File): print u'读取 LEO所有数据信息......' if not os.path.isfile(L1File): print 'Error: %s not found' % L1File sys.exit(1) try: fp = coda.open(L1File) except Exception, e: print 'Open file error<%s> .' % (e) return
def set_file_attr(self): """ seft self.file_att is dict """ data = dict() if self.resolution == 24000: satellite_type1 = ['METOP-A', 'METOP-B'] if self.satellite in satellite_type1: try: fp = coda.open(self.in_file) product_class = coda.get_product_class(fp) product_type = coda.get_product_type(fp) product_version = coda.get_product_version(fp) product_format = coda.get_product_format(fp) product_size = coda.get_product_file_size(fp) coda.close(fp) except Exception as e: print str(e) return data['class'] = product_class data['size'] = product_size data['type'] = product_type data['version'] = product_version data['format'] = product_format self.file_attr = data else: raise ValueError('Cant read this satellite`s data.: {}'.format( self.satellite)) elif self.resolution == 24001: satellite_type1 = ['METOP-A', 'METOP-B'] if self.satellite in satellite_type1: try: # 'NCETCDF4' ncr = Dataset(self.in_file, 'r', format='NETCDF3_CLASSIC') data = ncr.ncattrs() ncr.close() except Exception as e: print str(e) return self.file_attr = data else: raise ValueError('Cant read this satellite`s data.: {}'.format( self.satellite)) else: raise ValueError("Cant handle this resolution: ".format( self.resolution))
def get_sensor_azimuth(self): """ return sensor_azimuth """ if self.resolution == 40000: satellite_type1 = ['METOP-A', 'METOP-B'] if self.satellite in satellite_type1: try: fp = coda.open(self.in_file) angle = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'SAT_AZIMUTH') coda.close(fp) # angle = 30*3*32 = 1440, 取30*1*32 = 960, 取前959个 data_size = self.data_shape[0] data_row = angle.shape[0] data_col = angle[0].shape[1] data_pre = np.full(self.data_shape, -999.) for i in xrange(data_size): row, col = np.unravel_index(i, (data_row, data_col)) # print row, col, angle[row][1][col] data_pre[i] = angle[row][1][col] # 过滤无效值 invalid_index = np.logical_or(data_pre < 0, data_pre > 360) data_pre = data_pre.astype(np.float32) data_pre[invalid_index] = np.nan data = data_pre except Exception as e: print 'Open file error {}'.format(e) return else: raise ValueError('Cant read this satellite`s data.: {}'.format( self.satellite)) else: raise ValueError( 'Cant read this data, please check its resolution: {}'.format( self.in_file)) return data
def get_footprint(product): try: import coda except ImportError: return None path = "/METADATA/EOP_METADATA/om_featureOfInterest/eop_multiExtentOf/gml_surfaceMembers/gml_exterior@gml_posList" pf = coda.open(product) try: coord = coda.fetch(pf, path).split(' ') except coda.CodacError: return None finally: coda.close(pf) if len(coord) % 2 != 0: return None return Polygon([ LinearRing([ Point(float(lon), float(lat)) for lat, lon in zip(coord[0::2], coord[1::2]) ]) ])
def get_solar_zenith(self): """ return solar_zenith """ if self.resolution == 24000: satellite_type1 = ['METOP-A', 'METOP-B'] if self.satellite in satellite_type1: try: fp = coda.open(self.in_file) angle = coda.fetch(fp, 'MDR', -1, 'MDR', 'GGeoSondAnglesSUN') coda.close(fp) all_angle = [] for i in xrange(len(angle)): tmp_angle = angle[i].reshape(-1) if len(all_angle) == 0: all_angle = tmp_angle else: all_angle = np.concatenate((all_angle, tmp_angle)) s0 = self.data_shape[0] # 开始间隔一个取一个值,取偶数位 data_pre = (all_angle[0::2]).reshape(s0, 1) # 过滤无效值 invalid_index = np.logical_or(data_pre < 0, data_pre > 180) data_pre = data_pre.astype(np.float32) data_pre[invalid_index] = np.nan data = data_pre except Exception as e: print 'Open file error {}'.format(e) return else: raise ValueError('Cant read this satellite`s data.: {}'.format( self.satellite)) elif self.resolution == 24001: satellite_type1 = ['METOP-A', 'METOP-B'] if self.satellite in satellite_type1: try: ncr = Dataset(self.in_file, 'r', format='NETCDF3_CLASSIC') data_pre = ncr.variables['solar_zenith_angle'][:] ncr.close() # 过滤无效值 invalid_index = np.logical_or(data_pre < 0, data_pre > 180) data_pre = data_pre.astype(np.float32) data_pre[invalid_index] = np.nan data = data_pre.reshape(self.data_shape) except Exception as e: print 'Open file error {}'.format(e) return else: raise ValueError('Cant read this satellite`s data.: {}'.format( self.satellite)) else: raise ValueError( 'Cant read this data, please check its resolution: {}'.format( self.in_file)) return data
# default libRadtran input file text to be added to inp_file = open('default.inp', 'r') inp_default = inp_file.read() inp_file.close() # path and EPS file to read if len(sys.argv) != 2: text = '1 /path/file argument required and %d given.' % len(sys.argv) - 1 raise IOError(text) e_fn = sys.argv[1] if not (os.path.exists(e_fn)): text = 'The file %s does not exist.' % e_fn raise IOError(text) #e_fn = '/home/malapradej/Documents/PhD_UCL/Data/GOME-2/l1b/2007/01/GOME_xxx_1B_M02_20070125124939Z_20070125143057Z_R_O_20120209112017Z' # opening the file ef = coda.open(e_fn) # creates a base name without extension used in rest of code base_fn = os.path.splitext(e_fn)[0] # removes any extension from path base_fn = os.path.basename(base_fn) # selects only filename # path to DASF data to be saved path_dasf = '/home/ucfajrm/DATA/DASF/Data/DASF' if not (os.path.exists(path_dasf)): text = 'The path %s does not exist.' % path_dasf raise IOError(text) # leaf albedo file generated from PROSPECT # typical leaf level parameters generated based on metadata-analysis # see spreadsheet in PROSPECT folder l_lower = 580. # limits for plotting purposes
#!/usr/bin/env python # # mipl2geo.py import sys import coda if __name__ == "__main__": if len(sys.argv) < 2: print >> sys.stderr, "Usage: %s <mipas level 2 file> ..." % sys.argv[0] sys.exit(1) for f in sys.argv[1:]: pf = coda.open(f) product_class = coda.get_product_class(pf) product_type = coda.get_product_type(pf) if not product_class.startswith( "ENVISAT") or product_type != "MIP_NL__2P": print >> sys.stderr, "Error: file %s is not a MIPAS Level 2 product (product class = %s, product type = %s)" % ( sys.argv[1], product_class, product_type) sys.exit(1) print "Processing : %s" % f cursor = coda.Cursor() coda.cursor_set_product(cursor, pf) coda.cursor_goto_record_field_by_name(cursor, "scan_geolocation_ads")
def read(self, vars_to_retrieve=None, files=[], first_file=None, last_file=None, file_pattern=None, list_coda_paths=False, local_temp_dir=None): """Method that reads list of files as instance of :class:`UngriddedData` Parameters ---------- vars_to_retrieve : :obj:`list` or similar, optional, list containing variable IDs that are supposed to be read. If None, all variables in :attr:`PROVIDES_VARIABLES` are loaded files : :obj:`list`, optional list of files to be read. If None, then the file list is used that is returned on :func:`get_file_list`. first_file : :obj:`int`, optional index of first file in file list to read. If None, the very first file in the list is used. Note: is ignored if input parameter `file_pattern` is specified. last_file : :obj:`int`, optional index of last file in list to read. If None, the very last file in the list is used. Note: is ignored if input parameter `file_pattern` is specified. file_pattern : str, optional string pattern for file search (cf :func:`get_file_list`) :param local_temp_dir: Returns ------- UngriddedData data object Example: >>> import pyaerocom as pya >>> obj = pya.io.read_aeolus_l2a_data.ReadL2Data() >>> testfiles = [] >>> testfiles.append('/lustre/storeB/project/fou/kl/admaeolus/data.rev.2A02/download/2018-12/01/AE_OPER_ALD_U_N_2A_20181201T033526026_005423993_001590_0001.TGZ') >>> data=obj.read(files=testfiles) >>> data=obj.read(files=testfiles, vars_to_retrieve='ec355aer') """ import pathlib import tarfile import os import coda if local_temp_dir is None: local_temp_dir = self.LOCAL_TMP_DIR if vars_to_retrieve is None: vars_to_retrieve = self.DEFAULT_VARS elif isinstance(vars_to_retrieve, str): vars_to_retrieve = [vars_to_retrieve] if files is None: if len(self.files) == 0: self.get_file_list(pattern=file_pattern) files = self.files if file_pattern is None: if first_file is None: first_file = 0 if last_file is None: last_file = len(files) files = files[first_file:last_file] self.read_failed = [] temp_files = {} data_obj = UngriddedData(num_points=self._COLNO, chunksize=self._CHUNKSIZE) meta_key = 0.0 idx = 0 # check if the supplied file is a supported archive file (tar in this case) # and extract the files with supported suffixes to const._cachedir non_archive_files = [] for idx, _file in enumerate(sorted(files)): # temp = 'reading file: {}'.format(_file) self.logger.info('file: {}'.format(_file)) suffix = pathlib.Path(_file).suffix if suffix in self.SUPPORTED_ARCHIVE_SUFFIXES: temp = 'opening archive file; using {} as temp dir.'.format(local_temp_dir) self.logger.info(temp) # untar archive files first tarhandle = tarfile.open(_file) files_in_tar = tarhandle.getnames() for file_in_tar in files_in_tar: if pathlib.Path(file_in_tar).suffix in self.SUPPORTED_SUFFIXES: # extract file to tmp path member = tarhandle.getmember(file_in_tar) temp = 'extracting file {}...'.format(member.name) self.logger.info(temp) tarhandle.extract(member, path=local_temp_dir, set_attrs=False) extract_file = os.path.join(local_temp_dir, member.name) non_archive_files.append(extract_file) temp_files[extract_file] = True tarhandle.close() else: non_archive_files.append(_file) for idx, _file in enumerate(sorted(non_archive_files)): # list coda data paths in the 1st file in case the user asked for that if idx == 0 and list_coda_paths: pass coda_handle = coda.open(_file) root_field_names = coda.get_field_names(coda_handle) for field in root_field_names: print(field) coda.close(coda_handle) data_obj = None return data_obj file_data = self.read_file(_file, vars_to_retrieve=vars_to_retrieve, loglevel=logging.INFO, return_as='numpy') self.logger.info('{} points read'.format(file_data.shape[0])) # the metadata dict is left empty for L2 data # the location in the data set is time step dependant! if idx == 0: data_obj._data = file_data else: data_obj._data = np.append(data_obj._data, file_data, axis=0) data_obj._idx = data_obj._data.shape[0] + 1 file_data = None # remove file if it was temporary one if _file in temp_files: os.remove(_file) # pass # tmp_obj = UngriddedData() # tmp_obj._data = file_data # tmp_obj._idx = data_obj._data.shape[0] + 1 # data_obj.append(tmp_obj) self.logger.info('size of data object: {}'.format(data_obj._idx - 1)) return data_obj
if i < num_fields - 1: coda.cursor_goto_next_record_field(cursor) coda.cursor_goto_parent(cursor) if __name__ == "__main__": if len(sys.argv) != 2: print >> sys.stderr, "Usage: %s <envisat file>" % sys.argv[0] sys.exit(1) coda.set_option_perform_conversions(0) pf = coda.open(sys.argv[1]) product_class = coda.get_product_class(pf) if not product_class.startswith("ENVISAT"): print >>sys.stderr, "Error: file %s is not an ENVISAT product file (product class = %s)" % (sys.argv[1], product_class) sys.exit(1) cursor = coda.Cursor() coda.cursor_set_product(cursor, pf) print " MPH :" coda.cursor_goto_record_field_by_name(cursor, "mph") print_record(cursor) coda.cursor_goto_parent(cursor) print " SPH :"; coda.cursor_goto_record_field_by_name(cursor, "sph")
import os # Change this to the location of your AEOLUS codadef file os.putenv('CODA_DEFINITION', '/usr/local/share/coda/definitions') # You can also remove this line and set the CODA_DEFINITION environment variable globally on your system import coda from numpy import hstack, vstack # change this to the full path of your Aeolus L1B DBL file filename = "/path/to/AE_OPER_ALD_U_N_1B_20151002T001857059_005787000_046339_0001.DBL" product = coda.open(filename) ### Observation profiles ### # Observation profiles provide a single profile per BRC. # Reading all values will therefore return 'an array of arrays' (we will get an array for each '-1' in the coda.fetch()). # The first array is the list of BRCs, and the second array the list of points in the profile. # This array of arrays can be turned into a single 2D numpy array of shape [num_brc, num_vertical] by using 'vstack'. # Mie observation wind profiles print("Mie observation wind profiles") latitude = coda.fetch(product, 'geolocation', -1, 'observation_geolocation/observation_mie_geolocation', -1, 'latitude_of_height_bin') latitude = vstack(latitude) longitude = coda.fetch(product, 'geolocation', -1, 'observation_geolocation/observation_mie_geolocation', -1, 'longitude_of_height_bin') longitude = vstack(longitude)
def read_file(self, filename, vars_to_read=None, return_as='dict', loglevel=None): """method to read an ESA binary data file entirely Parameters ---------- filename : str absolute path to filename to read vars_to_read : list list of str with variable names to read; defaults to ['od550aer'] verbose : Bool set to True to increase verbosity Returns -------- Either: dictionary (default): keys are 'time', 'latitude', 'longitude', 'altitude' and the variable names 'ec550aer', 'bs550aer', 'sr', 'lod' if the whole file is read 'time' is a 1d array, while the other dict values are a another dict with the time as keys (the same ret['time']) and a numpy array as values. These values represent the profile. Note 1: latitude and longitude are height dependent due to the tilt of the measurement. Note 2: negative values indicate a NaN 2d ndarray of type float: representing a 'point cloud' with all points column 1: time in seconds since the Unix epoch with ms accuracy (same time for every height in a profile) column 2: latitude column 3: longitude column 4: altitude column 5: extinction column 6: backscatter column 7: sr column 8: lod Note: negative values are put to np.nan already The indexes are noted in pyaerocom.io.read_aeolus_l2a_data.ReadAeolusL2aData.<index_name> e.g. the time index is named pyaerocom.io.read_aeolus_l2a_data.ReadAeolusL2aData._TIMEINDEX have a look at the example to access the values This is whats in one DBL file codadump list /lustre/storeA/project/aerocom/aerocom1/ADM_CALIPSO_TEST/download/AE_OPER_ALD_U_N_2A_20070101T002249149_002772000_003606_0001.DBL /mph/product /mph/proc_stage /mph/ref_doc /mph/acquisition_station /mph/proc_center /mph/proc_time /mph/software_ver /mph/baseline /mph/sensing_start /mph/sensing_stop /mph/phase /mph/cycle /mph/rel_orbit /mph/abs_orbit /mph/state_vector_time /mph/delta_ut1 /mph/x_position /mph/y_position /mph/z_position /mph/x_velocity /mph/y_velocity /mph/z_velocity /mph/vector_source /mph/utc_sbt_time /mph/sat_binary_time /mph/clock_step /mph/leap_utc /mph/leap_sign /mph/leap_err /mph/product_err /mph/tot_size /mph/sph_size /mph/num_dsd /mph/dsd_size /mph/num_data_sets /sph/sph_descriptor /sph/intersect_start_lat /sph/intersect_start_long /sph/intersect_stop_lat /sph/intersect_stop_long /sph/sat_track /sph/num_brc /sph/num_meas_max_brc /sph/num_bins_per_meas /sph/num_prof_sca /sph/num_prof_ica /sph/num_prof_mca /sph/num_group_tot /dsd[?]/ds_name /dsd[?]/ds_type /dsd[?]/filename /dsd[?]/ds_offset /dsd[?]/ds_size /dsd[?]/num_dsr /dsd[?]/dsr_size /dsd[?]/byte_order /geolocation[?]/start_of_obs_time /geolocation[?]/num_meas_eff /geolocation[?]/measurement_geolocation[?]/centroid_time /geolocation[?]/measurement_geolocation[?]/mie_geolocation_height_bin[25]/longitude_of_height_bin /geolocation[?]/measurement_geolocation[?]/mie_geolocation_height_bin[25]/latitude_of_height_bin /geolocation[?]/measurement_geolocation[?]/mie_geolocation_height_bin[25]/altitude_of_height_bin /geolocation[?]/measurement_geolocation[?]/rayleigh_geolocation_height_bin[25]/longitude_of_height_bin /geolocation[?]/measurement_geolocation[?]/rayleigh_geolocation_height_bin[25]/latitude_of_height_bin /geolocation[?]/measurement_geolocation[?]/rayleigh_geolocation_height_bin[25]/altitude_of_height_bin /geolocation[?]/measurement_geolocation[?]/longitude_of_dem_intersection /geolocation[?]/measurement_geolocation[?]/latitude_of_dem_intersection /geolocation[?]/measurement_geolocation[?]/altitude_of_dem_intersection /geolocation[?]/geoid_separation /meas_pcd[?]/start_of_obs_time /meas_pcd[?]/l1b_input_screening/l1b_obs_screening /meas_pcd[?]/l1b_input_screening/l1b_obs_screening_flags[40] /meas_pcd[?]/l1b_input_screening/l1b_mie_meas_screening[?]/l1b_mie_meas_qc /meas_pcd[?]/l1b_input_screening/l1b_mie_meas_screening[?]/l1b_mie_meas_qc_flags[8] /meas_pcd[?]/l1b_input_screening/l1b_rayleigh_meas_screening[?]/l1b_rayleigh_meas_qc /meas_pcd[?]/l1b_input_screening/l1b_rayleigh_meas_screening[?]/l1b_rayleigh_meas_qc_flags[8] /meas_pcd[?]/l1b_cal_screening/cal_valid /meas_pcd[?]/l2a_processing_qc/sca_applied /meas_pcd[?]/l2a_processing_qc/ica_applied /meas_pcd[?]/l2a_processing_qc/mca_applied /meas_pcd[?]/l2a_processing_qc/feature_finder_indicators/layer_information[24]/bin_loaded /meas_pcd[?]/l2a_processing_qc/feature_finder_indicators/layer_information[24]/seed[30] /meas_pcd[?]/l2a_processing_qc/feature_finder_indicators/lowest_computable_bin[30] /sca_pcd[?]/starttime /sca_pcd[?]/firstmatchingbin /sca_pcd[?]/qc_flag /sca_pcd[?]/profile_pcd_bins[24]/extinction_variance /sca_pcd[?]/profile_pcd_bins[24]/backscatter_variance /sca_pcd[?]/profile_pcd_bins[24]/lod_variance /sca_pcd[?]/profile_pcd_bins[24]/processing_qc_flag /sca_pcd[?]/profile_pcd_mid_bins[23]/extinction_variance /sca_pcd[?]/profile_pcd_mid_bins[23]/backscatter_variance /sca_pcd[?]/profile_pcd_mid_bins[23]/lod_variance /sca_pcd[?]/profile_pcd_mid_bins[23]/ber_variance /sca_pcd[?]/profile_pcd_mid_bins[23]/processing_qc_flag /ica_pcd[?]/starttime /ica_pcd[?]/first_matching_bin /ica_pcd[?]/qc_flag /ica_pcd[?]/ica_processing_qc_flag_bin[24] /mca_pcd[?]/starttime /mca_pcd[?]/processing_qc_flag_bin[24] /amd_pcd[?]/starttime /amd_pcd[?]/l2b_amd_screening_qc /amd_pcd[?]/l2b_amd_screening_qc_flags /amd_pcd[?]/l2b_amd_collocations[?]/l2b_amd_collocation_qc /amd_pcd[?]/l2b_amd_collocations[?]/l2b_amd_collocation_qc_flags /group_pcd[?]/starttime /group_pcd[?]/brc_start /group_pcd[?]/measurement_start /group_pcd[?]/brc_end /group_pcd[?]/measurement_end /group_pcd[?]/height_bin_index /group_pcd[?]/upper_problem_flag /group_pcd[?]/particle_extinction_variance /group_pcd[?]/particle_backscatter_variance /group_pcd[?]/particle_lod_variance /group_pcd[?]/qc_flag /group_pcd[?]/mid_particle_extinction_variance_top /group_pcd[?]/mid_particle_backscatter_variance_top /group_pcd[?]/mid_particle_lod_variance_top /group_pcd[?]/mid_particle_ber_variance_top /group_pcd[?]/mid_particle_extinction_variance_bot /group_pcd[?]/mid_particle_backscatter_variance_bot /group_pcd[?]/mid_particle_lod_variance_bot /group_pcd[?]/mid_particle_ber_variance_bot /sca_optical_properties[?]/starttime /sca_optical_properties[?]/sca_optical_properties[24]/extinction /sca_optical_properties[?]/sca_optical_properties[24]/backscatter /sca_optical_properties[?]/sca_optical_properties[24]/lod /sca_optical_properties[?]/sca_optical_properties[24]/sr /sca_optical_properties[?]/geolocation_middle_bins[24]/longitude /sca_optical_properties[?]/geolocation_middle_bins[24]/latitude /sca_optical_properties[?]/geolocation_middle_bins[24]/altitude /sca_optical_properties[?]/sca_optical_properties_mid_bins[23]/extinction /sca_optical_properties[?]/sca_optical_properties_mid_bins[23]/backscatter /sca_optical_properties[?]/sca_optical_properties_mid_bins[23]/lod /sca_optical_properties[?]/sca_optical_properties_mid_bins[23]/ber /ica_optical_properties[?]/starttime /ica_optical_properties[?]/ica_optical_properties[24]/case /ica_optical_properties[?]/ica_optical_properties[24]/extinction /ica_optical_properties[?]/ica_optical_properties[24]/backscatter /ica_optical_properties[?]/ica_optical_properties[24]/lod /mca_optical_properties[?]/starttime /mca_optical_properties[?]/mca_optical_properties[24]/climber /mca_optical_properties[?]/mca_optical_properties[24]/extinction /mca_optical_properties[?]/mca_optical_properties[24]/lod /amd[?]/starttime /amd[?]/amd_properties[24]/pressure_fp /amd[?]/amd_properties[24]/temperature_fp /amd[?]/amd_properties[24]/frequencyshift_fp /amd[?]/amd_properties[24]/relativehumidity_fp /amd[?]/amd_properties[24]/molecularlod_fp /amd[?]/amd_properties[24]/molecularbackscatter_fp /amd[?]/amd_properties[24]/pressure_fiz /amd[?]/amd_properties[24]/temperature_fiz /amd[?]/amd_properties[24]/frequencyshift_fiz /amd[?]/amd_properties[24]/relativehumidity_fiz /amd[?]/amd_properties[24]/molecularlod_fiz /amd[?]/amd_properties[24]/molecularbackscatter_fiz /group_optical_properties[?]/starttime /group_optical_properties[?]/height_bin_index /group_optical_properties[?]/group_optical_property/group_extinction /group_optical_properties[?]/group_optical_property/group_backscatter /group_optical_properties[?]/group_optical_property/group_lod /group_optical_properties[?]/group_optical_property/group_sr /group_optical_properties[?]/group_geolocation_middle_bins/start_longitude /group_optical_properties[?]/group_geolocation_middle_bins/start_latitude /group_optical_properties[?]/group_geolocation_middle_bins/start_altitude /group_optical_properties[?]/group_geolocation_middle_bins/mid_longitude /group_optical_properties[?]/group_geolocation_middle_bins/mid_latitude /group_optical_properties[?]/group_geolocation_middle_bins/mid_altitude /group_optical_properties[?]/group_geolocation_middle_bins/stop_longitude /group_optical_properties[?]/group_geolocation_middle_bins/stop_latitude /group_optical_properties[?]/group_geolocation_middle_bins/stop_altitude /group_optical_properties[?]/group_optical_property_middle_bins/mid_extinction_top /group_optical_properties[?]/group_optical_property_middle_bins/mid_backscatter_top /group_optical_properties[?]/group_optical_property_middle_bins/mid_lod_top /group_optical_properties[?]/group_optical_property_middle_bins/mid_ber_top /group_optical_properties[?]/group_optical_property_middle_bins/mid_extinction_bot /group_optical_properties[?]/group_optical_property_middle_bins/mid_backscatter_bot /group_optical_properties[?]/group_optical_property_middle_bins/mid_lod_bot /group_optical_properties[?]/group_optical_property_middle_bins/mid_ber_bot /scene_classification[?]/starttime /scene_classification[?]/height_bin_index /scene_classification[?]/aladin_cloud_flag/clrh /scene_classification[?]/aladin_cloud_flag/clsr /scene_classification[?]/aladin_cloud_flag/downclber /scene_classification[?]/aladin_cloud_flag/topclber /scene_classification[?]/nwp_cloud_flag /scene_classification[?]/l2a_group_class_reliability The question mark indicates a variable size array It is not entirely clear what we actually have to look at. For simplicity the data of the group 'sca_optical_properties' is returned at this point Example ------- >>> import pyaerocom.io.read_aeolus_l2a_data >>> obj = pyaerocom.io.read_aeolus_l2a_data.ReadAeolusL2aData(verbose=True) >>> import os >>> os.environ['CODA_DEFINITION']='/lustre/storeA/project/aerocom/aerocom1/ADM_CALIPSO_TEST/' >>> filename = '/lustre/storeA/project/aerocom/aerocom1/ADM_CALIPSO_TEST/download/AE_OPER_ALD_U_N_2A_20070101T002249149_002772000_003606_0001.DBL' >>> # read returning a ndarray >>> filedata_numpy = obj.read_file(filename, vars_to_read=['ec550aer'], return_as='numpy') >>> time_as_numpy_datetime64 = filedata_numpy[0,obj._TIMEINDEX].astype('datetime64[s]') >>> print('time: {}'.format(time_as_numpy_datetime64)) >>> print('latitude: {}'.format(filedata_numpy[1,obj._LATINDEX])) >>> # read returning a dictionary >>> filedata = obj.read_file(filename, vars_to_read=['ec550aer']) >>> print('time: {}'.format(filedata['time'][0].astype('datetime64[s]'))) >>> print('all latitudes of 1st time step: {}'.format(filedata['latitude'][filedata['time'][0]])) """ import time import coda # coda uses 2000-01-01T00:00:00 as epoch unfortunately. # so calculate the difference in seconds to the Unix epoch seconds_to_add = np.datetime64('2000-01-01T00:00:00') - np.datetime64( '1970-01-01T00:00:00') seconds_to_add = seconds_to_add.astype(np.float_) # the same can be achieved using pandas, but we stick to numpy here # base_time = pd.DatetimeIndex(['2000-01-01']) # seconds_to_add = (base_time.view('int64') // pd.Timedelta(1, unit='s'))[0] start = time.perf_counter() file_data = {} self.logger.info('reading file {}'.format(filename)) # read file product = coda.open(filename) if vars_to_read is None: # read all variables vars_to_read = list(self.DATA_COLNAMES.keys()) vars_to_read.extend(list(self.METADATA_COLNAMES.keys())) # read data # start with the time because it is only stored once groups = self.TIME_PATH.split(self.GROUP_DELIMITER) file_data[self._TIME_NAME] = coda.fetch(product, groups[0], -1, groups[1]) # epoch is 1 January 2000 at ESA # so add offset to move that to 1 January 1970 # and save it into a np.datetime64[ms] object file_data[self._TIME_NAME] = \ ((file_data[self._TIME_NAME] + seconds_to_add) * 1.E3).astype(np.int).astype('datetime64[ms]') # read data in a simple dictionary for var in vars_to_read: groups = self._COLNAMES[var].split(self.GROUP_DELIMITER) if len(groups) == 3: file_data[var] = {} for idx, key in enumerate(file_data[self._TIME_NAME]): file_data[var][key] = coda.fetch(product, groups[0], idx, groups[1], -1, groups[2]) elif len(groups) == 2: file_data[var] = {} for idx, key in enumerate(file_data[self._TIME_NAME]): file_data[var][key] = coda.fetch(product, groups[0], -1, groups[1]) else: file_data[var] = {} for idx, key in enumerate(file_data[self._TIME_NAME]): file_data[var][key] = coda.fetch(product, groups[0]) if return_as == 'numpy': # return as one multidimensional numpy array that can be put into self.data directly # (column wise because the column numbers do not match) index_pointer = 0 data = np.empty([self._ROWNO, self._COLNO], dtype=np.float_) for idx, _time in enumerate(file_data['time'].astype(np.float_) / 1000.): # file_data['time'].astype(np.float_) is milliseconds after the (Unix) epoch # but we want to save the time as seconds since the epoch for _index in range( len(file_data['latitude'][file_data['time'][idx]])): # this works because all variables have to have the same size # (aka same number of height levels) # This loop could be avoided using numpy index slicing # do that in case we need more optimisations data[index_pointer, self._TIMEINDEX] = _time for var in vars_to_read: data[index_pointer, self.INDEX_DICT[var]] = file_data[var][ file_data['time'][idx]][_index] # put negative values to np.nan if the variable is not a metadata variable if data[index_pointer, self.INDEX_DICT[var]] == self.NAN_DICT[var]: data[index_pointer, self.INDEX_DICT[var]] = np.nan index_pointer += 1 if index_pointer >= self._ROWNO: # add another array chunk to self.data data = np.append(data, np.empty( [self._CHUNKSIZE, self._COLNO], dtype=np.float_), axis=0) self._ROWNO += self._CHUNKSIZE # return only the needed elements... file_data = data[0:index_pointer] end_time = time.perf_counter() elapsed_sec = end_time - start temp = 'time for single file read [s]: {:.3f}'.format(elapsed_sec) self.logger.info(temp) # self.logger.info('{} points read'.format(index_pointer)) return file_data
# If you repeatedly need to access data from the same file then this will be orders of magnitude faster. # import os # Change this to the location of your AEOLUS codadef file os.putenv('CODA_DEFINITION', '/usr/local/share/coda/definitions') # You can also remove this line and set the CODA_DEFINITION environment variable globally on your system import coda import numpy # change this to the full path of your Aeolus L1B DBL file filename = "AE_OPER_AUX_MET_12_20071107T090000_20071108T150000_0001.DBL" product = coda.open(filename) num_records = coda.fetch(product, '/sph/num_records_in_ds1') num_layers = coda.fetch(product, '/sph/num_of_model_layers') assert (num_records > 0 and num_layers > 0) amd_pnom = numpy.empty([num_records, num_layers]) amd_znom = numpy.empty([num_records, num_layers]) amd_t = numpy.empty([num_records, num_layers]) amd_u = numpy.empty([num_records, num_layers]) cursor = coda.Cursor() coda.cursor_set_product(cursor, product) coda.cursor_goto(cursor, '/met_off_nadir[0]') for i in range(num_records):
import coda pf = coda.open('/data/GOME2/GOME_xxx_1B/GOME_xxx_1B_M02_20100415122955Z_20100415123255Z_N_T_20100713095647Z') cursor = coda.Cursor() coda.cursor_set_product(cursor, pf) coda.cursor_goto(cursor, '/MDR') num_mdr = coda.cursor_get_num_elements(cursor) if num_mdr > 0: coda.cursor_goto_first_array_element(cursor) for i in xrange(num_mdr): index = coda.cursor_get_available_union_field_index(cursor) if index == 0: # Earthshine MDR # Note that fetching the full MDR is rather slow, since it converts # the full MDR to a hierarchicel set of Python structures mdr = coda.fetch(cursor, 'Earthshine') print mdr # If you want e.g. just the wavelength and band data of band 1b, you could use: # wavelength = coda.fetch(cursor, 'Earthshine', 'wavelength_1b') # rad = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'rad') # err = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'err_rad') # which will be much faster elif index == 1: # Calibration MDR pass elif index == 2: # Sun MDR mdr = coda.fetch(cursor, 'Sun') print mdr elif index == 3: # Moon MDR
# # mipl2geo.py import sys import coda if __name__ == "__main__": if len(sys.argv) < 2: print >>sys.stderr, "Usage: %s <mipas level 2 file> ..." % sys.argv[0] sys.exit(1) for f in sys.argv[1:]: pf = coda.open(f) product_class = coda.get_product_class(pf) product_type = coda.get_product_type(pf) if not product_class.startswith("ENVISAT") or product_type != "MIP_NL__2P": print >>sys.stderr, "Error: file %s is not a MIPAS Level 2 product (product class = %s, product type = %s)" % (sys.argv[1], product_class, product_type) sys.exit(1) print "Processing : %s" % f cursor = coda.Cursor() coda.cursor_set_product(cursor, pf) coda.cursor_goto_record_field_by_name(cursor, "scan_geolocation_ads") num_dsr = coda.cursor_get_num_elements(cursor)
def get_spectral_response(self): """ return 光谱波数和响应值,1维,2维 """ k = 1.98644746103858e-9 if self.resolution == 40000: satellite_type1 = ['METOP-A', 'METOP-B'] if self.satellite in satellite_type1: try: fp = coda.open(self.in_file) wave3 = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'WAVELENGTH_3') wave4 = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'WAVELENGTH_4') lambda_smr = coda.fetch(fp, 'VIADR_SMR', -1, 'LAMBDA_SMR') smr = coda.fetch(fp, 'VIADR_SMR', -1, 'SMR') sunz = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'SOLAR_ZENITH') coda.close(fp) # gome 959x4096 取后2048个点的辐射值 wavelens = self.record.wavelength[0, 2048:] response = self.record.spectral_radiance[:, 2048:] data_size = self.data_shape[0] data_row = sunz.shape[0] data_col = sunz[0].shape[1] # gome的辐亮度计算 wave3 wave4 是30*1024 for m in xrange(data_size): row, col = np.unravel_index(m, (data_row, data_col)) for i in xrange(2048): if i < 1024: response[m, i] = response[m, i] * k / wave3[row][i] else: response[m, i] = response[ m, i] * k / wave4[row][i - 1024] # 计算太阳辐亮度 sol_wavelens = np.zeros((2048, )) # 太阳辐亮度 sol_response = np.zeros((2048, )) # 太阳辐亮度对应的波长 for i in xrange(2048): if i < 1024: sol_response[i] = (smr[0][2][i] * k) / lambda_smr[0][2][i] sol_wavelens[i] = lambda_smr[0][2][i] else: sol_response[i] = (smr[0][3][i - 1024] * k) / lambda_smr[0][3][i - 1024] sol_wavelens[i] = lambda_smr[0][3][i - 1024] idx = np.where(response < 0) print len(idx[0]) if len(idx[0] > 0): response[idx] = 0. gome_wavelens, gome_response, solar_response = self.combine_gome_band34( wavelens, response, sol_wavelens, sol_response) s0, s1 = gome_response.shape gome_response = gome_response.reshape(s0, 1, s1) print gome_response.shape, solar_response.shape except Exception as e: print 'Open file error {}'.format(e) return self.solar_response = solar_response return gome_wavelens, gome_response
def _read(self, path, fields="all", return_header=False): tmpdira = config.conf["main"]["tmpdir"] tmpdirb = config.conf["main"]["tmpdirb"] tmpdir = (tmpdira if shutil.disk_usage(tmpdira).free > self.minspace else tmpdirb) with tempfile.NamedTemporaryFile(mode="wb", dir=tmpdir, delete=True) as tmpfile: with gzip.open(str(path), "rb") as gzfile: logging.debug("Decompressing {!s}".format(path)) gzcont = gzfile.read() logging.debug("Writing decompressed file to {!s}".format(tmpfile.name)) tmpfile.write(gzcont) del gzcont # All the hard work is in coda logging.debug("Reading {!s}".format(tmpfile.name)) cfp = coda.open(tmpfile.name) c = coda.fetch(cfp) logging.debug("Sorting info...") n_scanlines = c.MPHR.TOTAL_MDR start = datetime.datetime(*coda.time_double_to_parts_utc(c.MPHR.SENSING_START)) has_mdr = numpy.array([hasattr(m, 'MDR') for m in c.MDR], dtype=numpy.bool) bad = numpy.array([ (m.MDR.DEGRADED_PROC_MDR|m.MDR.DEGRADED_INST_MDR) if hasattr(m, 'MDR') else True for m in c.MDR], dtype=numpy.bool) dlt = numpy.concatenate( [m.MDR.OnboardUTC[:, numpy.newaxis] for m in c.MDR if hasattr(m, 'MDR')], 1) - c.MPHR.SENSING_START M = numpy.ma.zeros( dtype=self._dtype, shape=(n_scanlines, 30)) M["time"][has_mdr] = numpy.datetime64(start, "ms") + numpy.array(dlt*1e3, "m8[ms]").T specall = self.__obtain_from_mdr(c, "GS1cSpect").astype("f8") # apply scale factors first = c.MDR[0].MDR.IDefNsfirst1b last = c.MDR[0].MDR.IDefNslast1b for (slc_st, slc_fi, fact) in zip( filter(None, c.GIADR_ScaleFactors.IDefScaleSondNsfirst), c.GIADR_ScaleFactors.IDefScaleSondNslast, c.GIADR_ScaleFactors.IDefScaleSondScaleFactor): # Documented intervals are closed [a, b]; Python uses # half-open [a, b). specall[..., (slc_st-first):(slc_fi-first+1)] *= pow(10.0, -fact) M["spectral_radiance"][has_mdr] = specall locall = self.__obtain_from_mdr(c, "GGeoSondLoc") M["lon"][has_mdr] = locall[:, :, :, 0] M["lat"][has_mdr] = locall[:, :, :, 1] satangall = self.__obtain_from_mdr(c, "GGeoSondAnglesMETOP") M["satellite_zenith_angle"][has_mdr] = satangall[:, :, :, 0] M["satellite_azimuth_angle"][has_mdr] = satangall[:, :, :, 1] solangall = self.__obtain_from_mdr(c, "GGeoSondAnglesSUN") M["solar_zenith_angle"][has_mdr] = solangall[:, :, :, 0] M["solar_azimuth_angle"][has_mdr] = solangall[:, :, :, 1] for fld in M.dtype.names: M.mask[fld][~has_mdr, ...] = True M.mask[fld][bad, ...] = True m = c.MDR[0].MDR wavenumber = (m.IDefSpectDWn1b * numpy.arange(m.IDefNsfirst1b, m.IDefNslast1b+0.1) * (1/ureg.metre)) if self.wavenumber is None: self.wavenumber = wavenumber elif abs(self.wavenumber - wavenumber).max() > (0.05 * 1/(ureg.centimetre)): raise ValueError("Inconsistent wavenumbers") return M
def read(self, vars_to_retrieve=None, files=[], first_file=None, last_file=None, file_pattern=None, list_coda_paths=False, local_temp_dir=None, return_as='numpy', apply_quality_flag=0.0): """Method that reads list of files as instance of :class:`UngriddedData` Parameters ---------- vars_to_retrieve : :obj:`list` or similar, optional, list containing variable IDs that are supposed to be read. If None, all variables in :attr:`PROVIDES_VARIABLES` are loaded files : :obj:`list`, optional list of files to be read. If None, then the file list is used that is returned on :func:`get_file_list`. first_file : :obj:`int`, optional index of first file in file list to read. If None, the very first file in the list is used. Note: is ignored if input parameter `file_pattern` is specified. last_file : :obj:`int`, optional index of last file in list to read. If None, the very last file in the list is used. Note: is ignored if input parameter `file_pattern` is specified. file_pattern : str, optional string pattern for file search (cf :func:`get_file_list`) :param local_temp_dir: Returns ------- UngriddedData data object Example: >>> import pyaerocom as pya >>> obj = pya.io.read_aeolus_l2a_data.ReadL2Data() >>> testfiles = [] >>> testfiles.append('/lustre/storeB/project/fou/kl/admaeolus/data.rev.2A02/download/2018-12/01/AE_OPER_ALD_U_N_2A_20181201T033526026_005423993_001590_0001.TGZ') >>> data=obj.read(files=testfiles) >>> data=obj.read(files=testfiles, vars_to_retrieve='ec355aer') """ import pathlib import tarfile import os import coda if local_temp_dir is None: local_temp_dir = self.LOCAL_TMP_DIR if vars_to_retrieve is None: vars_to_retrieve = self.DEFAULT_VARS elif isinstance(vars_to_retrieve, str): vars_to_retrieve = [vars_to_retrieve] if files is None: if len(self.files) == 0: self.get_file_list(pattern=file_pattern) files = self.files if file_pattern is None: if first_file is None: first_file = 0 if last_file is None: last_file = len(files) files = files[first_file:last_file] self.read_failed = [] temp_files = {} data_obj = UngriddedData(num_points=self._CHUNKSIZE) meta_key = 0.0 idx = 0 # check if the supplied file is a supported archive file (tar in this case) # and extract the files with supported suffixes to const.CACHEDIR non_archive_files = [] for idx, _file in enumerate(sorted(files)): # temp = 'reading file: {}'.format(_file) self.logger.info('file: {}'.format(_file)) suffix = pathlib.Path(_file).suffix if suffix in self.SUPPORTED_ARCHIVE_SUFFIXES: temp = 'opening archive file; using {} as temp dir.'.format( local_temp_dir) self.logger.info(temp) # untar archive files first tarhandle = tarfile.open(_file) files_in_tar = tarhandle.getnames() for file_in_tar in files_in_tar: if pathlib.Path( file_in_tar).suffix in self.SUPPORTED_SUFFIXES: # extract file to tmp path member = tarhandle.getmember(file_in_tar) temp = 'extracting file {}...'.format(member.name) self.logger.info(temp) tarhandle.extract(member, path=local_temp_dir, set_attrs=False) extract_file = os.path.join(local_temp_dir, member.name) non_archive_files.append(extract_file) temp_files[extract_file] = True tarhandle.close() else: non_archive_files.append(_file) for idx, _file in enumerate(sorted(non_archive_files)): # list coda data paths in the 1st file in case the user asked for that if idx == 0 and list_coda_paths: pass coda_handle = coda.open(_file) root_field_names = coda.get_field_names(coda_handle) for field in root_field_names: print(field) coda.close(coda_handle) data_obj = None return data_obj file_data = self.read_file(_file, vars_to_retrieve=vars_to_retrieve, loglevel=logging.INFO, return_as=return_as) if return_as == 'numpy': self.logger.info('{} points read'.format(file_data.shape[0])) # the metadata dict is left empty for L2 data # the location in the data set is time step dependant! if idx == 0: data_obj._data = file_data else: data_obj._data = np.append(data_obj._data, file_data, axis=0) data_obj._idx = data_obj._data.shape[0] + 1 file_data = None # remove file if it was temporary one if _file in temp_files: os.remove(_file) # pass # tmp_obj = UngriddedData() # tmp_obj._data = file_data # tmp_obj._idx = data_obj._data.shape[0] + 1 # data_obj.append(tmp_obj) self.logger.info( 'size of data object: {}'.format(data_obj._idx - 1)) elif return_as == 'dict': if idx == 0: data_obj._data = {} shape_store = {} index_store = {} file_start_index_arr = [0] # apply quality flags if apply_quality_flag > 0.: qflags = file_data[self._QANAME] keep_indexes = np.where(qflags >= apply_quality_flag) elements_to_add = keep_indexes.size else: keep_indexes = np.arange(0, len(file_data[self._QANAME])) elements_to_add = file_data[self._QANAME].shape[0] for _key in file_data: # print('key: {}'.format(_key)) shape_store[_key] = file_data[_key].shape index_store[_key] = file_data[_key].shape[0] input_shape = list(file_data[_key].shape) input_shape[0] = self._ROWNO data_obj._data[_key] = np.empty(input_shape, dtype=np.float_) if len(input_shape) == 1: data_obj._data[_key][0:file_data[_key]. shape[0]] = file_data[_key] elif len(input_shape) == 2: data_obj._data[_key][0:file_data[_key]. shape[0], :] = file_data[_key] elif len(input_shape) == 3: data_obj._data[_key][ 0:file_data[_key]. shape[0], :, :] = file_data[_key] elif len(input_shape) == 4: data_obj._data[_key][ 0:file_data[_key]. shape[0], :, :, :] = file_data[_key] else: pass # 2nd + file else: if apply_quality_flag > 0.: qflags = file_data[self._QANAME] keep_indexes = np.where(qflags >= apply_quality_flag) elements_to_add = keep_indexes.size file_start_index_arr.append( file_data[self.TSSIZENAME].shape[0]) for _key in file_data: if _key in self.STATICFIELDNAMES: print('key: {}'.format(_key)) continue # shape_store[_key] = file_data[_key].shape elements_to_add = file_data[_key].shape[0] # extend data_obj._data[_key] if necessary if index_store[_key] + elements_to_add > data_obj._data[ _key].shape[0]: current_shape = list(data_obj._data[_key].shape) current_shape[ 0] = current_shape[0] + self._CHUNKSIZE tmp_data = np.empty(current_shape, dtype=np.float_) if len(current_shape) == 1: tmp_data[0:data_obj._data[_key]. shape[0]] = data_obj._data[_key] elif len(current_shape) == 2: tmp_data[0:data_obj._data[_key]. shape[0], :] = data_obj._data[_key] elif len(current_shape) == 3: tmp_data[0:data_obj._data[_key]. shape[0], :, :] = data_obj._data[_key] elif len(current_shape) == 4: tmp_data[ 0:data_obj._data[_key]. shape[0], :, :, :] = data_obj._data[_key] else: pass input_shape = list(file_data[_key].shape) if len(input_shape) == 1: data_obj._data[_key][ index_store[_key]:index_store[_key] + file_data[_key].shape[0]] = file_data[_key] elif len(input_shape) == 2: data_obj._data[_key][ index_store[_key]:index_store[_key] + file_data[_key].shape[0], :] = file_data[_key] elif len(input_shape) == 3: data_obj._data[_key][ index_store[_key]:index_store[_key] + file_data[_key]. shape[0], :, :] = file_data[_key] elif len(input_shape) == 4: data_obj._data[_key][ index_store[_key]:index_store[_key] + file_data[_key]. shape[0], :, :, :] = file_data[_key] else: pass index_store[_key] += elements_to_add file_data = None # remove file if it was temporary one if _file in temp_files: os.remove(_file) else: pass # now shorten the data dict to the necessary size if return_as == 'dict': for _key in data_obj._data: data_obj._data[_key] = data_obj._data[_key][:index_store[_key]] data_obj._data['file_indexes'] = file_start_index_arr # apply the quality flags if apply_quality_flag > 0.: pass return data_obj
import coda pf = coda.open( '/data/GOME2/GOME_xxx_1B/GOME_xxx_1B_M02_20100415122955Z_20100415123255Z_N_T_20100713095647Z' ) cursor = coda.Cursor() coda.cursor_set_product(cursor, pf) coda.cursor_goto(cursor, '/MDR') num_mdr = coda.cursor_get_num_elements(cursor) if num_mdr > 0: coda.cursor_goto_first_array_element(cursor) for i in xrange(num_mdr): index = coda.cursor_get_available_union_field_index(cursor) if index == 0: # Earthshine MDR # Note that fetching the full MDR is rather slow, since it converts # the full MDR to a hierarchicel set of Python structures mdr = coda.fetch(cursor, 'Earthshine') print mdr # If you want e.g. just the wavelength and band data of band 1b, you could use: # wavelength = coda.fetch(cursor, 'Earthshine', 'wavelength_1b') # rad = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'rad') # err = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'err_rad') # which will be much faster elif index == 1: # Calibration MDR pass elif index == 2: # Sun MDR mdr = coda.fetch(cursor, 'Sun') print mdr
def coda_open(filename): coda_handle = coda.open(filename) try: yield coda_handle finally: coda.close(coda_handle)