def Initialize(self, node): try: valueString = model.GetDataAsString(node.type, coda.fetch(node.cursor)) except (coda.CodaError, coda.CodacError) as ex: raise RenderError("[CODA] %s" % (str(ex),)) self.SetValue(valueString)
def _read(self, path, fields="all", return_header=False): tmpdira = config.conf["main"]["tmpdir"] tmpdirb = config.conf["main"]["tmpdirb"] tmpdir = (tmpdira if shutil.disk_usage(tmpdira).free > self.minspace else tmpdirb) with tempfile.NamedTemporaryFile(mode="wb", dir=tmpdir, delete=True) as tmpfile: with gzip.open(str(path), "rb") as gzfile: logging.debug("Decompressing {!s}".format(path)) gzcont = gzfile.read() logging.debug("Writing decompressed file to {!s}".format(tmpfile.name)) tmpfile.write(gzcont) del gzcont # All the hard work is in coda logging.debug("Reading {!s}".format(tmpfile.name)) cfp = coda.open(tmpfile.name) c = coda.fetch(cfp) logging.debug("Sorting info...") n_scanlines = c.MPHR.TOTAL_MDR start = datetime.datetime(*coda.time_double_to_parts_utc(c.MPHR.SENSING_START)) has_mdr = numpy.array([hasattr(m, 'MDR') for m in c.MDR], dtype=numpy.bool) bad = numpy.array([ (m.MDR.DEGRADED_PROC_MDR|m.MDR.DEGRADED_INST_MDR) if hasattr(m, 'MDR') else True for m in c.MDR], dtype=numpy.bool) dlt = numpy.concatenate( [m.MDR.OnboardUTC[:, numpy.newaxis] for m in c.MDR if hasattr(m, 'MDR')], 1) - c.MPHR.SENSING_START M = numpy.ma.zeros( dtype=self._dtype, shape=(n_scanlines, 30)) M["time"][has_mdr] = numpy.datetime64(start, "ms") + numpy.array(dlt*1e3, "m8[ms]").T specall = self.__obtain_from_mdr(c, "GS1cSpect") M["spectral_radiance"][has_mdr] = specall locall = self.__obtain_from_mdr(c, "GGeoSondLoc") M["lon"][has_mdr] = locall[:, :, :, 0] M["lat"][has_mdr] = locall[:, :, :, 1] satangall = self.__obtain_from_mdr(c, "GGeoSondAnglesMETOP") M["satellite_zenith_angle"][has_mdr] = satangall[:, :, :, 0] M["satellite_azimuth_angle"][has_mdr] = satangall[:, :, :, 1] solangall = self.__obtain_from_mdr(c, "GGeoSondAnglesSUN") M["solar_zenith_angle"][has_mdr] = solangall[:, :, :, 0] M["solar_azimuth_angle"][has_mdr] = solangall[:, :, :, 1] for fld in M.dtype.names: M.mask[fld][~has_mdr, ...] = True M.mask[fld][bad, ...] = True m = c.MDR[0].MDR wavenumber = (m.IDefSpectDWn1b * numpy.arange(m.IDefNsfirst1b, m.IDefNslast1b+0.1) * (1/ureg.metre)) if self.wavenumber is None: self.wavenumber = wavenumber elif abs(self.wavenumber - wavenumber).max() > (0.05 * 1/(ureg.centimetre)): raise ValueError("Inconsistent wavenumbers") return M
def Initialize(self, node): for i in range(0, len(node)): field = node[i] if not (coda.get_option_filter_record_fields() and (field.hidden or not field.available)): idx = self.InsertItem(i, str(i)) self.SetItem(idx, 1, field.real_name, model.GetNodeIcon(field, model.iconDictionary)) if field.available: if field.type == model.TYPE_RECORD: self.SetItem(idx, 2, "<record of %u field(s)>" % (len(field),)) elif field.type == model.TYPE_ARRAY: if field.isRankZero(): self.SetItem(idx, 2, "<rank-0 array of %s>" % (model.GetTypeAsString(field.base),)) else: dimensionString = "" for dim in field.dimensions[:-1]: dimensionString += "%u x " % (dim,) dimensionString += "%u" % (field.dimensions[-1],) self.SetItem(idx, 2, "[%s %s]" % (dimensionString, model.GetTypeAsString(field.base))) else: # a maximum size of _maxLineLength characters is specified, because # GDK may crash if the length of a string item gets too long. # also, if the line is really long the user will probably not want # to view as a single line in a ListCtrl anyway. try: data = coda.fetch(field.cursor) except (coda.CodaError, coda.CodacError) as ex: raise RenderError("[CODA] %s" % (str(ex),)) self.SetItem(idx, 2, model.GetDataAsString(field.type, data, frame._maxLineLength)) else: self.SetItem(idx, 2, "<unavailable>") # _AutoSize() sets the width of a column to the maximum of the # width of the longest item in the column and the width of the column # header. # I would like to _AutoSize() only column 0 and 1, and have column 2 # take up the rest of the available space (maybe constrained to a # minimum width, i.e. the width of the smallest item in the column). # This would be desirable, because sometimes there can be very long # items, which result in very small scrollbars. However, there does not # seem to be an easy way to achieve this. (Note that if an item is # longer than the column width, it is automatically clipped and # padded with '...' by the ListCtrl.) # # UPDATE: A specified maximum length (productbrowser.frame._maxLength) is # now used to avoid very long items. Therefore, we can now _AutoSize() # column 2 as well. However, _AutoSize()-ing only column 0 and 1, and # having column 2 take up the rest of the available space would still # be a better solution. self._AutoSize((0, 1, 2))
def get_solar_zenith(self): """ 获取太阳天顶角 :return: """ fp = coda.open(self.in_file) angles = coda.fetch(fp, 'MDR', -1, 'MDR', 'GGeoSondAnglesSUN') zenith = np.array([]) for i in angles: z = i.reshape(-1)[0::2] zenith = np.append(zenith, z) return zenith
def Initialize(self, node): assert node.type == model.TYPE_ARRAY, "PlotRenderer can only render node of type TYPE_ARRAY" assert not isinstance( node, model.ObjectArrayNode ), "PlotRenderer cannot render arrays of compound types" # coda.fetch() will promote a rank-0 array to a 1x1 array, # so a rank of 1 is the smallest possible rank. try: self.array = coda.fetch(node.cursor) except (coda.CodaError, coda.CodacError) as ex: raise RenderError("[CODA] %s" % (str(ex), )) # array sanity check ok = True if node.isRankZero(): ok = (self.array.ndim == 1) and (self.array.shape[0] == 1) else: ok = (self.array.ndim == len(node.dimensions)) and (self.array.size == len(node)) i = 0 while ok and i < len(node.dimensions): ok = ok and (self.array.shape[i] == node.dimensions[i]) i += 1 if not ok: raise RenderError( "[PlotRenderer] Array read from product does not match description." ) self.plot = PlotWindow(self, -1) sizer = wx.BoxSizer(wx.VERTICAL) sizer.Add(self.plot, 1, wx.EXPAND | wx.ALL, 5) if self.array.ndim > 1: self.slicer = NumpySlicer1D(self, -1, self.array, "select") sizer.Add(self.slicer, 0, wx.EXPAND | wx.LEFT | wx.RIGHT, 5) self.Bind(EVT_SLICE_CHANGED, self.OnSliceChanged) self.SetSizer(sizer) if self.array.ndim == 1: self.plot.AddDataSet(numpy.arange(self.array.shape[0]), self.array) else: slice = self.slicer.GetSlice() self.plot.AddDataSet(numpy.arange(slice.shape[0]), slice)
def Initialize(self, node): try: array = coda.fetch(node.cursor) except (coda.CodaError, coda.CodacError) as ex: raise RenderError("[CODA] %s" % (str(ex),)) # array sanity check if not array.ndim == 1: raise RenderError("[BytesRenderer] Array read from product does not match description.") if array.size == 0: raise RenderError("[BytesRenderer] Array read from product is empty.") width, height = self.GetTextExtent("FF") self.SetDefaultColSize(width + 10) self.SetDefaultRowSize(height + 10) self.SetDefaultCellAlignment(wx.ALIGN_CENTRE, wx.ALIGN_CENTRE) self.SetTable(HexNumpyWrapper(array), True)
def get_sensor_azimuth(self): """ return sensor_azimuth """ if self.resolution == 40000: satellite_type1 = ['METOP-A', 'METOP-B'] if self.satellite in satellite_type1: try: fp = coda.open(self.in_file) angle = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'SAT_AZIMUTH') coda.close(fp) # angle = 30*3*32 = 1440, 取30*1*32 = 960, 取前959个 data_size = self.data_shape[0] data_row = angle.shape[0] data_col = angle[0].shape[1] data_pre = np.full(self.data_shape, -999.) for i in xrange(data_size): row, col = np.unravel_index(i, (data_row, data_col)) # print row, col, angle[row][1][col] data_pre[i] = angle[row][1][col] # 过滤无效值 invalid_index = np.logical_or(data_pre < 0, data_pre > 360) data_pre = data_pre.astype(np.float32) data_pre[invalid_index] = np.nan data = data_pre except Exception as e: print 'Open file error {}'.format(e) return else: raise ValueError('Cant read this satellite`s data.: {}'.format( self.satellite)) else: raise ValueError( 'Cant read this data, please check its resolution: {}'.format( self.in_file)) return data
def get_footprint(product): try: import coda except ImportError: return None path = "/METADATA/EOP_METADATA/om_featureOfInterest/eop_multiExtentOf/gml_surfaceMembers/gml_exterior@gml_posList" pf = coda.open(product) try: coord = coda.fetch(pf, path).split(' ') except coda.CodacError: return None finally: coda.close(pf) if len(coord) % 2 != 0: return None return Polygon([ LinearRing([ Point(float(lon), float(lat)) for lat, lon in zip(coord[0::2], coord[1::2]) ]) ])
import os # Change this to the location of your AEOLUS codadef file os.putenv('CODA_DEFINITION', '/usr/local/share/coda/definitions') # You can also remove this line and set the CODA_DEFINITION environment variable globally on your system import coda import numpy # change this to the full path of your Aeolus L1B DBL file filename = "AE_OPER_AUX_MET_12_20071107T090000_20071108T150000_0001.DBL" product = coda.open(filename) num_records = coda.fetch(product, '/sph/num_records_in_ds1') num_layers = coda.fetch(product, '/sph/num_of_model_layers') assert (num_records > 0 and num_layers > 0) amd_pnom = numpy.empty([num_records, num_layers]) amd_znom = numpy.empty([num_records, num_layers]) amd_t = numpy.empty([num_records, num_layers]) amd_u = numpy.empty([num_records, num_layers]) cursor = coda.Cursor() coda.cursor_set_product(cursor, product) coda.cursor_goto(cursor, '/met_off_nadir[0]') for i in range(num_records): coda.cursor_goto(cursor, 'profile_data[0]') for j in range(num_layers - 1):
filename = "/path/to/AE_OPER_ALD_U_N_1B_20151002T001857059_005787000_046339_0001.DBL" product = coda.open(filename) ### Observation profiles ### # Observation profiles provide a single profile per BRC. # Reading all values will therefore return 'an array of arrays' (we will get an array for each '-1' in the coda.fetch()). # The first array is the list of BRCs, and the second array the list of points in the profile. # This array of arrays can be turned into a single 2D numpy array of shape [num_brc, num_vertical] by using 'vstack'. # Mie observation wind profiles print("Mie observation wind profiles") latitude = coda.fetch(product, 'geolocation', -1, 'observation_geolocation/observation_mie_geolocation', -1, 'latitude_of_height_bin') latitude = vstack(latitude) longitude = coda.fetch(product, 'geolocation', -1, 'observation_geolocation/observation_mie_geolocation', -1, 'longitude_of_height_bin') longitude = vstack(longitude) altitude = coda.fetch(product, 'geolocation', -1, 'observation_geolocation/observation_mie_geolocation', -1, 'altitude_of_height_bin') altitude = vstack(altitude) wind_velocity = coda.fetch(product, 'wind_velocity', -1, 'observation_wind_profile/mie_altitude_bin_wind_info', -1, 'wind_velocity') wind_velocity = vstack(wind_velocity) print(wind_velocity.shape) print(wind_velocity) # Rayleigh observation wind profiles
os.putenv('CODA_DEFINITION', '/usr/local/share/coda/definitions') # You can also remove this line and set the CODA_DEFINITION environment variable globally on your system import coda from numpy import hstack, vstack # change this to the full path of your Aeolus L2A DBL file filename = "/path/to/AE_OPER_ALD_U_N_2A_20071107T193753059_008688000_000578_0001.DBL" product = coda.open(filename) ### Standard Correct Algorithm (SCA) at middle bins ### if coda.get_field_available(product, 'sca_optical_properties'): print("SCA optical properties at middle bins") latitude = coda.fetch(product, 'sca_optical_properties', -1, 'geolocation_middle_bins', -1, 'latitude') latitude = vstack(latitude) longitude = coda.fetch(product, 'sca_optical_properties', -1, 'geolocation_middle_bins', -1, 'longitude') longitude = vstack(longitude) altitude = coda.fetch(product, 'sca_optical_properties', -1, 'geolocation_middle_bins', -1, 'altitude') altitude = vstack(altitude) extinction = coda.fetch(product, 'sca_optical_properties', -1, 'sca_optical_properties_mid_bins', -1, 'extinction') extinction = vstack(extinction) print(extinction.shape)
import os # Change this to the location of your AEOLUS codadef file os.putenv('CODA_DEFINITION', '/usr/local/share/coda/definitions') # You can also remove this line and set the CODA_DEFINITION environment variable globally on your system import coda import numpy # change this to the full path of your Aeolus L1B DBL file filename = "AE_OPER_AUX_MET_12_20071107T090000_20071108T150000_0001.DBL" product = coda.open(filename) num_records = coda.fetch(product, '/sph/num_records_in_ds1') num_layers = coda.fetch(product, '/sph/num_of_model_layers') assert(num_records > 0 and num_layers > 0) amd_pnom = numpy.empty([num_records, num_layers]) amd_znom = numpy.empty([num_records, num_layers]) amd_t = numpy.empty([num_records, num_layers]) amd_u = numpy.empty([num_records, num_layers]) cursor = coda.Cursor() coda.cursor_set_product(cursor, product) coda.cursor_goto(cursor, '/met_off_nadir[0]') for i in range(num_records): coda.cursor_goto(cursor, 'profile_data[0]') for j in range(num_layers - 1):
def extract_grib_metadata(gribfile): """ this will return a tuple containing: - ecmwfmars properties struct - levtype_options struct (see set_remote_url()) """ import coda @contextlib.contextmanager def coda_open(filename): coda_handle = coda.open(filename) try: yield coda_handle finally: coda.close(coda_handle) ecmwfmars = Struct() levtype_options = {} # TODO: add extraction of levtype_options with coda_open(gribfile) as coda_handle: cursor = coda.Cursor() coda.cursor_set_product(cursor, coda_handle) num_messages = coda.cursor_get_num_elements(cursor) coda.cursor_goto_first_array_element(cursor) for i in range(num_messages): index = coda.cursor_get_available_union_field_index(cursor) coda.cursor_goto_record_field_by_index(cursor, index) step = 0 if index == 0: # grib1 centuryOfReferenceTimeOfData = coda.fetch( cursor, "centuryOfReferenceTimeOfData") yearOfCentury = coda.fetch(cursor, "yearOfCentury") month = coda.fetch(cursor, "month") day = coda.fetch(cursor, "day") date = "%02d%02d-%02d-%02d" % (centuryOfReferenceTimeOfData - 1, yearOfCentury, month, day) hour = coda.fetch(cursor, "hour") minute = coda.fetch(cursor, "minute") time = "%02d:%02d:00" % (hour, minute) unitOfTimeRange = coda.fetch(cursor, "unitOfTimeRange") if unitOfTimeRange != 0: P1 = coda.fetch(cursor, "P1") if unitOfTimeRange == 1: step = P1 elif unitOfTimeRange == 2: step = 24 * P1 elif unitOfTimeRange == 10: step = 3 * P1 elif unitOfTimeRange == 11: step = 6 * P1 elif unitOfTimeRange == 13: step = 12 * P1 else: raise Error("unsupported unitOfTimeRange: %d" % (unitOfTimeRange, )) local = coda.fetch(cursor, "local") try: local = local[1:9].tobytes() except AttributeError: # workaround for older numpy versions local = local[1:9].tostring() marsclass, marstype, stream, expver = struct.unpack( '>BBH4s', local) else: # grib2 year = coda.fetch(cursor, "year") month = coda.fetch(cursor, "month") day = coda.fetch(cursor, "day") date = "%04d-%02d-%02d" % (year, month, day) hour = coda.fetch(cursor, "hour") minute = coda.fetch(cursor, "minute") second = coda.fetch(cursor, "second") time = "%02d:%02d:%02d" % (hour, minute, second) significanceOfReferenceTime = coda.fetch( cursor, "significanceOfReferenceTime") local = coda.fetch(cursor, "local[0]") try: local = local[2:12].tobytes() except AttributeError: # workaround for older numpy versions local = local[2:12].tostring() marsclass, marstype, stream, expver = struct.unpack( '>HHH4s', local) coda.cursor_goto_record_field_by_name(cursor, "data") num_data = coda.cursor_get_num_elements(cursor) coda.cursor_goto_first_array_element(cursor) prev_step = None for j in range(num_data): forecastTime = coda.fetch(cursor, "forecastTime") if forecastTime != 0: indicatorOfUnitOfTimeRange = coda.fetch( cursor, "indicatorOfUnitOfTimeRange") if indicatorOfUnitOfTimeRange == 0: # minutes step = 60 * forecastTime elif indicatorOfUnitOfTimeRange == 1: # hours step = 60 * 60 * forecastTime elif indicatorOfUnitOfTimeRange == 2: # days step = 24 * 60 * 60 * forecastTime elif indicatorOfUnitOfTimeRange == 10: # 3 hours step = 3 * 60 * 60 * forecastTime elif indicatorOfUnitOfTimeRange == 11: # 6 hours step = 6 * 60 * 60 * forecastTime elif indicatorOfUnitOfTimeRange == 12: # 12 hours step = 12 * 60 * 60 * forecastTime elif indicatorOfUnitOfTimeRange == 13: # seconds step = forecastTime step = int(step / 3600.) # convert seconds to hours if prev_step is None: prev_step = step elif step != prev_step: raise Error( "not all data has the same 'step' time (%d) (%d)" % (step, prev_step)) if j < num_data - 1: coda.cursor_goto_next_array_element(cursor) coda.cursor_goto_parent(cursor) coda.cursor_goto_parent(cursor) if marsclass not in MARSCLASSES: raise Error("unsupported MARS class (%d)" % (marsclass, )) marsclass = MARSCLASSES[marsclass] if marstype not in MARSTYPES: raise Error("unsupported MARS type (%d)" % (marstype, )) marstype = MARSTYPES[marstype] if stream not in MARSSTREAMS: raise Error("unsupported MARS stream (%d)" % (stream, )) stream = MARSSTREAMS[stream] if 'date' in ecmwfmars: if date != ecmwfmars.date: raise Error("not all data is for the same date (%s) (%s)" % (date, ecmwfmars.date)) if time != ecmwfmars.time: raise Error("not all data is for the same time (%s) (%s)" % (time, ecmwfmars.time)) if step != 0: if 'step' in ecmwfmars: if step != ecmwfmars.step: raise Error( "not all data has the same 'step' time (%d) (%d)" % (step, ecmwfmars.step)) else: raise Error("not all data has the same 'step' time") else: if 'step' in ecmwfmars and ecmwfmars.step != 0: raise Error("not all data has the same 'step' time") if marsclass != ecmwfmars.marsclass: raise Error( "not all data has the same MARS class (%s) (%s)" % (marsclass, ecmwfmars.marsclass)) if marstype != ecmwfmars.type: raise Error( "not all data has the same MARS type (%s) (%s)" % (marstype, ecmwfmars.type)) if stream != ecmwfmars.stream: raise Error( "not all data has the same MARS stream (%s) (%s)" % (stream, ecmwfmars.stream)) if expver != ecmwfmars.expver: raise Error( "not all data has the same MARS experiment version (%s) (%s)" % (expver, ecmwfmars.expver)) else: ecmwfmars.date = date ecmwfmars.time = time if step != 0: ecmwfmars.step = step ecmwfmars.marsclass = marsclass ecmwfmars.type = marstype ecmwfmars.stream = stream ecmwfmars.expver = expver if (marsclass, stream, expver, marstype) in PUBLIC_DATASETS: ecmwfmars.dataset = PUBLIC_DATASETS[(marsclass, stream, expver, marstype)] coda.cursor_goto_parent(cursor) if i < num_messages - 1: coda.cursor_goto_next_array_element(cursor) return ecmwfmars, levtype_options
os.putenv('CODA_DEFINITION', '/usr/local/share/coda/definitions') # You can also remove this line and set the CODA_DEFINITION environment variable globally on your system import coda from numpy import hstack, vstack # change this to the full path of your Aeolus L2A DBL file filename = "/path/to/AE_OPER_ALD_U_N_2A_20071107T193753059_008688000_000578_0001.DBL" product = coda.open(filename) ### Standard Correct Algorithm (SCA) at middle bins ### if coda.get_field_available(product, 'sca_optical_properties'): print("SCA optical properties at middle bins") latitude = coda.fetch(product, 'sca_optical_properties', -1, 'geolocation_middle_bins', -1, 'latitude') latitude = vstack(latitude) longitude = coda.fetch(product, 'sca_optical_properties', -1, 'geolocation_middle_bins', -1, 'longitude') longitude = vstack(longitude) altitude = coda.fetch(product, 'sca_optical_properties', -1, 'geolocation_middle_bins', -1, 'altitude') altitude = vstack(altitude) extinction = coda.fetch(product, 'sca_optical_properties', -1, 'sca_optical_properties_mid_bins', -1, 'extinction') extinction = vstack(extinction) print(extinction.shape) print(extinction) backscatter = coda.fetch(product, 'sca_optical_properties', -1, 'sca_optical_properties_mid_bins', -1, 'backscatter') backscatter = vstack(backscatter)
import coda pf = coda.open('/data/GOME2/GOME_xxx_1B/GOME_xxx_1B_M02_20100415122955Z_20100415123255Z_N_T_20100713095647Z') cursor = coda.Cursor() coda.cursor_set_product(cursor, pf) coda.cursor_goto(cursor, '/MDR') num_mdr = coda.cursor_get_num_elements(cursor) if num_mdr > 0: coda.cursor_goto_first_array_element(cursor) for i in xrange(num_mdr): index = coda.cursor_get_available_union_field_index(cursor) if index == 0: # Earthshine MDR # Note that fetching the full MDR is rather slow, since it converts # the full MDR to a hierarchicel set of Python structures mdr = coda.fetch(cursor, 'Earthshine') print mdr # If you want e.g. just the wavelength and band data of band 1b, you could use: # wavelength = coda.fetch(cursor, 'Earthshine', 'wavelength_1b') # rad = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'rad') # err = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'err_rad') # which will be much faster elif index == 1: # Calibration MDR pass elif index == 2: # Sun MDR mdr = coda.fetch(cursor, 'Sun') print mdr elif index == 3: # Moon MDR
#print("Individual Mie HLOS wind points") #latitude = coda.fetch(product, 'mie_geolocation', -1, 'windresult_geolocation/latitude_cog') #longitude = coda.fetch(product, 'mie_geolocation', -1, 'windresult_geolocation/longitude_cog') #altitude = coda.fetch(product, 'mie_geolocation', -1, 'windresult_geolocation/altitude_vcog') #mie_wind_velocity = coda.fetch(product, 'mie_hloswind', -1, 'windresult/mie_wind_velocity') #print(mie_wind_velocity.shape) #print(mie_wind_velocity) ### Rayleigh observation wind profiles points ### print("Individual Rayleight HLOS wind points") latitude = coda.fetch(product, 'rayleigh_geolocation', -1, 'windresult_geolocation/latitude_cog') longitude = coda.fetch(product, 'rayleigh_geolocation', -1, 'windresult_geolocation/longitude_cog') altitude = coda.fetch(product, 'rayleigh_geolocation', -1, 'windresult_geolocation/altitude_vcog') rayleigh_wind_velocity = coda.fetch(product, 'rayleigh_hloswind', -1, 'windresult/rayleigh_wind_velocity') print(latitude.shape) print(latitude) print(longitude.shape) print(longitude) print(altitude.shape) print(altitude) print(rayleigh_wind_velocity.shape) print(rayleigh_wind_velocity)
class CLASS_GOME_L1(): def __init__(self, BandLst): self.k = 1.98644746103858e-9 # 字典类型物理量 self.Ref = {} # 二维矩阵 self.Lons = [] self.Lats = [] self.Time = [] self.satAzimuth = [] self.satZenith = [] self.sunAzimuth = [] self.sunZenith = [] # 光谱信息 self.wavenumber = [] self.radiance = [] def Load(self, L1File): print u'读取 LEO所有数据信息......' if not os.path.isfile(L1File): print 'Error: %s not found' % L1File sys.exit(1) try: fp = coda.open(L1File) except Exception, e: print 'Open file error<%s> .' % (e) return try: # EPS = EUMETSAT Polar System atmospheric products (GOME-2 and IASI) # EPS = EUMETSAT极地大气系统产品(GOME-2和IASI)' # 获取文件头信息 product_class = coda.get_product_class(fp) product_type = coda.get_product_type(fp) product_version = coda.get_product_version(fp) product_format = coda.get_product_format(fp) product_size = coda.get_product_file_size(fp) print 'product_class ', product_class print 'product_type', product_type print 'product_version', product_version print 'product_format', product_format print 'product_size', product_size record = beatl2.ingest(L1File) WAVE_3 = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'WAVELENGTH_3') WAVE_4 = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'WAVELENGTH_4') LAMBDA_SMR = coda.fetch(fp, 'VIADR_SMR', -1, 'LAMBDA_SMR') SMR = coda.fetch(fp, 'VIADR_SMR', -1, 'SMR') print 'gome data is:' print record self.rec = record SUN_Z = coda.fetch( fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'SOLAR_ZENITH') SUN_A = coda.fetch( fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'SOLAR_AZIMUTH') SAT_Z = coda.fetch( fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'SAT_ZENITH') SAT_A = coda.fetch( fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'SAT_AZIMUTH') print '太阳方位角度长度', SUN_Z.shape, SUN_Z[0].shape # 数据观测总长度 dataLen = record.latitude.size dataCol = SUN_Z[0].shape[1] dataRow = SUN_Z.shape[0] # 角度存放内存 self.satZenith = np.full((dataLen, 1), -999.) self.satAzimuth = np.full((dataLen, 1), -999.) self.sunZenith = np.full((dataLen, 1), -999.) self.sunAzimuth = np.full((dataLen, 1), -999.) # 开始赋值 for i in xrange(dataLen): row, col = np.unravel_index(i, (dataRow, dataCol)) self.satAzimuth[i] = SAT_A[row][1][col] self.satZenith[i] = SAT_Z[row][1][col] self.sunAzimuth[i] = SUN_A[row][1][col] self.sunZenith[i] = SUN_Z[row][1][col] self.Lons = (record.longitude).reshape(dataLen, 1) self.Lats = (record.latitude).reshape(dataLen, 1) # 计算gome的辐亮度 self.radiance = record.spectral_radiance[:, 2048:] for m in xrange(dataLen): row, col = np.unravel_index(m, (dataRow, dataCol)) for i in xrange(2048): if i < 1024: self.radiance[m, i] = self.radiance[ m, i] * self.k / WAVE_3[row][i] else: self.radiance[m, i] = self.radiance[ m, i] * self.k / WAVE_4[row][i - 1024] # 计算太阳辐亮度 self.vec_Solar_L = np.zeros((2048,)) # 太阳辐亮度 self.vec_Solar_WL = np.zeros((2048,)) # 太阳辐亮度对应的波长 for i in xrange(2048): if i < 1024: self.vec_Solar_L[i] = ( SMR[0][2][i] * self.k) / LAMBDA_SMR[0][2][i] self.vec_Solar_WL[i] = LAMBDA_SMR[0][2][i] else: self.vec_Solar_L[i] = ( SMR[0][3][i - 1024] * self.k) / LAMBDA_SMR[0][3][i - 1024] self.vec_Solar_WL[i] = LAMBDA_SMR[0][3][i - 1024] print 'GOME数据观测长度 %d' % dataLen print '太阳辐亮度波长最小最大值' print np.min(self.vec_Solar_WL), self.vec_Solar_WL[0:3] print np.max(self.vec_Solar_WL) # 暂时取一个观测的光谱波数 self.wavenumber = record.wavelength[0, 2048:] print 'GOME辐亮度波长最小最大值,取第一个观测点' print np.min(self.wavenumber), self.wavenumber[0:3] print np.max(self.wavenumber) self.wavenumber = record.wavelength[9, 2048:] print 'GOME辐亮度波长最小最大值,取第十个观测点' print np.min(self.wavenumber), self.wavenumber[0:3] print np.max(self.wavenumber) v_ymd2seconds = np.vectorize(metop_ymd2seconds) T1 = v_ymd2seconds(record.time) self.Time = T1.reshape(dataLen, 1) # print time.gmtime(self.Time[0, 0]) except Exception as e: print str(e) sys.exit(1) finally: coda.close(fp)
raise IOError(text) # leaf albedo file generated from PROSPECT # typical leaf level parameters generated based on metadata-analysis # see spreadsheet in PROSPECT folder l_lower = 580. # limits for plotting purposes l_upper = 800. l_fn = 'leaf_spectrum.txt' leaf_alb_arr = np.genfromtxt(l_fn) leaf_alb_arr = leaf_alb_arr[np.intersect1d(np.array(np.where(leaf_alb_arr[:,0]\ >=l_lower)),np.array(np.where(leaf_alb_arr[:,0]<=l_upper)))] leaf_lam = leaf_alb_arr[:, 0] leaf_alb_arr = leaf_alb_arr[:, 1] + leaf_alb_arr[:, 2] # reading solar wavelength and irradiance in band 4 sol_lam = coda.fetch(ef, 'VIADR_SMR', 0, 'LAMBDA_SMR')[3] n_sol_lam = len(sol_lam) sol_irr = coda.fetch(ef, 'VIADR_SMR', 0, 'SMR')[3] sol_irr = phot_Watt(sol_irr, sol_lam) # reading total number of scans n_mdr = coda.fetch(ef, 'MPHR', 'TOTAL_MDR') # list for point lat, lon, radiance, irradiance, spectrum etc. Lat = [] Lon = [] Spec = [] Rad = [] Irr = [] Alb_sur = [] Alb_toa = []
pf = coda.open( '/data/GOME2/GOME_xxx_1B/GOME_xxx_1B_M02_20100415122955Z_20100415123255Z_N_T_20100713095647Z' ) cursor = coda.Cursor() coda.cursor_set_product(cursor, pf) coda.cursor_goto(cursor, '/MDR') num_mdr = coda.cursor_get_num_elements(cursor) if num_mdr > 0: coda.cursor_goto_first_array_element(cursor) for i in xrange(num_mdr): index = coda.cursor_get_available_union_field_index(cursor) if index == 0: # Earthshine MDR # Note that fetching the full MDR is rather slow, since it converts # the full MDR to a hierarchicel set of Python structures mdr = coda.fetch(cursor, 'Earthshine') print mdr # If you want e.g. just the wavelength and band data of band 1b, you could use: # wavelength = coda.fetch(cursor, 'Earthshine', 'wavelength_1b') # rad = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'rad') # err = coda.fetch(cursor, 'Earthshine', 'band_1b', [-1,-1], 'err_rad') # which will be much faster elif index == 1: # Calibration MDR pass elif index == 2: # Sun MDR mdr = coda.fetch(cursor, 'Sun') print mdr elif index == 3: # Moon MDR
product = coda.open(filename) # The L2B product stores all valid profile points (which are called 'results') as one big consecutive array # in the datasets mie_hloswind and rayleigh_hloswind. # The associated geolocation for those point results are stored in mie_geolocation and rayleigh_geolocation. # # The specification of which points belong together in which profile is kept in the mie_profile and rayleigh_profile # datasets. These datasets contain indices into the mie_hloswind and rayleigh hloswind datasets to specify which point # belongs at which level in a profile. ### Mie horizontal line of sight wind profile points ### print("Individual Mie HLOS wind points") latitude = coda.fetch(product, 'mie_geolocation', -1, 'windresult_geolocation/latitude_cog') longitude = coda.fetch(product, 'mie_geolocation', -1, 'windresult_geolocation/longitude_cog') altitude = coda.fetch(product, 'mie_geolocation', -1, 'windresult_geolocation/altitude_vcog') mie_wind_velocity = coda.fetch(product, 'mie_hloswind', -1, 'windresult/mie_wind_velocity') print(mie_wind_velocity.shape) print(mie_wind_velocity) ### Rayleigh observation wind profiles points ### print("Individual Rayleight HLOS wind points") latitude = coda.fetch(product, 'rayleigh_geolocation', -1, 'windresult_geolocation/latitude_cog')
product = coda.open(filename) ### Observation profiles ### # Observation profiles provide a single profile per BRC. # Reading all values will therefore return 'an array of arrays' (we will get an array for each '-1' in the coda.fetch()). # The first array is the list of BRCs, and the second array the list of points in the profile. # This array of arrays can be turned into a single 2D numpy array of shape [num_brc, num_vertical] by using 'vstack'. # Mie observation wind profiles print("Mie observation wind profiles") latitude = coda.fetch(product, 'geolocation', -1, 'observation_geolocation/observation_mie_geolocation', -1, 'latitude_of_height_bin') latitude = vstack(latitude) longitude = coda.fetch(product, 'geolocation', -1, 'observation_geolocation/observation_mie_geolocation', -1, 'longitude_of_height_bin') longitude = vstack(longitude) altitude = coda.fetch(product, 'geolocation', -1, 'observation_geolocation/observation_mie_geolocation', -1, 'altitude_of_height_bin') altitude = vstack(altitude) wind_velocity = coda.fetch( product, 'wind_velocity', -1,
def Initialize(self, node): assert node.type == model.TYPE_ARRAY, "ArrayRenderer can only render node of type TYPE_ARRAY" assert not isinstance(node, model.ObjectArrayNode), "ArrayRenderer cannot render arrays of compound types" # coda.fetch() will promote a rank-0 array to a 1x1 array, # so a rank of 1 is the smallest possible rank. try: self.array = coda.fetch(node.cursor) except (coda.CodaError, coda.CodacError) as ex: raise RenderError("[CODA] %s" % (str(ex),)) # array sanity check ok = True if node.isRankZero(): ok = (self.array.ndim == 1) and (self.array.shape[0] == 1) else: ok = (self.array.ndim == len(node.dimensions)) and (self.array.size == len(node)) i = 0 while ok and i < len(node.dimensions): ok = ok and (self.array.shape[i] == node.dimensions[i]) i += 1 if not ok: raise RenderError("[ArrayRenderer] Array read from product does not match description.") self.base = node.base self.slice = None # coda.fetch() will promote a rank-0 array to a 1x1 array, # so a rank of 1 is the smallest possible rank. if self.array.ndim == 1: self.array = self.array[:, numpy.newaxis] self.valueText = wx.TextCtrl(self, -1, "") valueSizer = wx.BoxSizer(wx.HORIZONTAL) valueSizer.Add(wx.StaticText(self, -1, "Value:"), 0, wx.ALIGN_CENTER, 0) valueSizer.Add((5, 5), 0, wx.EXPAND) valueSizer.Add(self.valueText, 1, 0, 0) self.sizer = wx.BoxSizer(wx.VERTICAL) self.grid = wx.grid.Grid(self, -1, style=wx.BORDER_SUNKEN) # TODO: use wxGrid::SetDefaultColSize() and row equiv. as an approx. # of AutoSize(). if self.array.ndim > 2: self.slicer = NumpySlicer2D(self, -1, self.array, ("row", "col")) self.Bind(EVT_SLICE_CHANGED, self.OnSliceChanged) self.UpdateGridData() # _really_ slow, so only used for string data (which can require a large cell width). if self.base == model.TYPE_STRING: self.grid.AutoSize() self.sizer.Add(valueSizer, 0, wx.EXPAND | wx.ALL, 5) self.sizer.Add(wx.StaticLine(self, -1, style=wx.LI_HORIZONTAL), 0, wx.EXPAND | wx.LEFT | wx.RIGHT, 5) self.sizer.Add(self.grid, 1, wx.EXPAND | wx.ALL, 5) if self.array.ndim > 2: self.sizer.Add(self.slicer, 0, wx.EXPAND | wx.LEFT | wx.RIGHT, 5) self.SetSizer(self.sizer) self.Bind(wx.grid.EVT_GRID_SELECT_CELL, self.OnSelectCell)
class CLASS_IASI_L1(): def __init__(self, BandLst): # 字典类型物理量 self.Tbb = {} self.Rad = {} # 二维矩阵 self.Lons = [] self.Lats = [] self.Time = [] self.satAzimuth = [] self.satZenith = [] self.sunAzimuth = [] self.sunZenith = [] # 光谱信息 self.wavenumber = [] self.radiance = [] def Load(self, L1File): print u'读取 LEO所有数据信息......' if not os.path.isfile(L1File): print 'Error: %s not found' % L1File sys.exit(1) try: fp = coda.open(L1File) except Exception, e: print 'Open file error<%s> .' % (e) return try: # EPS = EUMETSAT Polar System atmospheric products (GOME-2 and IASI) # EPS = EUMETSAT极地大气系统产品(GOME-2和IASI)' # 获取文件头信息 product_class = coda.get_product_class(fp) product_type = coda.get_product_type(fp) product_version = coda.get_product_version(fp) product_format = coda.get_product_format(fp) product_size = coda.get_product_file_size(fp) print 'product_class ', product_class print 'product_type', product_type print 'product_version', product_version print 'product_format', product_format print 'product_size', product_size record = beatl2.ingest(L1File) print record SAT_angle = coda.fetch(fp, 'MDR', -1, 'MDR', 'GGeoSondAnglesMETOP') SUN_angle = coda.fetch(fp, 'MDR', -1, 'MDR', 'GGeoSondAnglesSUN') all_sun_angle = [] all_sat_angle = [] for i in xrange(len(SAT_angle)): tmp_sat = SAT_angle[i].reshape(-1) tmp_sun = SUN_angle[i].reshape(-1) if len(all_sat_angle) == 0: all_sat_angle = tmp_sat all_sun_angle = tmp_sun else: all_sat_angle = np.concatenate((all_sat_angle, tmp_sat), 0) all_sun_angle = np.concatenate((all_sun_angle, tmp_sun), 0) iasiLen = len(record.longitude) self.satZenith = (all_sat_angle[0::2]).reshape(iasiLen, 1) self.satAzimuth = (all_sat_angle[1::2]).reshape(iasiLen, 1) self.sunZenith = (all_sun_angle[0::2]).reshape(iasiLen, 1) self.sunAzimuth = (all_sun_angle[1::2]).reshape(iasiLen, 1) self.Lons = (record.longitude).reshape(iasiLen, 1) self.Lats = (record.latitude).reshape(iasiLen, 1) self.radiance = record.spectral_radiance * 10**7 # 暂时取一个观测的光谱波数 self.wavenumber = record.wavenumber[0, :] v_ymd2seconds = np.vectorize(metop_ymd2seconds) T1 = v_ymd2seconds(record.time) self.Time = T1.reshape(iasiLen, 1) except Exception as e: print str(e) sys.exit(1) finally: coda.close(fp)
def get_spectral_response(self): """ return 光谱波数和响应值,1维,2维 """ k = 1.98644746103858e-9 if self.resolution == 40000: satellite_type1 = ['METOP-A', 'METOP-B'] if self.satellite in satellite_type1: try: fp = coda.open(self.in_file) wave3 = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'WAVELENGTH_3') wave4 = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'WAVELENGTH_4') lambda_smr = coda.fetch(fp, 'VIADR_SMR', -1, 'LAMBDA_SMR') smr = coda.fetch(fp, 'VIADR_SMR', -1, 'SMR') sunz = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'SOLAR_ZENITH') coda.close(fp) # gome 959x4096 取后2048个点的辐射值 wavelens = self.record.wavelength[0, 2048:] response = self.record.spectral_radiance[:, 2048:] data_size = self.data_shape[0] data_row = sunz.shape[0] data_col = sunz[0].shape[1] # gome的辐亮度计算 wave3 wave4 是30*1024 for m in xrange(data_size): row, col = np.unravel_index(m, (data_row, data_col)) for i in xrange(2048): if i < 1024: response[m, i] = response[m, i] * k / wave3[row][i] else: response[m, i] = response[ m, i] * k / wave4[row][i - 1024] # 计算太阳辐亮度 sol_wavelens = np.zeros((2048, )) # 太阳辐亮度 sol_response = np.zeros((2048, )) # 太阳辐亮度对应的波长 for i in xrange(2048): if i < 1024: sol_response[i] = (smr[0][2][i] * k) / lambda_smr[0][2][i] sol_wavelens[i] = lambda_smr[0][2][i] else: sol_response[i] = (smr[0][3][i - 1024] * k) / lambda_smr[0][3][i - 1024] sol_wavelens[i] = lambda_smr[0][3][i - 1024] idx = np.where(response < 0) print len(idx[0]) if len(idx[0] > 0): response[idx] = 0. gome_wavelens, gome_response, solar_response = self.combine_gome_band34( wavelens, response, sol_wavelens, sol_response) s0, s1 = gome_response.shape gome_response = gome_response.reshape(s0, 1, s1) print gome_response.shape, solar_response.shape except Exception as e: print 'Open file error {}'.format(e) return self.solar_response = solar_response return gome_wavelens, gome_response
def read_file(self, filename, vars_to_read=None, return_as='dict', loglevel=None): """method to read an ESA binary data file entirely Parameters ---------- filename : str absolute path to filename to read vars_to_read : list list of str with variable names to read; defaults to ['od550aer'] verbose : Bool set to True to increase verbosity Returns -------- Either: dictionary (default): keys are 'time', 'latitude', 'longitude', 'altitude' and the variable names 'ec550aer', 'bs550aer', 'sr', 'lod' if the whole file is read 'time' is a 1d array, while the other dict values are a another dict with the time as keys (the same ret['time']) and a numpy array as values. These values represent the profile. Note 1: latitude and longitude are height dependent due to the tilt of the measurement. Note 2: negative values indicate a NaN 2d ndarray of type float: representing a 'point cloud' with all points column 1: time in seconds since the Unix epoch with ms accuracy (same time for every height in a profile) column 2: latitude column 3: longitude column 4: altitude column 5: extinction column 6: backscatter column 7: sr column 8: lod Note: negative values are put to np.nan already The indexes are noted in pyaerocom.io.read_aeolus_l2a_data.ReadAeolusL2aData.<index_name> e.g. the time index is named pyaerocom.io.read_aeolus_l2a_data.ReadAeolusL2aData._TIMEINDEX have a look at the example to access the values This is whats in one DBL file codadump list /lustre/storeA/project/aerocom/aerocom1/ADM_CALIPSO_TEST/download/AE_OPER_ALD_U_N_2A_20070101T002249149_002772000_003606_0001.DBL /mph/product /mph/proc_stage /mph/ref_doc /mph/acquisition_station /mph/proc_center /mph/proc_time /mph/software_ver /mph/baseline /mph/sensing_start /mph/sensing_stop /mph/phase /mph/cycle /mph/rel_orbit /mph/abs_orbit /mph/state_vector_time /mph/delta_ut1 /mph/x_position /mph/y_position /mph/z_position /mph/x_velocity /mph/y_velocity /mph/z_velocity /mph/vector_source /mph/utc_sbt_time /mph/sat_binary_time /mph/clock_step /mph/leap_utc /mph/leap_sign /mph/leap_err /mph/product_err /mph/tot_size /mph/sph_size /mph/num_dsd /mph/dsd_size /mph/num_data_sets /sph/sph_descriptor /sph/intersect_start_lat /sph/intersect_start_long /sph/intersect_stop_lat /sph/intersect_stop_long /sph/sat_track /sph/num_brc /sph/num_meas_max_brc /sph/num_bins_per_meas /sph/num_prof_sca /sph/num_prof_ica /sph/num_prof_mca /sph/num_group_tot /dsd[?]/ds_name /dsd[?]/ds_type /dsd[?]/filename /dsd[?]/ds_offset /dsd[?]/ds_size /dsd[?]/num_dsr /dsd[?]/dsr_size /dsd[?]/byte_order /geolocation[?]/start_of_obs_time /geolocation[?]/num_meas_eff /geolocation[?]/measurement_geolocation[?]/centroid_time /geolocation[?]/measurement_geolocation[?]/mie_geolocation_height_bin[25]/longitude_of_height_bin /geolocation[?]/measurement_geolocation[?]/mie_geolocation_height_bin[25]/latitude_of_height_bin /geolocation[?]/measurement_geolocation[?]/mie_geolocation_height_bin[25]/altitude_of_height_bin /geolocation[?]/measurement_geolocation[?]/rayleigh_geolocation_height_bin[25]/longitude_of_height_bin /geolocation[?]/measurement_geolocation[?]/rayleigh_geolocation_height_bin[25]/latitude_of_height_bin /geolocation[?]/measurement_geolocation[?]/rayleigh_geolocation_height_bin[25]/altitude_of_height_bin /geolocation[?]/measurement_geolocation[?]/longitude_of_dem_intersection /geolocation[?]/measurement_geolocation[?]/latitude_of_dem_intersection /geolocation[?]/measurement_geolocation[?]/altitude_of_dem_intersection /geolocation[?]/geoid_separation /meas_pcd[?]/start_of_obs_time /meas_pcd[?]/l1b_input_screening/l1b_obs_screening /meas_pcd[?]/l1b_input_screening/l1b_obs_screening_flags[40] /meas_pcd[?]/l1b_input_screening/l1b_mie_meas_screening[?]/l1b_mie_meas_qc /meas_pcd[?]/l1b_input_screening/l1b_mie_meas_screening[?]/l1b_mie_meas_qc_flags[8] /meas_pcd[?]/l1b_input_screening/l1b_rayleigh_meas_screening[?]/l1b_rayleigh_meas_qc /meas_pcd[?]/l1b_input_screening/l1b_rayleigh_meas_screening[?]/l1b_rayleigh_meas_qc_flags[8] /meas_pcd[?]/l1b_cal_screening/cal_valid /meas_pcd[?]/l2a_processing_qc/sca_applied /meas_pcd[?]/l2a_processing_qc/ica_applied /meas_pcd[?]/l2a_processing_qc/mca_applied /meas_pcd[?]/l2a_processing_qc/feature_finder_indicators/layer_information[24]/bin_loaded /meas_pcd[?]/l2a_processing_qc/feature_finder_indicators/layer_information[24]/seed[30] /meas_pcd[?]/l2a_processing_qc/feature_finder_indicators/lowest_computable_bin[30] /sca_pcd[?]/starttime /sca_pcd[?]/firstmatchingbin /sca_pcd[?]/qc_flag /sca_pcd[?]/profile_pcd_bins[24]/extinction_variance /sca_pcd[?]/profile_pcd_bins[24]/backscatter_variance /sca_pcd[?]/profile_pcd_bins[24]/lod_variance /sca_pcd[?]/profile_pcd_bins[24]/processing_qc_flag /sca_pcd[?]/profile_pcd_mid_bins[23]/extinction_variance /sca_pcd[?]/profile_pcd_mid_bins[23]/backscatter_variance /sca_pcd[?]/profile_pcd_mid_bins[23]/lod_variance /sca_pcd[?]/profile_pcd_mid_bins[23]/ber_variance /sca_pcd[?]/profile_pcd_mid_bins[23]/processing_qc_flag /ica_pcd[?]/starttime /ica_pcd[?]/first_matching_bin /ica_pcd[?]/qc_flag /ica_pcd[?]/ica_processing_qc_flag_bin[24] /mca_pcd[?]/starttime /mca_pcd[?]/processing_qc_flag_bin[24] /amd_pcd[?]/starttime /amd_pcd[?]/l2b_amd_screening_qc /amd_pcd[?]/l2b_amd_screening_qc_flags /amd_pcd[?]/l2b_amd_collocations[?]/l2b_amd_collocation_qc /amd_pcd[?]/l2b_amd_collocations[?]/l2b_amd_collocation_qc_flags /group_pcd[?]/starttime /group_pcd[?]/brc_start /group_pcd[?]/measurement_start /group_pcd[?]/brc_end /group_pcd[?]/measurement_end /group_pcd[?]/height_bin_index /group_pcd[?]/upper_problem_flag /group_pcd[?]/particle_extinction_variance /group_pcd[?]/particle_backscatter_variance /group_pcd[?]/particle_lod_variance /group_pcd[?]/qc_flag /group_pcd[?]/mid_particle_extinction_variance_top /group_pcd[?]/mid_particle_backscatter_variance_top /group_pcd[?]/mid_particle_lod_variance_top /group_pcd[?]/mid_particle_ber_variance_top /group_pcd[?]/mid_particle_extinction_variance_bot /group_pcd[?]/mid_particle_backscatter_variance_bot /group_pcd[?]/mid_particle_lod_variance_bot /group_pcd[?]/mid_particle_ber_variance_bot /sca_optical_properties[?]/starttime /sca_optical_properties[?]/sca_optical_properties[24]/extinction /sca_optical_properties[?]/sca_optical_properties[24]/backscatter /sca_optical_properties[?]/sca_optical_properties[24]/lod /sca_optical_properties[?]/sca_optical_properties[24]/sr /sca_optical_properties[?]/geolocation_middle_bins[24]/longitude /sca_optical_properties[?]/geolocation_middle_bins[24]/latitude /sca_optical_properties[?]/geolocation_middle_bins[24]/altitude /sca_optical_properties[?]/sca_optical_properties_mid_bins[23]/extinction /sca_optical_properties[?]/sca_optical_properties_mid_bins[23]/backscatter /sca_optical_properties[?]/sca_optical_properties_mid_bins[23]/lod /sca_optical_properties[?]/sca_optical_properties_mid_bins[23]/ber /ica_optical_properties[?]/starttime /ica_optical_properties[?]/ica_optical_properties[24]/case /ica_optical_properties[?]/ica_optical_properties[24]/extinction /ica_optical_properties[?]/ica_optical_properties[24]/backscatter /ica_optical_properties[?]/ica_optical_properties[24]/lod /mca_optical_properties[?]/starttime /mca_optical_properties[?]/mca_optical_properties[24]/climber /mca_optical_properties[?]/mca_optical_properties[24]/extinction /mca_optical_properties[?]/mca_optical_properties[24]/lod /amd[?]/starttime /amd[?]/amd_properties[24]/pressure_fp /amd[?]/amd_properties[24]/temperature_fp /amd[?]/amd_properties[24]/frequencyshift_fp /amd[?]/amd_properties[24]/relativehumidity_fp /amd[?]/amd_properties[24]/molecularlod_fp /amd[?]/amd_properties[24]/molecularbackscatter_fp /amd[?]/amd_properties[24]/pressure_fiz /amd[?]/amd_properties[24]/temperature_fiz /amd[?]/amd_properties[24]/frequencyshift_fiz /amd[?]/amd_properties[24]/relativehumidity_fiz /amd[?]/amd_properties[24]/molecularlod_fiz /amd[?]/amd_properties[24]/molecularbackscatter_fiz /group_optical_properties[?]/starttime /group_optical_properties[?]/height_bin_index /group_optical_properties[?]/group_optical_property/group_extinction /group_optical_properties[?]/group_optical_property/group_backscatter /group_optical_properties[?]/group_optical_property/group_lod /group_optical_properties[?]/group_optical_property/group_sr /group_optical_properties[?]/group_geolocation_middle_bins/start_longitude /group_optical_properties[?]/group_geolocation_middle_bins/start_latitude /group_optical_properties[?]/group_geolocation_middle_bins/start_altitude /group_optical_properties[?]/group_geolocation_middle_bins/mid_longitude /group_optical_properties[?]/group_geolocation_middle_bins/mid_latitude /group_optical_properties[?]/group_geolocation_middle_bins/mid_altitude /group_optical_properties[?]/group_geolocation_middle_bins/stop_longitude /group_optical_properties[?]/group_geolocation_middle_bins/stop_latitude /group_optical_properties[?]/group_geolocation_middle_bins/stop_altitude /group_optical_properties[?]/group_optical_property_middle_bins/mid_extinction_top /group_optical_properties[?]/group_optical_property_middle_bins/mid_backscatter_top /group_optical_properties[?]/group_optical_property_middle_bins/mid_lod_top /group_optical_properties[?]/group_optical_property_middle_bins/mid_ber_top /group_optical_properties[?]/group_optical_property_middle_bins/mid_extinction_bot /group_optical_properties[?]/group_optical_property_middle_bins/mid_backscatter_bot /group_optical_properties[?]/group_optical_property_middle_bins/mid_lod_bot /group_optical_properties[?]/group_optical_property_middle_bins/mid_ber_bot /scene_classification[?]/starttime /scene_classification[?]/height_bin_index /scene_classification[?]/aladin_cloud_flag/clrh /scene_classification[?]/aladin_cloud_flag/clsr /scene_classification[?]/aladin_cloud_flag/downclber /scene_classification[?]/aladin_cloud_flag/topclber /scene_classification[?]/nwp_cloud_flag /scene_classification[?]/l2a_group_class_reliability The question mark indicates a variable size array It is not entirely clear what we actually have to look at. For simplicity the data of the group 'sca_optical_properties' is returned at this point Example ------- >>> import pyaerocom.io.read_aeolus_l2a_data >>> obj = pyaerocom.io.read_aeolus_l2a_data.ReadAeolusL2aData(verbose=True) >>> import os >>> os.environ['CODA_DEFINITION']='/lustre/storeA/project/aerocom/aerocom1/ADM_CALIPSO_TEST/' >>> filename = '/lustre/storeA/project/aerocom/aerocom1/ADM_CALIPSO_TEST/download/AE_OPER_ALD_U_N_2A_20070101T002249149_002772000_003606_0001.DBL' >>> # read returning a ndarray >>> filedata_numpy = obj.read_file(filename, vars_to_read=['ec550aer'], return_as='numpy') >>> time_as_numpy_datetime64 = filedata_numpy[0,obj._TIMEINDEX].astype('datetime64[s]') >>> print('time: {}'.format(time_as_numpy_datetime64)) >>> print('latitude: {}'.format(filedata_numpy[1,obj._LATINDEX])) >>> # read returning a dictionary >>> filedata = obj.read_file(filename, vars_to_read=['ec550aer']) >>> print('time: {}'.format(filedata['time'][0].astype('datetime64[s]'))) >>> print('all latitudes of 1st time step: {}'.format(filedata['latitude'][filedata['time'][0]])) """ import time import coda # coda uses 2000-01-01T00:00:00 as epoch unfortunately. # so calculate the difference in seconds to the Unix epoch seconds_to_add = np.datetime64('2000-01-01T00:00:00') - np.datetime64( '1970-01-01T00:00:00') seconds_to_add = seconds_to_add.astype(np.float_) # the same can be achieved using pandas, but we stick to numpy here # base_time = pd.DatetimeIndex(['2000-01-01']) # seconds_to_add = (base_time.view('int64') // pd.Timedelta(1, unit='s'))[0] start = time.perf_counter() file_data = {} self.logger.info('reading file {}'.format(filename)) # read file product = coda.open(filename) if vars_to_read is None: # read all variables vars_to_read = list(self.DATA_COLNAMES.keys()) vars_to_read.extend(list(self.METADATA_COLNAMES.keys())) # read data # start with the time because it is only stored once groups = self.TIME_PATH.split(self.GROUP_DELIMITER) file_data[self._TIME_NAME] = coda.fetch(product, groups[0], -1, groups[1]) # epoch is 1 January 2000 at ESA # so add offset to move that to 1 January 1970 # and save it into a np.datetime64[ms] object file_data[self._TIME_NAME] = \ ((file_data[self._TIME_NAME] + seconds_to_add) * 1.E3).astype(np.int).astype('datetime64[ms]') # read data in a simple dictionary for var in vars_to_read: groups = self._COLNAMES[var].split(self.GROUP_DELIMITER) if len(groups) == 3: file_data[var] = {} for idx, key in enumerate(file_data[self._TIME_NAME]): file_data[var][key] = coda.fetch(product, groups[0], idx, groups[1], -1, groups[2]) elif len(groups) == 2: file_data[var] = {} for idx, key in enumerate(file_data[self._TIME_NAME]): file_data[var][key] = coda.fetch(product, groups[0], -1, groups[1]) else: file_data[var] = {} for idx, key in enumerate(file_data[self._TIME_NAME]): file_data[var][key] = coda.fetch(product, groups[0]) if return_as == 'numpy': # return as one multidimensional numpy array that can be put into self.data directly # (column wise because the column numbers do not match) index_pointer = 0 data = np.empty([self._ROWNO, self._COLNO], dtype=np.float_) for idx, _time in enumerate(file_data['time'].astype(np.float_) / 1000.): # file_data['time'].astype(np.float_) is milliseconds after the (Unix) epoch # but we want to save the time as seconds since the epoch for _index in range( len(file_data['latitude'][file_data['time'][idx]])): # this works because all variables have to have the same size # (aka same number of height levels) # This loop could be avoided using numpy index slicing # do that in case we need more optimisations data[index_pointer, self._TIMEINDEX] = _time for var in vars_to_read: data[index_pointer, self.INDEX_DICT[var]] = file_data[var][ file_data['time'][idx]][_index] # put negative values to np.nan if the variable is not a metadata variable if data[index_pointer, self.INDEX_DICT[var]] == self.NAN_DICT[var]: data[index_pointer, self.INDEX_DICT[var]] = np.nan index_pointer += 1 if index_pointer >= self._ROWNO: # add another array chunk to self.data data = np.append(data, np.empty( [self._CHUNKSIZE, self._COLNO], dtype=np.float_), axis=0) self._ROWNO += self._CHUNKSIZE # return only the needed elements... file_data = data[0:index_pointer] end_time = time.perf_counter() elapsed_sec = end_time - start temp = 'time for single file read [s]: {:.3f}'.format(elapsed_sec) self.logger.info(temp) # self.logger.info('{} points read'.format(index_pointer)) return file_data
class GOME_COMM(): def __init__(self): self.k = 1.98644746103858e-9 # 中心 纬度 经度 self.centre_lat = np.zeros((30, 24)) self.centre_lon = np.zeros((30, 24)) self.centre_row = np.zeros((30, 24)) self.centre_col = np.zeros((30, 24)) # 角点 纬度 经度 self.corner_lat = np.zeros((30, 24, 4)) self.corner_lon = np.zeros((30, 24, 4)) self.corner_row = np.zeros((30, 24, 4)) self.corner_col = np.zeros((30, 24, 4)) self.sun_Z = np.zeros((30, 24)) # 太阳天顶角 self.sun_A = np.zeros((30, 24)) # 太阳方位角 self.sat_Z = np.zeros((30, 24)) # 卫星天顶角 self.sat_A = np.zeros((30, 24)) # 卫星方位角 self.band3 = np.zeros((30, 24, 1024)) # 辐射值 self.band4 = np.zeros((30, 24, 1024)) self.band3_ERR_RAD = np.zeros((30, 24, 1024)) self.band3_STOKES_FRACTION = np.zeros((30, 24, 1024)) self.band4_ERR_RAD = np.zeros((30, 24, 1024)) self.band4_STOKES_FRACTION = np.zeros((30, 24, 1024)) self.wave3 = np.zeros((30, 1024)) # 波长 self.wave4 = np.zeros((30, 1024)) # self.LAMBDA_SMR3 = np.zeros((1, 1024)) # self.LAMBDA_SMR4 = np.zeros((1, 1024)) # self.SMR3 = np.zeros((1, 1024)) # self.SMR4 = np.zeros((1, 1024)) # self.E_SMR3 = np.zeros((1, 1024)) # self.E_SMR4 = np.zeros((1, 1024)) # self.E_REL_SUN3 = np.zeros((1, 1024)) # self.E_REL_SUN4 = np.zeros((1, 1024)) self.E_SMR3 = np.zeros((1024)) self.E_SMR4 = np.zeros((1024)) self.E_REL_SUN3 = np.zeros((1024)) self.E_REL_SUN4 = np.zeros((1024)) self.arr_GOME_L = np.zeros((2048, 30, 24)) # 辐亮度 self.arr_GOME_WL = np.zeros((2048, 30, 24)) # 辐亮度对应的波长 self.vec_Solar_L = np.zeros((2, 1024)) # 太阳辐亮度 self.vec_Solar_WL = np.zeros((2, 1024)) # 太阳辐亮度对应的波长 # 如下变量是通过卷积计算的 self.vec_Radiance_Solar = np.zeros(15) self.vec_Radiance = np.zeros((15, 30, 24)) self.arr_Ref = np.zeros((15, 30, 24)) def init_gome(self, in_proj_cfg, des_sensor): ''' 读取yaml格式配置文件 ''' if not os.path.isfile(in_proj_cfg): print 'Not Found %s' % in_proj_cfg sys.exit(-1) with open(in_proj_cfg, 'r') as stream: cfg = yaml.load(stream) self.sat1 = cfg['INFO']['sat'] self.sensor1 = cfg['INFO']['sensor'] self.sensor2 = des_sensor self.ymd = cfg['INFO']['ymd'] self.ifile = cfg['PATH']['ipath'] self.ofile = cfg['PATH']['opath'] self.cmd = cfg['PROJ']['cmd'] self.col = cfg['PROJ']['col'] self.row = cfg['PROJ']['row'] self.res = cfg['PROJ']['res'] def read_gome(self, infile): # 打开gome文件 try: fp = coda.open(infile) except Exception, e: print 'Open file error<%s> .' % (e) return # 获取文件头信息 product_class = coda.get_product_class(fp) product_type = coda.get_product_type(fp) product_version = coda.get_product_version(fp) product_format = coda.get_product_format(fp) product_size = coda.get_product_file_size(fp) # EPS = EUMETSAT Polar System atmospheric products (GOME-2 and IASI) # EUMETSAT极地大气系统产品(GOME-2和IASI)' print 'product_class ', product_class print 'product_type', product_type print 'product_version', product_version print 'product_format', product_format print 'product_size', product_size CENTRE = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'CENTRE') CORNER = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'CORNER') SUN_Z = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'SOLAR_ZENITH') SUN_A = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'SOLAR_AZIMUTH') SAT_Z = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'SAT_ZENITH') SAT_A = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'GEO_EARTH', 'SAT_AZIMUTH') BAND_3 = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'BAND_3') BAND_4 = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'BAND_4') WAVE_3 = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'WAVELENGTH_3') WAVE_4 = coda.fetch(fp, 'MDR', -1, 'Earthshine', 'WAVELENGTH_4') LAMBDA_SMR = coda.fetch(fp, 'VIADR_SMR', -1, 'LAMBDA_SMR') SMR = coda.fetch(fp, 'VIADR_SMR', -1, 'SMR') E_SMR = coda.fetch(fp, 'VIADR_SMR', -1, 'E_SMR') E_REL_SUN = coda.fetch(fp, 'VIADR_SMR', -1, 'E_REL_SUN') # fp.close() # print 'CENTRE', CENTRE.shape, CENTRE[0].shape # print 'CORNER', CORNER.shape, CORNER[0].shape # print 'SUN_Z', SUN_Z.shape, SUN_Z[0].shape # print 'SUN_A', SUN_A.shape, SUN_A[0].shape # print 'SAT_Z', SAT_Z.shape, SAT_Z[0].shape # print 'SAT_A', SAT_A.shape, SAT_A[0].shape # print 'BAND_3', BAND_3.shape, BAND_3[0].shape # print 'BAND_4', BAND_4.shape, BAND_4[0].shape # print 'WAVE_3', WAVE_3.shape, WAVE_3[0].shape # print 'WAVE_4', WAVE_4.shape, WAVE_4[0].shape # print 'LAMBDA_SMR', LAMBDA_SMR.shape, LAMBDA_SMR[0].shape # print 'SMR', SMR.shape, SMR[0].shape # print 'E_SMR', E_SMR.shape, E_SMR[0].shape # print 'E_REL_SUN', E_REL_SUN.shape, E_REL_SUN[0].shape # print LAMBDA_SMR[0][0][1023] # print BAND_3[0][23][1023].RAD # print self.band3.shape for i in range(30): count = coda.get_size(fp, 'MDR', i, 'Earthshine', 'BAND_3') for j in range(int(count[0] * 0.75)): for m in range(1024): self.band3[i][j][m] = BAND_3[i][j][m].RAD self.band3_ERR_RAD[i][j][m] = BAND_3[i][j][m].ERR_RAD self.band3_STOKES_FRACTION[i][j][m] = BAND_3[i][j][ m].STOKES_FRACTION self.band4[i][j][m] = BAND_4[i][j][m].RAD self.band4_ERR_RAD[i][j][m] = BAND_4[i][j][m].ERR_RAD self.band4_STOKES_FRACTION[i][j][m] = BAND_4[i][j][ m].STOKES_FRACTION for m in range(2048): for i in range(30): count = coda.get_size(fp, 'MDR', i, 'Earthshine', 'BAND_3') for j in range(int(count[0] * 0.75)): if m < 1024: if BAND_3[i][j][m].RAD < 0: BAND_3[i][j][m].RAD = 0 self.arr_GOME_L[m][i][j] = (BAND_3[i][j][m].RAD * self.k) / WAVE_3[i][m] self.arr_GOME_WL[m][i][j] = WAVE_3[i][m] else: if BAND_4[i][j][m - 1024].RAD < 0: BAND_4[i][j][m - 1024].RAD = 0 self.arr_GOME_L[m][i][j] = ( BAND_4[i][j][m - 1024].RAD * self.k) / WAVE_4[i][m - 1024] self.arr_GOME_WL[m][i][j] = WAVE_4[i][m - 1024] for i in range(2): for j in range(1024): if i == 0: self.vec_Solar_L[i][j] = ( SMR[0][2][j] * self.k) / LAMBDA_SMR[0][2][j] # 太阳辐亮度 self.vec_Solar_WL[i][j] = LAMBDA_SMR[0][2][j] self.E_SMR3[j] = E_SMR[0][2][j] self.E_REL_SUN3[j] = E_REL_SUN[0][2][j] elif i == 1: self.vec_Solar_L[i][j] = ( SMR[0][3][j] * self.k) / LAMBDA_SMR[0][3][j] # 太阳辐亮度 self.vec_Solar_WL[i][j] = LAMBDA_SMR[0][3][j] self.E_SMR4[j] = E_SMR[0][3][j] self.E_REL_SUN4[j] = E_REL_SUN[0][3][j] for i in range(30): for j in range(24): self.sun_A[i][j] = SUN_A[i][1][j] self.sun_Z[i][j] = SUN_Z[i][1][j] self.sat_A[i][j] = SAT_A[i][1][j] self.sun_Z[i][j] = SAT_Z[i][1][j] self.centre_lat[i][j] = CENTRE[i][j].latitude self.centre_lon[i][j] = CENTRE[i][j].longitude for i in range(30): for j in range(24): for m in range(4): self.corner_lat[i][j][m] = CORNER[i][m][j].latitude self.corner_lon[i][j][m] = CORNER[i][m][j].longitude coda.close(fp)
def _read(self, path, fields="all", return_header=False): tmpdira = config.conf["main"]["tmpdir"] tmpdirb = config.conf["main"]["tmpdirb"] tmpdir = (tmpdira if shutil.disk_usage(tmpdira).free > self.minspace else tmpdirb) with tempfile.NamedTemporaryFile(mode="wb", dir=tmpdir, delete=True) as tmpfile: with gzip.open(str(path), "rb") as gzfile: logging.debug("Decompressing {!s}".format(path)) gzcont = gzfile.read() logging.debug("Writing decompressed file to {!s}".format(tmpfile.name)) tmpfile.write(gzcont) del gzcont # All the hard work is in coda logging.debug("Reading {!s}".format(tmpfile.name)) cfp = coda.open(tmpfile.name) c = coda.fetch(cfp) logging.debug("Sorting info...") n_scanlines = c.MPHR.TOTAL_MDR start = datetime.datetime(*coda.time_double_to_parts_utc(c.MPHR.SENSING_START)) has_mdr = numpy.array([hasattr(m, 'MDR') for m in c.MDR], dtype=numpy.bool) bad = numpy.array([ (m.MDR.DEGRADED_PROC_MDR|m.MDR.DEGRADED_INST_MDR) if hasattr(m, 'MDR') else True for m in c.MDR], dtype=numpy.bool) dlt = numpy.concatenate( [m.MDR.OnboardUTC[:, numpy.newaxis] for m in c.MDR if hasattr(m, 'MDR')], 1) - c.MPHR.SENSING_START M = numpy.ma.zeros( dtype=self._dtype, shape=(n_scanlines, 30)) M["time"][has_mdr] = numpy.datetime64(start, "ms") + numpy.array(dlt*1e3, "m8[ms]").T specall = self.__obtain_from_mdr(c, "GS1cSpect").astype("f8") # apply scale factors first = c.MDR[0].MDR.IDefNsfirst1b last = c.MDR[0].MDR.IDefNslast1b for (slc_st, slc_fi, fact) in zip( filter(None, c.GIADR_ScaleFactors.IDefScaleSondNsfirst), c.GIADR_ScaleFactors.IDefScaleSondNslast, c.GIADR_ScaleFactors.IDefScaleSondScaleFactor): # Documented intervals are closed [a, b]; Python uses # half-open [a, b). specall[..., (slc_st-first):(slc_fi-first+1)] *= pow(10.0, -fact) M["spectral_radiance"][has_mdr] = specall locall = self.__obtain_from_mdr(c, "GGeoSondLoc") M["lon"][has_mdr] = locall[:, :, :, 0] M["lat"][has_mdr] = locall[:, :, :, 1] satangall = self.__obtain_from_mdr(c, "GGeoSondAnglesMETOP") M["satellite_zenith_angle"][has_mdr] = satangall[:, :, :, 0] M["satellite_azimuth_angle"][has_mdr] = satangall[:, :, :, 1] solangall = self.__obtain_from_mdr(c, "GGeoSondAnglesSUN") M["solar_zenith_angle"][has_mdr] = solangall[:, :, :, 0] M["solar_azimuth_angle"][has_mdr] = solangall[:, :, :, 1] for fld in M.dtype.names: M.mask[fld][~has_mdr, ...] = True M.mask[fld][bad, ...] = True m = c.MDR[0].MDR wavenumber = (m.IDefSpectDWn1b * numpy.arange(m.IDefNsfirst1b, m.IDefNslast1b+0.1) * (1/ureg.metre)) if self.wavenumber is None: self.wavenumber = wavenumber elif abs(self.wavenumber - wavenumber).max() > (0.05 * 1/(ureg.centimetre)): raise ValueError("Inconsistent wavenumbers") return M
def get_solar_zenith(self): """ return solar_zenith """ if self.resolution == 24000: satellite_type1 = ['METOP-A', 'METOP-B'] if self.satellite in satellite_type1: try: fp = coda.open(self.in_file) angle = coda.fetch(fp, 'MDR', -1, 'MDR', 'GGeoSondAnglesSUN') coda.close(fp) all_angle = [] for i in xrange(len(angle)): tmp_angle = angle[i].reshape(-1) if len(all_angle) == 0: all_angle = tmp_angle else: all_angle = np.concatenate((all_angle, tmp_angle)) s0 = self.data_shape[0] # 开始间隔一个取一个值,取偶数位 data_pre = (all_angle[0::2]).reshape(s0, 1) # 过滤无效值 invalid_index = np.logical_or(data_pre < 0, data_pre > 180) data_pre = data_pre.astype(np.float32) data_pre[invalid_index] = np.nan data = data_pre except Exception as e: print 'Open file error {}'.format(e) return else: raise ValueError('Cant read this satellite`s data.: {}'.format( self.satellite)) elif self.resolution == 24001: satellite_type1 = ['METOP-A', 'METOP-B'] if self.satellite in satellite_type1: try: ncr = Dataset(self.in_file, 'r', format='NETCDF3_CLASSIC') data_pre = ncr.variables['solar_zenith_angle'][:] ncr.close() # 过滤无效值 invalid_index = np.logical_or(data_pre < 0, data_pre > 180) data_pre = data_pre.astype(np.float32) data_pre[invalid_index] = np.nan data = data_pre.reshape(self.data_shape) except Exception as e: print 'Open file error {}'.format(e) return else: raise ValueError('Cant read this satellite`s data.: {}'.format( self.satellite)) else: raise ValueError( 'Cant read this data, please check its resolution: {}'.format( self.in_file)) return data
def main(): parser = argparse.ArgumentParser( 'Read star reference spectra from a folder,' ' calculate mean, std, and quantiles (0.05, 0.50, 0.95),' ' save into a file in numpy format') parser.add_argument('--path', help="input folder") parser.add_argument('--out', help="output file") parser.add_argument( '--use-measures', action="store_true", help="use measurement data instead of the reference star spectrum") parser.add_argument('--plot-all', action="store_true", help="plot all spectra and also the result") parser.add_argument('--plot', action="store_true", help="plot the result") args = parser.parse_args() spectra = [] for file in os.listdir(args.path): if file[-3:] != '.N1': continue # - see https://github.com/stcorp/codadef-documentation for product_class, product_type, version # - to list field names: coda.get_field_names(h,""): [ # 'mph', 'sph', 'dsd', 'tra_summary_quality', 'tra_occultation_data', # 'tra_nom_wav_assignment', 'tra_ref_star_spectrum', 'tra_ref_atm_dens_profile', # 'tra_transmission', 'tra_satu_and_sfa_data', 'tra_auxiliary_data', # 'tra_geolocation' # ] # - meanings of the fields: http://envisat.esa.int/handbooks/gomos/CNTR2-2-3.html # file, product_class, product_type, version h = coda.open_as(os.path.join(args.path, file), 'ENVISAT_GOMOS', 'GOM_TRA_1P', 1) v = coda.fetch(h, 'sph') m = re.match(r'\d*(.*)', v[12].strip()) bayer = m[1] if m else v[12].strip() v = coda.fetch(h, 'tra_ref_star_spectrum') spe_ref_raw = np.array(v[0][1]).reshape( (1, -1)) # in electrons/pixel/0.5s if args.use_measures: # the actual measurement data: n = 10 v = coda.fetch(h, 'tra_transmission') spe_ref_raw = np.stack([np.array(r[2]) for r in v], axis=0)[:n, :] * spe_ref_raw v = coda.fetch(h, 'tra_nom_wav_assignment') lam = np.array(v[0][0]) # in nm v = coda.fetch(h, 'tra_occultation_data') sens_len = v[0][10] # how many elements the curve has sens_lam = np.array(v[0][11])[0:sens_len] # in nm sens_val = np.array(v[0][12])[ 0:sens_len] # in (photons/s/cm2/nm) / (electrons/pixel/0.5s) coda.close(h) # hack to alleviate a problem arising from interpolation and a sharp fall in sensitivity around 389nm dl = np.array([-0.3, 0, 0.3]) sens_lam = np.concatenate( (sens_lam[:12], sens_lam[12:13] + dl, sens_lam[13:])) sens_val = np.concatenate( (sens_val[:12], sens_val[11:14], sens_val[13:])) interp = interp1d(sens_lam, sens_val, kind='linear') spe_ref = spe_ref_raw * interp(lam).reshape((1, -1)) spectra.append(spe_ref) if args.plot_all: plt.plot( lam.reshape((1, -1)).repeat(len(spe_ref), axis=0), spe_ref) # plt.plot(lam, spe_ref_raw) # plt.plot(sens_lam, sens_val * 0.8*np.max(spe_ref)/np.max(sens_val), 'x-') plt.title(file) plt.xlabel('Wavelenth [nm]') plt.ylabel('Photon flux density [ph/s/cm2/nm]') plt.show() if len(spectra) == 0: raise Exception('no spectra found in folder %s' % args.path) spectra = np.concatenate(spectra, axis=0) quantiles = np.quantile(spectra, (0.05, 0.50, 0.95), axis=0) mean = np.mean(spectra, axis=0) std = np.std(spectra, axis=0) outfile = args.out.replace('{{bayer}}', bayer.lower().replace(' ', '_')) if outfile[-4:] != '.npz': outfile = outfile + '.npz' np.savez(outfile, bayer=bayer, lam=lam, mean=mean, std=std, q05=quantiles[0, :], q50=quantiles[1, :], q95=quantiles[2, :]) if args.plot or args.plot_all: plt.plot(lam, quantiles[1, :], 'C0-') plt.plot(lam, quantiles[0, :], 'C1:') plt.plot(lam, quantiles[2, :], 'C1:') plt.title('Median spectrum of star %s' % (bayer, )) plt.xlabel('Wavelength [nm]') plt.ylabel('Photon flux density [ph/s/cm2/nm]') plt.show()