def read_geo(self, key, info): """Read angles. """ if key.name in ['satellite_zenith_angle', 'satellite_azimuth_angle']: if self.senazi is None or self.senzen is None: self.senazi, self.senzen = self.angles("SatelliteAzimuthAngle", "SatelliteZenithAngle") if key.name == 'satellite_zenith_angle': return Projectable(self.senzen, copy=False, name=key.name, **self.mda) else: return Projectable(self.senazi, copy=False, name=key.name, **self.mda) if key.name in ['solar_zenith_angle', 'solar_azimuth_angle']: if self.solazi is None or self.solzen is None: self.solazi, self.solzen = self.angles("SolarAzimuthAngle", "SolarZenithAngle") if key.name == 'solar_zenith_angle': return Projectable(self.solzen, copy=False, name=key.name, **self.mda) else: return Projectable(self.solazi, copy=False, name=key.name, **self.mda)
def __call__(self, projectables, optional_datasets=None, **info): (band,) = projectables factor = 8 #proj = Projectable(band[::factor, ::factor], copy=False, **band.info) newshape = (band.shape[0] / factor, factor, band.shape[1] / factor, factor) proj = Projectable(band.reshape(newshape).mean(axis=3).mean(axis=1), copy=False, **band.info) self.apply_modifier_info(band, proj) old_area = proj.info['area'] proj.info['area'] = AreaDefinition(old_area.area_id, old_area.name, old_area.proj_id, old_area.proj_dict, old_area.x_size / factor, old_area.y_size / factor, old_area.area_extent) proj.info['resolution'] *= factor proj.info['id'] = DatasetID(name=proj.info['id'].name, resolution=proj.info['resolution'], wavelength=proj.info['id'].wavelength, polarization=proj.info['id'].polarization, calibration=proj.info['id'].calibration, modifiers=proj.info['id'].modifiers) return proj
def get_dataset(self, key, info): """Get a dataset from the file.""" if key.name in CHANNEL_NAMES: dataset = self.calibrate([key])[0] else: # Get sun-sat angles if key.name in ANGLES: if isinstance(getattr(self, ANGLES[key.name]), np.ndarray): dataset = Projectable( getattr(self, ANGLES[key.name]), copy=False) else: dataset = self.get_angles(key.name) else: logger.exception( "Not a supported sun-sensor viewing angle: %s", key.name) raise # TODO get metadata if self.lons is None or self.lats is None: self.navigate() if self.area is None: self.area = SwathDefinition(self.lons, self.lats) self.area.name = 'wla' if not self._shape: self._shape = dataset.shape dataset.info['area'] = self.area return dataset
def stack(datasets): """First dataset at the bottom.""" base = Projectable(datasets[0], copy=True) for dataset in datasets[1:]: base_mask = np.ma.getmaskarray(base) other_mask = np.ma.getmaskarray(dataset) base.mask = np.logical_and(base_mask, other_mask) not_masked = np.logical_not(other_mask) base[not_masked] = dataset[not_masked] return base
def get_dataset(self, key, info): avhrr_channel_index = {'1': 0, '2': 1, '3a': 2, '3b': 2, '4': 3, '5': 4} index = avhrr_channel_index[key.name] mask = False if key.name in ['3a', '3b'] and self._is3b is None: ch3a = bfield(self._data["id"]["id"], 10) self._is3b = np.logical_not(ch3a) if key.name == '3a': mask = np.tile(self._is3b, (1, 2048)) elif key.name == '3b': mask = np.tile(np.logical_not(self._is3b), (1, 2048)) data = self._data["image_data"][:, :, index] if key.calibration == 'counts': return Projectable(data, mask=mask, area=self.get_lonlats(), units='1') pg_spacecraft = ''.join(self.platform_name.split()).lower() jdays = (np.datetime64(self.start_time) - np.datetime64(str( self.year) + '-01-01T00:00:00Z')) / np.timedelta64(1, 'D') if index < 2 or key.name == '3a': data = calibrate_solar(data, index, self.year, jdays, pg_spacecraft) units = '%' if index > 2 or key.name == '3b': if self.times is None: self.times = time_seconds(self._data["timecode"], self.year) line_numbers = ( np.round((self.times - self.times[-1]) / np.timedelta64(166666667, 'ns'))).astype(np.int) line_numbers -= line_numbers[0] if self.prt is None: self.prt, self.ict, self.space = self.get_telemetry() chan = index + 1 data = calibrate_thermal(data, self.prt, self.ict[:, chan - 3], self.space[:, chan - 3], line_numbers, chan, pg_spacecraft) units = 'K' # TODO: check if entirely masked before returning return Projectable(data, mask=mask, area=self.get_lonlats(), units=units)
def step_impl(context): """ :type context: behave.runner.Context """ from satpy.scene import Scene from datetime import datetime from satpy.projectable import Projectable scn = Scene(platform_name="Suomi-NPP", sensor="viirs", start_time=datetime(2015, 3, 11, 11, 20), end_time=datetime(2015, 3, 11, 11, 26)) scn["MyDataset"] = Projectable([[1, 2], [3, 4]]) scn["MyDataset2"] = Projectable([[5, 6], [7, 8]]) context.scene = scn
def __call__(self, projectables, optional_datasets=None, **info): """CO2 correction of the brightness temperature of the MSG 3.9um channel. .. math:: T4_CO2corr = (BT(IR3.9)^4 + Rcorr)^0.25 Rcorr = BT(IR10.8)^4 - (BT(IR10.8)-dt_CO2)^4 dt_CO2 = (BT(IR10.8)-BT(IR13.4))/4.0 """ (ir_039, ir_108, ir_134) = projectables LOG.info('Applying CO2 correction') dt_co2 = (ir_108 - ir_134) / 4.0 rcorr = ir_108**4 - (ir_108 - dt_co2)**4 t4_co2corr = ir_039**4 + rcorr t4_co2corr = np.ma.where(t4_co2corr > 0.0, t4_co2corr, 0) t4_co2corr = t4_co2corr**0.25 info = ir_039.info.copy() proj = Projectable(t4_co2corr, mask=t4_co2corr.mask, **info) self.apply_modifier_info(ir_039, proj) return proj
def compute(self, data, weight_count=10000, weight_min=0.01, weight_distance_max=1.0, weight_sum_min=-1.0, maximum_weight_mode=False, **kwargs): rows = self.cache["rows"] cols = self.cache["cols"] # if the data is scan based then check its metadata or the passed kwargs # otherwise assume the entire input swath is one large "scanline" rows_per_scan = getattr(data, "info", kwargs).get("rows_per_scan", data.shape[0]) num_valid_points, out_arrs = fornav(cols, rows, self.target_area_def, data, rows_per_scan=rows_per_scan, weight_count=weight_count, weight_min=weight_min, weight_distance_max=weight_distance_max, weight_sum_min=weight_sum_min, maximum_weight_mode=maximum_weight_mode, **kwargs) num_valid_points = num_valid_points[0] out_arr = out_arrs[0] grid_covered_ratio = num_valid_points / float(out_arr.size) grid_covered = grid_covered_ratio > self.grid_coverage if not grid_covered: msg = "EWA resampling only found %f%% of the grid covered (need %f%%)" % (grid_covered_ratio * 100, self.grid_coverage * 100) raise RuntimeError(msg) LOG.debug("EWA resampling found %f%% of the grid covered" % (grid_covered_ratio * 100)) info = getattr(data, "info", {}) info["mask"] = np.isnan(out_arr) if np.issubdtype(data.dtype, np.floating) else out_arr == fill return Projectable(out_arr, **info)
def get_dataset(self, key, info): """Load a dataset """ if self.channel != key.name: return logger.debug('Reading %s.', key.name) variable = self.nc[self.channel + '_radiance'] radiances = (np.ma.masked_equal(variable[:], variable.attrs['_FillValue'], copy=False) * variable.attrs['scale_factor'] + variable.attrs['add_offset']) units = variable.attrs['units'] if key.calibration == 'reflectance': solar_flux = self.cal['solar_flux'][:] d_index = np.ma.masked_equal(self.cal['detector_index'][:], self.cal['detector_index'].attrs[ '_FillValue'], copy=False) idx = int(key.name[2:]) - 1 radiances /= solar_flux[idx, d_index] radiances *= np.pi * 100 units = '%' proj = Projectable(radiances, copy=False, units=units, platform_name=self.platform_name, sensor=self.sensor) return proj
def get_dataset(self, ds_key, ds_info, out=None): var_path = ds_info["file_key"] fill_value = ds_info.get("fill_value", 65535) data = self[var_path] dtype = ds_info.get("dtype", np.float32) mask = data == fill_value data = data.astype(dtype) * self[var_path + "/attr/SCALE FACTOR"] if ((ds_info.get('standard_name') == "longitude" or ds_info.get('standard_name') == "latitude") and ds_key.resolution == 10000): # FIXME: Lower frequency channels need CoRegistration parameters # applied data = data[:, ::2] mask = mask[:, ::2] ds_info.update({ "name": ds_key.name, "id": ds_key, "units": self[var_path + "/attr/UNIT"], "platform": self["/attr/PlatformShortName"].item(), "sensor": self["/attr/SensorShortName"].item(), "start_orbit": int(self["/attr/StartOrbitNumber"].item()), "end_orbit": int(self["/attr/StopOrbitNumber"].item()), }) return Projectable(data, mask=mask, **ds_info)
def __call__(self, projectables, nonprojectables=None, **info): if len(projectables) != 3: raise ValueError("Expected 3 datasets, got %d" % (len(projectables), )) try: the_data = np.rollaxis( np.ma.dstack([projectable for projectable in projectables]), axis=2) except ValueError: raise IncompatibleAreas # info = projectables[0].info.copy() # info.update(projectables[1].info) # info.update(projectables[2].info) info = combine_info(*projectables) info.update(self.info) info['id'] = DatasetID(self.info['name']) # FIXME: should this be done here ? info["wavelength_range"] = None info.pop("units", None) sensor = set() for projectable in projectables: current_sensor = projectable.info.get("sensor", None) if current_sensor: if isinstance(current_sensor, (str, bytes, six.text_type)): sensor.add(current_sensor) else: sensor |= current_sensor if len(sensor) == 0: sensor = None elif len(sensor) == 1: sensor = list(sensor)[0] info["sensor"] = sensor info["mode"] = "RGB" return Projectable(data=the_data, **info)
def get_angles(self, angle_id): """Get sun-satellite viewing angles""" tic = datetime.now() sunz40km = self._data["ang"][:, :, 0] * 1e-2 satz40km = self._data["ang"][:, :, 1] * 1e-2 azidiff40km = self._data["ang"][:, :, 2] * 1e-2 try: from geotiepoints.interpolator import Interpolator except ImportError: logger.warning("Could not interpolate sun-sat angles, " "python-geotiepoints missing.") self.sunz, self.satz, self.azidiff = sunz40km, satz40km, azidiff40km else: cols40km = np.arange(24, 2048, 40) cols1km = np.arange(2048) lines = sunz40km.shape[0] rows40km = np.arange(lines) rows1km = np.arange(lines) along_track_order = 1 cross_track_order = 3 satint = Interpolator( [sunz40km, satz40km, azidiff40km], (rows40km, cols40km), (rows1km, cols1km), along_track_order, cross_track_order) self.sunz, self.satz, self.azidiff = satint.interpolate() logger.debug("Interpolate sun-sat angles: time %s", str(datetime.now() - tic)) return Projectable(getattr(self, ANGLES[angle_id]), copy=False)
def get_dataset(self, key, info=None): """Load a dataset """ if key in self.cache: return self.cache[key] # Type dictionary typedict = { "af": "flash_accumulation", "afa": "accumulated_flash_area", "afr": "flash_radiance", "lgr": "radiance", "lef": "radiance", "lfl": "radiance" } #Get lightning data out of NetCDF container logger.debug("KEY: %s" % key.name) # Get grid dimensions from file refdim = self.nc['grid_position'][:] # Get number of lines and columns self.nlines = int(refdim[2]) self.ncols = int(refdim[3]) # Create reference grid grid = np.full((refdim[2], refdim[3]), np.NaN) # Get product value values = self.nc[typedict[key.name]][:] rows = self.nc['row'][:] cols = self.nc['column'][:] # Convert xy coordinates to flattend indices ids = np.ravel_multi_index([rows, cols], grid.shape) # Replace NaN values with data np.put(grid, ids, values) # Correct for bottom left origin in LI row/column indices. rotgrid = np.flipud(grid) logger.debug('DATA SHAPE: %s' % str(rotgrid.shape)) # Rotate the grid by 90 degree clockwise logger.warning("LI data has been roteted to fit to reference grid. \ Works only for test dataset") rotgrid = np.rot90(rotgrid, 3) logger.debug('[ Dimension ] : %s' % (refdim)) logger.debug("ROW/COLS: %d / %d" % (self.nlines, self.ncols)) logger.debug('[ Number of values ] : %d' % (len(values))) logger.debug('[Min/Max] : <%d> / <%d>' % (np.min(values), np.max(values))) logger.debug("START: %s" % self.start_time) logger.debug("END: %s" % self.end_time) ds = (np.ma.masked_invalid(rotgrid[:])) # Create projectable object out = Projectable(ds, dtype=np.float32) self.cache[key] = out return (out)
def get_dataset(self, key, info, out=None): to_return = out is None if out is None: nlines = int(self.data_info['number_of_lines']) ncols = int(self.data_info['number_of_columns']) out = Projectable(np.ma.empty((nlines, ncols), dtype=np.float32)) self.read_band(key, info, out) if to_return: from satpy.yaml_reader import Shuttle return Shuttle(out.data, out.mask, out.info)
def get_dataset(self, key, info): """Load a dataset """ logger.debug('Reading %s.', key.name) variable = self.nc[key.name] ds = (np.ma.masked_equal(variable[:], variable.attrs['_FillValue']) * (variable.attrs['scale_factor'] * 1.0) + variable.attrs.get('add_offset', 0)) proj = Projectable(ds, copy=False, **info) return proj
def __call__(self, projectables, optional_datasets=None, **info): """Boost vegetation effect thanks to NIR (0.8µm) band.""" (green, nir) = projectables LOG.info('Boosting vegetation on green band') proj = Projectable(green * 0.85 + nir * 0.15, copy=False, **green.info) self.apply_modifier_info(green, proj) return proj
def get_dataset(self, key, info): if key.name not in ['longitude', 'latitude']: return if self.lons is None or self.lats is None: lons_id = DatasetID('longitude', resolution=key.resolution) lats_id = DatasetID('latitude', resolution=key.resolution) lons, lats = self.load([lons_id, lats_id], interpolate=False, raw=True) from geotiepoints.geointerpolator import GeoInterpolator self.lons, self.lats = self._interpolate([lons, lats], self.resolution, lons_id.resolution, GeoInterpolator) if key.name == 'latitude': return Projectable(self.lats, id=key, **info) else: return Projectable(self.lons, id=key, **info)
def __call__(self, projectables, nonprojectables=None, **info): if len(projectables) != 3: raise ValueError("Expected 3 datasets, got %d" % (len(projectables), )) # Collect information that is the same between the projectables info = combine_info(*projectables) # Update that information with configured information (including name) info.update(self.info) # Force certain pieces of metadata that we *know* to be true info["wavelength"] = None info["mode"] = self.info.get("mode", "RGB") return Projectable(data=np.rollaxis(np.ma.dstack( [projectable for projectable in projectables]), axis=2), **info)
def get_dataset(self, key, info): """Read data from file and return the corresponding projectables. """ datadict = { 1000: [ 'EV_250_Aggr1km_RefSB', 'EV_500_Aggr1km_RefSB', 'EV_1KM_RefSB', 'EV_1KM_Emissive' ], 500: ['EV_250_Aggr500_RefSB', 'EV_500_RefSB'], 250: ['EV_250_RefSB'] } platform_name = self.mda['INVENTORYMETADATA'][ 'ASSOCIATEDPLATFORMINSTRUMENTSENSOR'][ 'ASSOCIATEDPLATFORMINSTRUMENTSENSORCONTAINER'][ 'ASSOCIATEDPLATFORMSHORTNAME']['VALUE'] if self.resolution != key.resolution: return datasets = datadict[self.resolution] for dataset in datasets: subdata = self.sd.select(dataset) band_names = subdata.attributes()["band_names"].split(",") # get the relative indices of the desired channel try: index = band_names.index(key.name) except ValueError: continue uncertainty = self.sd.select(dataset + "_Uncert_Indexes") if dataset.endswith('Emissive'): array = calibrate_tb(subdata, uncertainty, [index], band_names) else: array = calibrate_refl(subdata, uncertainty, [index]) projectable = Projectable(array[0], id=key, mask=array[0].mask) # if ((platform_name == 'Aqua' and key.name in ["6", "27", "36"]) or # (platform_name == 'Terra' and key.name in ["29"])): # height, width = projectable.shape # row_indices = projectable.mask.sum(1) == width # if row_indices.sum() != height: # projectable.mask[row_indices, :] = True return projectable
def get_dataset(self, ds_key, ds_info, out=None): var_path = ds_info["file_key"] fill_value = ds_info.get("fill_value", 65535) data = self[var_path] dtype = ds_info.get("dtype", np.float32) mask = data == fill_value data = data.astype(dtype) * self[var_path + "/attr/SCALE FACTOR"] ds_info.update({ "name": ds_key.name, "id": ds_key, "units": self[var_path + "/attr/UNIT"], "platform": self["/attr/PlatformShortName"].item(), "sensor": self["/attr/SensorShortName"].item(), "start_orbit": int(self["/attr/StartOrbitNumber"].item()), "end_orbit": int(self["/attr/StopOrbitNumber"].item()), }) return Projectable(data, mask=mask, **ds_info)
def get_dataset(self, key, info): """Load a dataset """ logger.debug('Reading %s.', key.name) print self.filename, info['file_key'] try: variable = self.nc[info['file_key']] except KeyError: return ds = (np.ma.masked_equal(variable[:], variable.attrs['_FillValue']) * (variable.attrs.get('scale_factor', 1) * 1.0) + variable.attrs.get('add_offset', 0)) ds.mask = np.ma.getmaskarray(ds) proj = Projectable(ds, copy=False, **info) return proj
def __call__(self, projectables, optional_datasets=None, **info): """Get the corrected reflectance when removing Rayleigh scattering. Uses pyspectral. """ from pyspectral.rayleigh import Rayleigh (vis, ) = projectables try: (sata, satz, suna, sunz) = optional_datasets except ValueError: from pyorbital.astronomy import get_alt_az, sun_zenith_angle from pyorbital.orbital import get_observer_look sunalt, suna = get_alt_az(vis.info['start_time'], *vis.info['area'].get_lonlats()) sunz = sun_zenith_angle(vis.info['start_time'], *vis.info['area'].get_lonlats()) lons, lats = vis.info['area'].get_lonlats() sata, satel = get_observer_look(vis.info['satellite_longitude'], vis.info['satellite_latitude'], vis.info['satellite_altitude'], vis.info['start_time'], lons, lats, 0) satz = 90 - satel LOG.info('Removing Rayleigh scattering') ssadiff = np.abs(suna - sata) ssadiff = np.where(np.greater(ssadiff, 180), 360 - ssadiff, ssadiff) corrector = Rayleigh(vis.info['platform_name'], vis.info['sensor'], atmosphere='us-standard', rural_aerosol=False) refl_cor_band = corrector.get_reflectance(sunz, satz, ssadiff, vis.info['id'].wavelength[1], vis) proj = Projectable(vis - refl_cor_band, copy=False, **vis.info) self.apply_modifier_info(vis, proj) return proj
def get_dataset(self, key, info): """Load a dataset """ if key.name not in self.datasets: return if self.nc is None: self.nc = h5netcdf.File(self.filename, 'r') logger.debug('Reading %s.', key.name) variable = self.nc[self.datasets[key.name]] values = (np.ma.masked_equal(variable[:], variable.attrs['_FillValue'], copy=False) * variable.attrs.get('scale_factor', 1) + variable.attrs.get('add_offset', 0)) units = variable.attrs['units'] l_step = self.nc.attrs['al_subsampling_factor'] c_step = self.nc.attrs['ac_subsampling_factor'] if c_step != 1 or l_step != 1: from geotiepoints.interpolator import Interpolator tie_lines = np.arange( 0, (values.shape[0] - 1) * l_step + 1, l_step) tie_cols = np.arange(0, (values.shape[1] - 1) * c_step + 1, c_step) lines = np.arange((values.shape[0] - 1) * l_step + 1) cols = np.arange((values.shape[1] - 1) * c_step + 1) along_track_order = 1 cross_track_order = 3 satint = Interpolator([values], (tie_lines, tie_cols), (lines, cols), along_track_order, cross_track_order) (values, ) = satint.interpolate() proj = Projectable(values, copy=False, units=units, platform_name=self.platform_name, sensor=self.sensor) return proj
def load(self, keys, interpolate=True, raw=False): projectables = [] for key in keys: dataset = self.sd.select(key.name.capitalize()) fill_value = dataset.attributes()["_FillValue"] try: scale_factor = dataset.attributes()["scale_factor"] except KeyError: scale_factor = 1 data = np.ma.masked_equal(dataset.get(), fill_value) * scale_factor # TODO: interpolate if needed if key.resolution is not None and key.resolution < self.resolution and interpolate: data = self._interpolate(data, self.resolution, key.resolution) if raw: projectables.append(data) else: projectables.append(Projectable(data, id=key)) return projectables
def __call__(self, projectables, optional_datasets=None, **info): """Get the reflectance part of an NIR channel. Not supposed to be used for wavelength outside [3, 4] µm. """ try: from pyspectral.near_infrared_reflectance import Calculator except ImportError: LOG.info("Couldn't load pyspectral") raise nir, tb11 = projectables LOG.info('Getting reflective part of %s', nir.info['name']) sun_zenith = None tb13_4 = None for dataset in optional_datasets: if dataset.info["standard_name"] == "solar_zenith_angle": sun_zenith = dataset elif (dataset.info['units'] == 'K' and "wavelengh" in dataset.info and dataset.info["wavelength"][0] <= 13.4 <= dataset.info["wavelength"][2]): tb13_4 = dataset # Check if the sun-zenith angle was provided: if sun_zenith is None: from pyorbital.astronomy import sun_zenith_angle as sza lons, lats = nir.info["area"].get_lonlats() sun_zenith = sza(nir.info['start_time'], lons, lats) refl39 = Calculator(nir.info['platform_name'], nir.info['sensor'], nir.info['name']) proj = Projectable(refl39.reflectance_from_tbs(sun_zenith, nir, tb11, tb13_4), **nir.info) self.apply_modifier_info(nir, proj) return proj
def get_dataset(self, key, info): """Load a dataset """ logger.debug('Reading in get_dataset%s.', key.name) variable = self.nc["Rad"] radiances = (np.ma.masked_equal( variable[:], variable.attrs['_FillValue'], copy=False) * variable.attrs['scale_factor'] + variable.attrs['add_offset']) units = variable.attrs['units'] self.calibrate(radiances, key) proj = Projectable(radiances, copy=False, units=units, platform_name=self.platform_name, sensor=self.sensor) self.nlines, self.ncols = radiances.shape return proj
def read_geo(self, key, info): """Read angles. """ if key.name in ['satellite_zenith_angle', 'satellite_azimuth_angle']: if self.senazi is None or self.senzen is None: self.senazi, self.senzen = self.angles("SatelliteAzimuthAngle", "SatelliteZenithAngle") if key.name == 'satellite_zenith_angle': return Projectable(self.senzen, copy=False, name=key.name, **self.mda) else: return Projectable(self.senazi, copy=False, name=key.name, **self.mda) if key.name in ['solar_zenith_angle', 'solar_azimuth_angle']: if self.solazi is None or self.solzen is None: self.solazi, self.solzen = self.angles("SolarAzimuthAngle", "SolarZenithAngle") if key.name == 'solar_zenith_angle': return Projectable(self.solzen, copy=False, name=key.name, **self.mda) else: return Projectable(self.solazi, copy=False, name=key.name, **self.mda) if info.get('standard_name') in ['latitude', 'longitude']: if self.lons is None or self.lats is None: self.lons, self.lats = self.navigate() mda = self.mda.copy() mda.update(info) if info['standard_name'] == 'longitude': return Projectable(self.lons, copy=False, id=key, **mda) else: return Projectable(self.lats, copy=False, id=key, **mda)
def load(self, dataset_keys, area=None, start_time=None, end_time=None): image_files = [] pattern = self.file_patterns[0] prologue_file = None epilogue_file = None for filename in self.info['filenames']: try: file_info = parse(pattern, os.path.basename(filename)) except ValueError: continue if file_info["segment"] == "EPI": epilogue_file = filename elif file_info["segment"] == "PRO": prologue_file = filename else: image_files.append(filename) start_times = set() datasets = DatasetDict() area_converted_to_extent = False area_extent = None for ds in dataset_keys: channel_files = [] for filename in image_files: file_info = parse(pattern, os.path.basename(filename)) if file_info["dataset_name"] == ds.name: channel_files.append(filename) start_times.add(file_info['start_time']) if not channel_files: continue kwargs = {} if 'platform_name' in self.info: kwargs['platform_name'] = self.info['platform_name'] # Convert area definitions to maximal area_extent if not area_converted_to_extent and area is not None: metadata = xrit.sat.load_files(prologue_file, channel_files, epilogue_file, only_metadata=True, **kwargs) # otherwise use the default value (MSG3 extent at # lon0=0.0), that is, do not pass default_extent=area_extent area_extent = area_defs_to_extent( [area], metadata.proj4_params) area_converted_to_extent = True try: calibrate = 1 if ds.calibration == 'counts': calibrate = 0 elif ds.calibration == 'radiance': calibrate = 2 image = xrit.sat.load_files(prologue_file, channel_files, epilogue_file, mask=True, calibrate=calibrate, **kwargs) if area_extent: metadata, data = image(area_extent) else: metadata, data = image() except CalibrationError: LOGGER.warning( "Loading non calibrated data since calibration failed.") image = xrit.sat.load_files(prologue_file, channel_files, epilogue_file, mask=True, calibrate=False, **kwargs) if area_extent: metadata, data = image(area_extent) else: metadata, data = image() except ReaderError as err: # if dataset can't be found, go on with next dataset LOGGER.error(str(err)) continue if len(metadata.instruments) != 1: sensor = None else: sensor = metadata.instruments[0] units = {'ALBEDO(%)': '%', 'KELVIN': 'K'} standard_names = {'1': 'counts', 'W m-2 sr-1 m-1': 'toa_outgoing_radiance_per_unit_wavelength', '%': 'toa_bidirectional_reflectance', 'K': 'toa_brightness_temperature'} unit = units.get(metadata.calibration_unit, metadata.calibration_unit) projectable = Projectable( data, name=ds.name, units=unit, standard_name=standard_names[unit], sensor=sensor, start_time=min(start_times), id=ds) # Build an area on the fly from the mipp metadata proj_params = getattr(metadata, "proj4_params").split(" ") proj_dict = {} for param in proj_params: key, val = param.split("=") proj_dict[key] = val if IS_PYRESAMPLE_LOADED: # Build area_def on-the-fly projectable.info["area"] = geometry.AreaDefinition( str(metadata.area_extent) + str(data.shape), "On-the-fly area", proj_dict["proj"], proj_dict, data.shape[1], data.shape[0], metadata.area_extent) else: LOGGER.info("Could not build area, pyresample missing...") datasets[ds] = projectable return datasets
def load(self, keys): """Read data from file and return the corresponding projectables. """ datadict = { 1000: ['EV_250_Aggr1km_RefSB', 'EV_500_Aggr1km_RefSB', 'EV_1KM_RefSB', 'EV_1KM_Emissive'], 500: ['EV_250_Aggr500_RefSB', 'EV_500_RefSB'], 250: ['EV_250_RefSB']} projectables = [] platform_name = self.mda['INVENTORYMETADATA']['ASSOCIATEDPLATFORMINSTRUMENTSENSOR'][ 'ASSOCIATEDPLATFORMINSTRUMENTSENSORCONTAINER']['ASSOCIATEDPLATFORMSHORTNAME']['VALUE'] keys = [key for key in keys if key.resolution == self.resolution] if len(keys) == 0: logger.debug("Nothing to read in %s.", self.filename) return projectables datasets = datadict[self.resolution] key_names = [key.name for key in keys] for dataset in datasets: subdata = self.sd.select(dataset) band_names = subdata.attributes()["band_names"].split(",") if len(set(key_names) & set(band_names)) > 0: # get the relative indices of the desired channels indices = [i for i, band in enumerate(band_names) if band in key_names] uncertainty = self.sd.select(dataset + "_Uncert_Indexes") if dataset.endswith('Emissive'): array = calibrate_tb( subdata, uncertainty, indices, band_names) else: array = calibrate_refl(subdata, uncertainty, indices) for (i, idx) in enumerate(indices): dsid = [key for key in keys if key.name == band_names[idx]][0] area = self.navigation_reader.get_lonlats(self.resolution) projectable = Projectable(array[i], id=dsid, area=area) if ((platform_name == 'Aqua' and dsid.name in ["6", "27", "36"]) or (platform_name == 'Terra' and dsid.name in ["29"])): height, width = projectable.shape row_indices = projectable.mask.sum(1) == width if row_indices.sum() == height: continue projectable.mask[indices, :] = True projectables.append(projectable) return projectables # Get the orbit number if not satscene.orbit: mda = self.data.attributes()["CoreMetadata.0"] orbit_idx = mda.index("ORBITNUMBER") satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116] # Get the geolocation # if resolution != 1000: # logger.warning("Cannot load geolocation at this resolution (yet).") # return for band_name in loaded_bands: lon, lat = self.get_lonlat(satscene[band_name].resolution, cores) area = geometry.SwathDefinition(lons=lon, lats=lat) satscene[band_name].area = area # Trimming out dead sensor lines (detectors) on aqua: # (in addition channel 21 is noisy) if satscene.satname == "aqua": for band in ["6", "27", "36"]: if not satscene[band].is_loaded() or satscene[band].data.mask.all(): continue width = satscene[band].data.shape[1] height = satscene[band].data.shape[0] indices = satscene[band].data.mask.sum(1) < width if indices.sum() == height: continue satscene[band] = satscene[band].data[indices, :] satscene[band].area = geometry.SwathDefinition( lons=satscene[band].area.lons[indices, :], lats=satscene[band].area.lats[indices, :]) # Trimming out dead sensor lines (detectors) on terra: # (in addition channel 27, 30, 34, 35, and 36 are nosiy) if satscene.satname == "terra": for band in ["29"]: if not satscene[band].is_loaded() or satscene[band].data.mask.all(): continue width = satscene[band].data.shape[1] height = satscene[band].data.shape[0] indices = satscene[band].data.mask.sum(1) < width if indices.sum() == height: continue satscene[band] = satscene[band].data[indices, :] satscene[band].area = geometry.SwathDefinition( lons=satscene[band].area.lons[indices, :], lats=satscene[band].area.lats[indices, :]) for band_name in loaded_bands: band_uid = hashlib.sha1(satscene[band_name].data.mask).hexdigest() satscene[band_name].area.area_id = ("swath_" + satscene.fullname + "_" + str(satscene.time_slot) + "_" + str(satscene[ band_name].shape) + "_" + str(band_uid)) satscene[band_name].area_id = satscene[band_name].area.area_id
def get_dataset(self, key, info): """Get calibrated channel data.""" if self.mdrs is None: self._read_all(self.filename) if key.name in ['longitude', 'latitude']: lons, lats = self.get_full_lonlats() # todo: make that datasets if key.name == 'longitude': return Projectable(lons, id=key, **info) else: return Projectable(lats, id=key, **info) if key.calibration == 'counts': raise ValueError('calibration=counts is not supported! ' + 'This reader cannot return counts') elif key.calibration not in [ 'reflectance', 'brightness_temperature', 'radiance' ]: raise ValueError('calibration type ' + str(key.calibration) + ' is not supported!') if key.name in ['3A', '3a'] and self.three_a_mask is None: self.three_a_mask = ((self["FRAME_INDICATOR"] & 2**16) != 2**16) if key.name in ['3B', '3b'] and self.three_b_mask is None: self.three_b_mask = ((self["FRAME_INDICATOR"] & 2**16) != 0) if key.name not in ["1", "2", "3a", "3A", "3b", "3B", "4", "5"]: LOG.info("Can't load channel in eps_l1b: " + str(key.name)) return if key.name == "1": if key.calibration == 'reflectance': array = np.ma.array( radiance_to_refl(self["SCENE_RADIANCES"][:, 0, :], self["CH1_SOLAR_FILTERED_IRRADIANCE"])) else: array = np.ma.array(self["SCENE_RADIANCES"][:, 0, :]) if key.name == "2": if key.calibration == 'reflectance': array = np.ma.array( radiance_to_refl(self["SCENE_RADIANCES"][:, 1, :], self["CH1_SOLAR_FILTERED_IRRADIANCE"])) else: array = np.ma.array(self["SCENE_RADIANCES"][:, 1, :]) if key.name.lower() == "3a": if key.calibration == 'reflectance': array = np.ma.array( radiance_to_refl(self["SCENE_RADIANCES"][:, 2, :], self["CH2_SOLAR_FILTERED_IRRADIANCE"])) else: array = np.ma.array(self["SCENE_RADIANCES"][:, 2, :]) mask = np.empty(array.shape, dtype=bool) mask[:, :] = self.three_a_mask[:, np.newaxis] array = np.ma.array(array, mask=mask, copy=False) if key.name.lower() == "3b": if key.calibration == 'brightness_temperature': array = np.array( radiance_to_bt(self["SCENE_RADIANCES"][:, 2, :], self["CH3B_CENTRAL_WAVENUMBER"], self["CH3B_CONSTANT1"], self["CH3B_CONSTANT2_SLOPE"])) else: array = self["SCENE_RADIANCES"][:, 2, :] mask = np.empty(array.shape, dtype=bool) mask[:, :] = self.three_b_mask[:, np.newaxis] array = np.ma.array(array, mask=mask, copy=False) if key.name == "4": if key.calibration == 'brightness_temperature': array = np.ma.array( radiance_to_bt(self["SCENE_RADIANCES"][:, 3, :], self["CH4_CENTRAL_WAVENUMBER"], self["CH4_CONSTANT1"], self["CH4_CONSTANT2_SLOPE"])) else: array = np.ma.array(self["SCENE_RADIANCES"][:, 3, :]) if key.name == "5": if key.calibration == 'brightness_temperature': array = np.ma.array( radiance_to_bt(self["SCENE_RADIANCES"][:, 4, :], self["CH5_CENTRAL_WAVENUMBER"], self["CH5_CONSTANT1"], self["CH5_CONSTANT2_SLOPE"])) else: array = np.ma.array(self["SCENE_RADIANCES"][:, 4, :]) proj = Projectable(array, mask=array.mask, id=key) return proj
def load(self, keys): """Read data from file and return the corresponding projectables. """ datadict = { 1000: [ 'EV_250_Aggr1km_RefSB', 'EV_500_Aggr1km_RefSB', 'EV_1KM_RefSB', 'EV_1KM_Emissive' ], 500: ['EV_250_Aggr500_RefSB', 'EV_500_RefSB'], 250: ['EV_250_RefSB'] } projectables = [] platform_name = self.mda['INVENTORYMETADATA'][ 'ASSOCIATEDPLATFORMINSTRUMENTSENSOR'][ 'ASSOCIATEDPLATFORMINSTRUMENTSENSORCONTAINER'][ 'ASSOCIATEDPLATFORMSHORTNAME']['VALUE'] keys = [key for key in keys if key.resolution == self.resolution] if len(keys) == 0: logger.debug("Nothing to read in %s.", self.filename) return projectables datasets = datadict[self.resolution] key_names = [key.name for key in keys] for dataset in datasets: subdata = self.sd.select(dataset) band_names = subdata.attributes()["band_names"].split(",") if len(set(key_names) & set(band_names)) > 0: # get the relative indices of the desired channels indices = [ i for i, band in enumerate(band_names) if band in key_names ] uncertainty = self.sd.select(dataset + "_Uncert_Indexes") if dataset.endswith('Emissive'): array = calibrate_tb(subdata, uncertainty, indices, band_names) else: array = calibrate_refl(subdata, uncertainty, indices) for (i, idx) in enumerate(indices): dsid = [ key for key in keys if key.name == band_names[idx] ][0] area = self.navigation_reader.get_lonlats(self.resolution) projectable = Projectable(array[i], id=dsid, area=area) if ((platform_name == 'Aqua' and dsid.name in ["6", "27", "36"]) or (platform_name == 'Terra' and dsid.name in ["29"])): height, width = projectable.shape row_indices = projectable.mask.sum(1) == width if row_indices.sum() == height: continue projectable.mask[indices, :] = True projectables.append(projectable) return projectables # Get the orbit number if not satscene.orbit: mda = self.data.attributes()["CoreMetadata.0"] orbit_idx = mda.index("ORBITNUMBER") satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116] # Get the geolocation # if resolution != 1000: # logger.warning("Cannot load geolocation at this resolution (yet).") # return for band_name in loaded_bands: lon, lat = self.get_lonlat(satscene[band_name].resolution, cores) area = geometry.SwathDefinition(lons=lon, lats=lat) satscene[band_name].area = area # Trimming out dead sensor lines (detectors) on aqua: # (in addition channel 21 is noisy) if satscene.satname == "aqua": for band in ["6", "27", "36"]: if not satscene[band].is_loaded( ) or satscene[band].data.mask.all(): continue width = satscene[band].data.shape[1] height = satscene[band].data.shape[0] indices = satscene[band].data.mask.sum(1) < width if indices.sum() == height: continue satscene[band] = satscene[band].data[indices, :] satscene[band].area = geometry.SwathDefinition( lons=satscene[band].area.lons[indices, :], lats=satscene[band].area.lats[indices, :]) # Trimming out dead sensor lines (detectors) on terra: # (in addition channel 27, 30, 34, 35, and 36 are nosiy) if satscene.satname == "terra": for band in ["29"]: if not satscene[band].is_loaded( ) or satscene[band].data.mask.all(): continue width = satscene[band].data.shape[1] height = satscene[band].data.shape[0] indices = satscene[band].data.mask.sum(1) < width if indices.sum() == height: continue satscene[band] = satscene[band].data[indices, :] satscene[band].area = geometry.SwathDefinition( lons=satscene[band].area.lons[indices, :], lats=satscene[band].area.lats[indices, :]) for band_name in loaded_bands: band_uid = hashlib.sha1(satscene[band_name].data.mask).hexdigest() satscene[band_name].area.area_id = ( "swath_" + satscene.fullname + "_" + str(satscene.time_slot) + "_" + str(satscene[band_name].shape) + "_" + str(band_uid)) satscene[band_name].area_id = satscene[band_name].area.area_id