def create(cls, product, factor, consistent_with): """ Return ConsistentProduct. Creates a ConsistentProduct from product with data multiplied by factor and adds consistent_with to the metadata. """ # Create the consistent product object consistent_product = cls( datetime=product.datetime, prodcode=product.prodcode, timeframe=product.timeframe, ) # Create the h5 datafile for it with product.get() as h5: data = h5['precipitation'] mask = np.equal(data, config.NODATAVALUE) data = dict(precipitation=np.ma.array(data, mask=mask) * factor) meta = dict(h5.attrs) meta.update(consistent_with=consistent_with) utils.save_dataset( data=data, meta=meta, path=consistent_product.path ) # get() will now work, so return the object. filepath = consistent_product.path filename = os.path.basename(filepath) logging.info('Created ConsistentProduct {}'.format(filename)) logging.debug('Created ConsistentProduct {}'.format(filepath)) return consistent_product
def create(cls, product, factor, consistent_with): """ Return ConsistentProduct. Creates a ConsistentProduct from product with data multiplied by factor and adds consistent_with to the metadata. """ # Create the consistent product object consistent_product = cls( datetime=product.datetime, prodcode=product.prodcode, timeframe=product.timeframe, ) # Create the h5 datafile for it with product.get() as h5: data = h5['precipitation'] mask = np.equal(data, config.NODATAVALUE) data = dict(precipitation=np.ma.array(data, mask=mask) * factor) meta = dict(h5.attrs) meta.update(consistent_with=consistent_with) utils.save_dataset(data=data, meta=meta, path=consistent_product.path) # get() will now work, so return the object. filepath = consistent_product.path filename = os.path.basename(filepath) logging.info('Created ConsistentProduct {}'.format(filename)) logging.debug('Created ConsistentProduct {}'.format(filepath)) return consistent_product
def _merge(self, aggregates): """ Return h5_dataset by merging iterable of aggregate objects. """ iterable = iter(map(make_aggregate, aggregates)) aggregate = iterable.next() h5 = aggregate.get() meta = dict(h5.attrs) available = [meta['available']] array = h5['precipitation'] fill_value = config.NODATAVALUE mask = np.equal(array, fill_value) masked_array = np.ma.array(array, mask=mask, fill_value=fill_value) h5.close() for aggregate in iterable: h5 = aggregate.get() array = h5['precipitation'] fill_value = config.NODATAVALUE mask = np.equal(array, fill_value) masked_array = np.ma.array([ masked_array, np.ma.array(array, mask=mask) ]).sum(0) for i in range(len(meta['radars'])): if meta['ranges'][i] == config.NODATAVALUE: meta['ranges'][i] = h5.attrs['ranges'][i] if (meta['locations'][i] == 2 * [config.NODATAVALUE]).all(): meta['locations'][i] = h5.attrs['locations'][i] available.append(meta['available']) meta['composite_count'] += h5.attrs.get( 'composite_count', ) meta['timestamp_last_composite'] = h5.attrs.get( 'timestamp_last_composite', ) h5.close() meta['available'] = np.vstack(available) data = dict(precipitation=masked_array) path = self.get_path() utils.save_dataset(data, meta, path) logging.info('Created aggregate {} ({})'.format( self.datetime, self.code ))
def _merge(self, aggregates): """ Return h5_dataset by merging iterable of aggregate objects. """ iterable = iter(map(make_aggregate, aggregates)) aggregate = iterable.next() h5 = aggregate.get() meta = dict(h5.attrs) available = [meta['available']] array = h5['precipitation'] fill_value = config.NODATAVALUE mask = np.equal(array, fill_value) masked_array = np.ma.array(array, mask=mask, fill_value=fill_value) h5.close() for aggregate in iterable: h5 = aggregate.get() array = h5['precipitation'] fill_value = config.NODATAVALUE mask = np.equal(array, fill_value) masked_array = np.ma.array( [masked_array, np.ma.array(array, mask=mask)]).sum(0) for i in range(len(meta['radars'])): if meta['ranges'][i] == config.NODATAVALUE: meta['ranges'][i] = h5.attrs['ranges'][i] if (meta['locations'][i] == 2 * [config.NODATAVALUE]).all(): meta['locations'][i] = h5.attrs['locations'][i] available.append(meta['available']) meta['composite_count'] += h5.attrs.get('composite_count', ) meta['timestamp_last_composite'] = h5.attrs.get( 'timestamp_last_composite', ) h5.close() meta['available'] = np.vstack(available) data = dict(precipitation=masked_array) path = self.get_path() utils.save_dataset(data, meta, path) logging.info('Created aggregate {} ({})'.format( self.datetime, self.code))
def make(self): aggregate = self._get_aggregate() aggregate.make() metafile = os.path.join(config.MISC_DIR, 'grondstations.csv') dataloader = DataLoader(metafile=metafile, aggregate=aggregate, timeframe=self.timeframe) try: dataloader.processdata() stations_count = len(dataloader.rainstations) cal_stations = [rs for rs in dataloader.rainstations if not rs.measurement == -999.] data_count = len(cal_stations) cal_station_ids = np.array([rs.station_id for rs in cal_stations], dtype='S20') cal_station_coords = [(rs.lon, rs.lat) for rs in cal_stations] cal_station_measurements = [rs.measurement for rs in cal_stations] logging.info('{} out of {} gauges have data for {}.'.format( data_count, stations_count, dataloader.date) ) except: logging.exception('Exception during calibration preprocessing:') stations_count = 0 cal_station_ids = [] cal_station_coords = [] cal_station_measurements = [] data_count = 0 interpolator = Interpolator(dataloader) # Calibrate, method depending on prodcode and timeframe precipitation_mask = np.equal( interpolator.precipitation, config.NODATAVALUE, ) if data_count == 0: logging.info('Calibrating is not useful without stations.') calibration_method = 'None' calibrated_radar = np.ma.array( interpolator.precipitation, mask=precipitation_mask, ) elif self.prodcode in ['a', 'u'] and self.timeframe in ['h', 'd']: logging.info('Calibrating using kriging.') calibration_method = 'Kriging External Drift' try: calibrated_radar = np.ma.where( precipitation_mask, interpolator.precipitation, interpolator.get_calibrated(), ) except: logging.exception('Exception during kriging:') calibrated_radar = None else: logging.info('Calibrating using idw.') calibration_method = 'Inverse Distance Weighting' try: factor = interpolator.get_correction_factor() calibrated_radar = np.ma.where( precipitation_mask, interpolator.precipitation, interpolator.precipitation * factor, ) except: logging.exception('Exception during idw:') calibrated_radar = None if calibrated_radar is None: logging.warn('Calibration failed.') calibration_method = 'None' self.calibrated = interpolator.precipitation else: mask = utils.get_countrymask() self.calibrated = (mask * calibrated_radar + (1 - mask) * interpolator.precipitation) self.metadata = dict(dataloader.dataset.attrs) dataloader.dataset.close() # Append metadata about the calibration self.metadata.update(dict( cal_station_ids=cal_station_ids, cal_station_coords=cal_station_coords, cal_station_measurements=cal_station_measurements, cal_stations_count=stations_count, cal_data_count=data_count, cal_method=calibration_method, )) calibrated_ma = np.ma.array( self.calibrated, mask=np.equal(self.calibrated, config.NODATAVALUE), ) logging.debug('Setting negative values to 0. Min was: {}'.format( calibrated_ma.min()), ) calibrated_ma[np.ma.less(calibrated_ma, 0)] = 0 utils.save_dataset(path=self.path, meta=self.metadata, data=dict(precipitation=calibrated_ma)) logging.info('Created CalibratedProduct {}'.format( os.path.basename(self.path) )) logging.debug(self.path)
def _create(self): """ Create a new dataset from the composite. """ # Create the composite dt_composite = self.datetime - self.TD[self.code] composite = Composite(compositedatetime=dt_composite, scancodes=self.radars, declutter=self.declutter, grid=self.grid, basedir=self.multiscandir).get() # Composite unit is mm/hr and covers 5 minutes. It must be in mm. fill_value = config.NODATAVALUE array = composite.GetRasterBand(1).ReadAsArray() mask = np.equal(array, fill_value) masked_array = np.ma.array( array, mask=mask, fill_value=fill_value, ) / 12 composite_meta = composite.GetMetadata() # Create the data for the h5 radars = sorted( [str(radar) for radar in json.loads(composite_meta['requested_stations'])], ) radar_list = zip(json.loads(composite_meta['stations']), json.loads(composite_meta['ranges']), json.loads(composite_meta['locations'])) locations_dict = {rad: loc for rad, rng, loc in radar_list} ranges_dict = {rad: rng for rad, rng, loc in radar_list} available = np.array([radar in ranges_dict for radar in radars]) ranges = [ranges_dict.get(radar, fill_value) for radar in radars] locations = np.array([locations_dict.get(radar, 2 * [fill_value]) for radar in radars]) h5_meta = dict( grid_projection=self.grid.projection, grid_extent=self.grid.extent, grid_size=self.grid.size, radars=radars, ranges=ranges, locations=locations, available=available, method=composite_meta['method'], declutter_history=float(composite_meta['declutter_history']), declutter_size=float(composite_meta['declutter_size']), timestamp_first_composite=composite_meta['timestamp'], timestamp_last_composite=composite_meta['timestamp'], composite_count=1, fill_value=fill_value, ) h5_data = dict( precipitation=masked_array, ) path = self.get_path() utils.save_dataset(h5_data, h5_meta, path) logging.info('Created aggregate {} ({})'.format( self.datetime, self.code ))
def _create(self): """ Create a new dataset from the composite. """ # Create the composite dt_composite = self.datetime - self.TD[self.code] composite = Composite(compositedatetime=dt_composite, scancodes=self.radars, declutter=self.declutter, grid=self.grid, basedir=self.multiscandir).get() # Composite unit is mm/hr and covers 5 minutes. It must be in mm. fill_value = config.NODATAVALUE array = composite.GetRasterBand(1).ReadAsArray() mask = np.equal(array, fill_value) masked_array = np.ma.array( array, mask=mask, fill_value=fill_value, ) / 12 composite_meta = composite.GetMetadata() # Create the data for the h5 radars = sorted([ str(radar) for radar in json.loads(composite_meta['requested_stations']) ], ) radar_list = zip(json.loads(composite_meta['stations']), json.loads(composite_meta['ranges']), json.loads(composite_meta['locations'])) locations_dict = {rad: loc for rad, rng, loc in radar_list} ranges_dict = {rad: rng for rad, rng, loc in radar_list} available = np.array([radar in ranges_dict for radar in radars]) ranges = [ranges_dict.get(radar, fill_value) for radar in radars] locations = np.array( [locations_dict.get(radar, 2 * [fill_value]) for radar in radars]) h5_meta = dict( grid_projection=self.grid.projection, grid_extent=self.grid.extent, grid_size=self.grid.size, radars=radars, ranges=ranges, locations=locations, available=available, method=composite_meta['method'], declutter_history=float(composite_meta['declutter_history']), declutter_size=float(composite_meta['declutter_size']), timestamp_first_composite=composite_meta['timestamp'], timestamp_last_composite=composite_meta['timestamp'], composite_count=1, fill_value=fill_value, ) h5_data = dict(precipitation=masked_array, ) path = self.get_path() utils.save_dataset(h5_data, h5_meta, path) logging.info('Created aggregate {} ({})'.format( self.datetime, self.code))
def make(self): aggregate = self._get_aggregate() aggregate.make() metafile = os.path.join(config.MISC_DIR, 'grondstations.csv') dataloader = DataLoader(metafile=metafile, aggregate=aggregate, timeframe=self.timeframe) try: dataloader.processdata() stations_count = len(dataloader.rainstations) data_count = len( [r for r in dataloader.rainstations if r.measurement != -999]) logging.info('{} out of {} gauges have data for {}.'.format( data_count, stations_count, dataloader.date)) except: logging.exception('Exception during calibration preprocessing:') stations_count = 0 data_count = 0 interpolator = Interpolator(dataloader) # Calibrate, method depending on prodcode and timeframe precipitation_mask = np.equal( interpolator.precipitation, config.NODATAVALUE, ) if data_count == 0: logging.info('Calibrating is not useful without stations.') calibration_method = 'None' calibrated_radar = np.ma.array( interpolator.precipitation, mask=precipitation_mask, ) elif self.prodcode in ['a', 'u'] and self.timeframe in ['h', 'd']: logging.info('Calibrating using kriging.') calibration_method = 'Kriging External Drift' try: calibrated_radar = np.ma.where( precipitation_mask, interpolator.precipitation, interpolator.get_calibrated(), ) except: logging.exception('Exception during kriging:') calibrated_radar = None else: logging.info('Calibrating using idw.') calibration_method = 'Inverse Distance Weighting' try: factor = interpolator.get_correction_factor() calibrated_radar = np.ma.where( precipitation_mask, interpolator.precipitation, interpolator.precipitation * factor, ) except: logging.exception('Exception during idw:') calibrated_radar = None if calibrated_radar is None: logging.warn('Calibration failed.') calibration_method = 'None' self.calibrated = interpolator.precipitation else: mask = utils.get_countrymask() self.calibrated = (mask * calibrated_radar + (1 - mask) * interpolator.precipitation) self.metadata = dict(dataloader.dataset.attrs) dataloader.dataset.close() # Append metadata about the calibration self.metadata.update( dict( cal_stations_count=stations_count, cal_data_count=data_count, cal_method=calibration_method, )) calibrated_ma = np.ma.array( self.calibrated, mask=np.equal(self.calibrated, config.NODATAVALUE), ) logging.debug( 'Setting negative values to 0. Min was: {}'.format( calibrated_ma.min()), ) calibrated_ma[np.ma.less(calibrated_ma, 0)] = 0 utils.save_dataset(path=self.path, meta=self.metadata, data=dict(precipitation=calibrated_ma)) logging.info('Created CalibratedProduct {}'.format( os.path.basename(self.path))) logging.debug(self.path)