def create(cls, product, factor, consistent_with):
"""
Return ConsistentProduct.
Creates a ConsistentProduct from product with data multiplied
by factor and adds consistent_with to the metadata.
"""
# Create the consistent product object
consistent_product = cls(
datetime=product.datetime,
prodcode=product.prodcode,
timeframe=product.timeframe,
)
# Create the h5 datafile for it
with product.get() as h5:
data = h5['precipitation']
mask = np.equal(data, config.NODATAVALUE)
data = dict(precipitation=np.ma.array(data, mask=mask) * factor)
meta = dict(h5.attrs)
meta.update(consistent_with=consistent_with)
utils.save_dataset(
data=data,
meta=meta,
path=consistent_product.path
)
# get() will now work, so return the object.
filepath = consistent_product.path
filename = os.path.basename(filepath)
logging.info('Created ConsistentProduct {}'.format(filename))
logging.debug('Created ConsistentProduct {}'.format(filepath))
return consistent_product
def create(cls, product, factor, consistent_with):
"""
Return ConsistentProduct.
Creates a ConsistentProduct from product with data multiplied
by factor and adds consistent_with to the metadata.
"""
# Create the consistent product object
consistent_product = cls(
datetime=product.datetime,
prodcode=product.prodcode,
timeframe=product.timeframe,
)
# Create the h5 datafile for it
with product.get() as h5:
data = h5['precipitation']
mask = np.equal(data, config.NODATAVALUE)
data = dict(precipitation=np.ma.array(data, mask=mask) * factor)
meta = dict(h5.attrs)
meta.update(consistent_with=consistent_with)
utils.save_dataset(data=data, meta=meta, path=consistent_product.path)
# get() will now work, so return the object.
filepath = consistent_product.path
filename = os.path.basename(filepath)
logging.info('Created ConsistentProduct {}'.format(filename))
logging.debug('Created ConsistentProduct {}'.format(filepath))
return consistent_product
def _merge(self, aggregates):
"""
Return h5_dataset by merging iterable of aggregate objects.
"""
iterable = iter(map(make_aggregate, aggregates))
aggregate = iterable.next()
h5 = aggregate.get()
meta = dict(h5.attrs)
available = [meta['available']]
array = h5['precipitation']
fill_value = config.NODATAVALUE
mask = np.equal(array, fill_value)
masked_array = np.ma.array(array, mask=mask, fill_value=fill_value)
h5.close()
for aggregate in iterable:
h5 = aggregate.get()
array = h5['precipitation']
fill_value = config.NODATAVALUE
mask = np.equal(array, fill_value)
masked_array = np.ma.array([
masked_array,
np.ma.array(array, mask=mask)
]).sum(0)
for i in range(len(meta['radars'])):
if meta['ranges'][i] == config.NODATAVALUE:
meta['ranges'][i] = h5.attrs['ranges'][i]
if (meta['locations'][i] == 2 * [config.NODATAVALUE]).all():
meta['locations'][i] = h5.attrs['locations'][i]
available.append(meta['available'])
meta['composite_count'] += h5.attrs.get(
'composite_count',
)
meta['timestamp_last_composite'] = h5.attrs.get(
'timestamp_last_composite',
)
h5.close()
meta['available'] = np.vstack(available)
data = dict(precipitation=masked_array)
path = self.get_path()
utils.save_dataset(data, meta, path)
logging.info('Created aggregate {} ({})'.format(
self.datetime, self.code
))
def _merge(self, aggregates):
"""
Return h5_dataset by merging iterable of aggregate objects.
"""
iterable = iter(map(make_aggregate, aggregates))
aggregate = iterable.next()
h5 = aggregate.get()
meta = dict(h5.attrs)
available = [meta['available']]
array = h5['precipitation']
fill_value = config.NODATAVALUE
mask = np.equal(array, fill_value)
masked_array = np.ma.array(array, mask=mask, fill_value=fill_value)
h5.close()
for aggregate in iterable:
h5 = aggregate.get()
array = h5['precipitation']
fill_value = config.NODATAVALUE
mask = np.equal(array, fill_value)
masked_array = np.ma.array(
[masked_array, np.ma.array(array, mask=mask)]).sum(0)
for i in range(len(meta['radars'])):
if meta['ranges'][i] == config.NODATAVALUE:
meta['ranges'][i] = h5.attrs['ranges'][i]
if (meta['locations'][i] == 2 * [config.NODATAVALUE]).all():
meta['locations'][i] = h5.attrs['locations'][i]
available.append(meta['available'])
meta['composite_count'] += h5.attrs.get('composite_count', )
meta['timestamp_last_composite'] = h5.attrs.get(
'timestamp_last_composite', )
h5.close()
meta['available'] = np.vstack(available)
data = dict(precipitation=masked_array)
path = self.get_path()
utils.save_dataset(data, meta, path)
logging.info('Created aggregate {} ({})'.format(
self.datetime, self.code))
def make(self):
aggregate = self._get_aggregate()
aggregate.make()
metafile = os.path.join(config.MISC_DIR, 'grondstations.csv')
dataloader = DataLoader(metafile=metafile,
aggregate=aggregate,
timeframe=self.timeframe)
try:
dataloader.processdata()
stations_count = len(dataloader.rainstations)
cal_stations = [rs for rs in dataloader.rainstations
if not rs.measurement == -999.]
data_count = len(cal_stations)
cal_station_ids = np.array([rs.station_id for rs in cal_stations],
dtype='S20')
cal_station_coords = [(rs.lon, rs.lat) for rs in cal_stations]
cal_station_measurements = [rs.measurement for rs in cal_stations]
logging.info('{} out of {} gauges have data for {}.'.format(
data_count, stations_count, dataloader.date)
)
except:
logging.exception('Exception during calibration preprocessing:')
stations_count = 0
cal_station_ids = []
cal_station_coords = []
cal_station_measurements = []
data_count = 0
interpolator = Interpolator(dataloader)
# Calibrate, method depending on prodcode and timeframe
precipitation_mask = np.equal(
interpolator.precipitation,
config.NODATAVALUE,
)
if data_count == 0:
logging.info('Calibrating is not useful without stations.')
calibration_method = 'None'
calibrated_radar = np.ma.array(
interpolator.precipitation,
mask=precipitation_mask,
)
elif self.prodcode in ['a', 'u'] and self.timeframe in ['h', 'd']:
logging.info('Calibrating using kriging.')
calibration_method = 'Kriging External Drift'
try:
calibrated_radar = np.ma.where(
precipitation_mask,
interpolator.precipitation,
interpolator.get_calibrated(),
)
except:
logging.exception('Exception during kriging:')
calibrated_radar = None
else:
logging.info('Calibrating using idw.')
calibration_method = 'Inverse Distance Weighting'
try:
factor = interpolator.get_correction_factor()
calibrated_radar = np.ma.where(
precipitation_mask,
interpolator.precipitation,
interpolator.precipitation * factor,
)
except:
logging.exception('Exception during idw:')
calibrated_radar = None
if calibrated_radar is None:
logging.warn('Calibration failed.')
calibration_method = 'None'
self.calibrated = interpolator.precipitation
else:
mask = utils.get_countrymask()
self.calibrated = (mask * calibrated_radar +
(1 - mask) * interpolator.precipitation)
self.metadata = dict(dataloader.dataset.attrs)
dataloader.dataset.close()
# Append metadata about the calibration
self.metadata.update(dict(
cal_station_ids=cal_station_ids,
cal_station_coords=cal_station_coords,
cal_station_measurements=cal_station_measurements,
cal_stations_count=stations_count,
cal_data_count=data_count,
cal_method=calibration_method,
))
calibrated_ma = np.ma.array(
self.calibrated,
mask=np.equal(self.calibrated, config.NODATAVALUE),
)
logging.debug('Setting negative values to 0. Min was: {}'.format(
calibrated_ma.min()),
)
calibrated_ma[np.ma.less(calibrated_ma, 0)] = 0
utils.save_dataset(path=self.path,
meta=self.metadata,
data=dict(precipitation=calibrated_ma))
logging.info('Created CalibratedProduct {}'.format(
os.path.basename(self.path)
))
logging.debug(self.path)
def _create(self):
"""
Create a new dataset from the composite.
"""
# Create the composite
dt_composite = self.datetime - self.TD[self.code]
composite = Composite(compositedatetime=dt_composite,
scancodes=self.radars,
declutter=self.declutter,
grid=self.grid,
basedir=self.multiscandir).get()
# Composite unit is mm/hr and covers 5 minutes. It must be in mm.
fill_value = config.NODATAVALUE
array = composite.GetRasterBand(1).ReadAsArray()
mask = np.equal(array, fill_value)
masked_array = np.ma.array(
array, mask=mask, fill_value=fill_value,
) / 12
composite_meta = composite.GetMetadata()
# Create the data for the h5
radars = sorted(
[str(radar)
for radar in json.loads(composite_meta['requested_stations'])],
)
radar_list = zip(json.loads(composite_meta['stations']),
json.loads(composite_meta['ranges']),
json.loads(composite_meta['locations']))
locations_dict = {rad: loc for rad, rng, loc in radar_list}
ranges_dict = {rad: rng for rad, rng, loc in radar_list}
available = np.array([radar in ranges_dict for radar in radars])
ranges = [ranges_dict.get(radar, fill_value) for radar in radars]
locations = np.array([locations_dict.get(radar, 2 * [fill_value])
for radar in radars])
h5_meta = dict(
grid_projection=self.grid.projection,
grid_extent=self.grid.extent,
grid_size=self.grid.size,
radars=radars,
ranges=ranges,
locations=locations,
available=available,
method=composite_meta['method'],
declutter_history=float(composite_meta['declutter_history']),
declutter_size=float(composite_meta['declutter_size']),
timestamp_first_composite=composite_meta['timestamp'],
timestamp_last_composite=composite_meta['timestamp'],
composite_count=1,
fill_value=fill_value,
)
h5_data = dict(
precipitation=masked_array,
)
path = self.get_path()
utils.save_dataset(h5_data, h5_meta, path)
logging.info('Created aggregate {} ({})'.format(
self.datetime, self.code
))
def _create(self):
"""
Create a new dataset from the composite.
"""
# Create the composite
dt_composite = self.datetime - self.TD[self.code]
composite = Composite(compositedatetime=dt_composite,
scancodes=self.radars,
declutter=self.declutter,
grid=self.grid,
basedir=self.multiscandir).get()
# Composite unit is mm/hr and covers 5 minutes. It must be in mm.
fill_value = config.NODATAVALUE
array = composite.GetRasterBand(1).ReadAsArray()
mask = np.equal(array, fill_value)
masked_array = np.ma.array(
array,
mask=mask,
fill_value=fill_value,
) / 12
composite_meta = composite.GetMetadata()
# Create the data for the h5
radars = sorted([
str(radar)
for radar in json.loads(composite_meta['requested_stations'])
], )
radar_list = zip(json.loads(composite_meta['stations']),
json.loads(composite_meta['ranges']),
json.loads(composite_meta['locations']))
locations_dict = {rad: loc for rad, rng, loc in radar_list}
ranges_dict = {rad: rng for rad, rng, loc in radar_list}
available = np.array([radar in ranges_dict for radar in radars])
ranges = [ranges_dict.get(radar, fill_value) for radar in radars]
locations = np.array(
[locations_dict.get(radar, 2 * [fill_value]) for radar in radars])
h5_meta = dict(
grid_projection=self.grid.projection,
grid_extent=self.grid.extent,
grid_size=self.grid.size,
radars=radars,
ranges=ranges,
locations=locations,
available=available,
method=composite_meta['method'],
declutter_history=float(composite_meta['declutter_history']),
declutter_size=float(composite_meta['declutter_size']),
timestamp_first_composite=composite_meta['timestamp'],
timestamp_last_composite=composite_meta['timestamp'],
composite_count=1,
fill_value=fill_value,
)
h5_data = dict(precipitation=masked_array, )
path = self.get_path()
utils.save_dataset(h5_data, h5_meta, path)
logging.info('Created aggregate {} ({})'.format(
self.datetime, self.code))
def make(self):
aggregate = self._get_aggregate()
aggregate.make()
metafile = os.path.join(config.MISC_DIR, 'grondstations.csv')
dataloader = DataLoader(metafile=metafile,
aggregate=aggregate,
timeframe=self.timeframe)
try:
dataloader.processdata()
stations_count = len(dataloader.rainstations)
data_count = len(
[r for r in dataloader.rainstations if r.measurement != -999])
logging.info('{} out of {} gauges have data for {}.'.format(
data_count, stations_count, dataloader.date))
except:
logging.exception('Exception during calibration preprocessing:')
stations_count = 0
data_count = 0
interpolator = Interpolator(dataloader)
# Calibrate, method depending on prodcode and timeframe
precipitation_mask = np.equal(
interpolator.precipitation,
config.NODATAVALUE,
)
if data_count == 0:
logging.info('Calibrating is not useful without stations.')
calibration_method = 'None'
calibrated_radar = np.ma.array(
interpolator.precipitation,
mask=precipitation_mask,
)
elif self.prodcode in ['a', 'u'] and self.timeframe in ['h', 'd']:
logging.info('Calibrating using kriging.')
calibration_method = 'Kriging External Drift'
try:
calibrated_radar = np.ma.where(
precipitation_mask,
interpolator.precipitation,
interpolator.get_calibrated(),
)
except:
logging.exception('Exception during kriging:')
calibrated_radar = None
else:
logging.info('Calibrating using idw.')
calibration_method = 'Inverse Distance Weighting'
try:
factor = interpolator.get_correction_factor()
calibrated_radar = np.ma.where(
precipitation_mask,
interpolator.precipitation,
interpolator.precipitation * factor,
)
except:
logging.exception('Exception during idw:')
calibrated_radar = None
if calibrated_radar is None:
logging.warn('Calibration failed.')
calibration_method = 'None'
self.calibrated = interpolator.precipitation
else:
mask = utils.get_countrymask()
self.calibrated = (mask * calibrated_radar +
(1 - mask) * interpolator.precipitation)
self.metadata = dict(dataloader.dataset.attrs)
dataloader.dataset.close()
# Append metadata about the calibration
self.metadata.update(
dict(
cal_stations_count=stations_count,
cal_data_count=data_count,
cal_method=calibration_method,
))
calibrated_ma = np.ma.array(
self.calibrated,
mask=np.equal(self.calibrated, config.NODATAVALUE),
)
logging.debug(
'Setting negative values to 0. Min was: {}'.format(
calibrated_ma.min()), )
calibrated_ma[np.ma.less(calibrated_ma, 0)] = 0
utils.save_dataset(path=self.path,
meta=self.metadata,
data=dict(precipitation=calibrated_ma))
logging.info('Created CalibratedProduct {}'.format(
os.path.basename(self.path)))
logging.debug(self.path)
