def ftp_publications(self, cascade=False):
""" Return product generator. """
if isinstance(self.datetimes, (list, tuple)):
datetimes = self.datetimes
else:
# Generate datetimes from rangetext string.
datetimes = utils.MultiDateRange(self.datetimes).iterdatetimes()
combinations = utils.get_product_combinations(
datetimes=datetimes,
prodcodes=self.prodcodes,
timeframes=self.timeframes,
)
for combination in combinations:
nowcast = combination.pop('nowcast')
if nowcast != self.nowcast:
continue
if nowcast:
yield products.CopiedProduct(datetime=combination['datetime'])
continue
consistent = utils.consistent_product_expected(
prodcode=combination['prodcode'],
timeframe=combination['timeframe'],
)
product = products.CalibratedProduct(**combination)
if not consistent:
yield product
if cascade:
rps = products.Consistifier.get_rescaled_products(product)
for rescaled_product in rps:
yield rescaled_product
def calibrate(result,
datetime,
prodcode,
timeframe,
nowcast,
radars,
declutter,
direct=False,
cascade=False):
""" Created calibrated aggregated composites. """
loghelper.setup_logging(logfile_name='radar_calibrate.log')
logging.info(20 * '-' + ' calibrate ' + 20 * '-')
try:
# Create products
if nowcast:
product = products.CopiedProduct(datetime)
else:
product = products.CalibratedProduct(
radars=radars,
prodcode=prodcode,
datetime=datetime,
timeframe=timeframe,
declutter=declutter,
)
product.make()
# Cascade when requested
if cascade:
combinations = utils.get_product_combinations(
datetimes=[datetime],
prodcodes=[prodcode],
timeframes=[timeframe],
)
for combination in combinations:
rescale_kwargs = dict(result=None,
direct=direct,
cascade=cascade)
extra_kwargs = {
k: v
for k, v in combination.items()
if k in ['datetime', 'prodcode', 'timeframe']
}
rescale_kwargs.update(extra_kwargs)
if direct:
rescale(**rescale_kwargs)
else:
rescale.delay(**rescale_kwargs)
except Exception as e:
logging.exception(e)
logging.info(20 * '-' + ' calibrate complete ' + 20 * '-')
def nowcast(result, datetime, timeframe, minutes):
"""
Create nowcast product.
"""
loghelper.setup_logging(logfile_name='radar_nowcast.log')
logging.info(20 * '-' + ' nowcast ' + 20 * '-')
# the result product is called the nowcast product
nowcast_product = products.NowcastProduct(
datetime=datetime,
timeframe=timeframe,
)
# the vector products (realtime, five minutes)
# are used to determine the translation vector.
vector_products = []
for vector_delay in minutes + 15, minutes:
vector_products.append(
products.CalibratedProduct(
prodcode='r',
timeframe='f',
datetime=datetime - timedelta(minutes=vector_delay),
))
# the base product is the product for which the data
# is shifted to arrive at a nowcasted product.
base_product = products.CalibratedProduct(prodcode='r',
timeframe='f',
datetime=datetime -
timedelta(minutes=minutes))
try:
nowcast_product.make(
base_product=base_product,
vector_products=vector_products,
)
except Exception as e:
logging.exception(e)
logging.info(20 * '-' + ' nowcast complete ' + 20 * '-')
def rescale(result,
datetime,
prodcode,
timeframe,
direct=False,
cascade=False):
""" Create rescaled products wherever possible. """
loghelper.setup_logging(logfile_name='radar_rescale.log')
logging.info(20 * '-' + ' rescale ' + 20 * '-')
try:
product = products.CalibratedProduct(prodcode=prodcode,
datetime=datetime,
timeframe=timeframe)
rescaleds = products.Consistifier.create_consistent_products(product)
if not rescaleds:
logging.info('Nothing to rescale.')
except Exception as e:
logging.exception(e)
logging.info(20 * '-' + ' rescale complete ' + 20 * '-')
def sandbox(**kwargs):
""" Print a value from various datafiles. """
i, j = map(int, kwargs['indices'].split(','))
datetimes = utils.MultiDateRange(kwargs['range']).iterdatetimes()
combinations = utils.get_product_combinations(
datetimes=datetimes,
prodcodes=kwargs['prodcodes'],
timeframes=kwargs['timeframes'],
)
for combination in combinations:
# The calibrate
calibrate = products.CalibratedProduct(**combination)
with calibrate.get() as calibrate_h5:
print('Calibrate: {}'.format(calibrate_h5['precipitation'][i, j]))
# The rescaled
if utils.consistent_product_expected(combination['prodcode'],
combination['timeframe']):
rescaled = products.ConsistentProduct(**combination)
print(rescaled.path)
with rescaled.get() as rescaled_h5:
print('Rescaled: {}'.format(rescaled_h5['precipitation'][i,
j], ))
# The thredds
thredds = products.ThreddsFile.get_for_product(calibrate)
t = thredds._index(calibrate)
with thredds.get() as thredds_h5:
print('Thredds: {}'.format(thredds_h5['precipitation'][i, j, t], ))
# The merged threddsfile
thredds_m = products.ThreddsFile.get_for_product(calibrate, merge=True)
t = thredds_m._index(calibrate)
with thredds_m.get() as thredds_m_h5:
print('Thredds(m): {} ({})'.format(
thredds_m_h5['precipitation'][i, j, t],
thredds_m_h5['available'][t]))
# Opendap
values = products.get_values_from_opendap(
x=i,
y=j,
start_date=combination['datetime'],
end_date=combination['datetime'],
)
print('Opendap: {}'.format(values))
def product_generator(product, prodcode, timeframe, datetimes, nowcast):
""" Return product generator. """
if product == 'a':
aggregate_kwargs = dict(declutter=None, radars=config.ALL_RADARS)
combinations = utils.get_aggregate_combinations(
datetimes=datetimes,
timeframes=timeframe,
)
for combination in combinations:
if nowcast != combination['nowcast']:
continue
if nowcast:
aggregate_kwargs.update(
dict(basedir=config.NOWCAST_AGGREGATE_DIR,
multiscandir=config.NOWCAST_MULTISCAN_DIR,
grid=scans.NOWCASTGRID,
datetime=combination['datetime'],
timeframe=combination['timeframe']))
else:
aggregate_kwargs.update(
dict(basedir=config.AGGREGATE_DIR,
multiscandir=config.MULTISCAN_DIR,
grid=scans.BASEGRID,
datetime=combination['datetime'],
timeframe=combination['timeframe']))
yield scans.Aggregate(**aggregate_kwargs)
else:
combinations = utils.get_product_combinations(prodcodes=prodcode,
datetimes=datetimes,
timeframes=timeframe)
for combination in combinations:
if nowcast != combination.pop('nowcast'):
continue
if nowcast:
yield products.CopiedProduct(datetime=combination['datetime'])
else:
if product == 'b':
yield products.CalibratedProduct(**combination)
else:
yield products.ConsistentProduct(**combination)
