def rescale(**kwargs):
""" Create aggregates given some commandline arguments. """
# Determine action
task = tasks.rescale
if kwargs['direct']:
action = task
else:
action = task.delay
# Determine derived arguments
datetimes = utils.MultiDateRange(kwargs['range']).iterdatetimes()
combinations = utils.get_product_combinations(
datetimes=datetimes,
prodcodes=kwargs['prodcodes'],
timeframes=kwargs['timeframes'])
# Execute or delay task
for combination in combinations:
action_kwargs = dict(result=None,
direct=kwargs['direct'],
cascade=kwargs['cascade'])
rescale_kwargs = {
k: v
for k, v in combination.items()
if k in ['datetime', 'prodcode', 'timeframe']
}
action_kwargs.update(rescale_kwargs)
action(**action_kwargs)
def ftp_publications(self, cascade=False):
""" Return product generator. """
if isinstance(self.datetimes, (list, tuple)):
datetimes = self.datetimes
else:
# Generate datetimes from rangetext string.
datetimes = utils.MultiDateRange(self.datetimes).iterdatetimes()
combinations = utils.get_product_combinations(
datetimes=datetimes,
prodcodes=self.prodcodes,
timeframes=self.timeframes,
)
for combination in combinations:
nowcast = combination.pop('nowcast')
if nowcast != self.nowcast:
continue
if nowcast:
yield products.CopiedProduct(datetime=combination['datetime'])
continue
consistent = utils.consistent_product_expected(
prodcode=combination['prodcode'],
timeframe=combination['timeframe'],
)
product = products.CalibratedProduct(**combination)
if not consistent:
yield product
if cascade:
rps = products.Consistifier.get_rescaled_products(product)
for rescaled_product in rps:
yield rescaled_product
def calibrate(**kwargs):
""" Create calibrated products given some commandline arguments. """
# Determine action
task = tasks.calibrate
if kwargs['direct']:
action = task
else:
action = task.delay
# Determine derived arguments
declutter = dict(
size=kwargs['declutter_size'],
history=kwargs['declutter_history'],
)
datetimes = utils.MultiDateRange(kwargs['range']).iterdatetimes()
combinations = utils.get_product_combinations(
datetimes=datetimes,
prodcodes=kwargs['prodcodes'],
timeframes=kwargs['timeframes'],
)
# Execute or delay task
for combination in combinations:
if kwargs['nowcast'] != combination['nowcast']:
continue
action_kwargs = dict(result=None,
declutter=declutter,
radars=kwargs['radars'],
direct=kwargs['direct'],
cascade=kwargs['cascade'])
action_kwargs.update(combination)
action(**action_kwargs)
def calibrate(**kwargs):
""" Create calibrated products given some commandline arguments. """
# Determine action
task = tasks.calibrate
if kwargs['direct']:
action = task
else:
action = task.delay
# Determine derived arguments
declutter = dict(
size=kwargs['declutter_size'],
history=kwargs['declutter_history'],
)
datetimes = utils.MultiDateRange(kwargs['range']).iterdatetimes()
combinations = utils.get_product_combinations(
datetimes=datetimes,
prodcodes=kwargs['prodcodes'],
timeframes=kwargs['timeframes'],
)
# Execute or delay task
for combination in combinations:
if kwargs['nowcast'] != combination['nowcast']:
continue
action_kwargs = dict(result=None,
declutter=declutter,
radars=kwargs['radars'],
direct=kwargs['direct'],
cascade=kwargs['cascade'])
action_kwargs.update(combination)
action(**action_kwargs)
def aggregate(result,
datetime,
timeframe,
nowcast,
radars,
declutter,
direct=False,
cascade=False):
""" Create aggregates and optionally cascade to depending products. """
loghelper.setup_logging(logfile_name='radar_aggregate.log')
logging.info(20 * '-' + ' aggregate ' + 20 * '-')
try:
# Create aggregates
aggregate_kwargs = dict(radars=radars,
declutter=declutter,
datetime=datetime,
timeframe=timeframe)
if nowcast:
aggregate_kwargs.update(
dict(basedir=config.NOWCAST_AGGREGATE_DIR,
multiscandir=config.NOWCAST_MULTISCAN_DIR,
grid=scans.NOWCASTGRID))
else:
aggregate_kwargs.update(
dict(basedir=config.AGGREGATE_DIR,
multiscandir=config.MULTISCAN_DIR,
grid=scans.BASEGRID))
aggregate = scans.Aggregate(**aggregate_kwargs)
aggregate.make()
# Cascade when requested
if cascade:
combinations = utils.get_product_combinations(
datetimes=[datetime],
timeframes=[timeframe],
)
for combination in combinations:
calibrate_kwargs = dict(result=None,
radars=radars,
declutter=declutter,
direct=direct,
cascade=cascade)
calibrate_kwargs.update(combination)
if direct:
calibrate(**calibrate_kwargs)
else:
calibrate.delay(**calibrate_kwargs)
except Exception as e:
logging.exception(e)
logging.info(20 * '-' + ' aggregate complete ' + 20 * '-')
def calibrate(result,
datetime,
prodcode,
timeframe,
nowcast,
radars,
declutter,
direct=False,
cascade=False):
""" Created calibrated aggregated composites. """
loghelper.setup_logging(logfile_name='radar_calibrate.log')
logging.info(20 * '-' + ' calibrate ' + 20 * '-')
try:
# Create products
if nowcast:
product = products.CopiedProduct(datetime)
else:
product = products.CalibratedProduct(
radars=radars,
prodcode=prodcode,
datetime=datetime,
timeframe=timeframe,
declutter=declutter,
)
product.make()
# Cascade when requested
if cascade:
combinations = utils.get_product_combinations(
datetimes=[datetime],
prodcodes=[prodcode],
timeframes=[timeframe],
)
for combination in combinations:
rescale_kwargs = dict(result=None,
direct=direct,
cascade=cascade)
extra_kwargs = {
k: v
for k, v in combination.items()
if k in ['datetime', 'prodcode', 'timeframe']
}
rescale_kwargs.update(extra_kwargs)
if direct:
rescale(**rescale_kwargs)
else:
rescale.delay(**rescale_kwargs)
except Exception as e:
logging.exception(e)
logging.info(20 * '-' + ' calibrate complete ' + 20 * '-')
def aggregate(result, datetime, timeframe, nowcast,
radars, declutter, direct=False, cascade=False):
""" Create aggregates and optionally cascade to depending products. """
loghelper.setup_logging(logfile_name='radar_aggregate.log')
logging.info(20 * '-' + ' aggregate ' + 20 * '-')
try:
# Create aggregates
aggregate_kwargs = dict(
radars=radars,
declutter=declutter,
datetime=datetime,
timeframe=timeframe
)
if nowcast:
aggregate_kwargs.update(dict(
basedir=config.NOWCAST_AGGREGATE_DIR,
multiscandir=config.NOWCAST_MULTISCAN_DIR,
grid=scans.NOWCASTGRID))
else:
aggregate_kwargs.update(dict(
basedir=config.AGGREGATE_DIR,
multiscandir=config.MULTISCAN_DIR,
grid=scans.BASEGRID))
aggregate = scans.Aggregate(**aggregate_kwargs)
aggregate.make()
# Cascade when requested
if cascade:
combinations = utils.get_product_combinations(
datetimes=[datetime], timeframes=[timeframe],
)
for combination in combinations:
calibrate_kwargs = dict(result=None,
radars=radars,
declutter=declutter,
direct=direct,
cascade=cascade)
calibrate_kwargs.update(combination)
if direct:
calibrate(**calibrate_kwargs)
else:
calibrate.delay(**calibrate_kwargs)
except Exception as e:
logging.exception(e)
logging.info(20 * '-' + ' aggregate complete ' + 20 * '-')
def sandbox(**kwargs):
""" Print a value from various datafiles. """
i, j = map(int, kwargs['indices'].split(','))
datetimes = utils.MultiDateRange(kwargs['range']).iterdatetimes()
combinations = utils.get_product_combinations(
datetimes=datetimes,
prodcodes=kwargs['prodcodes'],
timeframes=kwargs['timeframes'],
)
for combination in combinations:
# The calibrate
calibrate = products.CalibratedProduct(**combination)
with calibrate.get() as calibrate_h5:
print('Calibrate: {}'.format(calibrate_h5['precipitation'][i, j]))
# The rescaled
if utils.consistent_product_expected(combination['prodcode'],
combination['timeframe']):
rescaled = products.ConsistentProduct(**combination)
print(rescaled.path)
with rescaled.get() as rescaled_h5:
print('Rescaled: {}'.format(
rescaled_h5['precipitation'][i, j],
))
# The thredds
thredds = products.ThreddsFile.get_for_product(calibrate)
t = thredds._index(calibrate)
with thredds.get() as thredds_h5:
print('Thredds: {}'.format(
thredds_h5['precipitation'][i, j, t],
))
# The merged threddsfile
thredds_m = products.ThreddsFile.get_for_product(calibrate, merge=True)
t = thredds_m._index(calibrate)
with thredds_m.get() as thredds_m_h5:
print('Thredds(m): {} ({})'.format(
thredds_m_h5['precipitation'][i, j, t],
thredds_m_h5['available'][t]
))
# Opendap
values = products.get_values_from_opendap(
x=i,
y=j,
start_date=combination['datetime'],
end_date=combination['datetime'],
)
print('Opendap: {}'.format(values))
def product_generator(product, prodcode, timeframe, datetimes, nowcast):
""" Return product generator. """
if product == 'a':
aggregate_kwargs = dict(declutter=None,
radars=config.ALL_RADARS)
combinations = utils.get_aggregate_combinations(
datetimes=datetimes,
timeframes=timeframe,
)
for combination in combinations:
if nowcast != combination['nowcast']:
continue
if nowcast:
aggregate_kwargs.update(dict(
basedir=config.NOWCAST_AGGREGATE_DIR,
multiscandir=config.NOWCAST_MULTISCAN_DIR,
grid=scans.NOWCASTGRID,
datetime=combination['datetime'],
timeframe=combination['timeframe']))
else:
aggregate_kwargs.update(dict(
basedir=config.AGGREGATE_DIR,
multiscandir=config.MULTISCAN_DIR,
grid=scans.BASEGRID,
datetime=combination['datetime'],
timeframe=combination['timeframe']))
yield scans.Aggregate(**aggregate_kwargs)
else:
combinations = utils.get_product_combinations(prodcodes=prodcode,
datetimes=datetimes,
timeframes=timeframe)
for combination in combinations:
if nowcast != combination.pop('nowcast'):
continue
if nowcast:
yield products.CopiedProduct(datetime=combination['datetime'])
else:
if product == 'b':
yield products.CalibratedProduct(**combination)
else:
yield products.ConsistentProduct(**combination)
def sandbox(**kwargs):
""" Print a value from various datafiles. """
i, j = map(int, kwargs['indices'].split(','))
datetimes = utils.MultiDateRange(kwargs['range']).iterdatetimes()
combinations = utils.get_product_combinations(
datetimes=datetimes,
prodcodes=kwargs['prodcodes'],
timeframes=kwargs['timeframes'],
)
for combination in combinations:
# The calibrate
calibrate = products.CalibratedProduct(**combination)
with calibrate.get() as calibrate_h5:
print('Calibrate: {}'.format(calibrate_h5['precipitation'][i, j]))
# The rescaled
if utils.consistent_product_expected(combination['prodcode'],
combination['timeframe']):
rescaled = products.ConsistentProduct(**combination)
print(rescaled.path)
with rescaled.get() as rescaled_h5:
print('Rescaled: {}'.format(rescaled_h5['precipitation'][i,
j], ))
# The thredds
thredds = products.ThreddsFile.get_for_product(calibrate)
t = thredds._index(calibrate)
with thredds.get() as thredds_h5:
print('Thredds: {}'.format(thredds_h5['precipitation'][i, j, t], ))
# The merged threddsfile
thredds_m = products.ThreddsFile.get_for_product(calibrate, merge=True)
t = thredds_m._index(calibrate)
with thredds_m.get() as thredds_m_h5:
print('Thredds(m): {} ({})'.format(
thredds_m_h5['precipitation'][i, j, t],
thredds_m_h5['available'][t]))
# Opendap
values = products.get_values_from_opendap(
x=i,
y=j,
start_date=combination['datetime'],
end_date=combination['datetime'],
)
print('Opendap: {}'.format(values))
def product_generator(product, prodcode, timeframe, datetimes, nowcast):
""" Return product generator. """
if product == 'a':
aggregate_kwargs = dict(declutter=None, radars=config.ALL_RADARS)
combinations = utils.get_aggregate_combinations(
datetimes=datetimes,
timeframes=timeframe,
)
for combination in combinations:
if nowcast != combination['nowcast']:
continue
if nowcast:
aggregate_kwargs.update(
dict(basedir=config.NOWCAST_AGGREGATE_DIR,
multiscandir=config.NOWCAST_MULTISCAN_DIR,
grid=scans.NOWCASTGRID,
datetime=combination['datetime'],
timeframe=combination['timeframe']))
else:
aggregate_kwargs.update(
dict(basedir=config.AGGREGATE_DIR,
multiscandir=config.MULTISCAN_DIR,
grid=scans.BASEGRID,
datetime=combination['datetime'],
timeframe=combination['timeframe']))
yield scans.Aggregate(**aggregate_kwargs)
else:
combinations = utils.get_product_combinations(prodcodes=prodcode,
datetimes=datetimes,
timeframes=timeframe)
for combination in combinations:
if nowcast != combination.pop('nowcast'):
continue
if nowcast:
yield products.CopiedProduct(datetime=combination['datetime'])
else:
if product == 'b':
yield products.CalibratedProduct(**combination)
else:
yield products.ConsistentProduct(**combination)
def rescale(**kwargs):
""" Create aggregates given some commandline arguments. """
# Determine action
task = tasks.rescale
if kwargs['direct']:
action = task
else:
action = task.delay
# Determine derived arguments
datetimes = utils.MultiDateRange(kwargs['range']).iterdatetimes()
combinations = utils.get_product_combinations(
datetimes=datetimes,
prodcodes=kwargs['prodcodes'],
timeframes=kwargs['timeframes'])
# Execute or delay task
for combination in combinations:
action_kwargs = dict(result=None,
direct=kwargs['direct'],
cascade=kwargs['cascade'])
action_kwargs.update(combination)
action(**action_kwargs)
def calibrate(result, datetime, prodcode, timeframe, nowcast,
radars, declutter, direct=False, cascade=False):
""" Created calibrated aggregated composites. """
loghelper.setup_logging(logfile_name='radar_calibrate.log')
logging.info(20 * '-' + ' calibrate ' + 20 * '-')
try:
# Create products
if nowcast:
product = products.CopiedProduct(datetime)
else:
product = products.CalibratedProduct(
radars=radars,
prodcode=prodcode,
datetime=datetime,
timeframe=timeframe,
declutter=declutter,
)
product.make()
# Cascade when requested
if cascade:
combinations = utils.get_product_combinations(
datetimes=[datetime],
prodcodes=[prodcode],
timeframes=[timeframe],
)
for combination in combinations:
rescale_kwargs = dict(result=None,
direct=direct,
cascade=cascade)
extra_kwargs = {k: v
for k, v in combination.items()
if k in ['datetime', 'prodcode', 'timeframe']}
rescale_kwargs.update(extra_kwargs)
if direct:
rescale(**rescale_kwargs)
else:
rescale.delay(**rescale_kwargs)
except Exception as e:
logging.exception(e)
logging.info(20 * '-' + ' calibrate complete ' + 20 * '-')
