def aggregate(**kwargs):
""" Create aggregates given some commandline arguments. """
# Determine action
task = tasks.aggregate
if kwargs['direct']:
action = task
else:
action = task.delay
# Determine derived arguments
declutter = dict(
size=kwargs['declutter_size'],
history=kwargs['declutter_history'],
)
datetimes = utils.MultiDateRange(kwargs['range']).iterdatetimes()
combinations = utils.get_aggregate_combinations(
datetimes=datetimes, timeframes=kwargs['timeframes'],
)
# Execute or delay task
for combination in combinations:
if kwargs['nowcast'] != combination['nowcast']:
continue
action_kwargs = dict(result=None,
declutter=declutter,
radars=kwargs['radars'],
direct=kwargs['direct'],
cascade=kwargs['cascade'])
action_kwargs.update(combination)
action(**action_kwargs)
def product_generator(product, prodcode, timeframe, datetimes, nowcast):
""" Return product generator. """
if product == 'a':
aggregate_kwargs = dict(declutter=None,
radars=config.ALL_RADARS)
combinations = utils.get_aggregate_combinations(
datetimes=datetimes,
timeframes=timeframe,
)
for combination in combinations:
if nowcast != combination['nowcast']:
continue
if nowcast:
aggregate_kwargs.update(dict(
basedir=config.NOWCAST_AGGREGATE_DIR,
multiscandir=config.NOWCAST_MULTISCAN_DIR,
grid=scans.NOWCASTGRID,
datetime=combination['datetime'],
timeframe=combination['timeframe']))
else:
aggregate_kwargs.update(dict(
basedir=config.AGGREGATE_DIR,
multiscandir=config.MULTISCAN_DIR,
grid=scans.BASEGRID,
datetime=combination['datetime'],
timeframe=combination['timeframe']))
yield scans.Aggregate(**aggregate_kwargs)
else:
combinations = utils.get_product_combinations(prodcodes=prodcode,
datetimes=datetimes,
timeframes=timeframe)
for combination in combinations:
if nowcast != combination.pop('nowcast'):
continue
if nowcast:
yield products.CopiedProduct(datetime=combination['datetime'])
else:
if product == 'b':
yield products.CalibratedProduct(**combination)
else:
yield products.ConsistentProduct(**combination)
def product_generator(product, prodcode, timeframe, datetimes, nowcast):
""" Return product generator. """
if product == 'a':
aggregate_kwargs = dict(declutter=None, radars=config.ALL_RADARS)
combinations = utils.get_aggregate_combinations(
datetimes=datetimes,
timeframes=timeframe,
)
for combination in combinations:
if nowcast != combination['nowcast']:
continue
if nowcast:
aggregate_kwargs.update(
dict(basedir=config.NOWCAST_AGGREGATE_DIR,
multiscandir=config.NOWCAST_MULTISCAN_DIR,
grid=scans.NOWCASTGRID,
datetime=combination['datetime'],
timeframe=combination['timeframe']))
else:
aggregate_kwargs.update(
dict(basedir=config.AGGREGATE_DIR,
multiscandir=config.MULTISCAN_DIR,
grid=scans.BASEGRID,
datetime=combination['datetime'],
timeframe=combination['timeframe']))
yield scans.Aggregate(**aggregate_kwargs)
else:
combinations = utils.get_product_combinations(prodcodes=prodcode,
datetimes=datetimes,
timeframes=timeframe)
for combination in combinations:
if nowcast != combination.pop('nowcast'):
continue
if nowcast:
yield products.CopiedProduct(datetime=combination['datetime'])
else:
if product == 'b':
yield products.CalibratedProduct(**combination)
else:
yield products.ConsistentProduct(**combination)
def master(**kwargs):
""" Run the radar production chain for a single datetime. """
loghelper.setup_logging(logfile_name='radar_master.log')
logging.info(20 * '-' + ' master ' + 20 * '-')
# Determine the delivery datetime and if necessary wait for files.
if kwargs['range'] is not None:
datetimes = utils.DateRange(kwargs['range']).iterdatetimes()
for i, datetime_delivery in enumerate(datetimes):
if i > 0:
logging.warning('Range of datetimes given. Using the first.')
break
else:
datetime_delivery = utils.closest_time()
try:
files.sync_and_wait_for_files(dt_calculation=datetime_delivery)
except Exception as exception:
logging.exception(exception)
# Organize
files.organize_from_path(source_dir=config.SOURCE_DIR)
# Product datetime depends on delevery times
declutter = dict(
size=config.DECLUTTER_SIZE,
history=config.DECLUTTER_HISTORY,
)
radars = config.ALL_RADARS
# nowcast extent is from utils.get_combinations
delivery_times = [(p, t) for p, t in config.DELIVERY_TIMES if p in 'rnau']
# Submit tasks in a chain.
subtasks = [tasks.do_nothing.s()]
for prodcode, timedelta_delivery in delivery_times:
datetime_product = datetime_delivery - timedelta_delivery
combinations = utils.get_aggregate_combinations(
datetimes=[datetime_product], )
for combination in combinations:
# Add a separator between groups of tasks
logging.info(40 * '-')
# Nowcast combinations only proceed for prodcode r
if combination['nowcast'] and not prodcode == 'r':
continue
# Append aggregate subtask
aggregate_kwargs = dict(declutter=declutter, radars=radars)
aggregate_kwargs.update(combination)
subtasks.append(tasks.aggregate.s(**aggregate_kwargs))
tpl = 'Agg. task: {datetime} {timeframe} {nowcast}'
logging.info(tpl.format(**aggregate_kwargs))
# Append calibrate subtask
calibrate_kwargs = dict(prodcode=prodcode)
calibrate_kwargs.update(aggregate_kwargs)
subtasks.append(tasks.calibrate.s(**calibrate_kwargs))
tpl = 'Cal. task: {datetime} {timeframe} {prodcode} {nowcast}'
logging.info(tpl.format(**calibrate_kwargs))
# Append rescale subtask
rescale_kwargs = {
k: v
for k, v in calibrate_kwargs.items()
if k in ['datetime', 'prodcode', 'timeframe']
}
subtasks.append(tasks.rescale.s(**rescale_kwargs))
tpl = 'Res. task: {datetime} {timeframe} {prodcode}'
logging.info(tpl.format(**rescale_kwargs))
# Append publication subtask
subtasks.append(
tasks.publish.s(
datetimes=[calibrate_kwargs['datetime']],
prodcodes=[calibrate_kwargs['prodcode']],
timeframes=[calibrate_kwargs['timeframe']],
nowcast=calibrate_kwargs['nowcast'],
endpoints=['ftp', 'h5', 'local', 'image', 'h5m'],
cascade=True,
))
tpl = 'Pub. task: {datetime} {timeframe} {prodcode} {nowcast}'
logging.info(tpl.format(**calibrate_kwargs))
# Append subtask to create animated gif
subtasks.append(tasks.animate.s(datetime=datetime_delivery))
# Submit all subtask as a single chain
chain(*subtasks).apply_async()
logging.info(20 * '-' + ' master complete ' + 20 * '-')
def master(**kwargs):
""" Run the radar production chain for a single datetime. """
loghelper.setup_logging(logfile_name='radar_master.log')
logging.info(20 * '-' + ' master ' + 20 * '-')
# Determine the delivery datetime and if necessary wait for files.
if kwargs['range'] is not None:
datetimes = utils.DateRange(kwargs['range']).iterdatetimes()
for i, datetime_delivery in enumerate(datetimes):
if i > 0:
logging.warning('Range of datetimes given. Using the first.')
break
else:
datetime_delivery = utils.closest_time()
try:
files.sync_and_wait_for_files(dt_calculation=datetime_delivery)
except Exception as exception:
logging.exception(exception)
# Organize
files.organize_from_path(source_dir=config.SOURCE_DIR)
# Product datetime depends on delevery times
declutter = dict(
size=config.DECLUTTER_SIZE,
history=config.DECLUTTER_HISTORY,
)
radars = config.ALL_RADARS
delivery_times = (
('r', datetime.timedelta()),
('n', datetime.timedelta(hours=1)),
('a', datetime.timedelta(days=2)),
)
# Submit tasks in a chain.
subtasks = [tasks.do_nothing.s()]
for prodcode, timedelta_delivery in delivery_times:
datetime_product = datetime_delivery - timedelta_delivery
combinations = utils.get_aggregate_combinations(
datetimes=[datetime_product],
)
for combination in combinations:
# Add a separator between groups of tasks
logging.info(40 * '-')
# Nowcast combinations only proceed for prodcode r
if combination['nowcast'] and not prodcode == 'r':
continue
# Append aggregate subtask
aggregate_kwargs = dict(declutter=declutter, radars=radars)
aggregate_kwargs.update(combination)
subtasks.append(tasks.aggregate.s(**aggregate_kwargs))
tpl = 'Agg. task: {datetime} {timeframe} {nowcast}'
logging.info(tpl.format(**aggregate_kwargs))
# Append calibrate subtask
calibrate_kwargs = dict(prodcode=prodcode)
calibrate_kwargs.update(aggregate_kwargs)
subtasks.append(tasks.calibrate.s(**calibrate_kwargs))
tpl = 'Cal. task: {datetime} {timeframe} {prodcode} {nowcast}'
logging.info(tpl.format(**calibrate_kwargs))
# Append rescale subtask
rescale_kwargs = {k: v
for k, v in calibrate_kwargs.items()
if k in ['datetime', 'prodcode', 'timeframe']}
subtasks.append(tasks.rescale.s(**rescale_kwargs))
tpl = 'Res. task: {datetime} {timeframe} {prodcode}'
logging.info(tpl.format(**rescale_kwargs))
# Append publication subtask
subtasks.append(tasks.publish.s(
datetimes=[calibrate_kwargs['datetime']],
prodcodes=[calibrate_kwargs['prodcode']],
timeframes=[calibrate_kwargs['timeframe']],
nowcast=calibrate_kwargs['nowcast'],
endpoints=['ftp', 'h5', 'local', 'image', 'h5m'],
cascade=True,
))
tpl = 'Pub. task: {datetime} {timeframe} {prodcode} {nowcast}'
logging.info(tpl.format(**calibrate_kwargs))
# Append subtask to create animated gif
subtasks.append(tasks.animate.s(datetime=datetime_delivery))
# Submit all subtask as a single chain
chain(*subtasks).apply_async()
logging.info(20 * '-' + ' master complete ' + 20 * '-')
