def __init__(self, quality): # determine zones u = datetime.datetime.utcnow() - TOLERANCE self.r = utils.closest_time(timeframe='f', dt_close=u) self.n = utils.closest_time(timeframe='h', dt_close=u) - HOUR self.a = utils.closest_time(timeframe='d', dt_close=u) - HOUR * 48 self.quality = quality
def get_datetimes(): """ Return generator of datetimes. """ now = datetime.datetime.utcnow() five = datetime.timedelta(minutes=5) current = utils.closest_time() - datetime.timedelta(days=7) while current < now: yield current current += five
def master(**kwargs): """ Run the radar production chain for a single datetime. """ loghelper.setup_logging(logfile_name='radar_master.log') logging.info(20 * '-' + ' master ' + 20 * '-') # Determine the delivery datetime and if necessary wait for files. if kwargs['range'] is not None: datetimes = utils.DateRange(kwargs['range']).iterdatetimes() for i, datetime_delivery in enumerate(datetimes): if i > 0: logging.warning('Range of datetimes given. Using the first.') break else: datetime_delivery = utils.closest_time() try: files.sync_and_wait_for_files(dt_calculation=datetime_delivery) except Exception as exception: logging.exception(exception) # Organize files.organize_from_path(source_dir=config.SOURCE_DIR) # Product datetime depends on delevery times declutter = dict( size=config.DECLUTTER_SIZE, history=config.DECLUTTER_HISTORY, ) radars = config.ALL_RADARS # nowcast extent is from utils.get_combinations delivery_times = [(p, t) for p, t in config.DELIVERY_TIMES if p in 'rnau'] # Submit tasks in a chain. subtasks = [tasks.do_nothing.s()] for prodcode, timedelta_delivery in delivery_times: datetime_product = datetime_delivery - timedelta_delivery combinations = utils.get_aggregate_combinations( datetimes=[datetime_product], ) for combination in combinations: # Add a separator between groups of tasks logging.info(40 * '-') # Nowcast combinations only proceed for prodcode r if combination['nowcast'] and not prodcode == 'r': continue # Append aggregate subtask aggregate_kwargs = dict(declutter=declutter, radars=radars) aggregate_kwargs.update(combination) subtasks.append(tasks.aggregate.s(**aggregate_kwargs)) tpl = 'Agg. task: {datetime} {timeframe} {nowcast}' logging.info(tpl.format(**aggregate_kwargs)) # Append calibrate subtask calibrate_kwargs = dict(prodcode=prodcode) calibrate_kwargs.update(aggregate_kwargs) subtasks.append(tasks.calibrate.s(**calibrate_kwargs)) tpl = 'Cal. task: {datetime} {timeframe} {prodcode} {nowcast}' logging.info(tpl.format(**calibrate_kwargs)) # Append rescale subtask rescale_kwargs = { k: v for k, v in calibrate_kwargs.items() if k in ['datetime', 'prodcode', 'timeframe'] } subtasks.append(tasks.rescale.s(**rescale_kwargs)) tpl = 'Res. task: {datetime} {timeframe} {prodcode}' logging.info(tpl.format(**rescale_kwargs)) # Append publication subtask subtasks.append( tasks.publish.s( datetimes=[calibrate_kwargs['datetime']], prodcodes=[calibrate_kwargs['prodcode']], timeframes=[calibrate_kwargs['timeframe']], nowcast=calibrate_kwargs['nowcast'], endpoints=['ftp', 'h5', 'local', 'image', 'h5m'], cascade=True, )) tpl = 'Pub. task: {datetime} {timeframe} {prodcode} {nowcast}' logging.info(tpl.format(**calibrate_kwargs)) # Append subtask to create animated gif subtasks.append(tasks.animate.s(datetime=datetime_delivery)) # Submit all subtask as a single chain chain(*subtasks).apply_async() logging.info(20 * '-' + ' master complete ' + 20 * '-')
def master(**kwargs): """ Run the radar production chain for a single datetime. """ loghelper.setup_logging(logfile_name='radar_master.log') logging.info(20 * '-' + ' master ' + 20 * '-') # Determine the delivery datetime and if necessary wait for files. if kwargs['range'] is not None: datetimes = utils.DateRange(kwargs['range']).iterdatetimes() for i, datetime_delivery in enumerate(datetimes): if i > 0: logging.warning('Range of datetimes given. Using the first.') break else: datetime_delivery = utils.closest_time() try: files.sync_and_wait_for_files(dt_calculation=datetime_delivery) except Exception as exception: logging.exception(exception) # Organize files.organize_from_path(source_dir=config.SOURCE_DIR) # Product datetime depends on delevery times declutter = dict( size=config.DECLUTTER_SIZE, history=config.DECLUTTER_HISTORY, ) radars = config.ALL_RADARS delivery_times = ( ('r', datetime.timedelta()), ('n', datetime.timedelta(hours=1)), ('a', datetime.timedelta(days=2)), ) # Submit tasks in a chain. subtasks = [tasks.do_nothing.s()] for prodcode, timedelta_delivery in delivery_times: datetime_product = datetime_delivery - timedelta_delivery combinations = utils.get_aggregate_combinations( datetimes=[datetime_product], ) for combination in combinations: # Add a separator between groups of tasks logging.info(40 * '-') # Nowcast combinations only proceed for prodcode r if combination['nowcast'] and not prodcode == 'r': continue # Append aggregate subtask aggregate_kwargs = dict(declutter=declutter, radars=radars) aggregate_kwargs.update(combination) subtasks.append(tasks.aggregate.s(**aggregate_kwargs)) tpl = 'Agg. task: {datetime} {timeframe} {nowcast}' logging.info(tpl.format(**aggregate_kwargs)) # Append calibrate subtask calibrate_kwargs = dict(prodcode=prodcode) calibrate_kwargs.update(aggregate_kwargs) subtasks.append(tasks.calibrate.s(**calibrate_kwargs)) tpl = 'Cal. task: {datetime} {timeframe} {prodcode} {nowcast}' logging.info(tpl.format(**calibrate_kwargs)) # Append rescale subtask rescale_kwargs = {k: v for k, v in calibrate_kwargs.items() if k in ['datetime', 'prodcode', 'timeframe']} subtasks.append(tasks.rescale.s(**rescale_kwargs)) tpl = 'Res. task: {datetime} {timeframe} {prodcode}' logging.info(tpl.format(**rescale_kwargs)) # Append publication subtask subtasks.append(tasks.publish.s( datetimes=[calibrate_kwargs['datetime']], prodcodes=[calibrate_kwargs['prodcode']], timeframes=[calibrate_kwargs['timeframe']], nowcast=calibrate_kwargs['nowcast'], endpoints=['ftp', 'h5', 'local', 'image', 'h5m'], cascade=True, )) tpl = 'Pub. task: {datetime} {timeframe} {prodcode} {nowcast}' logging.info(tpl.format(**calibrate_kwargs)) # Append subtask to create animated gif subtasks.append(tasks.animate.s(datetime=datetime_delivery)) # Submit all subtask as a single chain chain(*subtasks).apply_async() logging.info(20 * '-' + ' master complete ' + 20 * '-')