コード例 #1
0
    def __init__(self, quality):
        # determine zones
        u = datetime.datetime.utcnow() - TOLERANCE
        self.r = utils.closest_time(timeframe='f', dt_close=u)
        self.n = utils.closest_time(timeframe='h', dt_close=u) - HOUR
        self.a = utils.closest_time(timeframe='d', dt_close=u) - HOUR * 48

        self.quality = quality
コード例 #2
0
ファイル: report.py プロジェクト: jaapschellekens/openradar
    def __init__(self, quality):
        # determine zones
        u = datetime.datetime.utcnow() - TOLERANCE
        self.r = utils.closest_time(timeframe='f', dt_close=u)
        self.n = utils.closest_time(timeframe='h', dt_close=u) - HOUR
        self.a = utils.closest_time(timeframe='d', dt_close=u) - HOUR * 48

        self.quality = quality
コード例 #3
0
def get_datetimes():
    """ Return generator of datetimes. """
    now = datetime.datetime.utcnow()
    five = datetime.timedelta(minutes=5)
    current = utils.closest_time() - datetime.timedelta(days=7)
    while current < now:
        yield current
        current += five
コード例 #4
0
ファイル: sync_radar_to_ftp.py プロジェクト: islenv/openradar
def get_datetimes():
    """ Return generator of datetimes. """
    now = datetime.datetime.utcnow()
    five = datetime.timedelta(minutes=5)
    current = utils.closest_time() - datetime.timedelta(days=7)
    while current < now:
        yield current
        current += five
コード例 #5
0
ファイル: master.py プロジェクト: islenv/openradar
def master(**kwargs):
    """ Run the radar production chain for a single datetime. """
    loghelper.setup_logging(logfile_name='radar_master.log')
    logging.info(20 * '-' + ' master ' + 20 * '-')

    # Determine the delivery datetime and if necessary wait for files.
    if kwargs['range'] is not None:
        datetimes = utils.DateRange(kwargs['range']).iterdatetimes()
        for i, datetime_delivery in enumerate(datetimes):
            if i > 0:
                logging.warning('Range of datetimes given. Using the first.')
                break
    else:
        datetime_delivery = utils.closest_time()
        try:
            files.sync_and_wait_for_files(dt_calculation=datetime_delivery)
        except Exception as exception:
            logging.exception(exception)

    # Organize
    files.organize_from_path(source_dir=config.SOURCE_DIR)

    # Product datetime depends on delevery times
    declutter = dict(
        size=config.DECLUTTER_SIZE,
        history=config.DECLUTTER_HISTORY,
    )
    radars = config.ALL_RADARS

    # nowcast extent is from utils.get_combinations
    delivery_times = [(p, t) for p, t in config.DELIVERY_TIMES if p in 'rnau']

    # Submit tasks in a chain.
    subtasks = [tasks.do_nothing.s()]
    for prodcode, timedelta_delivery in delivery_times:
        datetime_product = datetime_delivery - timedelta_delivery
        combinations = utils.get_aggregate_combinations(
            datetimes=[datetime_product], )
        for combination in combinations:
            # Add a separator between groups of tasks
            logging.info(40 * '-')

            # Nowcast combinations only proceed for prodcode r
            if combination['nowcast'] and not prodcode == 'r':
                continue

            # Append aggregate subtask
            aggregate_kwargs = dict(declutter=declutter, radars=radars)
            aggregate_kwargs.update(combination)
            subtasks.append(tasks.aggregate.s(**aggregate_kwargs))
            tpl = 'Agg. task: {datetime} {timeframe}   {nowcast}'
            logging.info(tpl.format(**aggregate_kwargs))

            # Append calibrate subtask
            calibrate_kwargs = dict(prodcode=prodcode)
            calibrate_kwargs.update(aggregate_kwargs)
            subtasks.append(tasks.calibrate.s(**calibrate_kwargs))
            tpl = 'Cal. task: {datetime} {timeframe} {prodcode} {nowcast}'
            logging.info(tpl.format(**calibrate_kwargs))

            # Append rescale subtask
            rescale_kwargs = {
                k: v
                for k, v in calibrate_kwargs.items()
                if k in ['datetime', 'prodcode', 'timeframe']
            }
            subtasks.append(tasks.rescale.s(**rescale_kwargs))
            tpl = 'Res. task: {datetime} {timeframe} {prodcode}'
            logging.info(tpl.format(**rescale_kwargs))

            # Append publication subtask
            subtasks.append(
                tasks.publish.s(
                    datetimes=[calibrate_kwargs['datetime']],
                    prodcodes=[calibrate_kwargs['prodcode']],
                    timeframes=[calibrate_kwargs['timeframe']],
                    nowcast=calibrate_kwargs['nowcast'],
                    endpoints=['ftp', 'h5', 'local', 'image', 'h5m'],
                    cascade=True,
                ))
            tpl = 'Pub. task: {datetime} {timeframe} {prodcode} {nowcast}'
            logging.info(tpl.format(**calibrate_kwargs))

    # Append subtask to create animated gif
    subtasks.append(tasks.animate.s(datetime=datetime_delivery))

    # Submit all subtask as a single chain
    chain(*subtasks).apply_async()

    logging.info(20 * '-' + ' master complete ' + 20 * '-')
コード例 #6
0
ファイル: master.py プロジェクト: jaapschellekens/openradar
def master(**kwargs):
    """ Run the radar production chain for a single datetime. """
    loghelper.setup_logging(logfile_name='radar_master.log')
    logging.info(20 * '-' + ' master ' + 20 * '-')

    # Determine the delivery datetime and if necessary wait for files.
    if kwargs['range'] is not None:
        datetimes = utils.DateRange(kwargs['range']).iterdatetimes()
        for i, datetime_delivery in enumerate(datetimes):
            if i > 0:
                logging.warning('Range of datetimes given. Using the first.')
                break
    else:
        datetime_delivery = utils.closest_time()
        try:
            files.sync_and_wait_for_files(dt_calculation=datetime_delivery)
        except Exception as exception:
            logging.exception(exception)

    # Organize
    files.organize_from_path(source_dir=config.SOURCE_DIR)

    # Product datetime depends on delevery times
    declutter = dict(
        size=config.DECLUTTER_SIZE,
        history=config.DECLUTTER_HISTORY,
    )
    radars = config.ALL_RADARS
    delivery_times = (
        ('r', datetime.timedelta()),
        ('n', datetime.timedelta(hours=1)),
        ('a', datetime.timedelta(days=2)),
    )

    # Submit tasks in a chain.
    subtasks = [tasks.do_nothing.s()]
    for prodcode, timedelta_delivery in delivery_times:
        datetime_product = datetime_delivery - timedelta_delivery
        combinations = utils.get_aggregate_combinations(
            datetimes=[datetime_product],
        )
        for combination in combinations:
            # Add a separator between groups of tasks
            logging.info(40 * '-')

            # Nowcast combinations only proceed for prodcode r
            if combination['nowcast'] and not prodcode == 'r':
                continue

            # Append aggregate subtask
            aggregate_kwargs = dict(declutter=declutter, radars=radars)
            aggregate_kwargs.update(combination)
            subtasks.append(tasks.aggregate.s(**aggregate_kwargs))
            tpl = 'Agg. task: {datetime} {timeframe}   {nowcast}'
            logging.info(tpl.format(**aggregate_kwargs))

            # Append calibrate subtask
            calibrate_kwargs = dict(prodcode=prodcode)
            calibrate_kwargs.update(aggregate_kwargs)
            subtasks.append(tasks.calibrate.s(**calibrate_kwargs))
            tpl = 'Cal. task: {datetime} {timeframe} {prodcode} {nowcast}'
            logging.info(tpl.format(**calibrate_kwargs))

            # Append rescale subtask
            rescale_kwargs = {k: v
                              for k, v in calibrate_kwargs.items()
                              if k in ['datetime', 'prodcode', 'timeframe']}
            subtasks.append(tasks.rescale.s(**rescale_kwargs))
            tpl = 'Res. task: {datetime} {timeframe} {prodcode}'
            logging.info(tpl.format(**rescale_kwargs))

            # Append publication subtask
            subtasks.append(tasks.publish.s(
                datetimes=[calibrate_kwargs['datetime']],
                prodcodes=[calibrate_kwargs['prodcode']],
                timeframes=[calibrate_kwargs['timeframe']],
                nowcast=calibrate_kwargs['nowcast'],
                endpoints=['ftp', 'h5', 'local', 'image', 'h5m'],
                cascade=True,
            ))
            tpl = 'Pub. task: {datetime} {timeframe} {prodcode} {nowcast}'
            logging.info(tpl.format(**calibrate_kwargs))

    # Append subtask to create animated gif
    subtasks.append(tasks.animate.s(datetime=datetime_delivery))

    # Submit all subtask as a single chain
    chain(*subtasks).apply_async()

    logging.info(20 * '-' + ' master complete ' + 20 * '-')