示例#1
0
def create_geotiff(dt_aggregate, code='5min'):

    pathhelper = utils.PathHelper(basedir=config.AGGREGATE_DIR,
                                  code=code,
                                  template='{code}_{timestamp}.h5')

    rasterlayerkwargs = utils.rain_kwargs(
        name='jet',
        max_rain=2,
        threshold=0.008,
    )

    aggregatepath = pathhelper.path(dt_aggregate)

    tifpath_rd = os.path.join(config.IMG_DIR, 'geotiff', 'rd',
                              dt_aggregate.strftime('%Y-%m-%d-%H-%M.tiff'))

    with h5py.File(aggregatepath, 'r') as h5:
        array = h5['precipitation']
        mask = np.equal(array, h5.attrs['fill_value'])
        masked_array = np.ma.array(array, mask=mask)

        # Create the rd tiff.
        utils.makedir(os.path.dirname(tifpath_rd))
        gridtools.RasterLayer(array=masked_array,
                              extent=h5.attrs['grid_extent'],
                              projection=h5.attrs['grid_projection'],
                              **rasterlayerkwargs).save(tifpath_rd, rgba=True)

    logging.info('saved {}.'.format(os.path.basename(tifpath_rd)))
    logging.debug('saved {}.'.format(tifpath_rd))
示例#2
0
def create_geotiff(dt_aggregate, code='5min'):

    pathhelper = utils.PathHelper(
        basedir=config.AGGREGATE_DIR,
        code=code,
        template='{code}_{timestamp}.h5'
    )

    rasterlayerkwargs = utils.rain_kwargs(
        name='jet', max_rain=2, threshold=0.008,
    )

    aggregatepath = pathhelper.path(dt_aggregate)

    tifpath_rd = os.path.join(
        config.IMG_DIR, 'geotiff', 'rd',
        dt_aggregate.strftime('%Y-%m-%d-%H-%M.tiff')
    )

    with h5py.File(aggregatepath, 'r') as h5:
        array = h5['precipitation']
        mask = np.equal(array, h5.attrs['fill_value'])
        masked_array = np.ma.array(array, mask=mask)

        # Create the rd tiff.
        utils.makedir(os.path.dirname(tifpath_rd))
        gridtools.RasterLayer(array=masked_array,
                              extent=h5.attrs['grid_extent'],
                              projection=h5.attrs['grid_projection'],
                              **rasterlayerkwargs).save(tifpath_rd, rgba=True)

    logging.info('saved {}.'.format(os.path.basename(tifpath_rd)))
    logging.debug('saved {}.'.format(tifpath_rd))
示例#3
0
def create_png(products, **kwargs):
    """
    Create image for products.

    This is a kind of sandbox version.
    """
    utils.makedir(config.IMG_DIR)

    # Load some images
    img_shape = shape_image()
    img_blue = plain_image(color=(0, 0, 127))
    img_shape_filled = shape_image_filled()

    # Get dutch time label
    tz_amsterdam = pytz.timezone('Europe/Amsterdam')
    tz_utc = pytz.timezone('UTC')

    # Loop products
    for product in products:
        utc = tz_utc.localize(product.datetime)
        amsterdam = utc.astimezone(tz_amsterdam)
        label = amsterdam.strftime('%Y-%m-%d %H:%M')
        label_from_kwargs = kwargs.get('label', '')
        if label_from_kwargs:
            label += ' ' + label_from_kwargs
        offset = 0.1, 0.9

        # Get data image
        try:
            with product.get() as h5:
                array = h5['precipitation'][...] / h5.attrs['composite_count']
                mask = np.equal(array, h5.attrs['fill_value'])
                img_radars = radars_image(h5=h5, label=label, offset=offset)
        except IOError:
            logging.debug('Does not exist: {}'.format(product.path))
            continue
        masked_array = np.ma.array(array, mask=mask)
        img_rain = data_image(masked_array, max_rain=2, threshold=0.008)

        timestamp = utils.datetime2timestamp(product.datetime)

        filename = '{}{}.{}'.format(
            timestamp,
            kwargs.get('postfix', ''),
            kwargs.get('format', 'png'),
        )
        # Merge and save
        path = os.path.join(config.IMG_DIR, filename)
        utils.merge([
            img_radars,
            img_rain,
            img_shape,
            img_shape_filled,
            img_blue,
        ]).save(path)

        logging.info('saved {}.'.format(os.path.basename(path)))
        logging.debug('saved {}.'.format(path))
示例#4
0
def create_png_for_animated_gif(products, **kwargs):
    """
    Create image for products.

    This is the tweaked version that creates the pngs for use in the
    animated gif.
    """
    utils.makedir(config.IMG_DIR)

    # Load some images
    img_mapbox = mapbox_image()
    img_osm = osm_image()
    img_shape = shape_image()
    img_blue = plain_image(color=(0, 0, 127))
    img_shape_filled = shape_image_filled()

    # Get dutch time label
    tz_amsterdam = pytz.timezone('Europe/Amsterdam')
    tz_utc = pytz.timezone('UTC')

    # Loop products
    for product in products:
        utc = tz_utc.localize(product.datetime)
        amsterdam = utc.astimezone(tz_amsterdam)
        #label = amsterdam.strftime('%Y-%m-%d %H:%M')
        label = amsterdam.strftime('%H:%M')
        offset = 0.25, 0.82

        # Get data image
        with product.get() as h5:
            array = h5['precipitation'][...] / h5.attrs['composite_count']
            mask = np.equal(array, h5.attrs['fill_value'])
            img_radars = radars_image(h5=h5, label=label, offset=offset)
        masked_array = np.ma.array(array, mask=mask)
        img_rain = data_image(masked_array, max_rain=2, threshold=0.008)

        timestamp = utils.datetime2timestamp(product.datetime)

        filename = '{}{}.{}'.format(
            timestamp,
            kwargs.get('postfix', ''),
            kwargs.get('format', 'png'),
        )
        # Merge and save
        path = os.path.join(config.IMG_DIR, filename)
        utils.merge([
            img_radars,
            img_rain,
            img_mapbox,
            img_shape,
            img_shape_filled,
            img_blue,
        ]).save(path)

        logging.info('saved {}.'.format(os.path.basename(path)))
        logging.debug('saved {}.'.format(path))
示例#5
0
def create_png(products, **kwargs):
    """
    Create image for products.

    This is a kind of sandbox version.
    """
    utils.makedir(config.IMG_DIR)

    # Load some images
    img_shape = shape_image()
    img_blue = plain_image(color=(0, 0, 127))
    img_shape_filled = shape_image_filled()

    # Get dutch time label
    tz_amsterdam = pytz.timezone('Europe/Amsterdam')
    tz_utc = pytz.timezone('UTC')

    # Loop products
    for product in products:
        utc = tz_utc.localize(product.datetime)
        amsterdam = utc.astimezone(tz_amsterdam)
        label = amsterdam.strftime('%Y-%m-%d %H:%M')
        label_from_kwargs = kwargs.get('label', '')
        if label_from_kwargs:
            label += ' ' + label_from_kwargs
        offset = 0.1, 0.9

        # Get data image
        try:
            with product.get() as h5:
                array = h5['precipitation'][...] / h5.attrs['composite_count']
                mask = np.equal(array, h5.attrs['fill_value'])
                img_radars = radars_image(h5=h5, label=label, offset=offset)
        except IOError:
            logging.debug('Does not exist: {}'.format(product.path))
            continue
        masked_array = np.ma.array(array, mask=mask)
        img_rain = data_image(masked_array, max_rain=2, threshold=0.008)

        timestamp = utils.datetime2timestamp(product.datetime)

        filename = '{}{}.{}'.format(
            timestamp, kwargs.get('postfix', ''), kwargs.get('format', 'png'),
        )
        # Merge and save
        path = os.path.join(config.IMG_DIR, filename)
        utils.merge([
            img_radars,
            img_rain,
            img_shape,
            img_shape_filled,
            img_blue,
        ]).save(path)

        logging.info('saved {}.'.format(os.path.basename(path)))
        logging.debug('saved {}.'.format(path))
示例#6
0
def create_png_for_animated_gif(products, **kwargs):
    """
    Create image for products.

    This is the tweaked version that creates the pngs for use in the
    animated gif.
    """
    utils.makedir(config.IMG_DIR)

    # Load some images
    img_mapbox = mapbox_image()
    img_osm = osm_image()
    img_shape = shape_image()
    img_blue = plain_image(color=(0, 0, 127))
    img_shape_filled = shape_image_filled()

    # Get dutch time label
    tz_amsterdam = pytz.timezone('Europe/Amsterdam')
    tz_utc = pytz.timezone('UTC')

    # Loop products
    for product in products:
        utc = tz_utc.localize(product.datetime)
        amsterdam = utc.astimezone(tz_amsterdam)
        #label = amsterdam.strftime('%Y-%m-%d %H:%M')
        label = amsterdam.strftime('%H:%M')
        offset = 0.25, 0.82

        # Get data image
        with product.get() as h5:
            array = h5['precipitation'][...] / h5.attrs['composite_count']
            mask = np.equal(array, h5.attrs['fill_value'])
            img_radars = radars_image(h5=h5, label=label, offset=offset)
        masked_array = np.ma.array(array, mask=mask)
        img_rain = data_image(masked_array, max_rain=2, threshold=0.008)

        timestamp = utils.datetime2timestamp(product.datetime)

        filename = '{}{}.{}'.format(
            timestamp, kwargs.get('postfix', ''), kwargs.get('format', 'png'),
        )
        # Merge and save
        path = os.path.join(config.IMG_DIR, filename)
        utils.merge([
            img_radars,
            img_rain,
            img_mapbox,
            img_shape,
            img_shape_filled,
            img_blue,
        ]).save(path)

        logging.info('saved {}.'.format(os.path.basename(path)))
        logging.debug('saved {}.'.format(path))
示例#7
0
    def __init__(self, datetime, max_age=86400):
        """
        Set datetime and empty connection dictionary. Max age is in
        seconds, measured from datetime.
        """
        utils.makedir(config.SOURCE_DIR)

        self.datetime = datetime
        self.max_age = max_age
        self.connections = {}
        # Check what is already there.
        self.arrived = []
示例#8
0
    def __init__(self, datetime, max_age=86400):
        """
        Set datetime and empty connection dictionary. Max age is in
        seconds, measured from datetime.
        """
        utils.makedir(config.SOURCE_DIR)

        self.datetime = datetime
        self.max_age = max_age
        self.connections = {}
        # Check what is already there.
        self.arrived = []
示例#9
0
def create_tif(products, image_dir=None, **kwargs):
    """ Save a tif with metadata """
    if image_dir:
        target_dir = image_dir
    else:
        target_dir = config.IMG_DIR
    utils.makedir(target_dir)

    # Loop products
    for product in products:

        # Get data
        try:
            with h5py.File(product.path, 'r') as h5:
                data = np.ma.masked_equal(h5['precipitation'],
                                          h5.attrs['fill_value'])
                attrs = dict(h5.attrs)
        except IOError:
            logging.info('Not found: {}, skipping.'.format(product))
            continue

        # make a dataset
        raster_layer = gridtools.RasterLayer(
            array=data,
            extent=attrs['grid_extent'],
            projection=attrs['grid_projection'],
        )
        mem = raster_layer.create_dataset(datatype=6)

        mem.SetProjection(PROJECTION_RD_WKT)
        band = mem.GetRasterBand(1)
        band.WriteArray(data.filled(band.GetNoDataValue()))
        for k, v in attrs.items():
            if hasattr(v, 'tolist'):
                v = json.dumps(v.tolist())
            band.SetMetadataItem(str(k), str(v))

        timestamp = utils.datetime2timestamp(product.datetime)
        filename = '{}{}.{}'.format(
            timestamp,
            kwargs.get('postfix', ''),
            kwargs.get('format', 'png'),
        )
        path = os.path.join(target_dir, filename)

        driver = gdal.GetDriverByName(b'gtiff')
        driver.CreateCopy(path, mem)

        logging.info('saved {}.'.format(os.path.basename(path)))
        logging.debug('saved {}.'.format(path))
示例#10
0
def create_tif(products, image_dir=None, **kwargs):
    """ Save a tif with metadata """
    if image_dir:
        target_dir = image_dir
    else:
        target_dir = config.IMG_DIR
    utils.makedir(target_dir)

    # Loop products
    for product in products:

        # Get data
        try:
            with h5py.File(product.path, 'r') as h5:
                data = np.ma.masked_equal(h5['precipitation'],
                                          h5.attrs['fill_value'])
                attrs = dict(h5.attrs)
        except IOError:
            logging.info('Not found: {}, skipping.'.format(product))
            continue

        # make a dataset
        raster_layer = gridtools.RasterLayer(
            array=data,
            extent=attrs['grid_extent'],
            projection=attrs['grid_projection'],
        )
        mem = raster_layer.create_dataset(datatype=6)

        mem.SetProjection(PROJECTION_RD_WKT)
        band = mem.GetRasterBand(1)
        band.WriteArray(data.filled(band.GetNoDataValue()))
        for k, v in attrs.items():
            if hasattr(v, 'tolist'):
                v = json.dumps(v.tolist())
            band.SetMetadataItem(str(k), str(v))

        timestamp = utils.datetime2timestamp(product.datetime)
        filename = '{}{}.{}'.format(
            timestamp, kwargs.get('postfix', ''), kwargs.get('format', 'png'),
        )
        path = os.path.join(target_dir, filename)

        driver = gdal.GetDriverByName(b'gtiff')
        driver.CreateCopy(path, mem)

        logging.info('saved {}.'.format(os.path.basename(path)))
        logging.debug('saved {}.'.format(path))
示例#11
0
文件: scans.py 项目: islenv/openradar
    def get(self):
        """
        Return a readonly hdf5 dataset for requested scans.

        If the dataset is not available, it is created.
        If it is available, but lacks some scan requested, it is appended.
        """
        utils.makedir(os.path.dirname(self.path))

        try:
            # Improperly closed h5 files cannot be opened.
            dataset = h5py.File(self.path, 'a')
        except IOError:
            dataset = h5py.File(self.path, 'w')

        if len(dataset):
            logging.debug(
                'Multiscan file already has {}.'.format(', '.join(dataset))
            )
        else:
            logging.debug('Starting with empty multiscan file.')

        for scancode in self.scancodes:
            scan = ScanSignature(
                scandatetime=self.multiscandatetime, scancode=scancode,
            ).get_scan(self.grid)
            if scan is None or scancode in dataset:
                continue
            if scan.is_readable():
                self._add(dataset=dataset, scan=scan)
            else:
                # Remove it.
                scanpath = scan.signature.get_scanpath()
                try:
                    shutil.move(scanpath, config.RADAR_DIR)
                    logging.warn('Removed corrupt scanfile {}'.format(
                        os.path.basename(scanpath),
                    ))
                except (OSError, IOError):
                    pass  # No permission.

        dataset.close()
        return h5py.File(self.path, 'r')
示例#12
0
    def get(self):
        """
        Return a readonly hdf5 dataset for requested scans.

        If the dataset is not available, it is created.
        If it is available, but lacks some scan requested, it is appended.
        """
        utils.makedir(os.path.dirname(self.path))

        try:
            # Improperly closed h5 files cannot be opened.
            dataset = h5py.File(self.path, 'a')
        except IOError:
            dataset = h5py.File(self.path, 'w')

        if len(dataset):
            logging.debug('Multiscan file already has {}.'.format(
                ', '.join(dataset)))
        else:
            logging.debug('Starting with empty multiscan file.')

        for scancode in self.scancodes:
            scan = ScanSignature(
                scandatetime=self.multiscandatetime,
                scancode=scancode,
            ).get_scan(self.grid)
            if scan is None or scancode in dataset:
                continue
            if scan.is_readable():
                self._add(dataset=dataset, scan=scan)
            else:
                # Remove it.
                scanpath = scan.signature.get_scanpath()
                try:
                    shutil.move(scanpath, config.RADAR_DIR)
                    logging.warn('Removed corrupt scanfile {}'.format(
                        os.path.basename(scanpath), ))
                except (OSError, IOError):
                    pass  # No permission.

        dataset.close()
        return h5py.File(self.path, 'r')
示例#13
0
 def publish_local(self, cascade=False):
     """ Publish to target dirs as configured in config. """
     # Prepare dirs
     logging.debug(
         'Preparing {} dirs.'.format(len(config.COPY_TARGET_DIRS)),
     )
     for target_dir in config.COPY_TARGET_DIRS:
         for path in [path
                      for d in config.PRODUCT_CODE.values()
                      for path in d.values()]:
             utils.makedir(os.path.join(target_dir, path))
     # Do the copying
     logging.debug('Copying publications.')
     for publication in self.publications(cascade=cascade):
         proddict = config.PRODUCT_CODE[publication.timeframe]
         for target_dir in config.COPY_TARGET_DIRS:
             target_subdir = os.path.join(
                 target_dir,
                 proddict[publication.prodcode],
             )
             shutil.copy(publication.path, target_subdir)
     logging.info('Local target dir copying complete.')
示例#14
0
    def create(self):
        """ Return newly created threddsfile. """
        utils.makedir(os.path.dirname(self.path))
        h5 = h5py.File(self.path)

        # East
        east = scans.BASEGRID.get_grid()[0][0]
        dataset = h5.create_dataset(
            'east', east.shape, east.dtype,
            compression='gzip', shuffle=True,
        )
        dataset[...] = east

        # North
        north = scans.BASEGRID.get_grid()[1][:, 0]
        dataset = h5.create_dataset(
            'north', north.shape, north.dtype,
            compression='gzip', shuffle=True,
        )
        dataset[...] = north

        # Time
        time = h5.create_dataset(
            'time', [self.timesteps], np.uint32,
            compression='gzip', shuffle=True,
        )
        time.attrs['standard_name'] = b'time'
        time.attrs['long_name'] = b'time'
        time.attrs['calendar'] = b'gregorian'
        time.attrs['units'] = self.datetime.strftime(
            'seconds since %Y-%m-%d'
        )
        time[...] = self._time()

        # Precipitation
        shape = scans.BASEGRID.get_shape() + tuple([self.timesteps])
        dataset = h5.create_dataset(
            'precipitation', shape, np.float32, fillvalue=config.NODATAVALUE,
            compression='gzip', shuffle=True, chunks=(20, 20, 24)
        )

        # Availability
        dataset = h5.create_dataset(
            'available', [self.timesteps], np.uint8, fillvalue=0,
            compression='gzip', shuffle=True,
        )
        dataset[...] = 0

        # Dimensions
        h5['precipitation'].dims.create_scale(h5['north'])
        h5['precipitation'].dims.create_scale(h5['east'])
        h5['precipitation'].dims.create_scale(h5['time'])

        h5['precipitation'].dims[0].attach_scale(h5['north'])
        h5['precipitation'].dims[1].attach_scale(h5['east'])
        h5['precipitation'].dims[2].attach_scale(h5['time'])

        h5['available'].dims.create_scale(h5['time'])
        h5['available'].dims[0].attach_scale(h5['time'])

        logging.info(
            'Created ThreddsFile {}'.format(os.path.basename(self.path)),
        )
        logging.debug(self.path)
        return h5
示例#15
0
    def create(self):
        """ Return newly created threddsfile. """
        utils.makedir(os.path.dirname(self.path))
        h5 = h5py.File(self.path)

        # East
        east = scans.BASEGRID.get_grid()[0][0]
        dataset = h5.create_dataset(
            'east',
            east.shape,
            east.dtype,
            compression='gzip',
            shuffle=True,
        )
        dataset[...] = east

        # North
        north = scans.BASEGRID.get_grid()[1][:, 0]
        dataset = h5.create_dataset(
            'north',
            north.shape,
            north.dtype,
            compression='gzip',
            shuffle=True,
        )
        dataset[...] = north

        # Time
        time = h5.create_dataset(
            'time',
            [self.timesteps],
            np.uint32,
            compression='gzip',
            shuffle=True,
        )
        time.attrs['standard_name'] = b'time'
        time.attrs['long_name'] = b'time'
        time.attrs['calendar'] = b'gregorian'
        time.attrs['units'] = self.datetime.strftime('seconds since %Y-%m-%d')
        time[...] = self._time()

        # Precipitation
        shape = scans.BASEGRID.get_shape() + tuple([self.timesteps])
        dataset = h5.create_dataset('precipitation',
                                    shape,
                                    np.float32,
                                    fillvalue=config.NODATAVALUE,
                                    compression='gzip',
                                    shuffle=True,
                                    chunks=(20, 20, 24))

        # Availability
        dataset = h5.create_dataset(
            'available',
            [self.timesteps],
            np.uint8,
            fillvalue=0,
            compression='gzip',
            shuffle=True,
        )
        dataset[...] = 0

        # Dimensions
        h5['precipitation'].dims.create_scale(h5['north'])
        h5['precipitation'].dims.create_scale(h5['east'])
        h5['precipitation'].dims.create_scale(h5['time'])

        h5['precipitation'].dims[0].attach_scale(h5['north'])
        h5['precipitation'].dims[1].attach_scale(h5['east'])
        h5['precipitation'].dims[2].attach_scale(h5['time'])

        h5['available'].dims.create_scale(h5['time'])
        h5['available'].dims[0].attach_scale(h5['time'])

        logging.info(
            'Created ThreddsFile {}'.format(os.path.basename(self.path)), )
        logging.debug(self.path)
        return h5