def _radar_dict_from_scan_name(self, scan_name):
"""
Return dict with radar parameters from ftp scansource.
Use this when judging files in ftp listing.
"""
for pattern in PATTERNS:
match = pattern.match(scan_name)
if match:
# handle id
if 'id' in pattern.groupindex:
radar_id = match.group('id') # dwd from 2013
else:
radar_id = ''
# handle code and timestamp
if 'code' in pattern.groupindex:
radar_code = match.group('code')
radar_timestamp = match.group('timestamp')
else:
radar_code = 'JAB'
name_timestamp = match.group('timestamp')
d1 = utils.timestamp2datetime(name_timestamp)
m = 5 * (d1.minute // 5)
d2 = d1.replace(minute=m, second=0, microsecond=0)
t = Timedelta(minutes=5)
d3 = d1 + min((d2 - d1), abs(d2 - d1 + t), key=abs) - t
radar_timestamp = utils.datetime2timestamp(d3)[:-2]
return {'id': radar_id,
'code': radar_code,
'timestamp': radar_timestamp}
message = 'No pattern matches {}'
raise ValueError(message.format(scan_name))
def create_png(products, **kwargs):
"""
Create image for products.
This is a kind of sandbox version.
"""
utils.makedir(config.IMG_DIR)
# Load some images
img_shape = shape_image()
img_blue = plain_image(color=(0, 0, 127))
img_shape_filled = shape_image_filled()
# Get dutch time label
tz_amsterdam = pytz.timezone('Europe/Amsterdam')
tz_utc = pytz.timezone('UTC')
# Loop products
for product in products:
utc = tz_utc.localize(product.datetime)
amsterdam = utc.astimezone(tz_amsterdam)
label = amsterdam.strftime('%Y-%m-%d %H:%M')
label_from_kwargs = kwargs.get('label', '')
if label_from_kwargs:
label += ' ' + label_from_kwargs
offset = 0.1, 0.9
# Get data image
try:
with product.get() as h5:
array = h5['precipitation'][...] / h5.attrs['composite_count']
mask = np.equal(array, h5.attrs['fill_value'])
img_radars = radars_image(h5=h5, label=label, offset=offset)
except IOError:
logging.debug('Does not exist: {}'.format(product.path))
continue
masked_array = np.ma.array(array, mask=mask)
img_rain = data_image(masked_array, max_rain=2, threshold=0.008)
timestamp = utils.datetime2timestamp(product.datetime)
filename = '{}{}.{}'.format(
timestamp,
kwargs.get('postfix', ''),
kwargs.get('format', 'png'),
)
# Merge and save
path = os.path.join(config.IMG_DIR, filename)
utils.merge([
img_radars,
img_rain,
img_shape,
img_shape_filled,
img_blue,
]).save(path)
logging.info('saved {}.'.format(os.path.basename(path)))
logging.debug('saved {}.'.format(path))
def create_png_for_animated_gif(products, **kwargs):
"""
Create image for products.
This is the tweaked version that creates the pngs for use in the
animated gif.
"""
utils.makedir(config.IMG_DIR)
# Load some images
img_mapbox = mapbox_image()
img_osm = osm_image()
img_shape = shape_image()
img_blue = plain_image(color=(0, 0, 127))
img_shape_filled = shape_image_filled()
# Get dutch time label
tz_amsterdam = pytz.timezone('Europe/Amsterdam')
tz_utc = pytz.timezone('UTC')
# Loop products
for product in products:
utc = tz_utc.localize(product.datetime)
amsterdam = utc.astimezone(tz_amsterdam)
#label = amsterdam.strftime('%Y-%m-%d %H:%M')
label = amsterdam.strftime('%H:%M')
offset = 0.25, 0.82
# Get data image
with product.get() as h5:
array = h5['precipitation'][...] / h5.attrs['composite_count']
mask = np.equal(array, h5.attrs['fill_value'])
img_radars = radars_image(h5=h5, label=label, offset=offset)
masked_array = np.ma.array(array, mask=mask)
img_rain = data_image(masked_array, max_rain=2, threshold=0.008)
timestamp = utils.datetime2timestamp(product.datetime)
filename = '{}{}.{}'.format(
timestamp,
kwargs.get('postfix', ''),
kwargs.get('format', 'png'),
)
# Merge and save
path = os.path.join(config.IMG_DIR, filename)
utils.merge([
img_radars,
img_rain,
img_mapbox,
img_shape,
img_shape_filled,
img_blue,
]).save(path)
logging.info('saved {}.'.format(os.path.basename(path)))
logging.debug('saved {}.'.format(path))
def create_png(products, **kwargs):
"""
Create image for products.
This is a kind of sandbox version.
"""
utils.makedir(config.IMG_DIR)
# Load some images
img_shape = shape_image()
img_blue = plain_image(color=(0, 0, 127))
img_shape_filled = shape_image_filled()
# Get dutch time label
tz_amsterdam = pytz.timezone('Europe/Amsterdam')
tz_utc = pytz.timezone('UTC')
# Loop products
for product in products:
utc = tz_utc.localize(product.datetime)
amsterdam = utc.astimezone(tz_amsterdam)
label = amsterdam.strftime('%Y-%m-%d %H:%M')
label_from_kwargs = kwargs.get('label', '')
if label_from_kwargs:
label += ' ' + label_from_kwargs
offset = 0.1, 0.9
# Get data image
try:
with product.get() as h5:
array = h5['precipitation'][...] / h5.attrs['composite_count']
mask = np.equal(array, h5.attrs['fill_value'])
img_radars = radars_image(h5=h5, label=label, offset=offset)
except IOError:
logging.debug('Does not exist: {}'.format(product.path))
continue
masked_array = np.ma.array(array, mask=mask)
img_rain = data_image(masked_array, max_rain=2, threshold=0.008)
timestamp = utils.datetime2timestamp(product.datetime)
filename = '{}{}.{}'.format(
timestamp, kwargs.get('postfix', ''), kwargs.get('format', 'png'),
)
# Merge and save
path = os.path.join(config.IMG_DIR, filename)
utils.merge([
img_radars,
img_rain,
img_shape,
img_shape_filled,
img_blue,
]).save(path)
logging.info('saved {}.'.format(os.path.basename(path)))
logging.debug('saved {}.'.format(path))
def create_png_for_animated_gif(products, **kwargs):
"""
Create image for products.
This is the tweaked version that creates the pngs for use in the
animated gif.
"""
utils.makedir(config.IMG_DIR)
# Load some images
img_mapbox = mapbox_image()
img_osm = osm_image()
img_shape = shape_image()
img_blue = plain_image(color=(0, 0, 127))
img_shape_filled = shape_image_filled()
# Get dutch time label
tz_amsterdam = pytz.timezone('Europe/Amsterdam')
tz_utc = pytz.timezone('UTC')
# Loop products
for product in products:
utc = tz_utc.localize(product.datetime)
amsterdam = utc.astimezone(tz_amsterdam)
#label = amsterdam.strftime('%Y-%m-%d %H:%M')
label = amsterdam.strftime('%H:%M')
offset = 0.25, 0.82
# Get data image
with product.get() as h5:
array = h5['precipitation'][...] / h5.attrs['composite_count']
mask = np.equal(array, h5.attrs['fill_value'])
img_radars = radars_image(h5=h5, label=label, offset=offset)
masked_array = np.ma.array(array, mask=mask)
img_rain = data_image(masked_array, max_rain=2, threshold=0.008)
timestamp = utils.datetime2timestamp(product.datetime)
filename = '{}{}.{}'.format(
timestamp, kwargs.get('postfix', ''), kwargs.get('format', 'png'),
)
# Merge and save
path = os.path.join(config.IMG_DIR, filename)
utils.merge([
img_radars,
img_rain,
img_mapbox,
img_shape,
img_shape_filled,
img_blue,
]).save(path)
logging.info('saved {}.'.format(os.path.basename(path)))
logging.debug('saved {}.'.format(path))
def create_tif(products, image_dir=None, **kwargs):
""" Save a tif with metadata """
if image_dir:
target_dir = image_dir
else:
target_dir = config.IMG_DIR
utils.makedir(target_dir)
# Loop products
for product in products:
# Get data
try:
with h5py.File(product.path, 'r') as h5:
data = np.ma.masked_equal(h5['precipitation'],
h5.attrs['fill_value'])
attrs = dict(h5.attrs)
except IOError:
logging.info('Not found: {}, skipping.'.format(product))
continue
# make a dataset
raster_layer = gridtools.RasterLayer(
array=data,
extent=attrs['grid_extent'],
projection=attrs['grid_projection'],
)
mem = raster_layer.create_dataset(datatype=6)
mem.SetProjection(PROJECTION_RD_WKT)
band = mem.GetRasterBand(1)
band.WriteArray(data.filled(band.GetNoDataValue()))
for k, v in attrs.items():
if hasattr(v, 'tolist'):
v = json.dumps(v.tolist())
band.SetMetadataItem(str(k), str(v))
timestamp = utils.datetime2timestamp(product.datetime)
filename = '{}{}.{}'.format(
timestamp,
kwargs.get('postfix', ''),
kwargs.get('format', 'png'),
)
path = os.path.join(target_dir, filename)
driver = gdal.GetDriverByName(b'gtiff')
driver.CreateCopy(path, mem)
logging.info('saved {}.'.format(os.path.basename(path)))
logging.debug('saved {}.'.format(path))
def command(text):
""" Check existence of realtime files. """
# prepare
period = periods.Period(text)
recently = Datetime.utcnow() - TOLERANCE
helper = utils.PathHelper(basedir=config.CALIBRATE_DIR, code="TF0005_R", template=config.PRODUCT_TEMPLATE)
# the check
for datetime in period:
if datetime > recently:
continue
path = helper.path(datetime)
if not os.path.exists(path):
timestamp = utils.datetime2timestamp(datetime)
logger.debug("bin/master -r {}".format(timestamp))
def create_tif(products, image_dir=None, **kwargs):
""" Save a tif with metadata """
if image_dir:
target_dir = image_dir
else:
target_dir = config.IMG_DIR
utils.makedir(target_dir)
# Loop products
for product in products:
# Get data
try:
with h5py.File(product.path, 'r') as h5:
data = np.ma.masked_equal(h5['precipitation'],
h5.attrs['fill_value'])
attrs = dict(h5.attrs)
except IOError:
logging.info('Not found: {}, skipping.'.format(product))
continue
# make a dataset
raster_layer = gridtools.RasterLayer(
array=data,
extent=attrs['grid_extent'],
projection=attrs['grid_projection'],
)
mem = raster_layer.create_dataset(datatype=6)
mem.SetProjection(PROJECTION_RD_WKT)
band = mem.GetRasterBand(1)
band.WriteArray(data.filled(band.GetNoDataValue()))
for k, v in attrs.items():
if hasattr(v, 'tolist'):
v = json.dumps(v.tolist())
band.SetMetadataItem(str(k), str(v))
timestamp = utils.datetime2timestamp(product.datetime)
filename = '{}{}.{}'.format(
timestamp, kwargs.get('postfix', ''), kwargs.get('format', 'png'),
)
path = os.path.join(target_dir, filename)
driver = gdal.GetDriverByName(b'gtiff')
driver.CreateCopy(path, mem)
logging.info('saved {}.'.format(os.path.basename(path)))
logging.debug('saved {}.'.format(path))
def command(text):
""" Check existence of realtime files. """
# prepare
period = periods.Period(text)
recently = Datetime.utcnow() - TOLERANCE
helper = utils.PathHelper(basedir=config.CALIBRATE_DIR,
code='TF0005_R',
template=config.PRODUCT_TEMPLATE)
# the check
for datetime in period:
if datetime > recently:
continue
path = helper.path(datetime)
if not os.path.exists(path):
timestamp = utils.datetime2timestamp(datetime)
logger.debug('bin/master -r {}'.format(timestamp))
def create_animated_gif(datetime):
""" Produces animated gif file from images in IMG_DIR. """
template = ('convert -set delay 20 -loop 0 -crop 340x370+80+40 '
'+repage {} {}')
step = config.TIMEFRAME_DELTA['f']
pngpaths = []
# Add files if they exist
for i in range(36):
timestamp = utils.datetime2timestamp(datetime - i * step)
pngpath = os.path.join(config.IMG_DIR, timestamp + '.png')
if os.path.exists(pngpath):
pngpaths = [pngpath] + pngpaths
logging.debug('Found {} suitable png files.'.format(len(pngpaths)))
# Create the gif
gifpath = os.path.join(config.IMG_DIR, 'radar.gif')
tempgifpath = gifpath.replace('radar.gif', 'radar_temp.gif')
command = template.format(' '.join(pngpaths), tempgifpath)
subprocess.call(shlex.split(command))
os.rename(tempgifpath, gifpath)
logging.debug(gifpath)
logging.info('Animation created at {}.'.format(gifpath))
def create_animated_gif(datetime):
""" Produces animated gif file from images in IMG_DIR. """
template = ('convert -set delay 20 -loop 0 -crop 340x370+80+40 '
'+repage {} {}')
step = config.TIMEFRAME_DELTA['f']
pngpaths = []
# Add files if they exist
for i in range(36):
timestamp = utils.datetime2timestamp(datetime - i * step)
pngpath = os.path.join(config.IMG_DIR, timestamp + '.png')
if os.path.exists(pngpath):
pngpaths = [pngpath] + pngpaths
logging.debug('Found {} suitable png files.'.format(len(pngpaths)))
# Create the gif
gifpath = os.path.join(config.IMG_DIR, 'radar.gif')
tempgifpath = gifpath.replace('radar.gif', 'radar_temp.gif')
command = template.format(' '.join(pngpaths), tempgifpath)
subprocess.call(shlex.split(command))
os.rename(tempgifpath, gifpath)
logging.debug(gifpath)
logging.info('Animation created at {}.'.format(gifpath))