def to_raster(tmpfn, now):
""" Convert the raw data into a RASTER Image
5 inch rain per hour is ~ 125 mm/hr, so over 5min that is 10 mm
Index 255 is missing
0 is zero
1 is 0.1 mm
254 is 25.4 mm
"""
data = np.loadtxt(tmpfn, skiprows=10)
# mm/hr to mm/5min
imgdata = data * 10.0 / 12.0
imgdata = np.where(imgdata < 0, 255, imgdata)
png = Image.fromarray(np.uint8(imgdata))
png.putpalette(mrms.make_colorramp())
meta = PngImagePlugin.PngInfo()
meta.add_text("title", now.strftime("%Y%m%d%H%M"), 0)
png.save("%s.png" % (tmpfn, ), pnginfo=meta)
del png
# Make worldfile
with open("%s.wld" % (tmpfn, ), "w") as fh:
fh.write("""0.004167
0.00
0.00
-0.004167
44.53785
-89.89942""")
def to_raster(tmpfn, now):
""" Convert the raw data into a RASTER Image
5 inch rain per hour is ~ 125 mm/hr, so over 5min that is 10 mm
Index 255 is missing
0 is zero
1 is 0.1 mm
254 is 25.4 mm
"""
data = np.loadtxt(tmpfn, skiprows=10)
# mm/hr to mm/5min
imgdata = (data * 10.0 / 12.0)
imgdata = np.where(imgdata < 0, 255, imgdata)
png = Image.fromarray(np.uint8(imgdata))
png.putpalette(mrms.make_colorramp())
meta = PngImagePlugin.PngInfo()
meta.add_text('title', now.strftime("%Y%m%d%H%M"), 0)
png.save('%s.png' % (tmpfn,), pnginfo=meta)
del png
# Make worldfile
o = open("%s.wld" % (tmpfn, ), 'w')
o.write("""0.004167
0.00
0.00
-0.004167
44.53785
-89.89942""")
o.close()
def main():
"""Go Main Go."""
data = numpy.zeros( (30,256), numpy.uint8)
for i in range(255):
data[0:15,i:i+1] = i
png = Image.fromarray( data )
#png.putpalette( make_gr_colorramp() )
ramp = list(mrms.make_colorramp())
ramp[-3:] = [255,255,255]
png.putpalette(ramp)
draw = ImageDraw.Draw(png)
#24h Precip
# 0-1 -> 100 - 0.01 res || 0 - 25 -> 100 - 0.25 mm 0
# 1-5 -> 80 - 0.05 res || 25 - 125 -> 80 - 1.25 mm 100
# 5-20 -> 75 - 0.20 res || 125 - 500 -> 75 - 5 mm 180
xs = [25, 50, 75, 100, 120, 140, 160, 180, 190, 205, 230]
ys = [0.25, 0.5, 0.75, 1, 2, 3, 4, 5, 7, 10, 15]
# 1hr precip
# 1 mm resolution
#xs = numpy.arange(25,255, 25)
#ys = numpy.arange(1,10)
for x,y in zip(xs, ys):
(w,h) = font.getsize(`y`)
draw.line( [x,17,x,10], fill=255)
draw.text( (x-(w/2),18), `y`,fill=255, font=font)
draw.text( (235,18), 'in',fill=255, font=font)
"""
#DBZ
for x in range(6,235,20):
dbz = int(x/2.0 - 33)
(w,h) = font.getsize(`dbz`)
draw.line( [x,17,x,10], fill=255)
draw.text( (x-(w/2),18), `dbz`,fill=255, font=font)
draw.text( (235,18), 'dBZ',fill=255, font=font)
"""
#draw.line( [0,0,255,0,255,29,0,29,0,0], fill=255)
png.save("test.png")
def doit(gts, hr):
"""
Actually generate a PNG file from the 8 NMQ tiles
"""
irealtime = is_realtime(gts)
routes = "ac" if irealtime else "a"
sts = gts - datetime.timedelta(hours=hr)
times = [gts]
if hr > 24:
times.append(gts - datetime.timedelta(hours=24))
if hr == 72:
times.append(gts - datetime.timedelta(hours=48))
metadata = {'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
'end_valid': gts.strftime("%Y-%m-%dT%H:%M:%SZ"),
'units': 'mm'}
total = None
mproduct = "RadarOnly_QPE_24H" if hr >= 24 else "RadarOnly_QPE_01H"
for now in times:
uri = now.strftime(("http://mtarchive.geol.iastate.edu/%Y/%m/%d/"
"mrms/ncep/" + mproduct + "/" + mproduct +
"_00.00_%Y%m%d-%H%M00.grib2.gz"))
gribfn = now.strftime((TMP + "/" + mproduct +
"_00.00_%Y%m%d-%H%M00.grib2.gz"))
if not os.path.isfile(gribfn):
res = requests.get(uri, timeout=60)
if res.status_code != 200:
# Cut down on logged errors
if gribfn in MISSED_FILES:
return
print(("make_mrms_rasters.py[%s] MISSING %s\n %s\n %s\n"
) % (hr, now.strftime("%Y-%m-%dT%H:%MZ"), uri, gribfn))
MISSED_FILES.append(gribfn)
return
o = open(gribfn, 'wb')
o.write(res.content)
o.close()
DOWNLOADED_FILES.append(gribfn)
fp = gzip.GzipFile(gribfn, 'rb')
(tmpfp, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
# careful here, how we deal with the two missing values!
if total is None:
total = grb['values']
else:
maxgrid = np.maximum(grb['values'], total)
total = np.where(np.logical_and(grb['values'] >= 0, total >= 0),
grb['values'] + total, maxgrid)
imgdata = convert_to_image(total)
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(imgdata.astype('u1'))
png.putpalette(mrms.make_colorramp())
png.save('%s.png' % (tmpfn,))
if irealtime:
# create a second PNG with null values set to black
imgdata = np.where(imgdata == 255, 0, imgdata)
png = Image.fromarray(imgdata.astype('u1'))
png.putpalette(mrms.make_colorramp())
png.save('%s_nn.png' % (tmpfn,))
# Now we need to generate the world file
mrms.write_worldfile('%s.wld' % (tmpfn,))
if irealtime:
mrms.write_worldfile('%s_nn.wld' % (tmpfn,))
# Inject WLD file
pqstr = ("%s -i -p 'plot %s %s "
"gis/images/4326/mrms/p%ih.wld GIS/mrms/p%ih_%s.wld wld' "
"%s.wld"
"") % (PQI, routes, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
if irealtime:
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.wld "
"GIS/mrms/p%ih_%s.wld wld' "
"%s_nn.wld"
"") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("%s -i -p 'plot %s %s "
"gis/images/4326/mrms/p%ih.png GIS/mrms/p%ih_%s.png png' "
"%s.png"
"") % (PQI, routes, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
if irealtime:
# Now we inject into LDM
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.png "
"GIS/mrms/p%ih_%s.png png' "
"%s_nn.png"
"") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
if irealtime:
# Create 900913 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s_nn.png %s_nn.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("%s -i -p 'plot c %s "
"gis/images/900913/mrms/p%ih.tif "
"GIS/mrms/p%ih_%s.tif tif' "
"%s.tif"
"") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
pqstr = ("%s -i -p 'plot c %s "
"gis/images/900913/mrms/p%ih_nn.tif "
"GIS/mrms/p%ih_%s.tif tif' "
"%s_nn.tif"
"") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn,), 'w')
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih.json "
"GIS/mrms/p%ih_%s.json json'"
" %s.json") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.json "
"GIS/mrms/p%ih_%s.json json'"
" %s.json") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
for suffix in ['tif', 'json', 'png', 'wld']:
fn = '%s.%s' % (tmpfn, suffix)
if os.path.isfile(fn):
os.unlink(fn)
if irealtime:
for suffix in ['tif', 'png', 'wld']:
fn = '%s_nn.%s' % (tmpfn, suffix)
if os.path.isfile(fn):
os.unlink(fn)
os.close(tmpfp)
os.unlink(tmpfn)
def doit(gts, hr):
"""
Actually generate a PNG file from the 8 NMQ tiles
"""
sts = gts - datetime.timedelta(hours=hr)
times = [gts]
if hr > 24:
times.append(gts - datetime.timedelta(hours=24))
if hr == 72:
times.append(gts - datetime.timedelta(hours=48))
metadata = {'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
'end_valid': gts.strftime("%Y-%m-%dT%H:%M:%SZ")}
# Create the image data
# imgdata = np.zeros( (szy, szx), 'u1')
# timestep = np.zeros( (szy, szx), 'f')
total = None
for now in times:
gribfn = now.strftime(("/mnt/a4/data/%Y/%m/%d/mrms/ncep/"
"RadarOnly_QPE_24H/"
"RadarOnly_QPE_24H_00.00_%Y%m%d-%H%M00"
".grib2.gz"))
if not os.path.isfile(gribfn):
print("mrms_raster_pXXh.py MISSING %s" % (gribfn,))
return
fp = gzip.GzipFile(gribfn, 'rb')
(tmpfp, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
# careful here, how we deal with the two missing values!
if total is None:
total = grb['values']
else:
maxgrid = np.maximum(grb['values'], total)
total = np.where(np.logical_and(grb['values'] >= 0, total >= 0),
grb['values'] + total, maxgrid)
"""
255 levels... wanna do 0 to 20 inches
index 255 is missing, index 0 is 0
0-1 -> 100 - 0.01 res || 0 - 25 -> 100 - 0.25 mm 0
1-5 -> 80 - 0.05 res || 25 - 125 -> 80 - 1.25 mm 100
5-20 -> 75 - 0.20 res || 125 - 500 -> 75 - 5 mm 180
"""
# total = np.flipud(total)
# Off scale gets index 254
imgdata = convert_to_image(total)
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(imgdata.astype('u1'))
png.putpalette(mrms.make_colorramp())
png.save('%s.png' % (tmpfn,))
# os.system("xv %s.png" % (tmpfn,))
# Now we need to generate the world file
mrms.write_worldfile('%s.wld' % (tmpfn,))
# Inject WLD file
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot ac %s "
"gis/images/4326/mrms/p%sh.wld GIS/mrms/p%sh_%s.wld wld' %s.wld"
"") % (gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot ac %s "
"gis/images/4326/mrms/p%sh.png GIS/mrms/p%sh_%s.png png' %s.png"
"") % (gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Create 900913 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s "
"gis/images/900913/mrms/p%sh.tif GIS/mrms/p%sh_%s.tif tif' %s.tif"
"") % (gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn,), 'w')
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s "
"gis/images/4326/mrms/p%sh.json GIS/mrms/p%sh_%s.json json'"
" %s.json") % (gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
for suffix in ['tif', 'json', 'png', 'wld']:
os.unlink('%s.%s' % (tmpfn, suffix))
os.close(tmpfp)
os.unlink(tmpfn)
def test_colorramp(self):
"""See what we can do with a colorramp"""
cmap = mrms.make_colorramp()
self.assertEqual(len(cmap), 256*3)
def workflow(now, realtime):
""" Generate for this timestep! """
szx = 7000
szy = 3500
# Create the image data
imgdata = np.zeros((szy, szx), "u1")
sts = now - datetime.timedelta(minutes=2)
metadata = {
"start_valid": sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
"end_valid": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
"product": "a2m",
"units": "0.02 mm",
}
gribfn = mrms.fetch("PrecipRate", now)
if gribfn is None:
LOG.info("NODATA for PrecipRate: %s", now.strftime("%Y-%m-%dT%H:%MZ"))
return
# http://www.nssl.noaa.gov/projects/mrms/operational/tables.php
# Says units are mm/hr
fp = gzip.GzipFile(gribfn, "rb")
(_, tmpfn) = tempfile.mkstemp()
with open(tmpfn, "wb") as fh:
fh.write(fp.read())
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
os.unlink(gribfn)
val = grb["values"]
# Convert into units of 0.1 mm accumulation
val = val / 60.0 * 2.0 * 50.0
val = np.where(val < 0.0, 255.0, val)
imgdata[:, :] = np.flipud(val.astype("int"))
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(np.flipud(imgdata))
png.putpalette(mrms.make_colorramp())
png.save("%s.png" % (tmpfn, ))
mrms.write_worldfile("%s.wld" % (tmpfn, ))
# Inject WLD file
routes = "c" if realtime else ""
prefix = "a2m"
pqstr = ("pqinsert -i -p 'plot a%s %s "
"gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld"
"") % (
routes,
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("pqinsert -i -p 'plot a%s %s "
"gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png"
"") % (
routes,
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
if realtime:
# Create 3857 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("pqinsert -i -p 'plot c %s "
"gis/images/3857/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif"
"") % (
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
with open("%s.json" % (tmpfn, ), "w") as fh:
fh.write(json.dumps(dict(meta=metadata)))
# Insert into LDM
pqstr = ("pqinsert -i -p 'plot c %s "
"gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' "
"%s.json") % (
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
for suffix in ["tif", "json", "png", "wld"]:
if os.path.isfile("%s.%s" % (tmpfn, suffix)):
os.unlink("%s.%s" % (tmpfn, suffix))
os.close(tmpfp)
os.unlink(tmpfn)
def doit(gts, hr):
"""
Actually generate a PNG file from the 8 NMQ tiles
"""
irealtime = is_realtime(gts)
routes = "ac" if irealtime else "a"
sts = gts - datetime.timedelta(hours=hr)
times = [gts]
if hr > 24:
times.append(gts - datetime.timedelta(hours=24))
if hr == 72:
times.append(gts - datetime.timedelta(hours=48))
metadata = {
"start_valid": sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
"end_valid": gts.strftime("%Y-%m-%dT%H:%M:%SZ"),
"units": "mm",
}
total = None
mproduct = "RadarOnly_QPE_24H" if hr >= 24 else "RadarOnly_QPE_01H"
for now in times:
gribfn = mrms.fetch(mproduct, now)
if gribfn is None:
print(
("make_mrms_rasters.py[%s] MISSING %s\n %s\n")
% (hr, now.strftime("%Y-%m-%dT%H:%MZ"), gribfn)
)
MISSED_FILES.append(gribfn)
return
DOWNLOADED_FILES.append(gribfn)
fp = gzip.GzipFile(gribfn, "rb")
(tmpfp, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, "wb")
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
# careful here, how we deal with the two missing values!
if total is None:
total = grb["values"]
else:
maxgrid = np.maximum(grb["values"], total)
total = np.where(
np.logical_and(grb["values"] >= 0, total >= 0),
grb["values"] + total,
maxgrid,
)
imgdata = convert_to_image(total)
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(imgdata.astype("u1"))
png.putpalette(mrms.make_colorramp())
png.save("%s.png" % (tmpfn,))
if irealtime:
# create a second PNG with null values set to black
imgdata = np.where(imgdata == 255, 0, imgdata)
png = Image.fromarray(imgdata.astype("u1"))
png.putpalette(mrms.make_colorramp())
png.save("%s_nn.png" % (tmpfn,))
# Now we need to generate the world file
mrms.write_worldfile("%s.wld" % (tmpfn,))
if irealtime:
mrms.write_worldfile("%s_nn.wld" % (tmpfn,))
# Inject WLD file
pqstr = (
"%s -i -p 'plot %s %s "
"gis/images/4326/mrms/p%ih.wld GIS/mrms/p%ih_%s.wld wld' "
"%s.wld"
""
) % (
PQI,
routes,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
if irealtime:
pqstr = (
"%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.wld "
"GIS/mrms/p%ih_%s.wld wld' "
"%s_nn.wld"
""
) % (
PQI,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = (
"%s -i -p 'plot %s %s "
"gis/images/4326/mrms/p%ih.png GIS/mrms/p%ih_%s.png png' "
"%s.png"
""
) % (
PQI,
routes,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
if irealtime:
# Now we inject into LDM
pqstr = (
"%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.png "
"GIS/mrms/p%ih_%s.png png' "
"%s_nn.png"
""
) % (
PQI,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
if irealtime:
# Create 900913 image
cmd = (
"gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif"
) % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
cmd = (
"gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s_nn.png %s_nn.tif"
) % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = (
"%s -i -p 'plot c %s "
"gis/images/900913/mrms/p%ih.tif "
"GIS/mrms/p%ih_%s.tif tif' "
"%s.tif"
""
) % (
PQI,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
pqstr = (
"%s -i -p 'plot c %s "
"gis/images/900913/mrms/p%ih_nn.tif "
"GIS/mrms/p%ih_%s.tif tif' "
"%s_nn.tif"
""
) % (
PQI,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn,), "w")
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = (
"%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih.json "
"GIS/mrms/p%ih_%s.json json'"
" %s.json"
) % (
PQI,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
pqstr = (
"%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.json "
"GIS/mrms/p%ih_%s.json json'"
" %s.json"
) % (
PQI,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
for suffix in ["tif", "json", "png", "wld"]:
fn = "%s.%s" % (tmpfn, suffix)
if os.path.isfile(fn):
os.unlink(fn)
if irealtime:
for suffix in ["tif", "png", "wld"]:
fn = "%s_nn.%s" % (tmpfn, suffix)
if os.path.isfile(fn):
os.unlink(fn)
os.close(tmpfp)
os.unlink(tmpfn)
def test_colorramp():
"""See what we can do with a colorramp"""
cmap = mrms.make_colorramp()
assert len(cmap) == 256 * 3
def test_colorramp(self):
"""See what we can do with a colorramp"""
c = mrms.make_colorramp()
self.assertEqual(len(c), 256 * 3)
def doit(gts, hr):
"""
Actually generate a PNG file from the 8 NMQ tiles
"""
irealtime = is_realtime(gts)
routes = "ac" if irealtime else "a"
sts = gts - datetime.timedelta(hours=hr)
times = [gts]
if hr > 24:
times.append(gts - datetime.timedelta(hours=24))
if hr == 72:
times.append(gts - datetime.timedelta(hours=48))
metadata = {
'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
'end_valid': gts.strftime("%Y-%m-%dT%H:%M:%SZ"),
'units': 'mm'
}
total = None
mproduct = "RadarOnly_QPE_24H" if hr >= 24 else "RadarOnly_QPE_01H"
for now in times:
uri = now.strftime(("http://mtarchive.geol.iastate.edu/%Y/%m/%d/"
"mrms/ncep/" + mproduct + "/" + mproduct +
"_00.00_%Y%m%d-%H%M00.grib2.gz"))
gribfn = now.strftime(
(TMP + "/" + mproduct + "_00.00_%Y%m%d-%H%M00.grib2.gz"))
if not os.path.isfile(gribfn):
res = requests.get(uri, timeout=60)
if res.status_code != 200:
# Cut down on logged errors
if gribfn in MISSED_FILES:
return
print(("make_mrms_rasters.py[%s] MISSING %s\n %s\n %s\n") %
(hr, now.strftime("%Y-%m-%dT%H:%MZ"), uri, gribfn))
MISSED_FILES.append(gribfn)
return
o = open(gribfn, 'wb')
o.write(res.content)
o.close()
DOWNLOADED_FILES.append(gribfn)
fp = gzip.GzipFile(gribfn, 'rb')
(tmpfp, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
# careful here, how we deal with the two missing values!
if total is None:
total = grb['values']
else:
maxgrid = np.maximum(grb['values'], total)
total = np.where(np.logical_and(grb['values'] >= 0, total >= 0),
grb['values'] + total, maxgrid)
imgdata = convert_to_image(total)
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(imgdata.astype('u1'))
png.putpalette(mrms.make_colorramp())
png.save('%s.png' % (tmpfn, ))
if irealtime:
# create a second PNG with null values set to black
imgdata = np.where(imgdata == 255, 0, imgdata)
png = Image.fromarray(imgdata.astype('u1'))
png.putpalette(mrms.make_colorramp())
png.save('%s_nn.png' % (tmpfn, ))
# Now we need to generate the world file
mrms.write_worldfile('%s.wld' % (tmpfn, ))
if irealtime:
mrms.write_worldfile('%s_nn.wld' % (tmpfn, ))
# Inject WLD file
pqstr = ("%s -i -p 'plot %s %s "
"gis/images/4326/mrms/p%ih.wld GIS/mrms/p%ih_%s.wld wld' "
"%s.wld"
"") % (PQI, routes, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
if irealtime:
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.wld "
"GIS/mrms/p%ih_%s.wld wld' "
"%s_nn.wld"
"") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("%s -i -p 'plot %s %s "
"gis/images/4326/mrms/p%ih.png GIS/mrms/p%ih_%s.png png' "
"%s.png"
"") % (PQI, routes, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
if irealtime:
# Now we inject into LDM
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.png "
"GIS/mrms/p%ih_%s.png png' "
"%s_nn.png"
"") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
if irealtime:
# Create 900913 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s_nn.png %s_nn.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("%s -i -p 'plot c %s "
"gis/images/900913/mrms/p%ih.tif "
"GIS/mrms/p%ih_%s.tif tif' "
"%s.tif"
"") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
pqstr = ("%s -i -p 'plot c %s "
"gis/images/900913/mrms/p%ih_nn.tif "
"GIS/mrms/p%ih_%s.tif tif' "
"%s_nn.tif"
"") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn, ), 'w')
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih.json "
"GIS/mrms/p%ih_%s.json json'"
" %s.json") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.json "
"GIS/mrms/p%ih_%s.json json'"
" %s.json") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
for suffix in ['tif', 'json', 'png', 'wld']:
fn = '%s.%s' % (tmpfn, suffix)
if os.path.isfile(fn):
os.unlink(fn)
if irealtime:
for suffix in ['tif', 'png', 'wld']:
fn = '%s_nn.%s' % (tmpfn, suffix)
if os.path.isfile(fn):
os.unlink(fn)
os.close(tmpfp)
os.unlink(tmpfn)
def test_colorramp():
"""See what we can do with a colorramp"""
cmap = mrms.make_colorramp()
assert len(cmap) == 256 * 3
c[1,:] = [0,0,0]
i = 2
for line in open('gr2ae.txt'):
c[i,:] = map(int, line.split()[-3:])
i+=1
c[255,:] = [255,255,255]
return tuple( c.ravel() )
data = numpy.zeros( (30,256), numpy.uint8)
for i in range(255):
data[0:15,i:i+1] = i
png = Image.fromarray( data )
#png.putpalette( make_gr_colorramp() )
ramp = list(mrms.make_colorramp())
ramp[-3:] = [255,255,255]
png.putpalette(ramp)
draw = ImageDraw.Draw(png)
#24h Precip
# 0-1 -> 100 - 0.01 res || 0 - 25 -> 100 - 0.25 mm 0
# 1-5 -> 80 - 0.05 res || 25 - 125 -> 80 - 1.25 mm 100
# 5-20 -> 75 - 0.20 res || 125 - 500 -> 75 - 5 mm 180
xs = [25, 50, 75, 100, 120, 140, 160, 180, 190, 205, 230]
ys = [0.25, 0.5, 0.75, 1, 2, 3, 4, 5, 7, 10, 15]
# 1hr precip
# 1 mm resolution
#xs = numpy.arange(25,255, 25)
#ys = numpy.arange(1,10)
def do(now, realtime, delta):
""" Generate for this timestep! """
szx = 7000
szy = 3500
# Create the image data
imgdata = np.zeros((szy, szx), 'u1')
sts = now - datetime.timedelta(minutes=2)
metadata = {
'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
'end_valid': now.strftime("%Y-%m-%dT%H:%M:%SZ"),
'product': 'a2m',
'units': '0.02 mm'
}
fn = now.strftime("PrecipRate_00.00_%Y%m%d-%H%M00.grib2.gz")
uri = now.strftime(("http://mtarchive.geol.iastate.edu/%Y/%m/%d/"
"mrms/ncep/PrecipRate/" + fn))
gribfn = "%s/%s" % (TMP, fn)
res = requests.get(uri, timeout=30)
if res.status_code != 200:
print("mrms_rainrate_comp.py MISSING %s" % (gribfn, ))
return
o = open(gribfn, 'wb')
o.write(res.content)
o.close()
# http://www.nssl.noaa.gov/projects/mrms/operational/tables.php
# Says units are mm/hr
fp = gzip.GzipFile(gribfn, 'rb')
(_, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
os.unlink(gribfn)
val = grb['values']
# Convert into units of 0.1 mm accumulation
val = val / 60.0 * 2.0 * 50.0
val = np.where(val < 0., 255., val)
imgdata[:, :] = np.flipud(val.astype('int'))
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(np.flipud(imgdata))
png.putpalette(mrms.make_colorramp())
png.save('%s.png' % (tmpfn, ))
mrms.write_worldfile('%s.wld' % (tmpfn, ))
# Inject WLD file
routes = "c" if realtime else ""
prefix = 'a2m'
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot a%s %s "
"gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld"
"") % (routes, now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot a%s %s "
"gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png"
"") % (routes, now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
if realtime:
# Create 900913 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s "
"gis/images/900913/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif"
"") % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn, ), 'w')
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s "
"gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' "
"%s.json") % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
for suffix in ['tif', 'json', 'png', 'wld']:
if os.path.isfile("%s.%s" % (tmpfn, suffix)):
os.unlink('%s.%s' % (tmpfn, suffix))
os.close(tmpfp)
os.unlink(tmpfn)
def do(now, hr ):
''' Generate for this timestep!
255 levels... wanna do 0 to 20 inches
index 255 is missing, index 0 is 0
0-1 -> 100 - 0.01 res || 0 - 25 -> 100 - 0.25 mm 0
1-5 -> 80 - 0.05 res || 25 - 125 -> 80 - 1.25 mm 100
5-20 -> 75 - 0.20 res || 125 - 500 -> 75 - 5 mm 180
'''
szx = 7000
szy = 3500
sts = now - datetime.timedelta(hours=hr)
metadata = {'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
'end_valid': now.strftime("%Y-%m-%dT%H:%M:%SZ"),
'product': 'lcref',
'units': 'mm' }
gribfn = now.strftime(("/mnt/a4/data/%Y/%m/%d/mrms/ncep/"
+"RadarOnly_QPE_01H/"
+"RadarOnly_QPE_01H_00.00_%Y%m%d-%H%M00.grib2.gz"))
if not os.path.isfile(gribfn):
print("mrms_raster_p1h.py MISSING %s" % (gribfn,))
return
fp = gzip.GzipFile(gribfn, 'rb')
(tmpfp, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
total = grb['values']
# Off scale gets index 254
imgdata = np.where(total >= 500, 254, 0)
imgdata = np.where(np.logical_and(total >= 125, total < 500),
180 + ((total - 125.) / 5.0), imgdata)
imgdata = np.where(np.logical_and(total >= 25, total < 125),
100 + ((total - 25.) / 1.25), imgdata)
imgdata = np.where(np.logical_and(total >= 0, total < 25),
total / 0.25, imgdata)
# -3 is no coverage -> 255
# -1 is misisng, so zero
# Index 255 is missing
imgdata = np.where( total < 0, 0, imgdata)
imgdata = np.where( total < -1, 255, imgdata)
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray( imgdata.astype('u1') )
png.putpalette(mrms.make_colorramp())
png.save('%s.png' % (tmpfn,))
# os.system('xv %s.png' % (tmpfn,))
mrms.write_worldfile('%s.wld' % (tmpfn,))
# Inject WLD file
prefix = 'p%sh' % (hr,)
pqstr = "/home/ldm/bin/pqinsert -i -p 'plot ac %s gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld" % (
now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn )
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = "/home/ldm/bin/pqinsert -p 'plot ac %s gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png" % (
now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn )
subprocess.call(pqstr, shell=True)
# Create 900913 image
cmd = "gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff -tr 1000.0 1000.0 %s.png %s.tif" % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = "/home/ldm/bin/pqinsert -p 'plot c %s gis/images/900913/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif" % (
now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn )
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn,), 'w')
j.write( json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = "/home/ldm/bin/pqinsert -p 'plot c %s gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' %s.json" % (
now.strftime("%Y%m%d%H%M"),prefix, prefix, now.strftime("%Y%m%d%H%M"), tmpfn )
subprocess.call(pqstr, shell=True)
for suffix in ['tif', 'json', 'png', 'wld']:
os.unlink('%s.%s' % (tmpfn, suffix))
os.close(tmpfp)
os.unlink(tmpfn)
def do(now, realtime, delta):
""" Generate for this timestep! """
szx = 7000
szy = 3500
# Create the image data
imgdata = np.zeros((szy, szx), 'u1')
sts = now - datetime.timedelta(minutes=2)
metadata = {'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
'end_valid': now.strftime("%Y-%m-%dT%H:%M:%SZ"),
'product': 'a2m',
'units': '0.1 mm'}
gribfn = now.strftime(("/mnt/a4/data/%Y/%m/%d/mrms/ncep/PrecipRate/"
"PrecipRate_00.00_%Y%m%d-%H%M00.grib2.gz"))
if not os.path.isfile(gribfn):
# Don't whine about old files being missing
if delta < 90:
print("mrms_rainrate_comp.py MISSING %s" % (gribfn,))
return
# http://www.nssl.noaa.gov/projects/mrms/operational/tables.php
# Says units are mm/hr
fp = gzip.GzipFile(gribfn, 'rb')
(_, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
val = grb['values']
# Convert into units of 0.1 mm accumulation
val = val / 60.0 * 2.0 * 10.0
val = np.where(val < 0., 255., val)
imgdata[:, :] = np.flipud(val.astype('int'))
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(np.flipud(imgdata))
png.putpalette(mrms.make_colorramp())
png.save('%s.png' % (tmpfn,))
mrms.write_worldfile('%s.wld' % (tmpfn,))
# Inject WLD file
routes = "c" if realtime else ""
prefix = 'a2m'
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot a%s %s "
"gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld"
"") % (routes, now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot a%s %s "
"gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png"
"") % (routes, now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
if realtime:
# Create 900913 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s "
"gis/images/900913/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif"
"") % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn,), 'w')
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s "
"gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' "
"%s.json") % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
for suffix in ['tif', 'json', 'png', 'wld']:
if os.path.isfile("%s.%s" % (tmpfn, suffix)):
os.unlink('%s.%s' % (tmpfn, suffix))
os.close(tmpfp)
os.unlink(tmpfn)
i = 2
for line in open('gr2ae.txt'):
c[i, :] = map(int, line.split()[-3:])
i += 1
c[255, :] = [255, 255, 255]
return tuple(c.ravel())
data = numpy.zeros((30, 256), numpy.uint8)
for i in range(255):
data[0:15, i:i + 1] = i
png = Image.fromarray(data)
#png.putpalette( make_gr_colorramp() )
ramp = list(mrms.make_colorramp())
ramp[-3:] = [255, 255, 255]
png.putpalette(ramp)
draw = ImageDraw.Draw(png)
#24h Precip
# 0-1 -> 100 - 0.01 res || 0 - 25 -> 100 - 0.25 mm 0
# 1-5 -> 80 - 0.05 res || 25 - 125 -> 80 - 1.25 mm 100
# 5-20 -> 75 - 0.20 res || 125 - 500 -> 75 - 5 mm 180
xs = [25, 50, 75, 100, 120, 140, 160, 180, 190, 205, 230]
ys = [0.25, 0.5, 0.75, 1, 2, 3, 4, 5, 7, 10, 15]
# 1hr precip
# 1 mm resolution
#xs = numpy.arange(25,255, 25)
#ys = numpy.arange(1,10)
