def test_fetch(self):
"""Can we fetch MRMS files? Yes we can!"""
product = "PrecipRate"
valid = datetime.datetime.utcnow() - datetime.timedelta(minutes=10)
valid -= datetime.timedelta(minutes=(valid.minute % 2))
fn = mrms.fetch(product, valid, tmpdir="/tmp")
if os.path.isfile(fn):
os.unlink(fn)
valid = valid.replace(tzinfo=pytz.utc) - datetime.timedelta(minutes=2)
fn = mrms.fetch(product, valid, tmpdir="/tmp")
if os.path.isfile(fn):
os.unlink(fn)
def ptype(grids, valid, iarchive):
"""MRMS Precip Type
http://www.nssl.noaa.gov/projects/mrms/operational/tables.php
-3 no coverage
0 no precipitation
1 warm stratiform rain
2 warm stratiform rain
3 snow
4 snow
5 reserved for future use
6 convective rain
7 rain mixed with hail
8 reserved for future use
9 flag no longer used
10 cold stratiform rain
91 tropical/stratiform rain mix
96 tropical/convective rain mix
"""
floor = datetime.datetime(2016, 1, 21)
floor = floor.replace(tzinfo=pytz.timezone("UTC"))
if valid < floor:
# Use hack for now
grids['ptype'] = np.where(grids['tmpc'] < 0, 3, 10)
return
fn = None
i = 0
while i < 10:
ts = valid - datetime.timedelta(minutes=i)
if ts.minute % 2 == 0:
testfn = mrms_util.fetch('PrecipFlag', ts)
if testfn is not None:
fn = testfn
break
i += 1
if fn is None:
print("Warning, no PrecipFlag data found!")
return
fp = gzip.GzipFile(fn, 'rb')
(_, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
map(os.unlink, [tmpfn, fn])
# 3500, 7000, starts in upper left
top = int((55. - reference.IA_NORTH) * 100.)
bottom = int((55. - reference.IA_SOUTH) * 100.)
right = int((reference.IA_EAST - -130.) * 100.) - 1
left = int((reference.IA_WEST - -130.) * 100.)
grids['ptype'] = np.flipud(grb['values'][top:bottom, left:right])
def pcpn(grids, valid, iarchive):
"""Attempt to use MRMS or stage IV pcpn here"""
floor = datetime.datetime(2014, 11, 1)
floor = floor.replace(tzinfo=pytz.timezone("UTC"))
if valid < floor:
# Use stageIV
ts = (valid + datetime.timedelta(minutes=60)).replace(minute=0)
gribfn = ts.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/stage4/ST4."
"%Y%m%d%H.01h.grib"))
if not os.path.isfile(gribfn):
return
grbs = pygrib.open(gribfn)
grib = grbs[1]
lats, lons = grib.latlons()
vals = grib.values
nn = NearestNDInterpolator((lons.flatten(), lats.flatten()),
vals.flatten())
grids['pcpn'] = nn(XI, YI)
return
fn = None
i = 0
while i < 10:
ts = valid - datetime.timedelta(minutes=i)
if ts.minute % 2 == 0:
testfn = mrms_util.fetch('PrecipRate', ts)
if testfn is not None:
fn = testfn
break
i += 1
if fn is None:
print("Warning, no PrecipRate data found!")
return
fp = gzip.GzipFile(fn, 'rb')
(_, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
values = grbs[1]['values']
# just set -3 (no coverage) to 0 for now
values = np.where(values < 0, 0, values)
map(os.unlink, [fn, tmpfn])
# 3500, 7000, starts in upper left
top = int((55. - reference.IA_NORTH) * 100.)
bottom = int((55. - reference.IA_SOUTH) * 100.)
right = int((reference.IA_EAST - -130.) * 100.) - 1
left = int((reference.IA_WEST - -130.) * 100.)
# two minute accumulation is in mm/hr / 60 * 5
# stage IV is mm/hr
grids['pcpn'] = np.flipud(values[top:bottom, left:right]) / 12.0
def do(now, realtime=False):
""" Generate for this timestep! """
szx = 7000
szy = 3500
# Create the image data
imgdata = np.zeros((szy, szx), 'u1')
metadata = {'start_valid': now.strftime("%Y-%m-%dT%H:%M:%SZ"),
'end_valid': now.strftime("%Y-%m-%dT%H:%M:%SZ"),
'product': 'lcref',
'units': '0.5 dBZ'}
gribfn = mrms.fetch('SeamlessHSR', now)
if gribfn is None:
print(("mrms_lcref_comp.py NODATA for SeamlessHSR: %s"
) % (now.strftime("%Y-%m-%dT%H:%MZ"),))
return
fp = gzip.GzipFile(gribfn, 'rb')
(_, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
os.unlink(gribfn)
val = grb['values']
# -999 is no coverage, go to 0
# -99 is missing , go to 255
val = np.where(val >= -32, (val + 32) * 2.0, val)
# val = np.where(val < -990., 0., val)
# val = np.where(val < -90., 255., val)
# This is an upstream BUG
val = np.where(val < 0., 0., val)
imgdata[:, :] = np.flipud(val.astype('int'))
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(np.flipud(imgdata))
png.putpalette(make_colorramp())
png.save('%s.png' % (tmpfn,))
mrms.write_worldfile('%s.wld' % (tmpfn,))
# Inject WLD file
prefix = 'lcref'
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot ac %s "
"gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld"
) % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot ac %s "
"gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png"
) % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Create 900913 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s "
"gis/images/900913/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif"
) % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn,), 'w')
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -p 'plot c %s "
"gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' %s.json"
) % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
for suffix in ['tif', 'json', 'png', 'wld']:
os.unlink('%s.%s' % (tmpfn, suffix))
os.close(tmpfp)
os.unlink(tmpfn)
def do(now, realtime=False):
""" Generate for this timestep! """
szx = 7000
szy = 3500
# Create the image data
imgdata = np.zeros((szy, szx), 'u1')
metadata = {'start_valid': now.strftime("%Y-%m-%dT%H:%M:%SZ"),
'end_valid': now.strftime("%Y-%m-%dT%H:%M:%SZ"),
'product': 'lcref',
'units': '0.5 dBZ'}
gribfn = mrms.fetch('SeamlessHSR', now)
if gribfn is None:
print(("mrms_lcref_comp.py NODATA for SeamlessHSR: %s"
) % (now.strftime("%Y-%m-%dT%H:%MZ"),))
return
fp = gzip.GzipFile(gribfn, 'rb')
(_, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
os.unlink(gribfn)
val = grb['values']
# -999 is no coverage, go to 0
# -99 is missing , go to 255
val = np.where(val >= -32, (val + 32) * 2.0, val)
# val = np.where(val < -990., 0., val)
# val = np.where(val < -90., 255., val)
# This is an upstream BUG
val = np.where(val < 0., 0., val)
imgdata[:, :] = np.flipud(val.astype('int'))
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(np.flipud(imgdata))
png.putpalette(make_colorramp())
png.save('%s.png' % (tmpfn,))
mrms.write_worldfile('%s.wld' % (tmpfn,))
# Inject WLD file
prefix = 'lcref'
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot ac %s "
"gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld"
) % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot ac %s "
"gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png"
) % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Create 900913 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s "
"gis/images/900913/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif"
) % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn,), 'w')
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -p 'plot c %s "
"gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' %s.json"
) % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
for suffix in ['tif', 'json', 'png', 'wld']:
os.unlink('%s.%s' % (tmpfn, suffix))
os.close(tmpfp)
os.unlink(tmpfn)
def doit(gts, hr):
"""
Actually generate a PNG file from the 8 NMQ tiles
"""
irealtime = is_realtime(gts)
routes = "ac" if irealtime else "a"
sts = gts - datetime.timedelta(hours=hr)
times = [gts]
if hr > 24:
times.append(gts - datetime.timedelta(hours=24))
if hr == 72:
times.append(gts - datetime.timedelta(hours=48))
metadata = {
"start_valid": sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
"end_valid": gts.strftime("%Y-%m-%dT%H:%M:%SZ"),
"units": "mm",
}
total = None
mproduct = "RadarOnly_QPE_24H" if hr >= 24 else "RadarOnly_QPE_01H"
for now in times:
gribfn = mrms.fetch(mproduct, now)
if gribfn is None:
print(("make_mrms_rasters.py[%s] MISSING %s\n %s\n") %
(hr, now.strftime("%Y-%m-%dT%H:%MZ"), gribfn))
MISSED_FILES.append(gribfn)
return
DOWNLOADED_FILES.append(gribfn)
fp = gzip.GzipFile(gribfn, "rb")
(tmpfp, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, "wb")
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
# careful here, how we deal with the two missing values!
if total is None:
total = grb["values"]
else:
maxgrid = np.maximum(grb["values"], total)
total = np.where(
np.logical_and(grb["values"] >= 0, total >= 0),
grb["values"] + total,
maxgrid,
)
imgdata = convert_to_image(total)
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(imgdata.astype("u1"))
png.putpalette(mrms.make_colorramp())
png.save("%s.png" % (tmpfn, ))
if irealtime:
# create a second PNG with null values set to black
imgdata = np.where(imgdata == 255, 0, imgdata)
png = Image.fromarray(imgdata.astype("u1"))
png.putpalette(mrms.make_colorramp())
png.save("%s_nn.png" % (tmpfn, ))
# Now we need to generate the world file
mrms.write_worldfile("%s.wld" % (tmpfn, ))
if irealtime:
mrms.write_worldfile("%s_nn.wld" % (tmpfn, ))
# Inject WLD file
pqstr = ("%s -i -p 'plot %s %s "
"gis/images/4326/mrms/p%ih.wld GIS/mrms/p%ih_%s.wld wld' "
"%s.wld"
"") % (
PQI,
routes,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
if irealtime:
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.wld "
"GIS/mrms/p%ih_%s.wld wld' "
"%s_nn.wld"
"") % (
PQI,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("%s -i -p 'plot %s %s "
"gis/images/4326/mrms/p%ih.png GIS/mrms/p%ih_%s.png png' "
"%s.png"
"") % (
PQI,
routes,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
if irealtime:
# Now we inject into LDM
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.png "
"GIS/mrms/p%ih_%s.png png' "
"%s_nn.png"
"") % (
PQI,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
if irealtime:
# Create 3857 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s_nn.png %s_nn.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("%s -i -p 'plot c %s "
"gis/images/3857/mrms/p%ih.tif "
"GIS/mrms/p%ih_%s.tif tif' "
"%s.tif"
"") % (
PQI,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
pqstr = ("%s -i -p 'plot c %s "
"gis/images/3857/mrms/p%ih_nn.tif "
"GIS/mrms/p%ih_%s.tif tif' "
"%s_nn.tif"
"") % (
PQI,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn, ), "w")
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih.json "
"GIS/mrms/p%ih_%s.json json'"
" %s.json") % (
PQI,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.json "
"GIS/mrms/p%ih_%s.json json'"
" %s.json") % (
PQI,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
for suffix in ["tif", "json", "png", "wld"]:
fn = "%s.%s" % (tmpfn, suffix)
if os.path.isfile(fn):
os.unlink(fn)
if irealtime:
for suffix in ["tif", "png", "wld"]:
fn = "%s_nn.%s" % (tmpfn, suffix)
if os.path.isfile(fn):
os.unlink(fn)
os.close(tmpfp)
os.unlink(tmpfn)
def workflow(now, realtime):
""" Generate for this timestep! """
szx = 7000
szy = 3500
# Create the image data
imgdata = np.zeros((szy, szx), "u1")
sts = now - datetime.timedelta(minutes=2)
metadata = {
"start_valid": sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
"end_valid": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
"product": "a2m",
"units": "0.02 mm",
}
gribfn = mrms.fetch("PrecipRate", now)
if gribfn is None:
print(("mrms_rainrate_comp.py NODATA for PrecipRate: %s") %
(now.strftime("%Y-%m-%dT%H:%MZ"), ))
return
# http://www.nssl.noaa.gov/projects/mrms/operational/tables.php
# Says units are mm/hr
fp = gzip.GzipFile(gribfn, "rb")
(_, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, "wb")
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
os.unlink(gribfn)
val = grb["values"]
# Convert into units of 0.1 mm accumulation
val = val / 60.0 * 2.0 * 50.0
val = np.where(val < 0.0, 255.0, val)
imgdata[:, :] = np.flipud(val.astype("int"))
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(np.flipud(imgdata))
png.putpalette(mrms.make_colorramp())
png.save("%s.png" % (tmpfn, ))
mrms.write_worldfile("%s.wld" % (tmpfn, ))
# Inject WLD file
routes = "c" if realtime else ""
prefix = "a2m"
pqstr = ("pqinsert -i -p 'plot a%s %s "
"gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld"
"") % (
routes,
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("pqinsert -i -p 'plot a%s %s "
"gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png"
"") % (
routes,
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
if realtime:
# Create 3857 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("pqinsert -i -p 'plot c %s "
"gis/images/3857/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif"
"") % (
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn, ), "w")
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = ("pqinsert -i -p 'plot c %s "
"gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' "
"%s.json") % (
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
for suffix in ["tif", "json", "png", "wld"]:
if os.path.isfile("%s.%s" % (tmpfn, suffix)):
os.unlink("%s.%s" % (tmpfn, suffix))
os.close(tmpfp)
os.unlink(tmpfn)
def doit(ts, hours):
"""
Create a plot of precipitation stage4 estimates for some day
"""
# Start at 1 AM
ts = ts.replace(minute=0, second=0, microsecond=0)
now = ts - datetime.timedelta(hours=hours - 1)
interval = datetime.timedelta(hours=1)
ets = datetime.datetime.utcnow()
ets = ets.replace(tzinfo=pytz.timezone("UTC"))
total = None
while now < ets:
gmt = now.astimezone(pytz.timezone("UTC"))
gribfn = None
for prefix in ['GaugeCorr', 'RadarOnly']:
gribfn = mrms.fetch(prefix + "_QPE_01H", gmt)
if gribfn is None:
continue
break
if gribfn is None:
print("q3_xhour.py[%s] MISSING %s" % (hours, now))
now += interval
continue
fp = gzip.GzipFile(gribfn, 'rb')
(tmpfp, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
# careful here, how we deal with the two missing values!
if total is None:
total = grb['values']
else:
maxgrid = np.maximum(grb['values'], total)
total = np.where(np.logical_and(grb['values'] >= 0, total >= 0),
grb['values'] + total, maxgrid)
now += interval
os.unlink(gribfn)
if total is None:
print("q3_xhour.py no data ts: %s hours: %s" % (ts, hours))
return
# Scale factor is 10
routes = "c"
if ts.minute == 0:
routes = "ac"
pqstr = "plot %s %s iowa_q2_%sh.png q2/iowa_q2_%sh_%s00.png png" % (
routes, ts.strftime("%Y%m%d%H%M"), hours, hours, ts.strftime("%H"))
lts = ts.astimezone(pytz.timezone("America/Chicago"))
subtitle = 'Total up to %s' % (lts.strftime("%d %B %Y %I:%M %p %Z"), )
mp = MapPlot(title=("NCEP MRMS Q3 (RADAR Only) %s Hour "
"Precipitation [inch]") % (hours, ),
subtitle=subtitle)
clevs = [0.01, 0.1, 0.3, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10]
mp.contourf(mrms.XAXIS,
mrms.YAXIS,
distance(np.flipud(total), 'MM').value('IN'),
clevs,
cmap=nwsprecip())
mp.drawcounties()
mp.postprocess(pqstr=pqstr, view=False)
mp.close()
def do(now, realtime, delta):
""" Generate for this timestep! """
szx = 7000
szy = 3500
# Create the image data
imgdata = np.zeros((szy, szx), 'u1')
sts = now - datetime.timedelta(minutes=2)
metadata = {'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
'end_valid': now.strftime("%Y-%m-%dT%H:%M:%SZ"),
'product': 'a2m',
'units': '0.02 mm'}
gribfn = mrms.fetch('PrecipRate', now)
if gribfn is None:
print(("mrms_rainrate_comp.py NODATA for PrecipRate: %s"
) % (now.strftime("%Y-%m-%dT%H:%MZ"),))
return
# http://www.nssl.noaa.gov/projects/mrms/operational/tables.php
# Says units are mm/hr
fp = gzip.GzipFile(gribfn, 'rb')
(_, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
os.unlink(gribfn)
val = grb['values']
# Convert into units of 0.1 mm accumulation
val = val / 60.0 * 2.0 * 50.0
val = np.where(val < 0., 255., val)
imgdata[:, :] = np.flipud(val.astype('int'))
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(np.flipud(imgdata))
png.putpalette(mrms.make_colorramp())
png.save('%s.png' % (tmpfn,))
mrms.write_worldfile('%s.wld' % (tmpfn,))
# Inject WLD file
routes = "c" if realtime else ""
prefix = 'a2m'
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot a%s %s "
"gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld"
"") % (routes, now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot a%s %s "
"gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png"
"") % (routes, now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
if realtime:
# Create 900913 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s "
"gis/images/900913/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif"
"") % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn,), 'w')
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s "
"gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' "
"%s.json") % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
for suffix in ['tif', 'json', 'png', 'wld']:
if os.path.isfile("%s.%s" % (tmpfn, suffix)):
os.unlink('%s.%s' % (tmpfn, suffix))
os.close(tmpfp)
os.unlink(tmpfn)
def run(ts):
"""Update netcdf file with the MRMS data
Args:
ts (datetime): timestamptz at midnight central time and we are running
forward in time
"""
nc = ncopen(iemre.get_daily_mrms_ncname(ts.year), "a", timeout=300)
offset = iemre.daily_offset(ts)
ncprecip = nc.variables["p01d"]
gmtts = ts.astimezone(pytz.UTC)
utcnow = utc()
total = None
lats = None
for _ in range(1, 25):
gmtts += datetime.timedelta(hours=1)
if gmtts > utcnow:
continue
gribfn = None
for prefix in ["MultiSensor_QPE_01H_Pass2", "RadarOnly_QPE_01H"]:
fn = mrms.fetch(prefix, gmtts)
if fn is None:
continue
fp = gzip.GzipFile(fn, "rb")
(_, gribfn) = tempfile.mkstemp()
tmpfp = open(gribfn, "wb")
tmpfp.write(fp.read())
tmpfp.close()
os.unlink(fn)
break
if gribfn is None:
if gmtts < utcnow:
LOG.info("MISSING %s", gmtts)
continue
grbs = pygrib.open(gribfn)
grb = grbs[1]
if lats is None:
lats, _ = grb.latlons()
os.unlink(gribfn)
val = grb["values"]
# Anything less than zero, we set to zero
val = np.where(val < 0, 0, val)
if total is None:
total = val
else:
total += val
if lats is None:
LOG.info("nodata for %s", ts.date())
return
# CAREFUL HERE! The MRMS grid is North to South
# set top (smallest y)
y0 = int((lats[0, 0] - iemre.NORTH) * 100.0)
y1 = int((lats[0, 0] - iemre.SOUTH) * 100.0)
x0 = int((iemre.WEST - mrms.WEST) * 100.0)
x1 = int((iemre.EAST - mrms.WEST) * 100.0)
# print(('y0:%s y1:%s x0:%s x1:%s lat0:%s offset:%s '
# ) % (y0, y1, x0, x1, lats[0, 0], offset))
ncprecip[offset, :, :] = np.flipud(total[y0:y1, x0:x1])
nc.close()
def do(now, realtime=False):
""" Generate for this timestep! """
szx = 7000
szy = 3500
# Create the image data
imgdata = np.zeros((szy, szx), "u1")
metadata = {
"start_valid": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
"end_valid": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
"product": "lcref",
"units": "0.5 dBZ",
}
gribfn = mrms.fetch("SeamlessHSR", now)
if gribfn is None:
print(("mrms_lcref_comp.py NODATA for SeamlessHSR: %s") %
(now.strftime("%Y-%m-%dT%H:%MZ"), ))
return
fp = gzip.GzipFile(gribfn, "rb")
(_, tmpfn) = tempfile.mkstemp()
with open(tmpfn, "wb") as fh:
fh.write(fp.read())
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
os.unlink(gribfn)
val = grb["values"]
# -999 is no coverage, go to 0
# -99 is missing , go to 255
val = np.where(val >= -32, (val + 32) * 2.0, val)
# val = np.where(val < -990., 0., val)
# val = np.where(val < -90., 255., val)
# This is an upstream BUG
val = np.where(val < 0.0, 0.0, val)
imgdata[:, :] = np.flipud(val.astype("int"))
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(np.flipud(imgdata))
png.putpalette(make_colorramp())
png.save("%s.png" % (tmpfn, ))
mrms.write_worldfile("%s.wld" % (tmpfn, ))
# Inject WLD file
prefix = "lcref"
routes = "ac" if realtime else "a"
pqstr = ("pqinsert -i -p 'plot %s %s "
"gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld") % (
routes,
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
LOG.debug(pqstr)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("pqinsert -i -p 'plot %s %s "
"gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png") % (
routes,
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
LOG.debug(pqstr)
subprocess.call(pqstr, shell=True)
# Create 3857 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("pqinsert -i -p 'plot c %s "
"gis/images/3857/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif") % (
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
if realtime:
LOG.debug(pqstr)
subprocess.call(pqstr, shell=True)
with open("%s.json" % (tmpfn, ), "w") as fh:
fh.write(json.dumps(dict(meta=metadata)))
# Insert into LDM
pqstr = (
"pqinsert -p 'plot c %s "
"gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' %s.json") % (
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
if realtime:
LOG.debug(pqstr)
subprocess.call(pqstr, shell=True)
for suffix in ["tif", "json", "png", "wld"]:
os.unlink(f"{tmpfn}.{suffix}")
os.close(tmpfp)
os.unlink(tmpfn)