def doit(gts, hr):
"""
Actually generate a PNG file from the 8 NMQ tiles
"""
sts = gts - datetime.timedelta(hours=hr)
times = [gts]
if hr > 24:
times.append(gts - datetime.timedelta(hours=24))
if hr == 72:
times.append(gts - datetime.timedelta(hours=48))
metadata = {'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
'end_valid': gts.strftime("%Y-%m-%dT%H:%M:%SZ")}
# Create the image data
# imgdata = np.zeros( (szy, szx), 'u1')
# timestep = np.zeros( (szy, szx), 'f')
total = None
for now in times:
gribfn = now.strftime(("/mnt/a4/data/%Y/%m/%d/mrms/ncep/"
"RadarOnly_QPE_24H/"
"RadarOnly_QPE_24H_00.00_%Y%m%d-%H%M00"
".grib2.gz"))
if not os.path.isfile(gribfn):
print("mrms_raster_pXXh.py MISSING %s" % (gribfn,))
return
fp = gzip.GzipFile(gribfn, 'rb')
(tmpfp, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
# careful here, how we deal with the two missing values!
if total is None:
total = grb['values']
else:
maxgrid = np.maximum(grb['values'], total)
total = np.where(np.logical_and(grb['values'] >= 0, total >= 0),
grb['values'] + total, maxgrid)
"""
255 levels... wanna do 0 to 20 inches
index 255 is missing, index 0 is 0
0-1 -> 100 - 0.01 res || 0 - 25 -> 100 - 0.25 mm 0
1-5 -> 80 - 0.05 res || 25 - 125 -> 80 - 1.25 mm 100
5-20 -> 75 - 0.20 res || 125 - 500 -> 75 - 5 mm 180
"""
# total = np.flipud(total)
# Off scale gets index 254
imgdata = convert_to_image(total)
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(imgdata.astype('u1'))
png.putpalette(mrms.make_colorramp())
png.save('%s.png' % (tmpfn,))
# os.system("xv %s.png" % (tmpfn,))
# Now we need to generate the world file
mrms.write_worldfile('%s.wld' % (tmpfn,))
# Inject WLD file
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot ac %s "
"gis/images/4326/mrms/p%sh.wld GIS/mrms/p%sh_%s.wld wld' %s.wld"
"") % (gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot ac %s "
"gis/images/4326/mrms/p%sh.png GIS/mrms/p%sh_%s.png png' %s.png"
"") % (gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Create 900913 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s "
"gis/images/900913/mrms/p%sh.tif GIS/mrms/p%sh_%s.tif tif' %s.tif"
"") % (gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn,), 'w')
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s "
"gis/images/4326/mrms/p%sh.json GIS/mrms/p%sh_%s.json json'"
" %s.json") % (gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
for suffix in ['tif', 'json', 'png', 'wld']:
os.unlink('%s.%s' % (tmpfn, suffix))
os.close(tmpfp)
os.unlink(tmpfn)
def do(now, realtime=False):
""" Generate for this timestep! """
szx = 7000
szy = 3500
# Create the image data
imgdata = np.zeros((szy, szx), 'u1')
metadata = {'start_valid': now.strftime("%Y-%m-%dT%H:%M:%SZ"),
'end_valid': now.strftime("%Y-%m-%dT%H:%M:%SZ"),
'product': 'lcref',
'units': '0.5 dBZ'}
gribfn = mrms.fetch('SeamlessHSR', now)
if gribfn is None:
print(("mrms_lcref_comp.py NODATA for SeamlessHSR: %s"
) % (now.strftime("%Y-%m-%dT%H:%MZ"),))
return
fp = gzip.GzipFile(gribfn, 'rb')
(_, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
os.unlink(gribfn)
val = grb['values']
# -999 is no coverage, go to 0
# -99 is missing , go to 255
val = np.where(val >= -32, (val + 32) * 2.0, val)
# val = np.where(val < -990., 0., val)
# val = np.where(val < -90., 255., val)
# This is an upstream BUG
val = np.where(val < 0., 0., val)
imgdata[:, :] = np.flipud(val.astype('int'))
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(np.flipud(imgdata))
png.putpalette(make_colorramp())
png.save('%s.png' % (tmpfn,))
mrms.write_worldfile('%s.wld' % (tmpfn,))
# Inject WLD file
prefix = 'lcref'
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot ac %s "
"gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld"
) % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot ac %s "
"gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png"
) % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Create 900913 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s "
"gis/images/900913/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif"
) % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn,), 'w')
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -p 'plot c %s "
"gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' %s.json"
) % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
for suffix in ['tif', 'json', 'png', 'wld']:
os.unlink('%s.%s' % (tmpfn, suffix))
os.close(tmpfp)
os.unlink(tmpfn)
def do(now, realtime=False):
""" Generate for this timestep! """
szx = 7000
szy = 3500
# Create the image data
imgdata = np.zeros((szy, szx), "u1")
metadata = {
"start_valid": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
"end_valid": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
"product": "lcref",
"units": "0.5 dBZ",
}
gribfn = mrms.fetch("SeamlessHSR", now)
if gribfn is None:
print(("mrms_lcref_comp.py NODATA for SeamlessHSR: %s") %
(now.strftime("%Y-%m-%dT%H:%MZ"), ))
return
fp = gzip.GzipFile(gribfn, "rb")
(_, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, "wb")
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
os.unlink(gribfn)
val = grb["values"]
# -999 is no coverage, go to 0
# -99 is missing , go to 255
val = np.where(val >= -32, (val + 32) * 2.0, val)
# val = np.where(val < -990., 0., val)
# val = np.where(val < -90., 255., val)
# This is an upstream BUG
val = np.where(val < 0.0, 0.0, val)
imgdata[:, :] = np.flipud(val.astype("int"))
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(np.flipud(imgdata))
png.putpalette(make_colorramp())
png.save("%s.png" % (tmpfn, ))
mrms.write_worldfile("%s.wld" % (tmpfn, ))
# Inject WLD file
prefix = "lcref"
pqstr = ("pqinsert -i -p 'plot ac %s "
"gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld") % (
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("pqinsert -i -p 'plot ac %s "
"gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png") % (
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
# Create 3857 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("pqinsert -i -p 'plot c %s "
"gis/images/3857/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif") % (
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn, ), "w")
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = (
"pqinsert -p 'plot c %s "
"gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' %s.json") % (
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
for suffix in ["tif", "json", "png", "wld"]:
os.unlink("%s.%s" % (tmpfn, suffix))
os.close(tmpfp)
os.unlink(tmpfn)
def test_write_worldfile(self):
"""see if we can write a world file"""
mrms.write_worldfile('/tmp/bah.wld')
self.assertTrue(os.path.isfile("/tmp/bah.wld"))
def doit(gts, hr):
"""
Actually generate a PNG file from the 8 NMQ tiles
"""
irealtime = is_realtime(gts)
routes = "ac" if irealtime else "a"
sts = gts - datetime.timedelta(hours=hr)
times = [gts]
if hr > 24:
times.append(gts - datetime.timedelta(hours=24))
if hr == 72:
times.append(gts - datetime.timedelta(hours=48))
metadata = {
"start_valid": sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
"end_valid": gts.strftime("%Y-%m-%dT%H:%M:%SZ"),
"units": "mm",
}
total = None
mproduct = "RadarOnly_QPE_24H" if hr >= 24 else "RadarOnly_QPE_01H"
for now in times:
gribfn = mrms.fetch(mproduct, now)
if gribfn is None:
print(
("make_mrms_rasters.py[%s] MISSING %s\n %s\n")
% (hr, now.strftime("%Y-%m-%dT%H:%MZ"), gribfn)
)
MISSED_FILES.append(gribfn)
return
DOWNLOADED_FILES.append(gribfn)
fp = gzip.GzipFile(gribfn, "rb")
(tmpfp, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, "wb")
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
# careful here, how we deal with the two missing values!
if total is None:
total = grb["values"]
else:
maxgrid = np.maximum(grb["values"], total)
total = np.where(
np.logical_and(grb["values"] >= 0, total >= 0),
grb["values"] + total,
maxgrid,
)
imgdata = convert_to_image(total)
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(imgdata.astype("u1"))
png.putpalette(mrms.make_colorramp())
png.save("%s.png" % (tmpfn,))
if irealtime:
# create a second PNG with null values set to black
imgdata = np.where(imgdata == 255, 0, imgdata)
png = Image.fromarray(imgdata.astype("u1"))
png.putpalette(mrms.make_colorramp())
png.save("%s_nn.png" % (tmpfn,))
# Now we need to generate the world file
mrms.write_worldfile("%s.wld" % (tmpfn,))
if irealtime:
mrms.write_worldfile("%s_nn.wld" % (tmpfn,))
# Inject WLD file
pqstr = (
"%s -i -p 'plot %s %s "
"gis/images/4326/mrms/p%ih.wld GIS/mrms/p%ih_%s.wld wld' "
"%s.wld"
""
) % (
PQI,
routes,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
if irealtime:
pqstr = (
"%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.wld "
"GIS/mrms/p%ih_%s.wld wld' "
"%s_nn.wld"
""
) % (
PQI,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = (
"%s -i -p 'plot %s %s "
"gis/images/4326/mrms/p%ih.png GIS/mrms/p%ih_%s.png png' "
"%s.png"
""
) % (
PQI,
routes,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
if irealtime:
# Now we inject into LDM
pqstr = (
"%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.png "
"GIS/mrms/p%ih_%s.png png' "
"%s_nn.png"
""
) % (
PQI,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
if irealtime:
# Create 900913 image
cmd = (
"gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif"
) % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
cmd = (
"gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s_nn.png %s_nn.tif"
) % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = (
"%s -i -p 'plot c %s "
"gis/images/900913/mrms/p%ih.tif "
"GIS/mrms/p%ih_%s.tif tif' "
"%s.tif"
""
) % (
PQI,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
pqstr = (
"%s -i -p 'plot c %s "
"gis/images/900913/mrms/p%ih_nn.tif "
"GIS/mrms/p%ih_%s.tif tif' "
"%s_nn.tif"
""
) % (
PQI,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn,), "w")
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = (
"%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih.json "
"GIS/mrms/p%ih_%s.json json'"
" %s.json"
) % (
PQI,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
pqstr = (
"%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.json "
"GIS/mrms/p%ih_%s.json json'"
" %s.json"
) % (
PQI,
gts.strftime("%Y%m%d%H%M"),
hr,
hr,
gts.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
for suffix in ["tif", "json", "png", "wld"]:
fn = "%s.%s" % (tmpfn, suffix)
if os.path.isfile(fn):
os.unlink(fn)
if irealtime:
for suffix in ["tif", "png", "wld"]:
fn = "%s_nn.%s" % (tmpfn, suffix)
if os.path.isfile(fn):
os.unlink(fn)
os.close(tmpfp)
os.unlink(tmpfn)
def workflow(now, realtime):
""" Generate for this timestep! """
szx = 7000
szy = 3500
# Create the image data
imgdata = np.zeros((szy, szx), "u1")
sts = now - datetime.timedelta(minutes=2)
metadata = {
"start_valid": sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
"end_valid": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
"product": "a2m",
"units": "0.02 mm",
}
gribfn = mrms.fetch("PrecipRate", now)
if gribfn is None:
LOG.info("NODATA for PrecipRate: %s", now.strftime("%Y-%m-%dT%H:%MZ"))
return
# http://www.nssl.noaa.gov/projects/mrms/operational/tables.php
# Says units are mm/hr
fp = gzip.GzipFile(gribfn, "rb")
(_, tmpfn) = tempfile.mkstemp()
with open(tmpfn, "wb") as fh:
fh.write(fp.read())
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
os.unlink(gribfn)
val = grb["values"]
# Convert into units of 0.1 mm accumulation
val = val / 60.0 * 2.0 * 50.0
val = np.where(val < 0.0, 255.0, val)
imgdata[:, :] = np.flipud(val.astype("int"))
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(np.flipud(imgdata))
png.putpalette(mrms.make_colorramp())
png.save("%s.png" % (tmpfn, ))
mrms.write_worldfile("%s.wld" % (tmpfn, ))
# Inject WLD file
routes = "c" if realtime else ""
prefix = "a2m"
pqstr = ("pqinsert -i -p 'plot a%s %s "
"gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld"
"") % (
routes,
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("pqinsert -i -p 'plot a%s %s "
"gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png"
"") % (
routes,
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
if realtime:
# Create 3857 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("pqinsert -i -p 'plot c %s "
"gis/images/3857/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif"
"") % (
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
with open("%s.json" % (tmpfn, ), "w") as fh:
fh.write(json.dumps(dict(meta=metadata)))
# Insert into LDM
pqstr = ("pqinsert -i -p 'plot c %s "
"gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' "
"%s.json") % (
now.strftime("%Y%m%d%H%M"),
prefix,
prefix,
now.strftime("%Y%m%d%H%M"),
tmpfn,
)
subprocess.call(pqstr, shell=True)
for suffix in ["tif", "json", "png", "wld"]:
if os.path.isfile("%s.%s" % (tmpfn, suffix)):
os.unlink("%s.%s" % (tmpfn, suffix))
os.close(tmpfp)
os.unlink(tmpfn)
def test_write_worldfile(self):
"""see if we can write a world file"""
mrms.write_worldfile("/tmp/bah.wld")
def test_write_worldfile():
"""see if we can write a world file"""
mrms.write_worldfile("/tmp/bah.wld")
assert os.path.isfile("/tmp/bah.wld")
def doit(gts, hr):
"""
Actually generate a PNG file from the 8 NMQ tiles
"""
irealtime = is_realtime(gts)
routes = "ac" if irealtime else "a"
sts = gts - datetime.timedelta(hours=hr)
times = [gts]
if hr > 24:
times.append(gts - datetime.timedelta(hours=24))
if hr == 72:
times.append(gts - datetime.timedelta(hours=48))
metadata = {'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
'end_valid': gts.strftime("%Y-%m-%dT%H:%M:%SZ"),
'units': 'mm'}
total = None
mproduct = "RadarOnly_QPE_24H" if hr >= 24 else "RadarOnly_QPE_01H"
for now in times:
uri = now.strftime(("http://mtarchive.geol.iastate.edu/%Y/%m/%d/"
"mrms/ncep/" + mproduct + "/" + mproduct +
"_00.00_%Y%m%d-%H%M00.grib2.gz"))
gribfn = now.strftime((TMP + "/" + mproduct +
"_00.00_%Y%m%d-%H%M00.grib2.gz"))
if not os.path.isfile(gribfn):
res = requests.get(uri, timeout=60)
if res.status_code != 200:
# Cut down on logged errors
if gribfn in MISSED_FILES:
return
print(("make_mrms_rasters.py[%s] MISSING %s\n %s\n %s\n"
) % (hr, now.strftime("%Y-%m-%dT%H:%MZ"), uri, gribfn))
MISSED_FILES.append(gribfn)
return
o = open(gribfn, 'wb')
o.write(res.content)
o.close()
DOWNLOADED_FILES.append(gribfn)
fp = gzip.GzipFile(gribfn, 'rb')
(tmpfp, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
# careful here, how we deal with the two missing values!
if total is None:
total = grb['values']
else:
maxgrid = np.maximum(grb['values'], total)
total = np.where(np.logical_and(grb['values'] >= 0, total >= 0),
grb['values'] + total, maxgrid)
imgdata = convert_to_image(total)
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(imgdata.astype('u1'))
png.putpalette(mrms.make_colorramp())
png.save('%s.png' % (tmpfn,))
if irealtime:
# create a second PNG with null values set to black
imgdata = np.where(imgdata == 255, 0, imgdata)
png = Image.fromarray(imgdata.astype('u1'))
png.putpalette(mrms.make_colorramp())
png.save('%s_nn.png' % (tmpfn,))
# Now we need to generate the world file
mrms.write_worldfile('%s.wld' % (tmpfn,))
if irealtime:
mrms.write_worldfile('%s_nn.wld' % (tmpfn,))
# Inject WLD file
pqstr = ("%s -i -p 'plot %s %s "
"gis/images/4326/mrms/p%ih.wld GIS/mrms/p%ih_%s.wld wld' "
"%s.wld"
"") % (PQI, routes, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
if irealtime:
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.wld "
"GIS/mrms/p%ih_%s.wld wld' "
"%s_nn.wld"
"") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("%s -i -p 'plot %s %s "
"gis/images/4326/mrms/p%ih.png GIS/mrms/p%ih_%s.png png' "
"%s.png"
"") % (PQI, routes, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
if irealtime:
# Now we inject into LDM
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.png "
"GIS/mrms/p%ih_%s.png png' "
"%s_nn.png"
"") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
if irealtime:
# Create 900913 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s_nn.png %s_nn.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("%s -i -p 'plot c %s "
"gis/images/900913/mrms/p%ih.tif "
"GIS/mrms/p%ih_%s.tif tif' "
"%s.tif"
"") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
pqstr = ("%s -i -p 'plot c %s "
"gis/images/900913/mrms/p%ih_nn.tif "
"GIS/mrms/p%ih_%s.tif tif' "
"%s_nn.tif"
"") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn,), 'w')
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih.json "
"GIS/mrms/p%ih_%s.json json'"
" %s.json") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.json "
"GIS/mrms/p%ih_%s.json json'"
" %s.json") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
for suffix in ['tif', 'json', 'png', 'wld']:
fn = '%s.%s' % (tmpfn, suffix)
if os.path.isfile(fn):
os.unlink(fn)
if irealtime:
for suffix in ['tif', 'png', 'wld']:
fn = '%s_nn.%s' % (tmpfn, suffix)
if os.path.isfile(fn):
os.unlink(fn)
os.close(tmpfp)
os.unlink(tmpfn)
def doit(gts, hr):
"""
Actually generate a PNG file from the 8 NMQ tiles
"""
irealtime = is_realtime(gts)
routes = "ac" if irealtime else "a"
sts = gts - datetime.timedelta(hours=hr)
times = [gts]
if hr > 24:
times.append(gts - datetime.timedelta(hours=24))
if hr == 72:
times.append(gts - datetime.timedelta(hours=48))
metadata = {
'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
'end_valid': gts.strftime("%Y-%m-%dT%H:%M:%SZ"),
'units': 'mm'
}
total = None
mproduct = "RadarOnly_QPE_24H" if hr >= 24 else "RadarOnly_QPE_01H"
for now in times:
uri = now.strftime(("http://mtarchive.geol.iastate.edu/%Y/%m/%d/"
"mrms/ncep/" + mproduct + "/" + mproduct +
"_00.00_%Y%m%d-%H%M00.grib2.gz"))
gribfn = now.strftime(
(TMP + "/" + mproduct + "_00.00_%Y%m%d-%H%M00.grib2.gz"))
if not os.path.isfile(gribfn):
res = requests.get(uri, timeout=60)
if res.status_code != 200:
# Cut down on logged errors
if gribfn in MISSED_FILES:
return
print(("make_mrms_rasters.py[%s] MISSING %s\n %s\n %s\n") %
(hr, now.strftime("%Y-%m-%dT%H:%MZ"), uri, gribfn))
MISSED_FILES.append(gribfn)
return
o = open(gribfn, 'wb')
o.write(res.content)
o.close()
DOWNLOADED_FILES.append(gribfn)
fp = gzip.GzipFile(gribfn, 'rb')
(tmpfp, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
# careful here, how we deal with the two missing values!
if total is None:
total = grb['values']
else:
maxgrid = np.maximum(grb['values'], total)
total = np.where(np.logical_and(grb['values'] >= 0, total >= 0),
grb['values'] + total, maxgrid)
imgdata = convert_to_image(total)
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(imgdata.astype('u1'))
png.putpalette(mrms.make_colorramp())
png.save('%s.png' % (tmpfn, ))
if irealtime:
# create a second PNG with null values set to black
imgdata = np.where(imgdata == 255, 0, imgdata)
png = Image.fromarray(imgdata.astype('u1'))
png.putpalette(mrms.make_colorramp())
png.save('%s_nn.png' % (tmpfn, ))
# Now we need to generate the world file
mrms.write_worldfile('%s.wld' % (tmpfn, ))
if irealtime:
mrms.write_worldfile('%s_nn.wld' % (tmpfn, ))
# Inject WLD file
pqstr = ("%s -i -p 'plot %s %s "
"gis/images/4326/mrms/p%ih.wld GIS/mrms/p%ih_%s.wld wld' "
"%s.wld"
"") % (PQI, routes, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
if irealtime:
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.wld "
"GIS/mrms/p%ih_%s.wld wld' "
"%s_nn.wld"
"") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("%s -i -p 'plot %s %s "
"gis/images/4326/mrms/p%ih.png GIS/mrms/p%ih_%s.png png' "
"%s.png"
"") % (PQI, routes, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
if irealtime:
# Now we inject into LDM
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.png "
"GIS/mrms/p%ih_%s.png png' "
"%s_nn.png"
"") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
if irealtime:
# Create 900913 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s_nn.png %s_nn.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("%s -i -p 'plot c %s "
"gis/images/900913/mrms/p%ih.tif "
"GIS/mrms/p%ih_%s.tif tif' "
"%s.tif"
"") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
pqstr = ("%s -i -p 'plot c %s "
"gis/images/900913/mrms/p%ih_nn.tif "
"GIS/mrms/p%ih_%s.tif tif' "
"%s_nn.tif"
"") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn, ), 'w')
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih.json "
"GIS/mrms/p%ih_%s.json json'"
" %s.json") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
pqstr = ("%s -i -p 'plot c %s "
"gis/images/4326/mrms/p%ih_nn.json "
"GIS/mrms/p%ih_%s.json json'"
" %s.json") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr,
gts.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
for suffix in ['tif', 'json', 'png', 'wld']:
fn = '%s.%s' % (tmpfn, suffix)
if os.path.isfile(fn):
os.unlink(fn)
if irealtime:
for suffix in ['tif', 'png', 'wld']:
fn = '%s_nn.%s' % (tmpfn, suffix)
if os.path.isfile(fn):
os.unlink(fn)
os.close(tmpfp)
os.unlink(tmpfn)
def do(now, realtime, delta):
""" Generate for this timestep! """
szx = 7000
szy = 3500
# Create the image data
imgdata = np.zeros((szy, szx), 'u1')
sts = now - datetime.timedelta(minutes=2)
metadata = {
'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
'end_valid': now.strftime("%Y-%m-%dT%H:%M:%SZ"),
'product': 'a2m',
'units': '0.02 mm'
}
fn = now.strftime("PrecipRate_00.00_%Y%m%d-%H%M00.grib2.gz")
uri = now.strftime(("http://mtarchive.geol.iastate.edu/%Y/%m/%d/"
"mrms/ncep/PrecipRate/" + fn))
gribfn = "%s/%s" % (TMP, fn)
res = requests.get(uri, timeout=30)
if res.status_code != 200:
print("mrms_rainrate_comp.py MISSING %s" % (gribfn, ))
return
o = open(gribfn, 'wb')
o.write(res.content)
o.close()
# http://www.nssl.noaa.gov/projects/mrms/operational/tables.php
# Says units are mm/hr
fp = gzip.GzipFile(gribfn, 'rb')
(_, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
os.unlink(gribfn)
val = grb['values']
# Convert into units of 0.1 mm accumulation
val = val / 60.0 * 2.0 * 50.0
val = np.where(val < 0., 255., val)
imgdata[:, :] = np.flipud(val.astype('int'))
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(np.flipud(imgdata))
png.putpalette(mrms.make_colorramp())
png.save('%s.png' % (tmpfn, ))
mrms.write_worldfile('%s.wld' % (tmpfn, ))
# Inject WLD file
routes = "c" if realtime else ""
prefix = 'a2m'
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot a%s %s "
"gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld"
"") % (routes, now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot a%s %s "
"gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png"
"") % (routes, now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
if realtime:
# Create 900913 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s "
"gis/images/900913/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif"
"") % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn, ), 'w')
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s "
"gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' "
"%s.json") % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
for suffix in ['tif', 'json', 'png', 'wld']:
if os.path.isfile("%s.%s" % (tmpfn, suffix)):
os.unlink('%s.%s' % (tmpfn, suffix))
os.close(tmpfp)
os.unlink(tmpfn)
def do(now, hr ):
''' Generate for this timestep!
255 levels... wanna do 0 to 20 inches
index 255 is missing, index 0 is 0
0-1 -> 100 - 0.01 res || 0 - 25 -> 100 - 0.25 mm 0
1-5 -> 80 - 0.05 res || 25 - 125 -> 80 - 1.25 mm 100
5-20 -> 75 - 0.20 res || 125 - 500 -> 75 - 5 mm 180
'''
szx = 7000
szy = 3500
sts = now - datetime.timedelta(hours=hr)
metadata = {'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
'end_valid': now.strftime("%Y-%m-%dT%H:%M:%SZ"),
'product': 'lcref',
'units': 'mm' }
gribfn = now.strftime(("/mnt/a4/data/%Y/%m/%d/mrms/ncep/"
+"RadarOnly_QPE_01H/"
+"RadarOnly_QPE_01H_00.00_%Y%m%d-%H%M00.grib2.gz"))
if not os.path.isfile(gribfn):
print("mrms_raster_p1h.py MISSING %s" % (gribfn,))
return
fp = gzip.GzipFile(gribfn, 'rb')
(tmpfp, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
total = grb['values']
# Off scale gets index 254
imgdata = np.where(total >= 500, 254, 0)
imgdata = np.where(np.logical_and(total >= 125, total < 500),
180 + ((total - 125.) / 5.0), imgdata)
imgdata = np.where(np.logical_and(total >= 25, total < 125),
100 + ((total - 25.) / 1.25), imgdata)
imgdata = np.where(np.logical_and(total >= 0, total < 25),
total / 0.25, imgdata)
# -3 is no coverage -> 255
# -1 is misisng, so zero
# Index 255 is missing
imgdata = np.where( total < 0, 0, imgdata)
imgdata = np.where( total < -1, 255, imgdata)
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray( imgdata.astype('u1') )
png.putpalette(mrms.make_colorramp())
png.save('%s.png' % (tmpfn,))
# os.system('xv %s.png' % (tmpfn,))
mrms.write_worldfile('%s.wld' % (tmpfn,))
# Inject WLD file
prefix = 'p%sh' % (hr,)
pqstr = "/home/ldm/bin/pqinsert -i -p 'plot ac %s gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld" % (
now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn )
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = "/home/ldm/bin/pqinsert -p 'plot ac %s gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png" % (
now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn )
subprocess.call(pqstr, shell=True)
# Create 900913 image
cmd = "gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff -tr 1000.0 1000.0 %s.png %s.tif" % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = "/home/ldm/bin/pqinsert -p 'plot c %s gis/images/900913/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif" % (
now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn )
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn,), 'w')
j.write( json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = "/home/ldm/bin/pqinsert -p 'plot c %s gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' %s.json" % (
now.strftime("%Y%m%d%H%M"),prefix, prefix, now.strftime("%Y%m%d%H%M"), tmpfn )
subprocess.call(pqstr, shell=True)
for suffix in ['tif', 'json', 'png', 'wld']:
os.unlink('%s.%s' % (tmpfn, suffix))
os.close(tmpfp)
os.unlink(tmpfn)
def do(now, realtime, delta):
""" Generate for this timestep! """
szx = 7000
szy = 3500
# Create the image data
imgdata = np.zeros((szy, szx), 'u1')
sts = now - datetime.timedelta(minutes=2)
metadata = {'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"),
'end_valid': now.strftime("%Y-%m-%dT%H:%M:%SZ"),
'product': 'a2m',
'units': '0.1 mm'}
gribfn = now.strftime(("/mnt/a4/data/%Y/%m/%d/mrms/ncep/PrecipRate/"
"PrecipRate_00.00_%Y%m%d-%H%M00.grib2.gz"))
if not os.path.isfile(gribfn):
# Don't whine about old files being missing
if delta < 90:
print("mrms_rainrate_comp.py MISSING %s" % (gribfn,))
return
# http://www.nssl.noaa.gov/projects/mrms/operational/tables.php
# Says units are mm/hr
fp = gzip.GzipFile(gribfn, 'rb')
(_, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
val = grb['values']
# Convert into units of 0.1 mm accumulation
val = val / 60.0 * 2.0 * 10.0
val = np.where(val < 0., 255., val)
imgdata[:, :] = np.flipud(val.astype('int'))
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(np.flipud(imgdata))
png.putpalette(mrms.make_colorramp())
png.save('%s.png' % (tmpfn,))
mrms.write_worldfile('%s.wld' % (tmpfn,))
# Inject WLD file
routes = "c" if realtime else ""
prefix = 'a2m'
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot a%s %s "
"gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld"
"") % (routes, now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot a%s %s "
"gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png"
"") % (routes, now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
if realtime:
# Create 900913 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s "
"gis/images/900913/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif"
"") % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn,), 'w')
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s "
"gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' "
"%s.json") % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
for suffix in ['tif', 'json', 'png', 'wld']:
if os.path.isfile("%s.%s" % (tmpfn, suffix)):
os.unlink('%s.%s' % (tmpfn, suffix))
os.close(tmpfp)
os.unlink(tmpfn)
def test_write_worldfile():
"""see if we can write a world file"""
mrms.write_worldfile('/tmp/bah.wld')
assert os.path.isfile("/tmp/bah.wld")
def do(now, realtime=False):
""" Generate for this timestep! """
szx = 7000
szy = 3500
# Create the image data
imgdata = np.zeros((szy, szx), 'u1')
metadata = {
'start_valid': now.strftime("%Y-%m-%dT%H:%M:%SZ"),
'end_valid': now.strftime("%Y-%m-%dT%H:%M:%SZ"),
'product': 'lcref',
'units': '0.5 dBZ'
}
gribfn = now.strftime(("/mnt/a4/data/%Y/%m/%d/mrms/ncep/SeamlessHSR/"
"SeamlessHSR_00.00_%Y%m%d-%H%M00.grib2.gz"))
if not os.path.isfile(gribfn):
print("mrms_lcref_comp.py MISSING %s" % (gribfn, ))
return
fp = gzip.GzipFile(gribfn, 'rb')
(_, tmpfn) = tempfile.mkstemp()
tmpfp = open(tmpfn, 'wb')
tmpfp.write(fp.read())
tmpfp.close()
grbs = pygrib.open(tmpfn)
grb = grbs[1]
os.unlink(tmpfn)
val = grb['values']
# -999 is no coverage, go to 0
# -99 is missing , go to 255
val = np.where(val >= -32, (val + 32) * 2.0, val)
# val = np.where(val < -990., 0., val)
# val = np.where(val < -90., 255., val)
# This is an upstream BUG
val = np.where(val < 0., 0., val)
imgdata[:, :] = np.flipud(val.astype('int'))
(tmpfp, tmpfn) = tempfile.mkstemp()
# Create Image
png = Image.fromarray(np.flipud(imgdata))
png.putpalette(make_colorramp())
png.save('%s.png' % (tmpfn, ))
mrms.write_worldfile('%s.wld' % (tmpfn, ))
# Inject WLD file
prefix = 'lcref'
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot ac %s "
"gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld") % (
now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Now we inject into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot ac %s "
"gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png") % (
now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
# Create 900913 image
cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff "
"-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn)
subprocess.call(cmd, shell=True)
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s "
"gis/images/900913/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif"
) % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
j = open("%s.json" % (tmpfn, ), 'w')
j.write(json.dumps(dict(meta=metadata)))
j.close()
# Insert into LDM
pqstr = ("/home/ldm/bin/pqinsert -p 'plot c %s "
"gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' %s.json"
) % (now.strftime("%Y%m%d%H%M"), prefix, prefix,
now.strftime("%Y%m%d%H%M"), tmpfn)
subprocess.call(pqstr, shell=True)
for suffix in ['tif', 'json', 'png', 'wld']:
os.unlink('%s.%s' % (tmpfn, suffix))
os.close(tmpfp)
os.unlink(tmpfn)
