def to_raster(tmpfn, now): """ Convert the raw data into a RASTER Image 5 inch rain per hour is ~ 125 mm/hr, so over 5min that is 10 mm Index 255 is missing 0 is zero 1 is 0.1 mm 254 is 25.4 mm """ data = np.loadtxt(tmpfn, skiprows=10) # mm/hr to mm/5min imgdata = data * 10.0 / 12.0 imgdata = np.where(imgdata < 0, 255, imgdata) png = Image.fromarray(np.uint8(imgdata)) png.putpalette(mrms.make_colorramp()) meta = PngImagePlugin.PngInfo() meta.add_text("title", now.strftime("%Y%m%d%H%M"), 0) png.save("%s.png" % (tmpfn, ), pnginfo=meta) del png # Make worldfile with open("%s.wld" % (tmpfn, ), "w") as fh: fh.write("""0.004167 0.00 0.00 -0.004167 44.53785 -89.89942""")
def to_raster(tmpfn, now): """ Convert the raw data into a RASTER Image 5 inch rain per hour is ~ 125 mm/hr, so over 5min that is 10 mm Index 255 is missing 0 is zero 1 is 0.1 mm 254 is 25.4 mm """ data = np.loadtxt(tmpfn, skiprows=10) # mm/hr to mm/5min imgdata = (data * 10.0 / 12.0) imgdata = np.where(imgdata < 0, 255, imgdata) png = Image.fromarray(np.uint8(imgdata)) png.putpalette(mrms.make_colorramp()) meta = PngImagePlugin.PngInfo() meta.add_text('title', now.strftime("%Y%m%d%H%M"), 0) png.save('%s.png' % (tmpfn,), pnginfo=meta) del png # Make worldfile o = open("%s.wld" % (tmpfn, ), 'w') o.write("""0.004167 0.00 0.00 -0.004167 44.53785 -89.89942""") o.close()
def main(): """Go Main Go.""" data = numpy.zeros( (30,256), numpy.uint8) for i in range(255): data[0:15,i:i+1] = i png = Image.fromarray( data ) #png.putpalette( make_gr_colorramp() ) ramp = list(mrms.make_colorramp()) ramp[-3:] = [255,255,255] png.putpalette(ramp) draw = ImageDraw.Draw(png) #24h Precip # 0-1 -> 100 - 0.01 res || 0 - 25 -> 100 - 0.25 mm 0 # 1-5 -> 80 - 0.05 res || 25 - 125 -> 80 - 1.25 mm 100 # 5-20 -> 75 - 0.20 res || 125 - 500 -> 75 - 5 mm 180 xs = [25, 50, 75, 100, 120, 140, 160, 180, 190, 205, 230] ys = [0.25, 0.5, 0.75, 1, 2, 3, 4, 5, 7, 10, 15] # 1hr precip # 1 mm resolution #xs = numpy.arange(25,255, 25) #ys = numpy.arange(1,10) for x,y in zip(xs, ys): (w,h) = font.getsize(`y`) draw.line( [x,17,x,10], fill=255) draw.text( (x-(w/2),18), `y`,fill=255, font=font) draw.text( (235,18), 'in',fill=255, font=font) """ #DBZ for x in range(6,235,20): dbz = int(x/2.0 - 33) (w,h) = font.getsize(`dbz`) draw.line( [x,17,x,10], fill=255) draw.text( (x-(w/2),18), `dbz`,fill=255, font=font) draw.text( (235,18), 'dBZ',fill=255, font=font) """ #draw.line( [0,0,255,0,255,29,0,29,0,0], fill=255) png.save("test.png")
def doit(gts, hr): """ Actually generate a PNG file from the 8 NMQ tiles """ irealtime = is_realtime(gts) routes = "ac" if irealtime else "a" sts = gts - datetime.timedelta(hours=hr) times = [gts] if hr > 24: times.append(gts - datetime.timedelta(hours=24)) if hr == 72: times.append(gts - datetime.timedelta(hours=48)) metadata = {'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"), 'end_valid': gts.strftime("%Y-%m-%dT%H:%M:%SZ"), 'units': 'mm'} total = None mproduct = "RadarOnly_QPE_24H" if hr >= 24 else "RadarOnly_QPE_01H" for now in times: uri = now.strftime(("http://mtarchive.geol.iastate.edu/%Y/%m/%d/" "mrms/ncep/" + mproduct + "/" + mproduct + "_00.00_%Y%m%d-%H%M00.grib2.gz")) gribfn = now.strftime((TMP + "/" + mproduct + "_00.00_%Y%m%d-%H%M00.grib2.gz")) if not os.path.isfile(gribfn): res = requests.get(uri, timeout=60) if res.status_code != 200: # Cut down on logged errors if gribfn in MISSED_FILES: return print(("make_mrms_rasters.py[%s] MISSING %s\n %s\n %s\n" ) % (hr, now.strftime("%Y-%m-%dT%H:%MZ"), uri, gribfn)) MISSED_FILES.append(gribfn) return o = open(gribfn, 'wb') o.write(res.content) o.close() DOWNLOADED_FILES.append(gribfn) fp = gzip.GzipFile(gribfn, 'rb') (tmpfp, tmpfn) = tempfile.mkstemp() tmpfp = open(tmpfn, 'wb') tmpfp.write(fp.read()) tmpfp.close() grbs = pygrib.open(tmpfn) grb = grbs[1] os.unlink(tmpfn) # careful here, how we deal with the two missing values! if total is None: total = grb['values'] else: maxgrid = np.maximum(grb['values'], total) total = np.where(np.logical_and(grb['values'] >= 0, total >= 0), grb['values'] + total, maxgrid) imgdata = convert_to_image(total) (tmpfp, tmpfn) = tempfile.mkstemp() # Create Image png = Image.fromarray(imgdata.astype('u1')) png.putpalette(mrms.make_colorramp()) png.save('%s.png' % (tmpfn,)) if irealtime: # create a second PNG with null values set to black imgdata = np.where(imgdata == 255, 0, imgdata) png = Image.fromarray(imgdata.astype('u1')) png.putpalette(mrms.make_colorramp()) png.save('%s_nn.png' % (tmpfn,)) # Now we need to generate the world file mrms.write_worldfile('%s.wld' % (tmpfn,)) if irealtime: mrms.write_worldfile('%s_nn.wld' % (tmpfn,)) # Inject WLD file pqstr = ("%s -i -p 'plot %s %s " "gis/images/4326/mrms/p%ih.wld GIS/mrms/p%ih_%s.wld wld' " "%s.wld" "") % (PQI, routes, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) if irealtime: pqstr = ("%s -i -p 'plot c %s " "gis/images/4326/mrms/p%ih_nn.wld " "GIS/mrms/p%ih_%s.wld wld' " "%s_nn.wld" "") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) # Now we inject into LDM pqstr = ("%s -i -p 'plot %s %s " "gis/images/4326/mrms/p%ih.png GIS/mrms/p%ih_%s.png png' " "%s.png" "") % (PQI, routes, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) if irealtime: # Now we inject into LDM pqstr = ("%s -i -p 'plot c %s " "gis/images/4326/mrms/p%ih_nn.png " "GIS/mrms/p%ih_%s.png png' " "%s_nn.png" "") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) if irealtime: # Create 900913 image cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff " "-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn) subprocess.call(cmd, shell=True) cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff " "-tr 1000.0 1000.0 %s_nn.png %s_nn.tif") % (tmpfn, tmpfn) subprocess.call(cmd, shell=True) # Insert into LDM pqstr = ("%s -i -p 'plot c %s " "gis/images/900913/mrms/p%ih.tif " "GIS/mrms/p%ih_%s.tif tif' " "%s.tif" "") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) pqstr = ("%s -i -p 'plot c %s " "gis/images/900913/mrms/p%ih_nn.tif " "GIS/mrms/p%ih_%s.tif tif' " "%s_nn.tif" "") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) j = open("%s.json" % (tmpfn,), 'w') j.write(json.dumps(dict(meta=metadata))) j.close() # Insert into LDM pqstr = ("%s -i -p 'plot c %s " "gis/images/4326/mrms/p%ih.json " "GIS/mrms/p%ih_%s.json json'" " %s.json") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) pqstr = ("%s -i -p 'plot c %s " "gis/images/4326/mrms/p%ih_nn.json " "GIS/mrms/p%ih_%s.json json'" " %s.json") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) for suffix in ['tif', 'json', 'png', 'wld']: fn = '%s.%s' % (tmpfn, suffix) if os.path.isfile(fn): os.unlink(fn) if irealtime: for suffix in ['tif', 'png', 'wld']: fn = '%s_nn.%s' % (tmpfn, suffix) if os.path.isfile(fn): os.unlink(fn) os.close(tmpfp) os.unlink(tmpfn)
def doit(gts, hr): """ Actually generate a PNG file from the 8 NMQ tiles """ sts = gts - datetime.timedelta(hours=hr) times = [gts] if hr > 24: times.append(gts - datetime.timedelta(hours=24)) if hr == 72: times.append(gts - datetime.timedelta(hours=48)) metadata = {'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"), 'end_valid': gts.strftime("%Y-%m-%dT%H:%M:%SZ")} # Create the image data # imgdata = np.zeros( (szy, szx), 'u1') # timestep = np.zeros( (szy, szx), 'f') total = None for now in times: gribfn = now.strftime(("/mnt/a4/data/%Y/%m/%d/mrms/ncep/" "RadarOnly_QPE_24H/" "RadarOnly_QPE_24H_00.00_%Y%m%d-%H%M00" ".grib2.gz")) if not os.path.isfile(gribfn): print("mrms_raster_pXXh.py MISSING %s" % (gribfn,)) return fp = gzip.GzipFile(gribfn, 'rb') (tmpfp, tmpfn) = tempfile.mkstemp() tmpfp = open(tmpfn, 'wb') tmpfp.write(fp.read()) tmpfp.close() grbs = pygrib.open(tmpfn) grb = grbs[1] os.unlink(tmpfn) # careful here, how we deal with the two missing values! if total is None: total = grb['values'] else: maxgrid = np.maximum(grb['values'], total) total = np.where(np.logical_and(grb['values'] >= 0, total >= 0), grb['values'] + total, maxgrid) """ 255 levels... wanna do 0 to 20 inches index 255 is missing, index 0 is 0 0-1 -> 100 - 0.01 res || 0 - 25 -> 100 - 0.25 mm 0 1-5 -> 80 - 0.05 res || 25 - 125 -> 80 - 1.25 mm 100 5-20 -> 75 - 0.20 res || 125 - 500 -> 75 - 5 mm 180 """ # total = np.flipud(total) # Off scale gets index 254 imgdata = convert_to_image(total) (tmpfp, tmpfn) = tempfile.mkstemp() # Create Image png = Image.fromarray(imgdata.astype('u1')) png.putpalette(mrms.make_colorramp()) png.save('%s.png' % (tmpfn,)) # os.system("xv %s.png" % (tmpfn,)) # Now we need to generate the world file mrms.write_worldfile('%s.wld' % (tmpfn,)) # Inject WLD file pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot ac %s " "gis/images/4326/mrms/p%sh.wld GIS/mrms/p%sh_%s.wld wld' %s.wld" "") % (gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) # Now we inject into LDM pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot ac %s " "gis/images/4326/mrms/p%sh.png GIS/mrms/p%sh_%s.png png' %s.png" "") % (gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) # Create 900913 image cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff " "-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn) subprocess.call(cmd, shell=True) # Insert into LDM pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s " "gis/images/900913/mrms/p%sh.tif GIS/mrms/p%sh_%s.tif tif' %s.tif" "") % (gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) j = open("%s.json" % (tmpfn,), 'w') j.write(json.dumps(dict(meta=metadata))) j.close() # Insert into LDM pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s " "gis/images/4326/mrms/p%sh.json GIS/mrms/p%sh_%s.json json'" " %s.json") % (gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) for suffix in ['tif', 'json', 'png', 'wld']: os.unlink('%s.%s' % (tmpfn, suffix)) os.close(tmpfp) os.unlink(tmpfn)
def test_colorramp(self): """See what we can do with a colorramp""" cmap = mrms.make_colorramp() self.assertEqual(len(cmap), 256*3)
def workflow(now, realtime): """ Generate for this timestep! """ szx = 7000 szy = 3500 # Create the image data imgdata = np.zeros((szy, szx), "u1") sts = now - datetime.timedelta(minutes=2) metadata = { "start_valid": sts.strftime("%Y-%m-%dT%H:%M:%SZ"), "end_valid": now.strftime("%Y-%m-%dT%H:%M:%SZ"), "product": "a2m", "units": "0.02 mm", } gribfn = mrms.fetch("PrecipRate", now) if gribfn is None: LOG.info("NODATA for PrecipRate: %s", now.strftime("%Y-%m-%dT%H:%MZ")) return # http://www.nssl.noaa.gov/projects/mrms/operational/tables.php # Says units are mm/hr fp = gzip.GzipFile(gribfn, "rb") (_, tmpfn) = tempfile.mkstemp() with open(tmpfn, "wb") as fh: fh.write(fp.read()) grbs = pygrib.open(tmpfn) grb = grbs[1] os.unlink(tmpfn) os.unlink(gribfn) val = grb["values"] # Convert into units of 0.1 mm accumulation val = val / 60.0 * 2.0 * 50.0 val = np.where(val < 0.0, 255.0, val) imgdata[:, :] = np.flipud(val.astype("int")) (tmpfp, tmpfn) = tempfile.mkstemp() # Create Image png = Image.fromarray(np.flipud(imgdata)) png.putpalette(mrms.make_colorramp()) png.save("%s.png" % (tmpfn, )) mrms.write_worldfile("%s.wld" % (tmpfn, )) # Inject WLD file routes = "c" if realtime else "" prefix = "a2m" pqstr = ("pqinsert -i -p 'plot a%s %s " "gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld" "") % ( routes, now.strftime("%Y%m%d%H%M"), prefix, prefix, now.strftime("%Y%m%d%H%M"), tmpfn, ) subprocess.call(pqstr, shell=True) # Now we inject into LDM pqstr = ("pqinsert -i -p 'plot a%s %s " "gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png" "") % ( routes, now.strftime("%Y%m%d%H%M"), prefix, prefix, now.strftime("%Y%m%d%H%M"), tmpfn, ) subprocess.call(pqstr, shell=True) if realtime: # Create 3857 image cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff " "-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn) subprocess.call(cmd, shell=True) # Insert into LDM pqstr = ("pqinsert -i -p 'plot c %s " "gis/images/3857/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif" "") % ( now.strftime("%Y%m%d%H%M"), prefix, prefix, now.strftime("%Y%m%d%H%M"), tmpfn, ) subprocess.call(pqstr, shell=True) with open("%s.json" % (tmpfn, ), "w") as fh: fh.write(json.dumps(dict(meta=metadata))) # Insert into LDM pqstr = ("pqinsert -i -p 'plot c %s " "gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' " "%s.json") % ( now.strftime("%Y%m%d%H%M"), prefix, prefix, now.strftime("%Y%m%d%H%M"), tmpfn, ) subprocess.call(pqstr, shell=True) for suffix in ["tif", "json", "png", "wld"]: if os.path.isfile("%s.%s" % (tmpfn, suffix)): os.unlink("%s.%s" % (tmpfn, suffix)) os.close(tmpfp) os.unlink(tmpfn)
def doit(gts, hr): """ Actually generate a PNG file from the 8 NMQ tiles """ irealtime = is_realtime(gts) routes = "ac" if irealtime else "a" sts = gts - datetime.timedelta(hours=hr) times = [gts] if hr > 24: times.append(gts - datetime.timedelta(hours=24)) if hr == 72: times.append(gts - datetime.timedelta(hours=48)) metadata = { "start_valid": sts.strftime("%Y-%m-%dT%H:%M:%SZ"), "end_valid": gts.strftime("%Y-%m-%dT%H:%M:%SZ"), "units": "mm", } total = None mproduct = "RadarOnly_QPE_24H" if hr >= 24 else "RadarOnly_QPE_01H" for now in times: gribfn = mrms.fetch(mproduct, now) if gribfn is None: print( ("make_mrms_rasters.py[%s] MISSING %s\n %s\n") % (hr, now.strftime("%Y-%m-%dT%H:%MZ"), gribfn) ) MISSED_FILES.append(gribfn) return DOWNLOADED_FILES.append(gribfn) fp = gzip.GzipFile(gribfn, "rb") (tmpfp, tmpfn) = tempfile.mkstemp() tmpfp = open(tmpfn, "wb") tmpfp.write(fp.read()) tmpfp.close() grbs = pygrib.open(tmpfn) grb = grbs[1] os.unlink(tmpfn) # careful here, how we deal with the two missing values! if total is None: total = grb["values"] else: maxgrid = np.maximum(grb["values"], total) total = np.where( np.logical_and(grb["values"] >= 0, total >= 0), grb["values"] + total, maxgrid, ) imgdata = convert_to_image(total) (tmpfp, tmpfn) = tempfile.mkstemp() # Create Image png = Image.fromarray(imgdata.astype("u1")) png.putpalette(mrms.make_colorramp()) png.save("%s.png" % (tmpfn,)) if irealtime: # create a second PNG with null values set to black imgdata = np.where(imgdata == 255, 0, imgdata) png = Image.fromarray(imgdata.astype("u1")) png.putpalette(mrms.make_colorramp()) png.save("%s_nn.png" % (tmpfn,)) # Now we need to generate the world file mrms.write_worldfile("%s.wld" % (tmpfn,)) if irealtime: mrms.write_worldfile("%s_nn.wld" % (tmpfn,)) # Inject WLD file pqstr = ( "%s -i -p 'plot %s %s " "gis/images/4326/mrms/p%ih.wld GIS/mrms/p%ih_%s.wld wld' " "%s.wld" "" ) % ( PQI, routes, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn, ) subprocess.call(pqstr, shell=True) if irealtime: pqstr = ( "%s -i -p 'plot c %s " "gis/images/4326/mrms/p%ih_nn.wld " "GIS/mrms/p%ih_%s.wld wld' " "%s_nn.wld" "" ) % ( PQI, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn, ) subprocess.call(pqstr, shell=True) # Now we inject into LDM pqstr = ( "%s -i -p 'plot %s %s " "gis/images/4326/mrms/p%ih.png GIS/mrms/p%ih_%s.png png' " "%s.png" "" ) % ( PQI, routes, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn, ) subprocess.call(pqstr, shell=True) if irealtime: # Now we inject into LDM pqstr = ( "%s -i -p 'plot c %s " "gis/images/4326/mrms/p%ih_nn.png " "GIS/mrms/p%ih_%s.png png' " "%s_nn.png" "" ) % ( PQI, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn, ) subprocess.call(pqstr, shell=True) if irealtime: # Create 900913 image cmd = ( "gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff " "-tr 1000.0 1000.0 %s.png %s.tif" ) % (tmpfn, tmpfn) subprocess.call(cmd, shell=True) cmd = ( "gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff " "-tr 1000.0 1000.0 %s_nn.png %s_nn.tif" ) % (tmpfn, tmpfn) subprocess.call(cmd, shell=True) # Insert into LDM pqstr = ( "%s -i -p 'plot c %s " "gis/images/900913/mrms/p%ih.tif " "GIS/mrms/p%ih_%s.tif tif' " "%s.tif" "" ) % ( PQI, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn, ) subprocess.call(pqstr, shell=True) pqstr = ( "%s -i -p 'plot c %s " "gis/images/900913/mrms/p%ih_nn.tif " "GIS/mrms/p%ih_%s.tif tif' " "%s_nn.tif" "" ) % ( PQI, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn, ) subprocess.call(pqstr, shell=True) j = open("%s.json" % (tmpfn,), "w") j.write(json.dumps(dict(meta=metadata))) j.close() # Insert into LDM pqstr = ( "%s -i -p 'plot c %s " "gis/images/4326/mrms/p%ih.json " "GIS/mrms/p%ih_%s.json json'" " %s.json" ) % ( PQI, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn, ) subprocess.call(pqstr, shell=True) pqstr = ( "%s -i -p 'plot c %s " "gis/images/4326/mrms/p%ih_nn.json " "GIS/mrms/p%ih_%s.json json'" " %s.json" ) % ( PQI, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn, ) subprocess.call(pqstr, shell=True) for suffix in ["tif", "json", "png", "wld"]: fn = "%s.%s" % (tmpfn, suffix) if os.path.isfile(fn): os.unlink(fn) if irealtime: for suffix in ["tif", "png", "wld"]: fn = "%s_nn.%s" % (tmpfn, suffix) if os.path.isfile(fn): os.unlink(fn) os.close(tmpfp) os.unlink(tmpfn)
def test_colorramp(): """See what we can do with a colorramp""" cmap = mrms.make_colorramp() assert len(cmap) == 256 * 3
def test_colorramp(self): """See what we can do with a colorramp""" c = mrms.make_colorramp() self.assertEqual(len(c), 256 * 3)
def doit(gts, hr): """ Actually generate a PNG file from the 8 NMQ tiles """ irealtime = is_realtime(gts) routes = "ac" if irealtime else "a" sts = gts - datetime.timedelta(hours=hr) times = [gts] if hr > 24: times.append(gts - datetime.timedelta(hours=24)) if hr == 72: times.append(gts - datetime.timedelta(hours=48)) metadata = { 'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"), 'end_valid': gts.strftime("%Y-%m-%dT%H:%M:%SZ"), 'units': 'mm' } total = None mproduct = "RadarOnly_QPE_24H" if hr >= 24 else "RadarOnly_QPE_01H" for now in times: uri = now.strftime(("http://mtarchive.geol.iastate.edu/%Y/%m/%d/" "mrms/ncep/" + mproduct + "/" + mproduct + "_00.00_%Y%m%d-%H%M00.grib2.gz")) gribfn = now.strftime( (TMP + "/" + mproduct + "_00.00_%Y%m%d-%H%M00.grib2.gz")) if not os.path.isfile(gribfn): res = requests.get(uri, timeout=60) if res.status_code != 200: # Cut down on logged errors if gribfn in MISSED_FILES: return print(("make_mrms_rasters.py[%s] MISSING %s\n %s\n %s\n") % (hr, now.strftime("%Y-%m-%dT%H:%MZ"), uri, gribfn)) MISSED_FILES.append(gribfn) return o = open(gribfn, 'wb') o.write(res.content) o.close() DOWNLOADED_FILES.append(gribfn) fp = gzip.GzipFile(gribfn, 'rb') (tmpfp, tmpfn) = tempfile.mkstemp() tmpfp = open(tmpfn, 'wb') tmpfp.write(fp.read()) tmpfp.close() grbs = pygrib.open(tmpfn) grb = grbs[1] os.unlink(tmpfn) # careful here, how we deal with the two missing values! if total is None: total = grb['values'] else: maxgrid = np.maximum(grb['values'], total) total = np.where(np.logical_and(grb['values'] >= 0, total >= 0), grb['values'] + total, maxgrid) imgdata = convert_to_image(total) (tmpfp, tmpfn) = tempfile.mkstemp() # Create Image png = Image.fromarray(imgdata.astype('u1')) png.putpalette(mrms.make_colorramp()) png.save('%s.png' % (tmpfn, )) if irealtime: # create a second PNG with null values set to black imgdata = np.where(imgdata == 255, 0, imgdata) png = Image.fromarray(imgdata.astype('u1')) png.putpalette(mrms.make_colorramp()) png.save('%s_nn.png' % (tmpfn, )) # Now we need to generate the world file mrms.write_worldfile('%s.wld' % (tmpfn, )) if irealtime: mrms.write_worldfile('%s_nn.wld' % (tmpfn, )) # Inject WLD file pqstr = ("%s -i -p 'plot %s %s " "gis/images/4326/mrms/p%ih.wld GIS/mrms/p%ih_%s.wld wld' " "%s.wld" "") % (PQI, routes, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) if irealtime: pqstr = ("%s -i -p 'plot c %s " "gis/images/4326/mrms/p%ih_nn.wld " "GIS/mrms/p%ih_%s.wld wld' " "%s_nn.wld" "") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) # Now we inject into LDM pqstr = ("%s -i -p 'plot %s %s " "gis/images/4326/mrms/p%ih.png GIS/mrms/p%ih_%s.png png' " "%s.png" "") % (PQI, routes, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) if irealtime: # Now we inject into LDM pqstr = ("%s -i -p 'plot c %s " "gis/images/4326/mrms/p%ih_nn.png " "GIS/mrms/p%ih_%s.png png' " "%s_nn.png" "") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) if irealtime: # Create 900913 image cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff " "-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn) subprocess.call(cmd, shell=True) cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff " "-tr 1000.0 1000.0 %s_nn.png %s_nn.tif") % (tmpfn, tmpfn) subprocess.call(cmd, shell=True) # Insert into LDM pqstr = ("%s -i -p 'plot c %s " "gis/images/900913/mrms/p%ih.tif " "GIS/mrms/p%ih_%s.tif tif' " "%s.tif" "") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) pqstr = ("%s -i -p 'plot c %s " "gis/images/900913/mrms/p%ih_nn.tif " "GIS/mrms/p%ih_%s.tif tif' " "%s_nn.tif" "") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) j = open("%s.json" % (tmpfn, ), 'w') j.write(json.dumps(dict(meta=metadata))) j.close() # Insert into LDM pqstr = ("%s -i -p 'plot c %s " "gis/images/4326/mrms/p%ih.json " "GIS/mrms/p%ih_%s.json json'" " %s.json") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) pqstr = ("%s -i -p 'plot c %s " "gis/images/4326/mrms/p%ih_nn.json " "GIS/mrms/p%ih_%s.json json'" " %s.json") % (PQI, gts.strftime("%Y%m%d%H%M"), hr, hr, gts.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) for suffix in ['tif', 'json', 'png', 'wld']: fn = '%s.%s' % (tmpfn, suffix) if os.path.isfile(fn): os.unlink(fn) if irealtime: for suffix in ['tif', 'png', 'wld']: fn = '%s_nn.%s' % (tmpfn, suffix) if os.path.isfile(fn): os.unlink(fn) os.close(tmpfp) os.unlink(tmpfn)
c[1,:] = [0,0,0] i = 2 for line in open('gr2ae.txt'): c[i,:] = map(int, line.split()[-3:]) i+=1 c[255,:] = [255,255,255] return tuple( c.ravel() ) data = numpy.zeros( (30,256), numpy.uint8) for i in range(255): data[0:15,i:i+1] = i png = Image.fromarray( data ) #png.putpalette( make_gr_colorramp() ) ramp = list(mrms.make_colorramp()) ramp[-3:] = [255,255,255] png.putpalette(ramp) draw = ImageDraw.Draw(png) #24h Precip # 0-1 -> 100 - 0.01 res || 0 - 25 -> 100 - 0.25 mm 0 # 1-5 -> 80 - 0.05 res || 25 - 125 -> 80 - 1.25 mm 100 # 5-20 -> 75 - 0.20 res || 125 - 500 -> 75 - 5 mm 180 xs = [25, 50, 75, 100, 120, 140, 160, 180, 190, 205, 230] ys = [0.25, 0.5, 0.75, 1, 2, 3, 4, 5, 7, 10, 15] # 1hr precip # 1 mm resolution #xs = numpy.arange(25,255, 25) #ys = numpy.arange(1,10)
def do(now, realtime, delta): """ Generate for this timestep! """ szx = 7000 szy = 3500 # Create the image data imgdata = np.zeros((szy, szx), 'u1') sts = now - datetime.timedelta(minutes=2) metadata = { 'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"), 'end_valid': now.strftime("%Y-%m-%dT%H:%M:%SZ"), 'product': 'a2m', 'units': '0.02 mm' } fn = now.strftime("PrecipRate_00.00_%Y%m%d-%H%M00.grib2.gz") uri = now.strftime(("http://mtarchive.geol.iastate.edu/%Y/%m/%d/" "mrms/ncep/PrecipRate/" + fn)) gribfn = "%s/%s" % (TMP, fn) res = requests.get(uri, timeout=30) if res.status_code != 200: print("mrms_rainrate_comp.py MISSING %s" % (gribfn, )) return o = open(gribfn, 'wb') o.write(res.content) o.close() # http://www.nssl.noaa.gov/projects/mrms/operational/tables.php # Says units are mm/hr fp = gzip.GzipFile(gribfn, 'rb') (_, tmpfn) = tempfile.mkstemp() tmpfp = open(tmpfn, 'wb') tmpfp.write(fp.read()) tmpfp.close() grbs = pygrib.open(tmpfn) grb = grbs[1] os.unlink(tmpfn) os.unlink(gribfn) val = grb['values'] # Convert into units of 0.1 mm accumulation val = val / 60.0 * 2.0 * 50.0 val = np.where(val < 0., 255., val) imgdata[:, :] = np.flipud(val.astype('int')) (tmpfp, tmpfn) = tempfile.mkstemp() # Create Image png = Image.fromarray(np.flipud(imgdata)) png.putpalette(mrms.make_colorramp()) png.save('%s.png' % (tmpfn, )) mrms.write_worldfile('%s.wld' % (tmpfn, )) # Inject WLD file routes = "c" if realtime else "" prefix = 'a2m' pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot a%s %s " "gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld" "") % (routes, now.strftime("%Y%m%d%H%M"), prefix, prefix, now.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) # Now we inject into LDM pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot a%s %s " "gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png" "") % (routes, now.strftime("%Y%m%d%H%M"), prefix, prefix, now.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) if realtime: # Create 900913 image cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff " "-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn) subprocess.call(cmd, shell=True) # Insert into LDM pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s " "gis/images/900913/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif" "") % (now.strftime("%Y%m%d%H%M"), prefix, prefix, now.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) j = open("%s.json" % (tmpfn, ), 'w') j.write(json.dumps(dict(meta=metadata))) j.close() # Insert into LDM pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s " "gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' " "%s.json") % (now.strftime("%Y%m%d%H%M"), prefix, prefix, now.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) for suffix in ['tif', 'json', 'png', 'wld']: if os.path.isfile("%s.%s" % (tmpfn, suffix)): os.unlink('%s.%s' % (tmpfn, suffix)) os.close(tmpfp) os.unlink(tmpfn)
def do(now, hr ): ''' Generate for this timestep! 255 levels... wanna do 0 to 20 inches index 255 is missing, index 0 is 0 0-1 -> 100 - 0.01 res || 0 - 25 -> 100 - 0.25 mm 0 1-5 -> 80 - 0.05 res || 25 - 125 -> 80 - 1.25 mm 100 5-20 -> 75 - 0.20 res || 125 - 500 -> 75 - 5 mm 180 ''' szx = 7000 szy = 3500 sts = now - datetime.timedelta(hours=hr) metadata = {'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"), 'end_valid': now.strftime("%Y-%m-%dT%H:%M:%SZ"), 'product': 'lcref', 'units': 'mm' } gribfn = now.strftime(("/mnt/a4/data/%Y/%m/%d/mrms/ncep/" +"RadarOnly_QPE_01H/" +"RadarOnly_QPE_01H_00.00_%Y%m%d-%H%M00.grib2.gz")) if not os.path.isfile(gribfn): print("mrms_raster_p1h.py MISSING %s" % (gribfn,)) return fp = gzip.GzipFile(gribfn, 'rb') (tmpfp, tmpfn) = tempfile.mkstemp() tmpfp = open(tmpfn, 'wb') tmpfp.write(fp.read()) tmpfp.close() grbs = pygrib.open(tmpfn) grb = grbs[1] os.unlink(tmpfn) total = grb['values'] # Off scale gets index 254 imgdata = np.where(total >= 500, 254, 0) imgdata = np.where(np.logical_and(total >= 125, total < 500), 180 + ((total - 125.) / 5.0), imgdata) imgdata = np.where(np.logical_and(total >= 25, total < 125), 100 + ((total - 25.) / 1.25), imgdata) imgdata = np.where(np.logical_and(total >= 0, total < 25), total / 0.25, imgdata) # -3 is no coverage -> 255 # -1 is misisng, so zero # Index 255 is missing imgdata = np.where( total < 0, 0, imgdata) imgdata = np.where( total < -1, 255, imgdata) (tmpfp, tmpfn) = tempfile.mkstemp() # Create Image png = Image.fromarray( imgdata.astype('u1') ) png.putpalette(mrms.make_colorramp()) png.save('%s.png' % (tmpfn,)) # os.system('xv %s.png' % (tmpfn,)) mrms.write_worldfile('%s.wld' % (tmpfn,)) # Inject WLD file prefix = 'p%sh' % (hr,) pqstr = "/home/ldm/bin/pqinsert -i -p 'plot ac %s gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld" % ( now.strftime("%Y%m%d%H%M"), prefix, prefix, now.strftime("%Y%m%d%H%M"), tmpfn ) subprocess.call(pqstr, shell=True) # Now we inject into LDM pqstr = "/home/ldm/bin/pqinsert -p 'plot ac %s gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png" % ( now.strftime("%Y%m%d%H%M"), prefix, prefix, now.strftime("%Y%m%d%H%M"), tmpfn ) subprocess.call(pqstr, shell=True) # Create 900913 image cmd = "gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff -tr 1000.0 1000.0 %s.png %s.tif" % (tmpfn, tmpfn) subprocess.call(cmd, shell=True) # Insert into LDM pqstr = "/home/ldm/bin/pqinsert -p 'plot c %s gis/images/900913/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif" % ( now.strftime("%Y%m%d%H%M"), prefix, prefix, now.strftime("%Y%m%d%H%M"), tmpfn ) subprocess.call(pqstr, shell=True) j = open("%s.json" % (tmpfn,), 'w') j.write( json.dumps(dict(meta=metadata))) j.close() # Insert into LDM pqstr = "/home/ldm/bin/pqinsert -p 'plot c %s gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' %s.json" % ( now.strftime("%Y%m%d%H%M"),prefix, prefix, now.strftime("%Y%m%d%H%M"), tmpfn ) subprocess.call(pqstr, shell=True) for suffix in ['tif', 'json', 'png', 'wld']: os.unlink('%s.%s' % (tmpfn, suffix)) os.close(tmpfp) os.unlink(tmpfn)
def do(now, realtime, delta): """ Generate for this timestep! """ szx = 7000 szy = 3500 # Create the image data imgdata = np.zeros((szy, szx), 'u1') sts = now - datetime.timedelta(minutes=2) metadata = {'start_valid': sts.strftime("%Y-%m-%dT%H:%M:%SZ"), 'end_valid': now.strftime("%Y-%m-%dT%H:%M:%SZ"), 'product': 'a2m', 'units': '0.1 mm'} gribfn = now.strftime(("/mnt/a4/data/%Y/%m/%d/mrms/ncep/PrecipRate/" "PrecipRate_00.00_%Y%m%d-%H%M00.grib2.gz")) if not os.path.isfile(gribfn): # Don't whine about old files being missing if delta < 90: print("mrms_rainrate_comp.py MISSING %s" % (gribfn,)) return # http://www.nssl.noaa.gov/projects/mrms/operational/tables.php # Says units are mm/hr fp = gzip.GzipFile(gribfn, 'rb') (_, tmpfn) = tempfile.mkstemp() tmpfp = open(tmpfn, 'wb') tmpfp.write(fp.read()) tmpfp.close() grbs = pygrib.open(tmpfn) grb = grbs[1] os.unlink(tmpfn) val = grb['values'] # Convert into units of 0.1 mm accumulation val = val / 60.0 * 2.0 * 10.0 val = np.where(val < 0., 255., val) imgdata[:, :] = np.flipud(val.astype('int')) (tmpfp, tmpfn) = tempfile.mkstemp() # Create Image png = Image.fromarray(np.flipud(imgdata)) png.putpalette(mrms.make_colorramp()) png.save('%s.png' % (tmpfn,)) mrms.write_worldfile('%s.wld' % (tmpfn,)) # Inject WLD file routes = "c" if realtime else "" prefix = 'a2m' pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot a%s %s " "gis/images/4326/mrms/%s.wld GIS/mrms/%s_%s.wld wld' %s.wld" "") % (routes, now.strftime("%Y%m%d%H%M"), prefix, prefix, now.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) # Now we inject into LDM pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot a%s %s " "gis/images/4326/mrms/%s.png GIS/mrms/%s_%s.png png' %s.png" "") % (routes, now.strftime("%Y%m%d%H%M"), prefix, prefix, now.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) if realtime: # Create 900913 image cmd = ("gdalwarp -s_srs EPSG:4326 -t_srs EPSG:3857 -q -of GTiff " "-tr 1000.0 1000.0 %s.png %s.tif") % (tmpfn, tmpfn) subprocess.call(cmd, shell=True) # Insert into LDM pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s " "gis/images/900913/mrms/%s.tif GIS/mrms/%s_%s.tif tif' %s.tif" "") % (now.strftime("%Y%m%d%H%M"), prefix, prefix, now.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) j = open("%s.json" % (tmpfn,), 'w') j.write(json.dumps(dict(meta=metadata))) j.close() # Insert into LDM pqstr = ("/home/ldm/bin/pqinsert -i -p 'plot c %s " "gis/images/4326/mrms/%s.json GIS/mrms/%s_%s.json json' " "%s.json") % (now.strftime("%Y%m%d%H%M"), prefix, prefix, now.strftime("%Y%m%d%H%M"), tmpfn) subprocess.call(pqstr, shell=True) for suffix in ['tif', 'json', 'png', 'wld']: if os.path.isfile("%s.%s" % (tmpfn, suffix)): os.unlink('%s.%s' % (tmpfn, suffix)) os.close(tmpfp) os.unlink(tmpfn)
i = 2 for line in open('gr2ae.txt'): c[i, :] = map(int, line.split()[-3:]) i += 1 c[255, :] = [255, 255, 255] return tuple(c.ravel()) data = numpy.zeros((30, 256), numpy.uint8) for i in range(255): data[0:15, i:i + 1] = i png = Image.fromarray(data) #png.putpalette( make_gr_colorramp() ) ramp = list(mrms.make_colorramp()) ramp[-3:] = [255, 255, 255] png.putpalette(ramp) draw = ImageDraw.Draw(png) #24h Precip # 0-1 -> 100 - 0.01 res || 0 - 25 -> 100 - 0.25 mm 0 # 1-5 -> 80 - 0.05 res || 25 - 125 -> 80 - 1.25 mm 100 # 5-20 -> 75 - 0.20 res || 125 - 500 -> 75 - 5 mm 180 xs = [25, 50, 75, 100, 120, 140, 160, 180, 190, 205, 230] ys = [0.25, 0.5, 0.75, 1, 2, 3, 4, 5, 7, 10, 15] # 1hr precip # 1 mm resolution #xs = numpy.arange(25,255, 25) #ys = numpy.arange(1,10)