Пример #1
0
def do_month(year, month, routes):
    """ Generate a MRMS plot for the month!"""

    sts = datetime.datetime(year, month, 1)
    ets = sts + datetime.timedelta(days=35)
    ets = ets.replace(day=1)

    today = datetime.datetime.now()
    if ets > today:
        ets = today

    idx0 = iemre.daily_offset(sts)
    idx1 = iemre.daily_offset(ets)

    nc = ncopen(iemre.get_daily_mrms_ncname(year), 'r')

    lats = nc.variables['lat'][:]
    lons = nc.variables['lon'][:]
    p01d = distance(np.sum(nc.variables['p01d'][idx0:idx1, :, :], 0),
                    'MM').value('IN')
    nc.close()

    mp = MapPlot(sector='iowa', title='MRMS %s - %s Total Precipitation' % (
                 sts.strftime("%-d %b"),
                 (ets - datetime.timedelta(days=1)).strftime("%-d %b %Y")),
                 subtitle='Data from NOAA MRMS Project')
    x, y = np.meshgrid(lons, lats)
    bins = [0.01, 0.1, 0.5, 1, 1.5, 2, 3, 4, 5, 6, 7, 8, 12, 16, 20]
    mp.pcolormesh(x, y, p01d, bins, units='inches')
    mp.drawcounties()
    currentfn = "summary/iowa_mrms_q3_month.png"
    archivefn = sts.strftime("%Y/%m/summary/iowa_mrms_q3_month.png")
    pqstr = "plot %s %s00 %s %s png" % (
                routes, sts.strftime("%Y%m%d%H"), currentfn, archivefn)
    mp.postprocess(pqstr=pqstr)
Пример #2
0
def main():
    """Go Main"""
    total = None
    years = 0.
    for yr in range(1981, 2018):
        print(yr)
        ncfn = "/mesonet/data/prism/%s_daily.nc" % (yr, )
        nc = netCDF4.Dataset(ncfn)
        if total is None:
            lons = nc.variables['lon'][:]
            lats = nc.variables['lat'][:]
            total = np.zeros(nc.variables['tmax'].shape[1:], np.float)
        days = np.zeros(nc.variables['tmax'].shape[1:], np.float)
        sidx = daily_offset(datetime.date(yr, 1, 1))
        eidx = daily_offset(datetime.date(yr, 7, 4))
        for idx in range(sidx, eidx):
            days += np.where(nc.variables['tmax'][idx, :, :] > THRESHOLD,
                             1, 0)
        nc.close()
        years += 1.
        total += days

    val = days - (total / years)
    print(np.max(val))
    print(np.min(val))
    mp = MapPlot(sector='conus',
                 title=("OSU PRISM 2017 Days with High >= 90$^\circ$F "
                        "Departure"),
                 subtitle=("2017 thru 4 July against 1981-2016 "
                           "Year to Date Average"))
    mp.contourf(lons, lats, val, np.arange(-25, 26, 5),
                units='days', cmap=plt.get_cmap('seismic'))
    mp.postprocess(filename='test.png')
Пример #3
0
def do_month(year, month, routes):
    """ Generate a MRMS plot for the month!"""

    sts = datetime.datetime(year,month,1)
    ets = sts + datetime.timedelta(days=35)
    ets = ets.replace(day=1)

    today = datetime.datetime.now()
    if ets > today:
        ets = today

    idx0 = iemre.daily_offset(sts)
    idx1 = iemre.daily_offset(ets)

    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_mrms_daily.nc" % (year,),
                          'r')

    lats = nc.variables['lat'][:]
    lons = nc.variables['lon'][:]
    p01d = np.sum(nc.variables['p01d'][idx0:idx1,:,:],0) / 24.5
    nc.close()

    m = MapPlot(sector='iowa', title='MRMS %s - %s Total Precipitation' % (
            sts.strftime("%-d %b"), 
            (ets - datetime.timedelta(days=1)).strftime("%-d %b %Y")),
            subtitle='Data from NOAA MRMS Project')
    x,y = np.meshgrid(lons, lats)
    bins = [0.01, 0.1, 0.5, 1, 1.5, 2, 3, 4, 5, 6, 7, 8, 12, 16, 20]
    m.pcolormesh(x, y, p01d, bins, units='inches')
    m.drawcounties()
    currentfn = "summary/iowa_mrms_q3_month.png"
    archivefn = sts.strftime("%Y/%m/summary/iowa_mrms_q3_month.png")
    pqstr = "plot %s %s00 %s %s png" % (
                routes, sts.strftime("%Y%m%d%H"), currentfn, archivefn)
    m.postprocess(pqstr=pqstr)
Пример #4
0
def main():
    """Do Something Fun!"""
    form = cgi.FormContent()
    ts = datetime.datetime.strptime(form["date"][0], "%Y-%m-%d")
    lat = float(form["lat"][0])
    lon = float(form["lon"][0])
    fmt = form["format"][0]
    if fmt != 'json':
        sys.stdout.write("Content-type: text/plain\n\n")
        sys.stdout.write("ERROR: Service only emits json at this time")
        return

    i, j = iemre.find_ij(lon, lat)
    offset = iemre.daily_offset(ts)

    res = {'data': [], }

    fn = "/mesonet/data/iemre/%s_mw_daily.nc" % (ts.year,)

    sys.stdout.write('Content-type: application/json\n\n')
    if not os.path.isfile(fn):
        sys.stdout.write(json.dumps(res))
        sys.exit()

    if i is None or j is None:
        sys.stdout.write(json.dumps({'error': 'Coordinates outside of domain'}
                                    ))
        return

    nc = netCDF4.Dataset(fn, 'r')

    c2000 = ts.replace(year=2000)
    coffset = iemre.daily_offset(c2000)

    cnc = netCDF4.Dataset("/mesonet/data/iemre/mw_dailyc.nc", 'r')

    res['data'].append({
        'daily_high_f': myrounder(
           datatypes.temperature(
                nc.variables['high_tmpk'][offset, j, i], 'K').value('F'), 1),
        'climate_daily_high_f': myrounder(
           datatypes.temperature(
                cnc.variables['high_tmpk'][coffset, j, i], 'K').value("F"), 1),
        'daily_low_f': myrounder(
           datatypes.temperature(
                nc.variables['low_tmpk'][offset, j, i], 'K').value("F"), 1),
        'climate_daily_low_f': myrounder(
           datatypes.temperature(
                cnc.variables['low_tmpk'][coffset, j, i], 'K').value("F"), 1),
        'daily_precip_in': myrounder(
           nc.variables['p01d'][offset, j, i] / 25.4, 2),
        'climate_daily_precip_in': myrounder(
           cnc.variables['p01d'][coffset, j, i] / 25.4, 2),
      })
    nc.close()
    cnc.close()

    sys.stdout.write(json.dumps(res))
Пример #5
0
def main():
    """Go Main Go"""
    ets = datetime.datetime.now() - datetime.timedelta(days=1)
    sts = datetime.datetime(ets.year, 1, 1)

    # Get the normal accumm
    with ncopen(iemre.get_dailyc_ncname()) as cnc:
        lons = cnc.variables["lon"][:]
        lats = cnc.variables["lat"][:]
        index0 = iemre.daily_offset(sts)
        index1 = iemre.daily_offset(ets)
        clprecip = np.sum(cnc.variables["p01d"][index0:index1, :, :], 0)

    with ncopen(iemre.get_daily_ncname(sts.year)) as nc:
        obprecip = np.sum(nc.variables["p01d"][index0:index1, :, :], 0)

    lons, lats = np.meshgrid(lons, lats)

    # Plot departure from normal
    mp = MapPlot(
        sector="midwest",
        title=("Precipitation Departure %s - %s") %
        (sts.strftime("%b %d %Y"), ets.strftime("%b %d %Y")),
        subtitle="based on IEM Estimates",
    )

    mp.pcolormesh(lons, lats, (obprecip - clprecip) / 25.4,
                  np.arange(-10, 10, 1))
    mp.postprocess(
        pqstr="plot c 000000000000 summary/year/stage4_diff.png bogus png")
    mp.close()

    # Plot normals
    mp = MapPlot(
        sector="midwest",
        title=("Normal Precipitation:: %s - %s") %
        (sts.strftime("%b %d %Y"), ets.strftime("%b %d %Y")),
        subtitle="based on IEM Estimates",
    )

    mp.pcolormesh(lons, lats, (clprecip) / 25.4, np.arange(0, 30, 2))
    mp.postprocess(
        pqstr="plot c 000000000000 summary/year/stage4_normals.png bogus png")
    mp.close()

    # Plot Obs
    mp = MapPlot(
        sector="midwest",
        title=("Estimated Precipitation:: %s - %s") %
        (sts.strftime("%b %d %Y"), ets.strftime("%b %d %Y")),
        subtitle="based on IEM Estimates",
    )

    mp.pcolormesh(lons, lats, (obprecip) / 25.4, np.arange(0, 30, 2))
    mp.postprocess(
        pqstr="plot c 000000000000 summary/year/stage4obs.png bogus png")
    mp.close()
Пример #6
0
def plotter(fdict):
    """ Go """
    import matplotlib
    matplotlib.use('agg')
    import matplotlib.pyplot as plt
    from pyiem.plot.geoplot import MapPlot

    ctx = util.get_autoplot_context(fdict, get_description())
    date = ctx['date']
    sector = ctx['sector']
    threshold = ctx['threshold']
    threshold_mm = distance(threshold, 'IN').value('MM')
    window_sts = date - datetime.timedelta(days=90)
    if window_sts.year != date.year:
        raise Exception('Sorry, do not support multi-year plots yet!')

    idx0 = iemre.daily_offset(window_sts)
    idx1 = iemre.daily_offset(date)
    ncfn = "/mesonet/data/iemre/%s_mw_mrms_daily.nc" % (date.year, )
    ncvar = 'p01d'
    if not os.path.isfile(ncfn):
        raise Exception("No data for that year, sorry.")
    nc = netCDF4.Dataset(ncfn, 'r')

    grid = np.zeros((len(nc.dimensions['lat']),
                     len(nc.dimensions['lon'])))
    total = np.zeros((len(nc.dimensions['lat']),
                      len(nc.dimensions['lon'])))
    for i, idx in enumerate(range(idx1, idx1-90, -1)):
        total += nc.variables[ncvar][idx, :, :]
        grid = np.where(np.logical_and(grid == 0,
                                       total > threshold_mm), i, grid)
    lon = np.append(nc.variables['lon'][:], [-80.5])
    lat = np.append(nc.variables['lat'][:], [49.])
    nc.close()

    mp = MapPlot(sector='state', state=sector, titlefontsize=14,
                 subtitlefontsize=12,
                 title=("NOAA MRMS Q3: Number of Recent Days "
                        "till Accumulating %s\" of Precip"
                        ) % (threshold, ),
                 subtitle=("valid %s: based on per calendar day "
                           "estimated preciptation, GaugeCorr and "
                           "RadarOnly products"
                           ) % (date.strftime("%-d %b %Y"), ))
    x, y = np.meshgrid(lon, lat)
    cmap = plt.get_cmap('terrain')
    cmap.set_over('k')
    cmap.set_under('white')
    mp.pcolormesh(x, y, grid,
                  np.arange(0, 81, 10), cmap=cmap, units='days')
    mp.drawcounties()
    mp.drawcities()

    return mp.fig
Пример #7
0
def test_daily_offset():
    """ Compute the offsets """
    ts = utc(2013, 1, 1, 0, 0)
    offset = iemre.daily_offset(ts)
    assert offset == 0

    ts = datetime.date(2013, 2, 1)
    offset = iemre.daily_offset(ts)
    assert offset == 31

    ts = utc(2013, 1, 5, 12, 0)
    offset = iemre.daily_offset(ts)
    assert offset == 4
Пример #8
0
def test_daily_offset():
    """ Compute the offsets """
    ts = utc(2013, 1, 1, 0, 0)
    offset = iemre.daily_offset(ts)
    assert offset == 0

    ts = datetime.date(2013, 2, 1)
    offset = iemre.daily_offset(ts)
    assert offset == 31

    ts = utc(2013, 1, 5, 12, 0)
    offset = iemre.daily_offset(ts)
    assert offset == 4
Пример #9
0
    def test_daily_offset(self):
        """ Compute the offsets """
        ts = datetime.datetime(2013, 1, 1, 0, 0)
        ts = ts.replace(tzinfo=pytz.timezone("UTC"))
        offset = iemre.daily_offset(ts)
        self.assertEqual(offset, 0)

        ts = datetime.date(2013, 2, 1)
        offset = iemre.daily_offset(ts)
        self.assertEqual(offset, 31)

        ts = datetime.datetime(2013, 1, 5, 12, 0)
        ts = ts.replace(tzinfo=pytz.timezone("UTC"))
        offset = iemre.daily_offset(ts)
        self.assertEqual(offset, 4)
Пример #10
0
    def test_daily_offset(self):
        """ Compute the offsets """
        ts = datetime.datetime(2013, 1, 1, 0, 0)
        ts = ts.replace(tzinfo=pytz.timezone("UTC"))
        offset = iemre.daily_offset(ts)
        self.assertEqual(offset, 0)

        ts = datetime.date(2013, 2, 1)
        offset = iemre.daily_offset(ts)
        self.assertEqual(offset, 31)

        ts = datetime.datetime(2013, 1, 5, 12, 0)
        ts = ts.replace(tzinfo=pytz.timezone("UTC"))
        offset = iemre.daily_offset(ts)
        self.assertEqual(offset, 4)
Пример #11
0
def replace_cfs(nc, valid, islice, jslice):
    """Copy CFS data into the given year."""
    tidx0 = (valid - datetime.date(valid.year, 1, 1)).days
    tidx1 = (
        datetime.date(valid.year, 12, 31) - datetime.date(valid.year, 1, 1)
    ).days
    cfsnc = ncopen(valid.strftime("/mesonet/data/iemre/cfs_%Y%m%d%H.nc"))
    tidx = iemre.daily_offset(valid + datetime.timedelta(days=1))
    tslice = slice(tidx0 + 1, tidx1 + 1)
    # print("replace_cfs filling %s from %s" % (tslice, tidx))
    # CFS is W m-2, we want MJ
    nc.variables["srad"][tslice, :, :] = (
        cfsnc.variables["srad"][tidx:, jslice, islice] * 86400.0 / 1000000.0
    )
    highc = temperature(
        cfsnc.variables["high_tmpk"][tidx:, jslice, islice], "K"
    ).value("C")
    lowc = temperature(
        cfsnc.variables["low_tmpk"][tidx:, jslice, islice], "K"
    ).value("C")
    nc.variables["tmax"][tslice, :, :] = highc
    nc.variables["tmin"][tslice, :, :] = lowc
    nc.variables["gdd_f"][tslice, :, :] = gdd(
        temperature(highc, "C"), temperature(lowc, "C")
    )
    nc.variables["prcp"][tslice, :, :] = cfsnc.variables["p01d"][
        tidx:, jslice, islice
    ]
    cfsnc.close()
Пример #12
0
def do_coop(ts):
    """Use COOP solar radiation data"""
    pgconn = psycopg2.connect(database='coop', host='iemdb', user='******')
    cursor = pgconn.cursor()

    cursor.execute(
        """SELECT ST_x(geom), ST_y(geom),
        coalesce(narr_srad, merra_srad) from alldata a JOIN stations t
        ON (a.station = t.id) WHERE
        day = %s and t.network ~* 'CLIMATE' and substr(id, 3, 1) != 'C'
        and substr(id, 3, 4) != '0000'
    """, (ts.strftime("%Y-%m-%d"), ))
    lons = []
    lats = []
    vals = []
    for row in cursor:
        if row[2] is None or row[2] < 0:
            continue
        lons.append(row[0])
        lats.append(row[1])
        vals.append(row[2])

    nn = NearestNDInterpolator((np.array(lons), np.array(lats)),
                               np.array(vals))
    xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS)

    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_daily.nc" % (ts.year, ),
                         'a')
    offset = iemre.daily_offset(ts)
    # Data above is MJ / d / m-2, we want W / m-2
    nc.variables['rsds'][offset, :, :] = nn(xi, yi) * 1000000. / 86400.
    nc.close()
Пример #13
0
def grid_day(nc, ts):
    """
    """
    offset = iemre.daily_offset(ts)
    icursor.execute("""
       SELECT ST_x(s.geom) as lon, ST_y(s.geom) as lat, 
       (CASE WHEN pday >= 0 then pday else null end) as precipdata,
       (CASE WHEN max_tmpf > -50 and max_tmpf < 130 then max_tmpf else null end) as highdata,
       (CASE WHEN min_tmpf > -50 and min_tmpf < 95 then min_tmpf else null end) as lowdata 
       from summary_%s c, stations s WHERE day = '%s' and 
       s.network in ('IA_ASOS', 'MN_ASOS', 'WI_ASOS', 'IL_ASOS', 'MO_ASOS',
        'KS_ASOS', 'NE_ASOS', 'SD_ASOS', 'ND_ASOS', 'KY_ASOS', 'MI_ASOS',
        'OH_ASOS', 'AWOS') and c.iemid = s.iemid
        """ % (ts.year, ts.strftime("%Y-%m-%d")))

    if icursor.rowcount > 4:
        res = generic_gridder(icursor, 'highdata')
        nc.variables['high_tmpk'][offset] = datatypes.temperature(res, 'F').value('K')
        icursor.scroll(0, mode='absolute')
        res = generic_gridder(icursor, 'lowdata')
        nc.variables['low_tmpk'][offset] = datatypes.temperature(res, 'F').value('K')
        icursor.scroll(0, mode='absolute')
        #res = generic_gridder(icursor, 'precipdata')
        #nc.variables['p01d'][offset] = res * 25.4
    else:
        print "%s has %02i entries, FAIL" % (ts.strftime("%Y-%m-%d"), 
            icursor.rowcount)
Пример #14
0
def do_precip12(nc, ts):
    """Compute the 24 Hour precip at 12 UTC, we do some more tricks though"""
    offset = iemre.daily_offset(ts)
    ets = datetime.datetime(ts.year, ts.month, ts.day, 12)
    ets = ets.replace(tzinfo=pytz.timezone("UTC"))
    sts = ets - datetime.timedelta(hours=24)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    if ts.month == 1 and ts.day == 1:
        print(("p01d_12z  for %s [idx:%s] %s(%s)->%s(%s) SPECIAL"
               ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
                    ets.strftime("%Y%m%d%H"), offset2))
        ncfn = "/mesonet/data/iemre/%s_mw_hourly.nc" % (ets.year,)
        if not os.path.isfile(ncfn):
            print("Missing %s" % (ncfn,))
            return
        hnc = netCDF4.Dataset(ncfn)
        phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0)
        hnc.close()
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (
            sts.year,))
        phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0)
        hnc.close()
    else:
        print(("p01d_12z  for %s [idx:%s] %s(%s)->%s(%s)"
               ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
                    ets.strftime("%Y%m%d%H"), offset2))
        ncfn = "/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year,)
        if not os.path.isfile(ncfn):
            print("Missing %s" % (ncfn,))
            return
        hnc = netCDF4.Dataset(ncfn)
        phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0)
        hnc.close()
    nc.variables['p01d_12z'][offset] = phour
Пример #15
0
def grid_day(nc, ts):
    """
    I proctor the gridding of data on an hourly basis
    @param ts Timestamp of the analysis, we'll consider a 20 minute window
    """
    cursor = COOP.cursor(cursor_factory=psycopg2.extras.DictCursor)
    offset = iemre.daily_offset(ts)
    if ts.day == 29 and ts.month == 2:
        ts = datetime.datetime(2000, 3, 1)

    sql = """SELECT * from ncdc_climate71 WHERE valid = '%s' and
             substr(station,3,4) != '0000' and substr(station,3,1) != 'C'
             """ % (ts.strftime("%Y-%m-%d"), )
    cursor.execute(sql)
    if cursor.rowcount > 4:
        res = generic_gridder(nc, cursor, 'high')
        if res is not None:
            nc.variables['tmax'][offset] = datatypes.temperature(
                res, 'F').value('C')
        cursor.scroll(0, mode='absolute')
        res = generic_gridder(nc, cursor, 'low')
        if res is not None:
            nc.variables['tmin'][offset] = datatypes.temperature(
                res, 'F').value('C')
        cursor.scroll(0, mode='absolute')
        res = generic_gridder(nc, cursor, 'precip')
        if res is not None:
            nc.variables['ppt'][offset] = datatypes.distance(res,
                                                             'IN').value('MM')
    else:
        print(("%s has %02i entries, FAIL") %
              (ts.strftime("%Y-%m-%d"), cursor.rowcount))
Пример #16
0
def estimate_precip(df, ts):
    """Estimate precipitation based on IEMRE"""
    idx = iemre.daily_offset(ts)
    nc = ncopen(iemre.get_daily_ncname(ts.year), 'r', timeout=300)
    grid12 = distance(nc.variables['p01d_12z'][idx, :, :],
                      'MM').value("IN").filled(0)
    grid00 = distance(nc.variables['p01d'][idx, :, :],
                      "MM").value("IN").filled(0)
    nc.close()

    for sid, row in df.iterrows():
        if not pd.isnull(row['precip']):
            continue
        if row['precip24_hour'] in [0, 22, 23]:
            precip = grid00[row['gridj'], row['gridi']]
        else:
            precip = grid12[row['gridj'], row['gridi']]
        # denote trace
        if precip > 0 and precip < 0.01:
            df.at[sid, 'precip'] = TRACE_VALUE
        elif precip < 0:
            df.at[sid, 'precip'] = 0
        elif np.isnan(precip) or np.ma.is_masked(precip):
            df.at[sid, 'precip'] = 0
        else:
            df.at[sid, 'precip'] = "%.2f" % (precip,)
Пример #17
0
def estimate_hilo(df, ts):
    """Estimate the High and Low Temperature based on gridded data"""
    idx = iemre.daily_offset(ts)
    nc = ncopen(iemre.get_daily_ncname(ts.year), 'r', timeout=300)
    highgrid12 = temperature(nc.variables['high_tmpk_12z'][idx, :, :],
                             'K').value('F')
    lowgrid12 = temperature(nc.variables['low_tmpk_12z'][idx, :, :],
                            'K').value('F')
    highgrid00 = temperature(nc.variables['high_tmpk'][idx, :, :],
                             'K').value('F')
    lowgrid00 = temperature(nc.variables['low_tmpk'][idx, :, :],
                            'K').value('F')
    nc.close()

    for sid, row in df.iterrows():
        if pd.isnull(row['high']):
            if row['temp24_hour'] in [0, 22, 23]:
                val = highgrid00[row['gridj'], row['gridi']]
            else:
                val = highgrid12[row['gridj'], row['gridi']]
            if sid == 'IA1402':
                print(row['temp24_hour'])
            if not np.ma.is_masked(val):
                df.at[sid, 'high'] = val
        if pd.isnull(row['low']):
            if row['temp24_hour'] in [0, 22, 23]:
                val = lowgrid00[row['gridj'], row['gridi']]
            else:
                val = lowgrid12[row['gridj'], row['gridi']]
            if not np.ma.is_masked(val):
                df.at[sid, 'low'] = val
Пример #18
0
def run(ts):
    ''' Actually do the work, please '''
    nc = netCDF4.Dataset(
        '/mesonet/data/iemre/%s_mw_mrms_daily.nc' % (ts.year, ), 'a')
    offset = iemre.daily_offset(ts)
    ncprecip = nc.variables['p01d']
    ts += datetime.timedelta(hours=24)
    gmtts = ts.astimezone(pytz.timezone("UTC"))

    fn = gmtts.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/GIS/q2/"
                         "p24h_%Y%m%d%H00.png"))
    img = Image.open(fn)
    data = np.asarray(img)
    # data is 3500,7000 , starting at upper L
    data = np.flipud(data)
    # Anything over 254 is bad
    res = np.where(data > 254, 0, data)
    res = np.where(np.logical_and(data >= 0, data < 100), data * 0.25, res)
    res = np.where(np.logical_and(data >= 100, data < 180),
                   25. + ((data - 100) * 1.25), res)
    res = np.where(np.logical_and(data >= 180, data < 255),
                   125. + ((data - 180) * 5.), res)

    y1 = (iemre.NORTH - mrms.SOUTH) * 100.0
    y0 = (iemre.SOUTH - mrms.SOUTH) * 100.0
    x0 = (iemre.WEST - mrms.WEST) * 100.0
    x1 = (iemre.EAST - mrms.WEST) * 100.0
    ncprecip[offset, :, :] = res[y0:y1, x0:x1]
    nc.close()
Пример #19
0
def grid_day(nc, ts):
    """
    """
    offset = iemre.daily_offset(ts)
    print(('cal hi/lo for %s [idx:%s]') % (ts, offset))
    sql = """
       SELECT ST_x(s.geom) as lon, ST_y(s.geom) as lat, s.state,
       s.name, s.id as station,
       (CASE WHEN pday >= 0 then pday else null end) as precipdata,
       (CASE WHEN max_tmpf > -50 and max_tmpf < 130
           then max_tmpf else null end) as highdata,
       (CASE WHEN min_tmpf > -50 and min_tmpf < 95
           then min_tmpf else null end) as lowdata
       from summary_%s c, stations s WHERE day = '%s' and
       s.network in ('IA_ASOS', 'MN_ASOS', 'WI_ASOS', 'IL_ASOS', 'MO_ASOS',
        'KS_ASOS', 'NE_ASOS', 'SD_ASOS', 'ND_ASOS', 'KY_ASOS', 'MI_ASOS',
        'OH_ASOS', 'AWOS') and c.iemid = s.iemid
        """ % (ts.year, ts.strftime("%Y-%m-%d"))
    df = read_sql(sql, pgconn)

    if len(df.index) > 4:
        res = generic_gridder(df, 'highdata')
        nc.variables['high_tmpk'][offset] = datatypes.temperature(
                                                res, 'F').value('K')
        res = generic_gridder(df, 'lowdata')
        nc.variables['low_tmpk'][offset] = datatypes.temperature(
                                            res, 'F').value('K')
    else:
        print "%s has %02i entries, FAIL" % (ts.strftime("%Y-%m-%d"),
                                             cursor.rowcount)
Пример #20
0
def run(ts):
    """Actually do the work, please"""
    nc = ncopen(iemre.get_daily_mrms_ncname(ts.year), "a", timeout=300)
    offset = iemre.daily_offset(ts)
    ncprecip = nc.variables["p01d"]
    ts += datetime.timedelta(hours=24)
    gmtts = ts.astimezone(pytz.utc)
    fn = findfile(gmtts)
    if fn is None:
        print("merge_mrms_q2 failed to find file for time: %s" % (gmtts, ))
        return
    img = Image.open(fn)
    data = np.asarray(img)
    # data is 3500,7000 , starting at upper L
    data = np.flipud(data)
    # Anything over 254 is bad
    res = np.where(data > 254, 0, data)
    res = np.where(np.logical_and(data >= 0, data < 100), data * 0.25, res)
    res = np.where(
        np.logical_and(data >= 100, data < 180),
        25.0 + ((data - 100) * 1.25),
        res,
    )
    res = np.where(
        np.logical_and(data >= 180, data < 255),
        125.0 + ((data - 180) * 5.0),
        res,
    )

    y1 = int((iemre.NORTH - mrms.SOUTH) * 100.0)
    y0 = int((iemre.SOUTH - mrms.SOUTH) * 100.0)
    x0 = int((iemre.WEST - mrms.WEST) * 100.0)
    x1 = int((iemre.EAST - mrms.WEST) * 100.0)
    ncprecip[offset, :, :] = res[y0:y1, x0:x1]
    nc.close()
Пример #21
0
def grid_day(nc, ts):
    """
    I proctor the gridding of data on an hourly basis
    @param ts Timestamp of the analysis, we'll consider a 20 minute window
    """
    offset = iemre.daily_offset(ts)
    if ts.day == 29 and ts.month == 2:
        ts = datetime.datetime(2000, 3, 1)

    sql = """SELECT * from ncdc_climate71 WHERE valid = '%s' and
             substr(station,3,4) != '0000' and substr(station,3,1) != 'C'
             """ % (ts.strftime("%Y-%m-%d"), )
    cursor.execute(sql)
    if cursor.rowcount > 4:
        res = generic_gridder(cursor, 'high')
        if res is not None:
            nc.variables['high_tmpk'][offset] = datatypes.temperature(res, 'F').value('K')
        cursor.scroll(0, mode='absolute')
        res = generic_gridder(cursor, 'low')
        if res is not None:
            nc.variables['low_tmpk'][offset] = datatypes.temperature(res, 'F').value('K')
        cursor.scroll(0, mode='absolute')
        res = generic_gridder(cursor, 'precip')
        if res is not None:
            nc.variables['p01d'][offset] = res * 25.4
    else:
        print "%s has %02i entries, FAIL" % (ts.strftime("%Y-%m-%d"),
                                             cursor.rowcount)
Пример #22
0
def do_precip12(nc, ts):
    """Compute the 24 Hour precip at 12 UTC, we do some more tricks though"""
    offset = iemre.daily_offset(ts)
    ets = datetime.datetime(ts.year, ts.month, ts.day, 12)
    ets = ets.replace(tzinfo=pytz.timezone("UTC"))
    sts = ets - datetime.timedelta(hours=24)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    if ts.month == 1 and ts.day == 1:
        print(("p01d_12z  for %s [idx:%s] %s(%s)->%s(%s) SPECIAL") %
              (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
               ets.strftime("%Y%m%d%H"), offset2))
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" %
                              (ets.year, ))
        phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0)
        hnc.close()
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" %
                              (sts.year, ))
        phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0)
        hnc.close()
    else:
        print(("p01d_12z  for %s [idx:%s] %s(%s)->%s(%s)") %
              (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
               ets.strftime("%Y%m%d%H"), offset2))
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" %
                              (ts.year, ))
        phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0)
        hnc.close()
    nc.variables['p01d_12z'][offset] = phour
Пример #23
0
def run(ts):
    ''' Actually do the work, please '''
    nc = netCDF4.Dataset('/mesonet/data/iemre/%s_mw_mrms_daily.nc' % (
                                                            ts.year,),
                         'a')
    offset = iemre.daily_offset(ts)
    ncprecip = nc.variables['p01d']
    ts += datetime.timedelta(hours=24)
    gmtts = ts.astimezone(pytz.timezone("UTC"))

    fn = gmtts.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/GIS/q2/"
                        "p24h_%Y%m%d%H00.png"))
    img = Image.open(fn)
    data = np.asarray(img)
    # data is 3500,7000 , starting at upper L
    data = np.flipud(data)
    # Anything over 254 is bad
    res = np.where(data > 254, 0, data)
    res = np.where(np.logical_and(data >= 0, data < 100), data * 0.25, res)
    res = np.where(np.logical_and(data >= 100, data < 180),
                   25. + ((data - 100) * 1.25), res)
    res = np.where(np.logical_and(data >= 180, data < 255),
                   125. + ((data - 180) * 5.), res)

    y1 = (iemre.NORTH - mrms.SOUTH) * 100.0
    y0 = (iemre.SOUTH - mrms.SOUTH) * 100.0
    x0 = (iemre.WEST - mrms.WEST) * 100.0
    x1 = (iemre.EAST - mrms.WEST) * 100.0
    ncprecip[offset, :, :] = res[y0:y1, x0:x1]
    nc.close()
Пример #24
0
def estimate_hilo(ts):
    """Estimate the High and Low Temperature based on gridded data"""
    idx = iemre.daily_offset(ts)
    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_daily.nc" % (ts.year, ),
                         'r')
    highgrid12 = temperature(nc.variables['high_tmpk_12z'][idx, :, :],
                             'K').value('F')
    lowgrid12 = temperature(nc.variables['low_tmpk_12z'][idx, :, :],
                            'K').value('F')
    highgrid00 = temperature(nc.variables['high_tmpk'][idx, :, :],
                             'K').value('F')
    lowgrid00 = temperature(nc.variables['low_tmpk'][idx, :, :],
                            'K').value('F')
    nc.close()

    for sid in nt.sts.keys():
        if nt.sts[sid]['temp24_hour'] in [0, 22, 23]:
            val = highgrid00[nt.sts[sid]['gridj'], nt.sts[sid]['gridi']]
        else:
            val = highgrid12[nt.sts[sid]['gridj'], nt.sts[sid]['gridi']]
        if val > -80 and val < 140:
            nt.sts[sid]['high'] = "%.0f" % (val, )

        if nt.sts[sid]['temp24_hour'] in [0, 22, 23]:
            val = lowgrid00[nt.sts[sid]['gridj'], nt.sts[sid]['gridi']]
        else:
            val = lowgrid12[nt.sts[sid]['gridj'], nt.sts[sid]['gridi']]
        if val > -80 and val < 140:
            nt.sts[sid]['low'] = "%.0f" % (val, )
Пример #25
0
def plot_daily(date, interval, plotvar, mc, mckey):
    """ Generate the plot, please """
    opts = PLOT_OPS[plotvar]
    offset = iemre.daily_offset(date)

    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_daily.nc" % (date.year, ))
    data = nc.variables[opts['ncvar_daily']][offset] / 25.4  # inches
    lons = nc.variables['lon'][:]
    lats = nc.variables['lat'][:]
    extra = lons[-1] + (lons[-1] - lons[-2])
    lons = np.concatenate([lons, [
        extra,
    ]])

    extra = lats[-1] + (lats[-1] - lats[-2])
    lats = np.concatenate([lats, [
        extra,
    ]])
    x, y = np.meshgrid(lons, lats)

    nc.close()

    p = plot.MapPlot(
        sector='midwest',
        title='%s IEM Reanalysis %s [%s]' %
        (date.strftime("%-d %b %Y"), opts['title'], opts['units']))
    p.pcolormesh(x, y, data, opts['clevs'], units=opts['units'])
    p.postprocess(web=True, memcache=mc, memcachekey=mckey, memcacheexpire=0)
Пример #26
0
def do_hrrr(ts):
    """Convert the hourly HRRR data to IEMRE grid"""
    total = None
    xaxis = None
    yaxis = None
    for hr in range(5, 23):  # Only need 5 AM to 10 PM for solar
        utc = ts.replace(hour=hr).astimezone(pytz.timezone("UTC"))
        fn = utc.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/model/hrrr/%H/"
                           "hrrr.t%Hz.3kmf00.grib2"))
        if not os.path.isfile(fn):
            # print 'HRRR file %s missing' % (fn,)
            continue
        grbs = pygrib.open(fn)
        try:
            if utc >= SWITCH_DATE:
                grb = grbs.select(name='Downward short-wave radiation flux')
            else:
                grb = grbs.select(parameterNumber=192)
        except ValueError:
            print 'coop/hrrr_solarrad.py %s had no solar rad' % (fn,)
            continue
        if len(grb) == 0:
            print 'Could not find SWDOWN in HRR %s' % (fn,)
            continue
        g = grb[0]
        if total is None:
            total = g.values
            lat1 = g['latitudeOfFirstGridPointInDegrees']
            lon1 = g['longitudeOfFirstGridPointInDegrees']
            llcrnrx, llcrnry = LCC(lon1, lat1)
            nx = g['Nx']
            ny = g['Ny']
            dx = g['DxInMetres']
            dy = g['DyInMetres']
            xaxis = llcrnrx + dx * np.arange(nx)
            yaxis = llcrnry + dy * np.arange(ny)
        else:
            total += g.values

    if total is None:
        print 'coop/hrrr_solarrad.py found no HRRR data for %s' % (
                                                    ts.strftime("%d %b %Y"), )
        return

    # We wanna store as W m-2, so we just average out the data by hour
    total = total / 24.0

    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_daily.nc" % (ts.year,),
                         'a')
    offset = iemre.daily_offset(ts)
    data = nc.variables['rsds'][offset, :, :]
    for i, lon in enumerate(iemre.XAXIS):
        for j, lat in enumerate(iemre.YAXIS):
            (x, y) = LCC(lon, lat)
            i2 = np.digitize([x], xaxis)[0]
            j2 = np.digitize([y], yaxis)[0]
            data[j, i] = total[j2, i2]

    nc.variables['rsds'][offset] = data
    nc.close()
Пример #27
0
def do_coop(ts):
    """Use COOP solar radiation data"""
    pgconn = psycopg2.connect(database='coop', host='iemdb', user='******')
    cursor = pgconn.cursor()

    cursor.execute("""SELECT ST_x(geom), ST_y(geom),
        coalesce(narr_srad, merra_srad) from alldata a JOIN stations t
        ON (a.station = t.id) WHERE
        day = %s and t.network ~* 'CLIMATE' and substr(id, 3, 1) != 'C'
        and substr(id, 3, 4) != '0000'
    """, (ts.strftime("%Y-%m-%d"), ))
    lons = []
    lats = []
    vals = []
    for row in cursor:
        if row[2] is None or row[2] < 0:
            continue
        lons.append(row[0])
        lats.append(row[1])
        vals.append(row[2])

    nn = NearestNDInterpolator((np.array(lons), np.array(lats)),
                               np.array(vals))
    xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS)

    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_daily.nc" % (ts.year,),
                         'a')
    offset = iemre.daily_offset(ts)
    # Data above is MJ / d / m-2, we want W / m-2
    nc.variables['rsds'][offset, :, :] = nn(xi, yi) * 1000000. / 86400.
    nc.close()
Пример #28
0
def do_precip(nc, ts):
    """Compute the 6 UTC to 6 UTC precip

    We need to be careful here as the timestamp sent to this app is today,
    we are actually creating the analysis for yesterday
    """
    sts = datetime.datetime(ts.year, ts.month, ts.day, 6)
    sts = sts.replace(tzinfo=pytz.timezone("UTC"))
    ets = sts + datetime.timedelta(hours=24)
    offset = iemre.daily_offset(ts)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    if ts.month == 12 and ts.day == 31:
        print(("p01d      for %s [idx:%s] %s(%s)->%s(%s) SPECIAL") %
              (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
               ets.strftime("%Y%m%d%H"), offset2))
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" %
                              (ets.year, ))
        phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0)
        hnc.close()
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" %
                              (sts.year, ))
        phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0)
        hnc.close()
    else:
        print(("p01d      for %s [idx:%s] %s(%s)->%s(%s)") %
              (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
               ets.strftime("%Y%m%d%H"), offset2))
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" %
                              (sts.year, ))
        phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0)
        hnc.close()
    nc.variables['p01d'][offset] = phour
Пример #29
0
def do_precip12(ts, ds):
    """Compute the 24 Hour precip at 12 UTC, we do some more tricks though"""
    offset = iemre.daily_offset(ts)
    ets = utc(ts.year, ts.month, ts.day, 12)
    sts = ets - datetime.timedelta(hours=24)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    if ts.month == 1 and ts.day == 1:
        if sts.year >= 1900:
            LOG.warning(
                "p01d_12z  for %s [idx:%s] %s(%s)->%s(%s) SPECIAL",
                ts,
                offset,
                sts.strftime("%Y%m%d%H"),
                offset1,
                ets.strftime("%Y%m%d%H"),
                offset2,
            )
        ncfn = iemre.get_hourly_ncname(ets.year)
        if not os.path.isfile(ncfn):
            LOG.warning("Missing %s", ncfn)
            return
        with ncopen(ncfn, timeout=600) as hnc:
            phour = np.sum(hnc.variables["p01m"][:offset2, :, :], 0)
        with ncopen(iemre.get_hourly_ncname(sts.year), timeout=600) as hnc:
            phour += np.sum(hnc.variables["p01m"][offset1:, :, :], 0)
    else:
        ncfn = iemre.get_hourly_ncname(ts.year)
        if not os.path.isfile(ncfn):
            LOG.warning("Missing %s", ncfn)
            return
        with ncopen(ncfn, timeout=600) as hnc:
            phour = np.sum(hnc.variables["p01m"][offset1:offset2, :, :], 0)
    ds["p01d_12z"].values = np.where(phour < 0, 0, phour)
Пример #30
0
def plot_daily(date, interval, plotvar, mc, mckey):
    """ Generate the plot, please """
    opts = PLOT_OPS[plotvar]
    offset = iemre.daily_offset(date)
    
    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_daily.nc" % (date.year,))
    data = nc.variables[opts['ncvar_daily']][offset] / 25.4 # inches
    lons = nc.variables['lon'][:]
    lats = nc.variables['lat'][:]
    extra = lons[-1] + (lons[-1] - lons[-2])
    lons = np.concatenate([lons, [extra,]])

    extra = lats[-1] + (lats[-1] - lats[-2])
    lats = np.concatenate([lats, [extra,]])
    x,y = np.meshgrid(lons, lats)

    nc.close()

    p = plot.MapPlot(sector='midwest',
                 title='%s IEM Reanalysis %s [%s]' % (date.strftime("%-d %b %Y"),
                                                 opts['title'], opts['units'])
                 )
    p.pcolormesh(x, y, data, opts['clevs'],
                 units=opts['units'])
    p.postprocess(web=True, memcache=mc, memcachekey=mckey, memcacheexpire=0)
Пример #31
0
def do_precip(nc, ts):
    """Compute the 6 UTC to 6 UTC precip

    We need to be careful here as the timestamp sent to this app is today,
    we are actually creating the analysis for yesterday
    """
    sts = datetime.datetime(ts.year, ts.month, ts.day, 6)
    sts = sts.replace(tzinfo=pytz.timezone("UTC"))
    ets = sts + datetime.timedelta(hours=24)
    offset = iemre.daily_offset(ts)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    if ts.month == 12 and ts.day == 31:
        print(("p01d      for %s [idx:%s] %s(%s)->%s(%s) SPECIAL"
               ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
                    ets.strftime("%Y%m%d%H"), offset2))
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (
            ets.year,))
        phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0)
        hnc.close()
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (
            sts.year,))
        phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0)
        hnc.close()
    else:
        print(("p01d      for %s [idx:%s] %s(%s)->%s(%s)"
               ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
                    ets.strftime("%Y%m%d%H"), offset2))
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (
            sts.year,))
        phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0)
        hnc.close()
    nc.variables['p01d'][offset] = phour
Пример #32
0
def do_hrrr(ts):
    """Convert the hourly HRRR data to IEMRE grid"""
    total = None
    xaxis = None
    yaxis = None
    for hr in range(5, 23):  # Only need 5 AM to 10 PM for solar
        utc = ts.replace(hour=hr).astimezone(pytz.timezone("UTC"))
        fn = utc.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/model/hrrr/%H/"
                           "hrrr.t%Hz.3kmf00.grib2"))
        if not os.path.isfile(fn):
            # print 'HRRR file %s missing' % (fn,)
            continue
        grbs = pygrib.open(fn)
        try:
            if utc >= SWITCH_DATE:
                grb = grbs.select(name='Downward short-wave radiation flux')
            else:
                grb = grbs.select(parameterNumber=192)
        except ValueError:
            print 'coop/hrrr_solarrad.py %s had no solar rad' % (fn, )
            continue
        if len(grb) == 0:
            print 'Could not find SWDOWN in HRR %s' % (fn, )
            continue
        g = grb[0]
        if total is None:
            total = g.values
            lat1 = g['latitudeOfFirstGridPointInDegrees']
            lon1 = g['longitudeOfFirstGridPointInDegrees']
            llcrnrx, llcrnry = LCC(lon1, lat1)
            nx = g['Nx']
            ny = g['Ny']
            dx = g['DxInMetres']
            dy = g['DyInMetres']
            xaxis = llcrnrx + dx * np.arange(nx)
            yaxis = llcrnry + dy * np.arange(ny)
        else:
            total += g.values

    if total is None:
        print 'coop/hrrr_solarrad.py found no HRRR data for %s' % (
            ts.strftime("%d %b %Y"), )
        return

    # We wanna store as W m-2, so we just average out the data by hour
    total = total / 24.0

    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_daily.nc" % (ts.year, ),
                         'a')
    offset = iemre.daily_offset(ts)
    data = nc.variables['rsds'][offset, :, :]
    for i, lon in enumerate(iemre.XAXIS):
        for j, lat in enumerate(iemre.YAXIS):
            (x, y) = LCC(lon, lat)
            i2 = np.digitize([x], xaxis)[0]
            j2 = np.digitize([y], yaxis)[0]
            data[j, i] = total[j2, i2]

    nc.variables['rsds'][offset] = data
    nc.close()
Пример #33
0
def grid_day(nc, ts):
    """
    I proctor the gridding of data on an hourly basis
    @param ts Timestamp of the analysis, we'll consider a 20 minute window
    """
    cursor = COOP.cursor(cursor_factory=psycopg2.extras.DictCursor)
    offset = iemre.daily_offset(ts)
    if ts.day == 29 and ts.month == 2:
        ts = datetime.datetime(2000, 3, 1)

    sql = """SELECT * from climate51 WHERE valid = '%s' and
             substr(station,3,4) != '0000' and substr(station,3,1) != 'C'
             """ % (ts.strftime("%Y-%m-%d"), )
    cursor.execute(sql)
    res = generic_gridder(nc, cursor, "high")
    nc.variables["high_tmpk"][offset] = datatypes.temperature(res,
                                                              "F").value("K")

    cursor.scroll(0, mode="absolute")
    res = generic_gridder(nc, cursor, "low")
    nc.variables["low_tmpk"][offset] = datatypes.temperature(res,
                                                             "F").value("K")

    cursor.scroll(0, mode="absolute")
    res = generic_gridder(nc, cursor, "precip")
    nc.variables["p01d"][offset] = datatypes.distance(res, "IN").value("MM")

    cursor.scroll(0, mode="absolute")
    res = generic_gridder(nc, cursor, "gdd50")
    nc.variables["gdd50"][offset] = res
Пример #34
0
def merge(nc, valid, gribname, vname):
    """Merge in the grib data"""
    fn = valid.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/model/cfs/%H/" +
                         gribname + ".01.%Y%m%d%H.daily.grib2"))
    if not os.path.isfile(fn):
        print("cfs2iemre missing %s, abort" % (fn, ))
        sys.exit()
    grbs = pygrib.open(fn)
    lats = None
    lons = None
    xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS)
    for grib in tqdm(grbs,
                     total=grbs.messages,
                     desc=vname,
                     disable=not sys.stdout.isatty()):
        ftime = valid + datetime.timedelta(hours=grib.forecastTime)
        # move us safely back to get into the proper date
        cst = ftime - datetime.timedelta(hours=7)
        if cst.year != valid.year:
            continue
        if lats is None:
            lats, lons = grib.latlons()
        vals = grib.values
        nn = NearestNDInterpolator((lons.flat, lats.flat), vals.flat)
        vals = nn(xi, yi)
        tstep = iemre.daily_offset(cst.date())
        current = nc.variables[vname][tstep, :, :]
        if current.mask.all():
            current[:, :] = DEFAULTS[vname]
        nc.variables[vname][tstep, :, :] = AGGFUNC[vname](current, vals)
Пример #35
0
def estimate_hilo(ts):
    """Estimate the High and Low Temperature based on gridded data"""
    idx = iemre.daily_offset(ts)
    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_daily.nc" % (ts.year, ),
                         'r')
    highgrid12 = temperature(nc.variables['high_tmpk_12z'][idx, :, :],
                             'K').value('F')
    lowgrid12 = temperature(nc.variables['low_tmpk_12z'][idx, :, :],
                            'K').value('F')
    highgrid00 = temperature(nc.variables['high_tmpk'][idx, :, :],
                             'K').value('F')
    lowgrid00 = temperature(nc.variables['low_tmpk'][idx, :, :],
                            'K').value('F')
    nc.close()

    for sid in nt.sts.keys():
        if nt.sts[sid]['temp24_hour'] in [0, 22, 23]:
            val = highgrid00[nt.sts[sid]['gridj'], nt.sts[sid]['gridi']]
        else:
            val = highgrid12[nt.sts[sid]['gridj'], nt.sts[sid]['gridi']]
        if val > -80 and val < 140:
            nt.sts[sid]['high'] = "%.0f" % (val, )

        if nt.sts[sid]['temp24_hour'] in [0, 22, 23]:
            val = lowgrid00[nt.sts[sid]['gridj'], nt.sts[sid]['gridi']]
        else:
            val = lowgrid12[nt.sts[sid]['gridj'], nt.sts[sid]['gridi']]
        if val > -80 and val < 140:
            nt.sts[sid]['low'] = "%.0f" % (val, )
Пример #36
0
def do_precip12(ts):
    """Compute the 24 Hour precip at 12 UTC, we do some more tricks though"""
    offset = iemre.daily_offset(ts)
    ets = utc(ts.year, ts.month, ts.day, 12)
    sts = ets - datetime.timedelta(hours=24)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    if ts.month == 1 and ts.day == 1:
        if sts.year >= 1900:
            print(("p01d_12z  for %s [idx:%s] %s(%s)->%s(%s) SPECIAL") %
                  (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
                   ets.strftime("%Y%m%d%H"), offset2))
        ncfn = iemre.get_hourly_ncname(ets.year)
        if not os.path.isfile(ncfn):
            print("Missing %s" % (ncfn, ))
            return
        hnc = ncopen(ncfn, timeout=600)
        phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0)
        hnc.close()
        hnc = ncopen(iemre.get_hourly_ncname(sts.year), timeout=600)
        phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0)
        hnc.close()
    else:
        ncfn = iemre.get_hourly_ncname(ts.year)
        if not os.path.isfile(ncfn):
            print("Missing %s" % (ncfn, ))
            return
        hnc = ncopen(ncfn, timeout=600)
        phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0)
        hnc.close()
    write_grid(ts, 'p01d_12z', np.where(phour < 0, 0, phour))
Пример #37
0
def do_var(varname):
    """
    Run our estimator for a given variable
    """
    currentnc = None
    sql = """select day, station from alldata_%s WHERE %s is null
        and day >= '1893-01-01' ORDER by day ASC""" % (state.lower(), varname)
    ccursor.execute(sql)
    for row in ccursor:
        day = row[0]
        station = row[1]
        if station not in nt.sts:
            continue

        sql = """
            SELECT station, %s from alldata_%s WHERE %s is not NULL
            and station in %s and day = '%s'
            """ % (varname, state, varname, tuple(friends[station]), day)
        ccursor2.execute(sql)
        weight = []
        value = []
        for row2 in ccursor2:
            idx = friends[station].index(row2[0])
            weight.append(weights[station][idx])
            value.append(row2[1])

        if len(weight) < 3:
            # Nearest neighbors failed, so lets look at our grided analysis
            # and sample from it
            if currentnc is None or currentnc.title.find(str(day.year)) == -1:
                currentnc = netCDF4.Dataset(("/mesonet/data/iemre/"
                                             "%s_mw_daily.nc") % (day.year, ))
            tidx = iemre.daily_offset(
                datetime.datetime(day.year, day.month, day.day))
            iidx, jidx = iemre.find_ij(nt.sts[station]['lon'],
                                       nt.sts[station]['lat'])
            iemreval = currentnc.variables[vnameconv[varname]][tidx, jidx,
                                                               iidx]
            if varname in ('high', 'low'):
                interp = temperature(iemreval, 'K').value('F')
            else:
                interp = iemreval / 24.5
            print '--> Neighbor failure, %s %s %s' % (station, day, varname)
        else:
            mass = sum(weight)
            interp = np.sum(np.array(weight) * np.array(value) / mass)

        dataformat = '%.2f'
        if varname in ['high', 'low']:
            dataformat = '%.0f'
        print(('Set station: %s day: %s varname: %s value: %s') %
              (station, day, varname, dataformat % (interp, )))
        sql = """
            UPDATE alldata_%s SET estimated = true, %s = %s WHERE
            station = '%s' and day = '%s'
            """ % (state.lower(), varname, dataformat %
                   (interp, ), station, day)
        sql = sql.replace(' nan ', ' null ')
        ccursor2.execute(sql)
Пример #38
0
def doday(ts, realtime):
    """
    Create a plot of precipitation stage4 estimates for some day
    """
    lts = utc(ts.year, ts.month, ts.day, 12)
    lts = lts.astimezone(pytz.timezone("America/Chicago"))
    # make assumptions about the last valid MRMS data
    if realtime:
        # Up until :59 after of the last hour
        lts = (datetime.datetime.now() -
               datetime.timedelta(hours=1)).replace(minute=59)
    else:
        lts = lts.replace(year=ts.year,
                          month=ts.month,
                          day=ts.day,
                          hour=23,
                          minute=59)

    idx = iemre.daily_offset(ts)
    ncfn = iemre.get_daily_mrms_ncname(ts.year)
    if not os.path.isfile(ncfn):
        LOG.info("File %s missing, abort.", ncfn)
        return
    with ncopen(ncfn, timeout=300) as nc:
        precip = nc.variables['p01d'][idx, :, :]
        lats = nc.variables['lat'][:]
        lons = nc.variables['lon'][:]
    subtitle = "Total between 12:00 AM and %s" % (
        lts.strftime("%I:%M %p %Z"), )
    routes = 'ac'
    if not realtime:
        routes = 'a'

    # clevs = np.arange(0, 0.25, 0.05)
    # clevs = np.append(clevs, np.arange(0.25, 3., 0.25))
    # clevs = np.append(clevs, np.arange(3., 10.0, 1))
    clevs = [
        0.01, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10
    ]

    (xx, yy) = np.meshgrid(lons, lats)
    for sector in ['iowa', 'midwest']:
        pqstr = ("plot %s %s00 %s_q2_1d.png %s_q2_1d.png png") % (
            routes, ts.strftime("%Y%m%d%H"), sector, sector)
        mp = MapPlot(title=("%s NCEP MRMS Q3 Today's Precipitation") %
                     (ts.strftime("%-d %b %Y"), ),
                     subtitle=subtitle,
                     sector=sector)

        mp.pcolormesh(xx,
                      yy,
                      distance(precip, 'MM').value('IN'),
                      clevs,
                      cmap=nwsprecip(),
                      units='inch')
        if sector == 'iowa':
            mp.drawcounties()
        mp.postprocess(pqstr=pqstr, view=False)
        mp.close()
Пример #39
0
def workflow(valid):
    """Our workflow"""
    if valid.month == 1 and valid.day == 1:
        print("prism_adjust_stage4, sorry Jan 1 processing is a TODO!")
        return
    # read prism
    tidx = daily_offset(valid)
    nc = netCDF4.Dataset("/mesonet/data/prism/%s_daily.nc" % (valid.year, ),
                         'r')
    ppt = nc.variables['ppt'][tidx, :, :]
    # missing as zero
    ppt = np.where(ppt.mask, 0, ppt)
    lons = nc.variables['lon'][:]
    lats = nc.variables['lat'][:]
    nc.close()
    (lons, lats) = np.meshgrid(lons, lats)

    # Interpolate this onto the stage4 grid
    nc = netCDF4.Dataset(
        ("/mesonet/data/stage4/%s_stage4_hourly.nc") % (valid.year, ), 'a')
    p01m = nc.variables['p01m']
    p01m_status = nc.variables['p01m_status']
    s4lons = nc.variables['lon'][:]
    s4lats = nc.variables['lat'][:]
    # Values are in the hourly arrears, so start at -23 and thru current hour
    sts_tidx = hourly_offset(valid - datetime.timedelta(hours=23))
    ets_tidx = hourly_offset(valid + datetime.timedelta(hours=1))
    s4total = np.sum(p01m[sts_tidx:ets_tidx, :, :], axis=0)
    # make sure the s4total does not have zeros
    s4total = np.where(s4total < 0.001, 0.001, s4total)

    nn = NearestNDInterpolator((lons.flat, lats.flat), ppt.flat)
    prism_on_s4grid = nn(s4lons, s4lats)
    multiplier = prism_on_s4grid / s4total

    # Do the work now, we should not have to worry about the scale factor
    for tidx in range(sts_tidx, ets_tidx):
        newval = p01m[tidx, :, :] * multiplier
        p01m[tidx, :, :] = newval
        # make sure have data
        if np.ma.max(newval) > 0:
            p01m_status[tidx] = 2
        else:
            print(("prism_adjust_stage4 NOOP for time %s[idx:%s]") %
                  ((datetime.datetime(valid.year, 1, 1, 0) +
                    datetime.timedelta(hours=tidx)).strftime("%Y-%m-%dT%H"),
                   tidx))
    """
    s4total_v2 = np.sum(p01m[sts_tidx:ets_tidx, :, :], axis=0)

    from pyiem.plot.geoplot import MapPlot
    import matplotlib.pyplot as plt
    mp = MapPlot(sector='iowa')
    mp.pcolormesh(s4lons, s4lats, s4total_v2,
                  np.arange(-10, 11, 1), cmap=plt.get_cmap("BrBG"))
    mp.postprocess(filename='test.png')
    mp.close()
    """
    nc.close()
Пример #40
0
def do_var(varname):
    """
    Run our estimator for a given variable
    """
    currentnc = None
    sql = """select day, station from alldata_%s WHERE %s is null
        and day >= '1893-01-01' ORDER by day ASC""" % (state.lower(), varname)
    ccursor.execute(sql)
    for row in ccursor:
        day = row[0]
        station = row[1]
        if station not in nt.sts:
            continue

        sql = """
            SELECT station, %s from alldata_%s WHERE %s is not NULL
            and station in %s and day = '%s'
            """ % (varname, state, varname, tuple(friends[station]), day)
        ccursor2.execute(sql)
        weight = []
        value = []
        for row2 in ccursor2:
            idx = friends[station].index(row2[0])
            weight.append(weights[station][idx])
            value.append(row2[1])

        if len(weight) < 3:
            # Nearest neighbors failed, so lets look at our grided analysis
            # and sample from it
            if currentnc is None or currentnc.title.find(str(day.year)) == -1:
                currentnc = netCDF4.Dataset(("/mesonet/data/iemre/"
                                             "%s_mw_daily.nc") % (day.year,))
            tidx = iemre.daily_offset(datetime.datetime(day.year, day.month,
                                                        day.day))
            iidx, jidx = iemre.find_ij(nt.sts[station]['lon'],
                                       nt.sts[station]['lat'])
            iemreval = currentnc.variables[vnameconv[varname]][tidx, jidx,
                                                               iidx]
            if varname in ('high', 'low'):
                interp = temperature(iemreval, 'K').value('F')
            else:
                interp = distance(iemreval, 'MM').value('IN')
            print '--> Neighbor failure, %s %s %s' % (station, day, varname)
        else:
            mass = sum(weight)
            interp = np.sum(np.array(weight) * np.array(value) / mass)

        dataformat = '%.2f'
        if varname in ['high', 'low']:
            dataformat = '%.0f'
        print(('Set station: %s day: %s varname: %s value: %s'
               ) % (station, day, varname, dataformat % (interp,)))
        sql = """
            UPDATE alldata_%s SET estimated = true, %s = %s WHERE
            station = '%s' and day = '%s'
            """ % (state.lower(), varname,
                   dataformat % (interp,), station, day)
        sql = sql.replace(' nan ', ' null ')
        ccursor2.execute(sql)
Пример #41
0
def do_process(valid, fn):
    """Process this file, please """
    # shape of data is (1, 621, 1405)
    data = rasterio.open(fn).read()
    varname = fn.split("_")[1]
    idx = daily_offset(valid)
    with ncopen("/mesonet/data/prism/%s_daily.nc" % (valid.year, ), 'a') as nc:
        nc.variables[varname][idx] = np.flipud(data[0])
Пример #42
0
def do_process(valid, fn):
    """Process this file, please """
    # shape of data is (1, 621, 1405)
    data = rasterio.open(fn).read()
    varname = fn.split("_")[1]
    nc = netCDF4.Dataset("/mesonet/data/prism/%s_daily.nc" % (valid.year,),
                         'a')
    idx = daily_offset(valid)
    nc.variables[varname][idx] = np.flipud(data[0])
    nc.close()
Пример #43
0
def doday(ts, realtime):
    """
    Create a plot of precipitation stage4 estimates for some day
    """
    lts = datetime.datetime.utcnow().replace(tzinfo=pytz.timezone("UTC"))
    lts = lts.astimezone(pytz.timezone("America/Chicago"))
    # make assumptions about the last valid MRMS data
    if realtime:
        # Up until :59 after of the last hour
        lts = (lts - datetime.timedelta(hours=1)).replace(minute=59)
    else:
        lts = lts.replace(year=ts.year,
                          month=ts.month,
                          day=ts.day,
                          hour=23,
                          minute=59)

    idx = iemre.daily_offset(ts)
    ncfn = "/mesonet/data/iemre/%s_mw_mrms_daily.nc" % (ts.year, )
    nc = netCDF4.Dataset(ncfn)
    precip = nc.variables['p01d'][idx, :, :]
    lats = nc.variables['lat'][:]
    lons = nc.variables['lon'][:]
    subtitle = "Total between 12:00 AM and %s" % (
        lts.strftime("%I:%M %p %Z"), )
    routes = 'ac'
    if not realtime:
        routes = 'a'

    # clevs = np.arange(0, 0.25, 0.05)
    # clevs = np.append(clevs, np.arange(0.25, 3., 0.25))
    # clevs = np.append(clevs, np.arange(3., 10.0, 1))
    clevs = [
        0.01, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10
    ]

    sector = 'iowa'
    pqstr = ("plot %s %s00 %s_q2_1d.png %s_q2_1d.png png") % (
        routes, ts.strftime("%Y%m%d%H"), sector, sector)
    m = MapPlot(title=("%s NCEP MRMS Q3 Today's Precipitation") %
                (ts.strftime("%-d %b %Y"), ),
                subtitle=subtitle,
                sector=sector)

    (x, y) = np.meshgrid(lons, lats)

    m.pcolormesh(x,
                 y,
                 distance(precip, 'MM').value('IN'),
                 clevs,
                 cmap=nwsprecip(),
                 units='inch')
    m.drawcounties()
    m.postprocess(pqstr=pqstr, view=False)
    m.close()
Пример #44
0
def plotter(fdict):
    """ Go """
    import matplotlib
    matplotlib.use('agg')
    from pyiem.plot import MapPlot
    ptype = fdict.get('ptype', 'c')
    date = datetime.datetime.strptime(fdict.get('date', '2015-01-01'),
                                      '%Y-%m-%d')
    varname = fdict.get('var', 'rsds')

    idx0 = iemre.daily_offset(date)
    nc = netCDF4.Dataset(("/mesonet/data/iemre/%s_mw_daily.nc"
                          ) % (date.year, ), 'r')
    lats = nc.variables['lat'][:]
    lons = nc.variables['lon'][:]
    if varname == 'rsds':
        # Value is in W m**-2, we want MJ
        data = nc.variables[varname][idx0, :, :] * 86400. / 1000000.
        units = 'MJ d-1'
        clevs = np.arange(0, 37, 3.)
        clevs[0] = 0.01
        clevstride = 1
    elif varname in ['p01d', 'p01d_12z']:
        # Value is in W m**-2, we want MJ
        data = nc.variables[varname][idx0, :, :] / 25.4
        units = 'inch'
        clevs = np.arange(0, 0.25, 0.05)
        clevs = np.append(clevs, np.arange(0.25, 3., 0.25))
        clevs = np.append(clevs, np.arange(3., 10.0, 1))
        clevs[0] = 0.01
        clevstride = 1
    elif varname in ['high_tmpk', 'low_tmpk', 'high_tmpk_12z', 'low_tmpk_12z']:
        # Value is in W m**-2, we want MJ
        data = temperature(nc.variables[varname][idx0, :, :], 'K').value('F')
        units = 'F'
        clevs = np.arange(-30, 120, 2)
        clevstride = 5
    nc.close()

    title = date.strftime("%-d %B %Y")
    m = MapPlot(sector='midwest', axisbg='white', nocaption=True,
                title='IEM Reanalysis of %s for %s' % (PDICT.get(varname),
                                                       title),
                subtitle='Data derived from various NOAA datasets'
                )
    if np.ma.is_masked(np.max(data)):
        return 'Data Unavailable'
    x, y = np.meshgrid(lons, lats)
    if ptype == 'c':
        m.contourf(x, y, data, clevs, clevstride=clevstride, units=units)
    else:
        m.pcolormesh(x, y, data, clevs, clevstride=clevstride, units=units)

    return m.fig
Пример #45
0
def do_precip(nc, ts):
    """Compute the precip totals based on the hourly analysis totals"""
    offset = iemre.daily_offset(ts)
    ets = ts.replace(hour=12, tzinfo=pytz.timezone("UTC"))
    sts = ets - datetime.timedelta(hours=24)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year,))
    phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0)
    nc.variables['p01d'][offset] = phour
    hnc.close()
Пример #46
0
def main(argv):
    """Go Main Go."""
    year = int(argv[1])
    ets = min([datetime.date(year, 12, 31), datetime.date.today()])
    queue = []
    for x0 in np.arange(iemre.WEST, iemre.EAST, 5.):
        for y0 in np.arange(iemre.SOUTH, iemre.NORTH, 5.):
            queue.append([x0, y0])
    for x0, y0 in tqdm(queue, disable=not sys.stdout.isatty()):
        url = (
            "https://power.larc.nasa.gov/cgi-bin/v1/DataAccess.py?"
            "request=execute&identifier=Regional&"
            "parameters=ALLSKY_SFC_SW_DWN&"
            "startDate=%s0101&endDate=%s&userCommunity=SSE&"
            "tempAverage=DAILY&bbox=%s,%s,%s,%s&user=anonymous&"
            "outputList=NETCDF"
        ) % (year, ets.strftime("%Y%m%d"), y0, x0,
             min([y0 + 5., iemre.NORTH]) - 0.1,
             min([x0 + 5., iemre.EAST]) - 0.1)
        req = requests.get(url, timeout=60)
        js = req.json()
        if 'outputs' not in js:
            print(url)
            print(js)
            continue
        fn = js['outputs']['netcdf']
        req = requests.get(fn, timeout=60, stream=True)
        ncfn = '/tmp/power%s.nc' % (year, )
        with open(ncfn, 'wb') as fh:
            for chunk in req.iter_content(chunk_size=1024):
                if chunk:
                    fh.write(chunk)
            fh.close()
        nc = ncopen(ncfn)
        for day, _ in enumerate(nc.variables['time'][:]):
            date = datetime.date(year, 1, 1) + datetime.timedelta(days=day)
            # kwh to MJ/d  3600 * 1000 / 1e6
            data = nc.variables['ALLSKY_SFC_SW_DWN'][day, :, :] * 3.6
            # Sometimes there are missing values?
            if np.ma.is_masked(data):
                data[data.mask] = np.mean(data)
            i, j = iemre.find_ij(x0, y0)
            # resample data is 0.5, iemre is 0.125
            data = np.repeat(np.repeat(data, 4, axis=0), 4, axis=1)
            shp = np.shape(data)
            # print("i: %s j: %s shp: %s" % (i, j, shp))
            renc = ncopen(iemre.get_daily_ncname(year), 'a')
            renc.variables['power_swdn'][
                iemre.daily_offset(date),
                slice(j, j+shp[0]), slice(i, i+shp[1])
            ] = data
            renc.close()
        nc.close()
Пример #47
0
def write_grid(valid, vname, grid):
    """Write data to backend netcdf"""
    offset = iemre.daily_offset(valid)
    nc = ncopen(iemre.get_daily_ncname(valid.year), 'a', timeout=600)
    if nc is None:
        print("daily_analysis#write_grid first open attempt failed, try #2")
        nc = ncopen(iemre.get_daily_ncname(valid.year), 'a', timeout=600)
    print(("%13s [idx:%s] min: %6.2f max: %6.2f [%s]") %
          (vname, offset, np.nanmin(grid), np.nanmax(grid),
           nc.variables[vname].units))
    nc.variables[vname][offset] = grid
    nc.close()
Пример #48
0
def run(ts):
    """ Actually do the work, please """
    nc = netCDF4.Dataset(('/mesonet/data/iemre/%s_mw_mrms_daily.nc'
                          '') % (ts.year,), 'a')
    offset = iemre.daily_offset(ts)
    ncprecip = nc.variables['p01d']

    # We want this mrms variable to replicate the netcdf file, so the
    # origin is the southwestern corner
    ts += datetime.timedelta(hours=24)
    gmtts = ts.astimezone(pytz.timezone("UTC"))

    gribfn = gmtts.strftime(("/mnt/a4/data/%Y/%m/%d/mrms/ncep/"
                             "RadarOnly_QPE_24H/"
                             "RadarOnly_QPE_24H_00.00_%Y%m%d-%H%M00.grib2.gz"))
    if not os.path.isfile(gribfn):
        print("merge_mrms_q3.py MISSING %s" % (gribfn,))
        return

    fp = gzip.GzipFile(gribfn, 'rb')
    (_, tmpfn) = tempfile.mkstemp()
    tmpfp = open(tmpfn, 'wb')
    tmpfp.write(fp.read())
    tmpfp.close()
    grbs = pygrib.open(tmpfn)
    grb = grbs[1]
    lats, _ = grb.latlons()
    os.unlink(tmpfn)

    val = grb['values']
    # Anything less than zero, we set to zero
    val = np.where(val < 0, 0, val)

    # CAREFUL HERE!  The MRMS grid is North to South
    # set top (smallest y)
    y0 = int((lats[0, 0] - iemre.NORTH) * 100.0)
    y1 = int((lats[0, 0] - iemre.SOUTH) * 100.0)
    x0 = int((iemre.WEST - mrms.WEST) * 100.0)
    x1 = int((iemre.EAST - mrms.WEST) * 100.0)
    # print 'y0:%s y1:%s x0:%s x1:%s' % (y0, y1, x0, x1)
    ncprecip[offset, :, :] = np.flipud(val[y0:y1, x0:x1])
    # m = MapPlot(sector='midwest')
    # x, y = np.meshgrid(nc.variables['lon'][:], nc.variables['lat'][:])
    # m.pcolormesh(x, y, ncprecip[offset,:,:], range(10), latlon=True)
    # m.postprocess(filename='test.png')
    # (fig, ax) = plt.subplots()
    # ax.imshow(mrms)
    # fig.savefig('test.png')
    # (fig, ax) = plt.subplots()
    # ax.imshow(mrms[y0:y1,x0:x1])
    # fig.savefig('test2.png')
    nc.close()
Пример #49
0
def load_iemre():
    """Use IEM Reanalysis for non-precip data

    24km product is smoothed down to the 0.01 degree grid
    """
    printt("load_iemre() called")
    xaxis = np.arange(MYWEST, MYEAST, 0.01)
    yaxis = np.arange(MYSOUTH, MYNORTH, 0.01)
    xi, yi = np.meshgrid(xaxis, yaxis)

    fn = iemre.get_daily_ncname(VALID.year)
    if not os.path.isfile(fn):
        printt("Missing %s for load_solar, aborting" % (fn, ))
        sys.exit()
    with ncopen(fn) as nc:
        offset = iemre.daily_offset(VALID)
        lats = nc.variables["lat"][:]
        lons = nc.variables["lon"][:]
        lons, lats = np.meshgrid(lons, lats)

        # Storage is W m-2, we want langleys per day
        data = nc.variables["rsds"][offset, :, :] * 86400.0 / 1000000.0 * 23.9
        # Default to a value of 300 when this data is missing, for some reason
        nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)),
                                   np.ravel(data))
        SOLAR[:] = iemre_bounds_check("rsds", nn(xi, yi), 0, 1000)

        data = temperature(nc.variables["high_tmpk"][offset, :, :],
                           "K").value("C")
        nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)),
                                   np.ravel(data))
        HIGH_TEMP[:] = iemre_bounds_check("high_tmpk", nn(xi, yi), -60, 60)

        data = temperature(nc.variables["low_tmpk"][offset, :, :],
                           "K").value("C")
        nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)),
                                   np.ravel(data))
        LOW_TEMP[:] = iemre_bounds_check("low_tmpk", nn(xi, yi), -60, 60)

        data = temperature(nc.variables["avg_dwpk"][offset, :, :],
                           "K").value("C")
        nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)),
                                   np.ravel(data))
        DEWPOINT[:] = iemre_bounds_check("avg_dwpk", nn(xi, yi), -60, 60)

        data = nc.variables["wind_speed"][offset, :, :]
        nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)),
                                   np.ravel(data))
        WIND[:] = iemre_bounds_check("wind_speed", nn(xi, yi), 0, 30)
    printt("load_iemre() finished")
Пример #50
0
def do_day(valid):
    """ Process a day please """
    idx = iemre.daily_offset(valid)
    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_daily.nc" % (valid.year, ),
                         'r')
    high = temperature(nc.variables['high_tmpk_12z'][idx, :, :],
                       'K').value('F')
    low = temperature(nc.variables['low_tmpk_12z'][idx, :, :], 'K').value('F')
    precip = nc.variables['p01d_12z'][idx, :, :] / 25.4
    nc.close()
    for state in ('IA', 'NE', 'MN', 'WI', 'MI', 'OH', 'IN', 'IL', 'MO', 'KS',
                  'KY', 'ND', 'SD'):
        do_state_day(state, valid, high, low, precip)
        do_climdiv_day(state, valid, high, low, precip)
Пример #51
0
def load_iemre():
    """Use IEM Reanalysis for non-precip data

    24km product is smoothed down to the 0.01 degree grid
    """
    printt("load_iemre() called")
    xaxis = np.arange(MYWEST, MYEAST, 0.01)
    yaxis = np.arange(MYSOUTH, MYNORTH, 0.01)
    xi, yi = np.meshgrid(xaxis, yaxis)

    fn = iemre.get_daily_ncname(VALID.year)
    if not os.path.isfile(fn):
        printt("Missing %s for load_solar, aborting" % (fn,))
        sys.exit()
    nc = netCDF4.Dataset(fn, 'r')
    offset = iemre.daily_offset(VALID)
    lats = nc.variables['lat'][:]
    lons = nc.variables['lon'][:]
    lons, lats = np.meshgrid(lons, lats)

    # Storage is W m-2, we want langleys per day
    data = nc.variables['rsds'][offset, :, :] * 86400. / 1000000. * 23.9
    # Default to a value of 300 when this data is missing, for some reason
    nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)),
                               np.ravel(data))
    SOLAR[:] = iemre_bounds_check('rsds', nn(xi, yi), 0, 1000)

    data = temperature(nc.variables['high_tmpk'][offset, :, :], 'K').value('C')
    nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)),
                               np.ravel(data))
    HIGH_TEMP[:] = iemre_bounds_check('high_tmpk', nn(xi, yi), -60, 60)

    data = temperature(nc.variables['low_tmpk'][offset, :, :], 'K').value('C')
    nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)),
                               np.ravel(data))
    LOW_TEMP[:] = iemre_bounds_check('low_tmpk', nn(xi, yi), -60, 60)

    data = temperature(nc.variables['avg_dwpk'][offset, :, :], 'K').value('C')
    nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)),
                               np.ravel(data))
    DEWPOINT[:] = iemre_bounds_check('avg_dwpk', nn(xi, yi), -60, 60)

    data = nc.variables['wind_speed'][offset, :, :]
    nn = NearestNDInterpolator((np.ravel(lons), np.ravel(lats)),
                               np.ravel(data))
    WIND[:] = iemre_bounds_check('wind_speed', nn(xi, yi), 0, 30)

    nc.close()
    printt("load_iemre() finished")
Пример #52
0
def do_precip12(nc, ts):
    """Compute the 24 Hour precip at 12 UTC, we do some more tricks though"""
    offset = iemre.daily_offset(ts)
    ets = datetime.datetime(ts.year, ts.month, ts.day, 12)
    ets = ets.replace(tzinfo=pytz.timezone("UTC"))
    sts = ets - datetime.timedelta(hours=24)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    print(("p01d_12z  for %s [idx:%s] %s(%s)->%s(%s)"
           ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
                ets.strftime("%Y%m%d%H"), offset2))
    hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year,))
    phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0)
    nc.variables['p01d_12z'][offset] = phour
    hnc.close()
Пример #53
0
def do_day(valid):
    """ Process a day please """
    idx = iemre.daily_offset(valid)
    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_daily.nc" % (valid.year, ),
                         'r')
    high = temperature(nc.variables['high_tmpk_12z'][idx, :, :],
                       'K').value('F')
    low = temperature(nc.variables['low_tmpk_12z'][idx, :, :],
                      'K').value('F')
    precip = nc.variables['p01d_12z'][idx, :, :] / 25.4
    nc.close()
    for state in ('IA', 'NE', 'MN', 'WI', 'MI', 'OH', 'IN', 'IL', 'MO',
                  'KS', 'KY', 'ND', 'SD'):
        do_state_day(state, valid, high, low, precip)
        do_climdiv_day(state, valid, high, low, precip)
Пример #54
0
def estimate_snow(ts):
    """Estimate the Snow based on COOP reports"""
    idx = iemre.daily_offset(ts)
    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_daily.nc" % (ts.year, ),
                         'r')
    snowgrid12 = nc.variables['snow_12z'][idx, :, :] / 25.4
    snowdgrid12 = nc.variables['snowd_12z'][idx, :, :] / 25.4
    nc.close()

    for sid in nt.sts.keys():
        val = snowgrid12[nt.sts[sid]['gridj'], nt.sts[sid]['gridi']]
        if val >= 0 and val < 100:
            nt.sts[sid]['snow'] = "%.1f" % (val, )
        val = snowdgrid12[nt.sts[sid]['gridj'], nt.sts[sid]['gridi']]
        if val >= 0 and val < 140:
            nt.sts[sid]['snowd'] = "%.1f" % (val, )
Пример #55
0
def doday(ts, realtime):
    """
    Create a plot of precipitation stage4 estimates for some day
    """
    lts = datetime.datetime.utcnow().replace(
                tzinfo=pytz.timezone("UTC"))
    lts = lts.astimezone(pytz.timezone("America/Chicago"))
    # make assumptions about the last valid MRMS data
    if realtime:
        # Up until :59 after of the last hour
        lts = (lts - datetime.timedelta(hours=1)).replace(minute=59)
    else:
        lts = lts.replace(year=ts.year, month=ts.month, day=ts.day,
                          hour=23, minute=59)

    idx = iemre.daily_offset(ts)
    ncfn = "/mesonet/data/iemre/%s_mw_mrms_daily.nc" % (ts.year,)
    nc = netCDF4.Dataset(ncfn)
    precip = nc.variables['p01d'][idx, :, :]
    lats = nc.variables['lat'][:]
    lons = nc.variables['lon'][:]
    subtitle = "Total between 12:00 AM and %s" % (
                                            lts.strftime("%I:%M %p %Z"),)
    routes = 'ac'
    if not realtime:
        routes = 'a'

    # clevs = np.arange(0, 0.25, 0.05)
    # clevs = np.append(clevs, np.arange(0.25, 3., 0.25))
    # clevs = np.append(clevs, np.arange(3., 10.0, 1))
    clevs = [0.01, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8,
             10]

    sector = 'iowa'
    pqstr = ("plot %s %s00 %s_q2_1d.png %s_q2_1d.png png"
             ) % (routes, ts.strftime("%Y%m%d%H"), sector, sector)
    m = MapPlot(title=("%s NCEP MRMS Q3 Today's Precipitation"
                       ) % (ts.strftime("%-d %b %Y"),),
                subtitle=subtitle, sector=sector)

    (x, y) = np.meshgrid(lons, lats)

    m.pcolormesh(x, y, distance(precip, 'MM').value('IN'), clevs,
                 cmap=nwsprecip(), units='inch')
    m.drawcounties()
    m.postprocess(pqstr=pqstr, view=False)
    m.close()
Пример #56
0
def plotter(fdict):
    """ Go """
    import matplotlib
    matplotlib.use('agg')
    import matplotlib.pyplot as plt
    import matplotlib.dates as mdates
    year = int(fdict.get('year', 2014))
    threshold = float(fdict.get('threshold', 1))
    period = int(fdict.get('period', 7))
    state = fdict.get('state', 'IA')[:2]

    nc2 = netCDF4.Dataset("/mesonet/data/iemre/state_weights.nc")
    iowa = nc2.variables[state][:]
    iowapts = np.sum(np.where(iowa > 0, 1, 0))
    nc2.close()

    nc = netCDF4.Dataset('/mesonet/data/iemre/%s_mw_daily.nc' % (year, ))
    precip = nc.variables['p01d']

    now = datetime.datetime(year, 1, 1)
    now += datetime.timedelta(days=(period-1))
    ets = datetime.datetime(year, 12, 31)
    days = []
    coverage = []
    while now < ets:
        idx = iemre.daily_offset(now)
        sevenday = np.sum(precip[(idx-period):idx, :, :], 0)
        pday = np.where(iowa > 0, sevenday[:, :], -1)
        tots = np.sum(np.where(pday >= (threshold * 25.4), 1, 0))
        days.append(now)
        coverage.append(tots / float(iowapts) * 100.0)

        now += datetime.timedelta(days=1)
    df = pd.DataFrame(dict(day=pd.Series(days),
                           coverage=pd.Series(coverage)))

    (fig, ax) = plt.subplots(1, 1)
    ax.bar(days, coverage, fc='g', ec='g')
    ax.set_title(("IEM Estimated Areal Coverage Percent of %s\n"
                  " receiving %.2f inches of rain over trailing %s day period"
                  ) % (STATES[state], threshold, period))
    ax.set_ylabel("Areal Coverage [%]")
    ax.xaxis.set_major_formatter(mdates.DateFormatter('%b\n%Y'))
    ax.grid(True)
    return fig, df
Пример #57
0
def grid_day12(nc, ts):
    """Use the COOP data for gridding
    """
    offset = iemre.daily_offset(ts)
    print(('12z hi/lo for %s [idx:%s]') % (ts, offset))
    sql = """
       SELECT ST_x(s.geom) as lon, ST_y(s.geom) as lat, s.state,
       s.id as station, s.name as name,
       (CASE WHEN pday >= 0 then pday else null end) as precipdata,
       (CASE WHEN snow >= 0 then snow else null end) as snowdata,
       (CASE WHEN snowd >= 0 then snowd else null end) as snowddata,
       (CASE WHEN max_tmpf > -50 and max_tmpf < 130
           then max_tmpf else null end) as highdata,
       (CASE WHEN min_tmpf > -50 and min_tmpf < 95
           then min_tmpf else null end) as lowdata
       from summary_%s c, stations s WHERE day = '%s' and
       s.network in ('IA_COOP', 'MN_COOP', 'WI_COOP', 'IL_COOP', 'MO_COOP',
        'KS_COOP', 'NE_COOP', 'SD_COOP', 'ND_COOP', 'KY_COOP', 'MI_COOP',
        'OH_COOP') and c.iemid = s.iemid and
        extract(hour from c.coop_valid) between 4 and 11
        """ % (ts.year, ts.strftime("%Y-%m-%d"))
    df = read_sql(sql, pgconn)

    if len(df.index) > 4:
        res = generic_gridder(df, 'highdata')
        nc.variables['high_tmpk_12z'][offset] = datatypes.temperature(
                                                res, 'F').value('K')

        res = generic_gridder(df, 'lowdata')
        nc.variables['low_tmpk_12z'][offset] = datatypes.temperature(
                                            res, 'F').value('K')

        res = generic_gridder(df, 'snowdata')
        nc.variables['snow_12z'][offset] = res * 25.4

        res = generic_gridder(df, 'snowddata')
        nc.variables['snowd_12z'][offset] = res * 25.4
    else:
        print "%s has %02i entries, FAIL" % (ts.strftime("%Y-%m-%d"),
                                             len(df.index))
Пример #58
0
def plotter(fdict):
    """ Go """
    import matplotlib
    matplotlib.use('agg')
    from pyiem.plot import MapPlot
    ptype = fdict.get('ptype', 'c')
    date = datetime.datetime.strptime(fdict.get('date', '2015-01-01'),
                                      '%Y-%m-%d')
    varname = fdict.get('var', 'rsds')

    idx0 = iemre.daily_offset(date)
    nc = netCDF4.Dataset(("/mesonet/data/iemre/%s_mw_daily.nc"
                          ) % (date.year, ), 'r')
    lats = nc.variables['lat'][:]
    lons = nc.variables['lon'][:]
    if varname == 'rsds':
        # Value is in W m**-2, we want MJ
        data = nc.variables['rsds'][idx0, :, :] * 86400. / 1000000.
        units = 'MJ d-1'
    nc.close()

    title = date.strftime("%-d %B %Y")
    m = MapPlot(sector='midwest', axisbg='white', nocaption=True,
                title='IEM Reanalysis of %s for %s' % (PDICT.get(varname),
                                                       title),
                subtitle='Data derived from various NOAA datasets'
                )
    if np.ma.is_masked(np.max(data)):
        return 'Data Unavailable'
    clevs = np.arange(0, 37, 3.)
    clevs[0] = 0.01
    x, y = np.meshgrid(lons, lats)
    if ptype == 'c':
        m.contourf(x, y, data, clevs, units=units)
    else:
        m.pcolormesh(x, y, data, clevs, units=units)

    return m.fig
Пример #59
0
def doday(ts, realtime):
    """
    Create a plot of precipitation stage4 estimates for some day
    """
    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_ifc_daily.nc" % (ts.year,))
    idx = daily_offset(ts)
    xaxis = nc.variables['lon'][:]
    yaxis = nc.variables['lat'][:]
    total = nc.variables['p01d'][idx, :, :]
    nc.close()
    lastts = datetime.datetime(ts.year, ts.month, ts.day, 23, 59)
    if realtime:
        now = datetime.datetime.now() - datetime.timedelta(minutes=60)
        lastts = now.replace(minute=59)
    subtitle = "Total between 12:00 AM and %s" % (
                                                  lastts.strftime("%I:%M %p"),)
    routes = 'ac'
    if not realtime:
        routes = 'a'

    clevs = [0.01, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8,
             10]

    pqstr = ("plot %s %s00 iowa_ifc_1d.png iowa_ifc_1d.png png"
             ) % (routes, ts.strftime("%Y%m%d%H"))
    m = MapPlot(title=("%s Iowa Flood Center Today's Precipitation"
                       ) % (ts.strftime("%-d %b %Y"),),
                subtitle=subtitle, sector='custom',
                west=xaxis[0], east=xaxis[-1],
                south=yaxis[0], north=yaxis[-1])

    (x, y) = np.meshgrid(xaxis, yaxis)

    m.pcolormesh(x, y, distance(total, 'MM').value("IN"), clevs,
                 cmap=nwsprecip(), units='inch')
    m.drawcounties()
    m.postprocess(pqstr=pqstr, view=False)
    m.close()
Пример #60
0
def estimate_precip(ts):
    """Estimate precipitation based on IEMRE"""
    idx = iemre.daily_offset(ts)
    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_daily.nc" % (ts.year, ),
                         'r')
    grid12 = nc.variables['p01d_12z'][idx, :, :] / 25.4
    grid00 = nc.variables['p01d'][idx, :, :] / 25.4
    nc.close()

    for sid in nt.sts.keys():
        if nt.sts[sid]['precip24_hour'] in [0, 22, 23]:
            precip = grid00[nt.sts[sid]['gridj'], nt.sts[sid]['gridi']]
        else:
            precip = grid12[nt.sts[sid]['gridj'], nt.sts[sid]['gridi']]
        # denote trace
        if precip > 0 and precip < 0.01:
            nt.sts[sid]['precip'] = 0.0001
        elif precip < 0:
            nt.sts[sid]['precip'] = 0
        elif np.isnan(precip) or np.ma.is_masked(precip):
            nt.sts[sid]['precip'] = None
        else:
            nt.sts[sid]['precip'] = "%.2f" % (precip,)