Ejemplo n.º 1
0
def estimate_precip(df, ts):
    """Estimate precipitation based on IEMRE"""
    idx = iemre.daily_offset(ts)
    nc = ncopen(iemre.get_daily_ncname(ts.year), 'r', timeout=300)
    grid12 = distance(nc.variables['p01d_12z'][idx, :, :],
                      'MM').value("IN").filled(0)
    grid00 = distance(nc.variables['p01d'][idx, :, :],
                      "MM").value("IN").filled(0)
    nc.close()

    for sid, row in df.iterrows():
        if not pd.isnull(row['precip']):
            continue
        if row['precip24_hour'] in [0, 22, 23]:
            precip = grid00[row['gridj'], row['gridi']]
        else:
            precip = grid12[row['gridj'], row['gridi']]
        # denote trace
        if precip > 0 and precip < 0.01:
            df.at[sid, 'precip'] = TRACE_VALUE
        elif precip < 0:
            df.at[sid, 'precip'] = 0
        elif np.isnan(precip) or np.ma.is_masked(precip):
            df.at[sid, 'precip'] = 0
        else:
            df.at[sid, 'precip'] = "%.2f" % (precip,)
Ejemplo n.º 2
0
def replace_obs_iem(df, location):
    """Replace dataframe data with obs for this location

    Tricky part, if the baseline already provides data for this year, we should
    use it!
    """
    pgconn = get_dbconn("iem", user="******")
    cursor = pgconn.cursor()
    station = XREF[location]["station"]
    today = datetime.date.today()
    jan1 = today.replace(month=1, day=1)
    years = [
        int(y) for y in np.arange(df.index.values.min().year,
                                  df.index.values.max().year + 1)
    ]

    table = "summary_%s" % (jan1.year, )
    cursor.execute(
        """
        select day, max_tmpf, min_tmpf, srad_mj, pday
        from """ + table + """ s JOIN stations t on (s.iemid = t.iemid)
        WHERE t.id = %s and max_tmpf is not null
        and day < 'TODAY' ORDER by day ASC
        """,
        (station, ),
    )
    rcols = ["maxt", "mint", "radn", "gdd", "rain"]
    replaced = []
    for row in cursor:
        valid = row[0]
        # Does our df currently have data for this date?  If so, we shall do
        # no more
        dont_replace = not np.isnan(df.at[valid, "mint"])
        if not dont_replace:
            replaced.append(valid)
        _gdd = gdd(temperature(row[1], "F"), temperature(row[2], "F"))
        for year in years:
            if valid.month == 2 and valid.day == 29 and year % 4 != 0:
                continue
            if dont_replace:
                df.loc[valid.replace(year=year), rcols[3:]] = (
                    _gdd,
                    distance(row[4], "in").value("mm"),
                )
                continue
            df.loc[valid.replace(year=year), rcols] = (
                temperature(row[1], "F").value("C"),
                temperature(row[2], "F").value("C"),
                row[3],
                _gdd,
                distance(row[4], "in").value("mm"),
            )
    if replaced:
        LOG.info(
            "  used IEM Access %s from %s->%s",
            station,
            replaced[0],
            replaced[-1],
        )
Ejemplo n.º 3
0
def update_precip(date, station, hdf):
    """Do the update work"""
    sts = datetime.datetime(date.year, date.month, date.day, 7)
    sts = sts.replace(tzinfo=pytz.utc)
    ets = sts + datetime.timedelta(hours=24)
    ldf = hdf[(hdf['station'] == station) & (hdf['valid'] >= sts) &
              (hdf['valid'] < ets)]

    newpday = distance(ldf['precip_in'].sum(), 'IN')
    # update iemaccess
    pgconn = get_dbconn('iem')
    cursor = pgconn.cursor()
    cursor.execute(
        """
    UPDATE summary s
    SET pday = %s
    FROM stations t
    WHERE t.iemid = s.iemid and s.day = %s and t.id = %s and
    t.network = 'ISUSM'
    """, (newpday.value('IN'), date, station))
    cursor.close()
    pgconn.commit()
    # update isusm
    pgconn = get_dbconn('isuag')
    cursor = pgconn.cursor()
    # daily
    cursor.execute(
        """
    UPDATE sm_daily
    SET rain_mm_tot_qc = %s, rain_mm_tot_f = 'E'
    WHERE valid = %s and station = %s
    """, (newpday.value('MM'), date, station))
    for _, row in ldf.iterrows():
        # hourly
        total = distance(row['precip_in'], 'IN').value('MM')
        cursor.execute(
            """
        UPDATE sm_hourly
        SET rain_mm_tot_qc = %s, rain_mm_tot_f = 'E'
        WHERE valid = %s and station = %s
        """, (total, row['valid'], station))
        # 15minute
        for minute in [45, 30, 15, 0]:
            cursor.execute(
                """
            UPDATE sm_15minute
            SET rain_mm_tot_qc = %s, rain_mm_tot_f = 'E'
            WHERE valid = %s and station = %s
            """, (total / 4.,
                  row['valid'] - datetime.timedelta(minutes=minute), station))
    cursor.close()
    pgconn.commit()
Ejemplo n.º 4
0
def update_precip(date, station, hdf):
    """Do the update work"""
    sts = datetime.datetime(date.year, date.month, date.day, 7)
    sts = sts.replace(tzinfo=pytz.utc)
    ets = sts + datetime.timedelta(hours=24)
    ldf = hdf[
        (hdf["station"] == station)
        & (hdf["valid"] >= sts)
        & (hdf["valid"] < ets)
    ]

    newpday = distance(ldf["precip_in"].sum(), "IN")
    set_iemacces(station, date, newpday.value("IN"))
    # update isusm
    pgconn = get_dbconn("isuag")
    cursor = pgconn.cursor()
    # daily
    cursor.execute(
        """
        UPDATE sm_daily
        SET rain_mm_tot_qc = %s, rain_mm_tot_f = 'E'
        WHERE valid = %s and station = %s
    """,
        (newpday.value("MM"), date, station),
    )
    for _, row in ldf.iterrows():
        # hourly
        total = distance(row["precip_in"], "IN").value("MM")
        cursor.execute(
            """
            UPDATE sm_hourly
            SET rain_mm_tot_qc = %s, rain_mm_tot_f = 'E'
            WHERE valid = %s and station = %s
        """,
            (total, row["valid"], station),
        )
        # minute
        for minute in range(60):
            cursor.execute(
                """
                UPDATE sm_minute
                SET rain_in_tot_qc = %s, rain_in_tot_f = 'E'
                WHERE valid = %s and station = %s
            """,
                (
                    total / 60.0 / 25.4,
                    row["valid"] - datetime.timedelta(minutes=minute),
                    station,
                ),
            )
    cursor.close()
    pgconn.commit()
Ejemplo n.º 5
0
def varconv(val, element):
    """Convert NCDC to something we use in the database"""
    ret = None
    if element in ["TMAX", "TMIN"]:
        fv = round(temperature(float(val) / 10.0, "C").value("F"), 0)
        ret = None if (fv < -100 or fv > 150) else fv
    elif element in ["PRCP"]:
        fv = round(distance(float(val) / 10.0, "MM").value("IN"), 2)
        ret = None if fv < 0 else fv
    elif element in ["SNOW", "SNWD"]:
        fv = round(distance(float(val), "MM").value("IN"), 1)
        ret = None if fv < 0 else fv
    return ret
Ejemplo n.º 6
0
def varconv(val, element):
    """Convert NCDC to something we use in the database"""
    ret = None
    if element in ['TMAX', 'TMIN']:
        fv = round(temperature(float(val) / 10.0, 'C').value('F'), 0)
        ret = None if (fv < -100 or fv > 150) else fv
    elif element in ['PRCP']:
        fv = round(distance(float(val) / 10.0, 'MM').value('IN'), 2)
        ret = None if fv < 0 else fv
    elif element in ['SNOW', 'SNWD']:
        fv = round(distance(float(val), 'MM').value('IN'), 1)
        ret = None if fv < 0 else fv
    return ret
Ejemplo n.º 7
0
def estimate_snow(df, ts):
    """Estimate the Snow based on COOP reports"""
    idx = iemre.daily_offset(ts)
    nc = ncopen(iemre.get_daily_ncname(ts.year), 'r', timeout=300)
    snowgrid12 = distance(nc.variables['snow_12z'][idx, :, :],
                          'MM').value('IN').filled(0)
    snowdgrid12 = distance(nc.variables['snowd_12z'][idx, :, :],
                           'MM').value('IN').filled(0)
    nc.close()

    for sid, row in df.iterrows():
        if pd.isnull(row['snow']):
            df.at[sid, 'snow'] = snowgrid12[row['gridj'], row['gridi']]
        if pd.isnull(row['snowd']):
            df.at[sid, 'snowd'] = snowdgrid12[row['gridj'], row['gridi']]
Ejemplo n.º 8
0
def replace_forecast(df, location):
    """Replace dataframe data with forecast for this location"""
    pgconn = psycopg2.connect(database='coop', host='iemdb', user='******')
    cursor = pgconn.cursor()
    today = datetime.date.today()
    nextjan1 = datetime.date(today.year + 1, 1, 1)
    coop = XREF[location]['climodat']
    years = [int(y) for y in np.arange(df.index.values.min().year,
                                       df.index.values.max().year + 1)]
    cursor.execute("""
        SELECT day, high, low, precip from alldata_forecast WHERE
        modelid = (SELECT id from forecast_inventory WHERE model = 'NDFD'
        ORDER by modelts DESC LIMIT 1) and station = %s and day >= %s
    """, (coop, today))
    rcols = ['maxt', 'mint', 'rain']
    for row in cursor:
        valid = row[0]
        maxc = temperature(row[1], 'F').value('C')
        minc = temperature(row[2], 'F').value('C')
        rain = distance(row[3], 'IN').value('MM')
        for year in years:
            df.loc[valid.replace(year=year), rcols] = (maxc, minc, rain)

    # Need to get radiation from CFS
    cursor.execute("""
        SELECT day, srad from alldata_forecast WHERE
        modelid = (SELECT id from forecast_inventory WHERE model = 'CFS'
        ORDER by modelts DESC LIMIT 1) and station = %s and day >= %s
        and day < %s
    """, (coop, today, nextjan1))
    for row in cursor:
        valid = row[0]
        for year in years:
            df.loc[valid.replace(year=year), 'radn'] = row[1]
Ejemplo n.º 9
0
def main(argv):
    """Do Stuff"""
    fn = argv[1]
    year = int(argv[2])
    df = read_cli(fn)
    df["pcpn_in"] = distance(df["pcpn"].values, "MM").value("IN")
    do_qc(fn, df, year)
Ejemplo n.º 10
0
def load(dirname, location, sdate):
    """ Read a file please """
    data = []
    idx = []
    mindoy = int(sdate.strftime("%j"))
    for line in open("%s/%s.met" % (dirname, location)):
        line = line.strip()
        if not line.startswith('19') and not line.startswith('20'):
            continue
        tokens = line.split()
        if int(tokens[1]) < mindoy:
            continue
        data.append(tokens)
        ts = (datetime.date(int(tokens[0]), 1, 1) +
              datetime.timedelta(days=int(tokens[1])-1))
        idx.append(ts)
    if len(data[0]) < 10:
        cols = ['year', 'doy', 'radn', 'maxt', 'mint', 'rain']
    else:
        cols = ['year', 'doy', 'radn', 'maxt', 'mint',
                'rain', 'gdd', 'st4', 'st12', 'st24',
                'st50', 'sm12', 'sm24', 'sm50']
    df = pd.DataFrame(data, index=idx,
                      columns=cols)
    for col in cols:
        df[col] = pd.to_numeric(df[col], errors='coerce')
    if len(data[0]) < 10:
        df['gdd'] = gdd(temperature(df['maxt'].values, 'C'),
                        temperature(df['mint'].values, 'C'))
    df['gddcum'] = df.groupby(['year'])['gdd'].apply(lambda x: x.cumsum())
    df['raincum'] = distance(
        df.groupby(['year'])['rain'].apply(lambda x: x.cumsum()),
        'MM').value('IN')
    return df
Ejemplo n.º 11
0
def replace_cfs(df, location):
    """Replace the CFS data for this year!"""
    pgconn = get_dbconn('coop', user='******')
    cursor = pgconn.cursor()
    coop = XREF[location]['climodat']
    today = datetime.date.today() + datetime.timedelta(days=3)
    dec31 = today.replace(day=31, month=12)
    cursor.execute(
        """
        SELECT day, high, low, precip, srad from alldata_forecast WHERE
        modelid = (SELECT id from forecast_inventory WHERE model = 'CFS'
        ORDER by modelts DESC LIMIT 1) and station = %s and day >= %s
        and day <= %s ORDER by day ASC
    """, (coop, today, dec31))
    rcols = ['maxt', 'mint', 'rain', 'radn']
    if cursor.rowcount == 0:
        print("  replace_cfs found zero rows!")
        return
    for row in cursor:
        maxt = temperature(row[1], 'F').value('C')
        mint = temperature(row[2], 'F').value('C')
        rain = distance(row[3], 'IN').value('MM')
        radn = row[4]
        df.loc[row[0], rcols] = [maxt, mint, rain, radn]

    if row[0] == dec31:
        return
    now = row[0] + datetime.timedelta(days=1)
    # OK, if our last row does not equal dec31, we have some more work to do
    print(("  Replacing %s->%s with previous year's data") % (now, dec31))
    while now <= dec31:
        lastyear = now.replace(year=(now.year - 1))
        df.loc[now, rcols] = df.loc[lastyear, rcols]
        now += datetime.timedelta(days=1)
Ejemplo n.º 12
0
def do(lon, lat, station):
    """ Process this station and geography """
    idx = np.digitize([lon, ], lons)[0]
    jdx = np.digitize([lat, ], lats)[0]
    print("--> Processing %s i:%s j:%s" % (station, idx, jdx))

    pdata = pr_nc.variables['pr'][:, jdx, idx]
    xdata = tasmax_nc.variables['tmax'][:, jdx, idx]
    ndata = tasmin_nc.variables['tmin'][:, jdx, idx]

    highs = temperature(xdata, 'C').value('F')
    lows = temperature(ndata, 'C').value('F')
    precips = distance(pdata, 'MM').value('IN')

    now = basets
    high = low = precip = None
    for k, _ in enumerate(tmdata):
        now += datetime.timedelta(days=1)
        if now.month == 2 and now.day == 29:
            # Insert missing data
            insert(station, now, high, low, precip)
            now += datetime.timedelta(days=1)
        high = fix(highs[k])
        low = fix(lows[k])
        if low is not None and high is not None and low > high:
            # Swap, sigh
            print(('%s %s high: %.1f low: %.1f was swapped'
                   ) % (now.strftime("%m-%d-%Y"), station, high, low))
            high2 = high
            high = low
            low = high2
        precip = fix(precips[k])
        insert(station, now, high, low, precip)
Ejemplo n.º 13
0
def main(argv):
    """Do Stuff"""
    fn = argv[1]
    year = int(argv[2])
    df = read_cli(fn)
    df['pcpn_in'] = distance(df['pcpn'].values, 'MM').value('IN')
    do_qc(fn, df, year)
Ejemplo n.º 14
0
def dowork(form):
    """Do work!"""
    date = datetime.datetime.strptime(form.getfirst('valid'), '%Y-%m-%d')
    lat = float(form.getfirst("lat"))
    lon = float(form.getfirst("lon"))

    # We want data for the UTC date and timestamps are in the rears, so from
    # 1z through 1z
    sts = utc(date.year, date.month, date.day, 1)
    ets = sts + datetime.timedelta(hours=24)
    sidx = iemre.hourly_offset(sts)
    eidx = iemre.hourly_offset(ets)

    ncfn = "/mesonet/data/stage4/%s_stage4_hourly.nc" % (date.year, )
    res = {'gridi': -1, 'gridj': -1, 'data': []}
    if not os.path.isfile(ncfn):
        return json.dumps(res)
    with ncopen(ncfn) as nc:
        dist = ((nc.variables['lon'][:] - lon)**2 +
                (nc.variables['lat'][:] - lat)**2)**0.5
        (j, i) = np.unravel_index(dist.argmin(), dist.shape)
        res['gridi'] = int(i)
        res['gridj'] = int(j)

        ppt = nc.variables['p01m'][sidx:eidx, j, i]

    for tx, pt in enumerate(ppt):
        valid = sts + datetime.timedelta(hours=tx)
        res['data'].append({
            'end_valid': valid.strftime("%Y-%m-%dT%H:00:00Z"),
            'precip_in': myrounder(
                        datatypes.distance(pt, 'MM').value('IN'), 2)
            })

    return json.dumps(res)
Ejemplo n.º 15
0
def compute_stage4(lon, lat, year):
    """Build a daily dataframe for the stage4 data"""
    nc = netCDF4.Dataset("/mesonet/data/stage4/%s_stage4_hourly.nc" % (year,))
    lons = nc.variables['lon'][:]
    lats = nc.variables['lat'][:]
    dist = ((lons - lon)**2 + (lats - lat)**2)**0.5
    (yidx, xidx) = np.unravel_index(dist.argmin(), dist.shape)
    print(("Computed stage4 nclon:%.2f nclat:%.2f yidx:%s xidx:%s "
           ) % (lons[yidx, xidx], lats[yidx, xidx], yidx, xidx))
    p01i = distance(nc.variables['p01m'][:, yidx, xidx], 'MM').value('IN')
    nc.close()
    df = pd.DataFrame({'precip': 0.0},
                      index=pd.date_range('%s-01-01' % (year, ),
                                          '%s-12-31' % (year, ),
                                          tz='America/Chicago'))
    for date in df.index.values:
        date2 = datetime.datetime.utcfromtimestamp(date.tolist()/1e9)
        ts = datetime.datetime(date2.year, date2.month, date2.day, 6)
        ts = ts.replace(tzinfo=pytz.utc)
        ts = ts.astimezone(pytz.timezone("America/Chicago"))
        ts = ts.replace(hour=0)
        ts = ts.astimezone(pytz.utc)
        tidx = hourly_offset(ts)
        # values are in the rears
        val = np.ma.sum(p01i[tidx+1:tidx+25])
        if val > 0:
            df.at[date, 'precip'] = val  # close enough
    return df
Ejemplo n.º 16
0
def main(argv):
    """Do Stuff"""
    fn = argv[1]
    year = int(argv[2])
    df = read_cli(fn)
    df['pcpn_in'] = distance(df['pcpn'].values, 'MM').value('IN')
    do_qc(fn, df, year)
Ejemplo n.º 17
0
def main():
    """Go!"""
    title = 'NOAA MRMS Q3: RADAR + Guage Corrected Rainfall Estimates + NWS Storm Reports'
    mp = MapPlot(sector='custom',
                 north=42.3, east=-93.0, south=41.65, west=-94.1,
                 axisbg='white',
                 titlefontsize=14,
                 title=title,
                 subtitle='Valid: 14 June 2018')

    shp = shapefile.Reader('cities.shp')
    for record in shp.shapeRecords():
        geo = shape(record.shape)
        mp.ax.add_geometries([geo], ccrs.PlateCarree(), zorder=Z_OVERLAY2,
                             facecolor='None', edgecolor='k', lw=2)

    grbs = pygrib.open('MRMS_GaugeCorr_QPE_24H_00.00_20180614-200000.grib2')
    grb = grbs.message(1)
    pcpn = distance(grb['values'], 'MM').value('IN')
    lats, lons = grb.latlons()
    lons -= 360.
    clevs = [0.01, 0.1, 0.3, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10]
    cmap = nwsprecip()
    cmap.set_over('k')

    mp.pcolormesh(lons, lats, pcpn, clevs, cmap=cmap, latlon=True,
                  units='inch')
    lons, lats, vals, labels = get_data()
    mp.drawcounties()
    mp.plot_values(lons, lats, vals, "%s", labels=labels,
                   labelbuffer=1, labelcolor='white')

    mp.drawcities(labelbuffer=5, minarea=0.2)
    mp.postprocess(filename='test.png')
Ejemplo n.º 18
0
def main():
    """Go Main Go"""
    os.chdir("baseline")
    for fn in glob.glob("*.met"):
        location = fn[:-4]
        output = open("%s.csv" % (location, ), "w")
        output.write("date,high[F],low[F],precip[inch],gdd[F]\n")
        for line in open(fn):
            line = line.strip()
            if (not line.startswith("2012") and not line.startswith("2015")
                    and not line.startswith("2016")):
                continue
            tokens = line.split()
            valid = datetime.date(int(
                tokens[0]), 1, 1) + datetime.timedelta(days=int(tokens[1]) - 1)
            high = temperature(float(tokens[3]), "C")
            low = temperature(float(tokens[4]), "C")
            gdd = met.gdd(high, low, 50, 86)
            precip = distance(float(tokens[5]), "MM")
            output.write(("%s,%.1f,%.1f,%.2f,%.1f\n") % (
                valid.strftime("%Y-%m-%d"),
                high.value("F"),
                low.value("F"),
                precip.value("IN"),
                gdd,
            ))
        output.close()
Ejemplo n.º 19
0
def one():
    """option 1"""
    icursor = ISUAG.cursor()
    iemcursor = IEM.cursor()
    icursor.execute(
        """
        SELECT station, valid, ws_mps_s_wvt, winddir_d1_wvt, rain_mm_tot,
        tair_c_max, tair_c_min
        from sm_daily
    """
    )

    for row in icursor:
        avg_sknt = speed(row[2], "MPS").value("KT")
        avg_drct = row[3]
        pday = distance(row[4], "MM").value("IN")
        high = temperature(row[5], "C").value("F")
        low = temperature(row[6], "C").value("F")
        iemcursor.execute(
            """
        UPDATE summary SET avg_sknt = %s, vector_avg_drct = %s, pday = %s,
        max_tmpf = %s, min_tmpf = %s
        WHERE
        iemid = (select iemid from stations WHERE network = 'ISUSM' and
        id = %s) and day = %s
        """,
            (avg_sknt, avg_drct, pday, high, low, row[0], row[1]),
        )
    iemcursor.close()
    IEM.commit()
    IEM.close()
Ejemplo n.º 20
0
def compute_stage4(lon, lat, year):
    """Build a daily dataframe for the stage4 data"""
    nc = netCDF4.Dataset("/mesonet/data/stage4/%s_stage4_hourly.nc" % (year, ))
    lons = nc.variables['lon'][:]
    lats = nc.variables['lat'][:]
    dist = ((lons - lon)**2 + (lats - lat)**2)**0.5
    (yidx, xidx) = np.unravel_index(dist.argmin(), dist.shape)
    print(("Computed stage4 nclon:%.2f nclat:%.2f yidx:%s xidx:%s ") %
          (lons[yidx, xidx], lats[yidx, xidx], yidx, xidx))
    p01i = distance(nc.variables['p01m'][:, yidx, xidx], 'MM').value('IN')
    nc.close()
    df = pd.DataFrame({'precip': 0.0},
                      index=pd.date_range('%s-01-01' % (year, ),
                                          '%s-12-31' % (year, ),
                                          tz='America/Chicago'))
    for date in df.index.values:
        date2 = datetime.datetime.utcfromtimestamp(date.tolist() / 1e9)
        ts = datetime.datetime(date2.year, date2.month, date2.day, 6)
        ts = ts.replace(tzinfo=pytz.utc)
        ts = ts.astimezone(pytz.timezone("America/Chicago"))
        ts = ts.replace(hour=0)
        ts = ts.astimezone(pytz.utc)
        tidx = hourly_offset(ts)
        # values are in the rears
        val = np.ma.sum(p01i[tidx + 1:tidx + 25])
        if val > 0:
            df.at[date, 'precip'] = val  # close enough
    return df
Ejemplo n.º 21
0
def grid_day(nc, ts):
    """
    I proctor the gridding of data on an hourly basis
    @param ts Timestamp of the analysis, we'll consider a 20 minute window
    """
    cursor = COOP.cursor(cursor_factory=psycopg2.extras.DictCursor)
    offset = iemre.daily_offset(ts)
    if ts.day == 29 and ts.month == 2:
        ts = datetime.datetime(2000, 3, 1)

    sql = """SELECT * from climate51 WHERE valid = '%s' and
             substr(station,3,4) != '0000' and substr(station,3,1) != 'C'
             """ % (ts.strftime("%Y-%m-%d"), )
    cursor.execute(sql)
    res = generic_gridder(nc, cursor, "high")
    nc.variables["high_tmpk"][offset] = datatypes.temperature(res,
                                                              "F").value("K")

    cursor.scroll(0, mode="absolute")
    res = generic_gridder(nc, cursor, "low")
    nc.variables["low_tmpk"][offset] = datatypes.temperature(res,
                                                             "F").value("K")

    cursor.scroll(0, mode="absolute")
    res = generic_gridder(nc, cursor, "precip")
    nc.variables["p01d"][offset] = datatypes.distance(res, "IN").value("MM")

    cursor.scroll(0, mode="absolute")
    res = generic_gridder(nc, cursor, "gdd50")
    nc.variables["gdd50"][offset] = res
Ejemplo n.º 22
0
def dowork(form):
    """Do work!"""
    date = datetime.datetime.strptime(form.getfirst('valid'), '%Y-%m-%d')
    lat = float(form.getfirst("lat"))
    lon = float(form.getfirst("lon"))

    # We want data for the UTC date and timestamps are in the rears, so from
    # 1z through 1z
    sts = datetime.datetime(date.year, date.month, date.day, 1)
    sts = sts.replace(tzinfo=pytz.utc)
    ets = sts + datetime.timedelta(hours=24)
    sidx = iemre.hourly_offset(sts)
    eidx = iemre.hourly_offset(ets)

    ncfn = "/mesonet/data/stage4/%s_stage4_hourly.nc" % (date.year, )
    nc = netCDF4.Dataset(ncfn, 'r')

    dist = ((nc.variables['lon'][:] - lon)**2 +
            (nc.variables['lat'][:] - lat)**2)**0.5
    (j, i) = np.unravel_index(dist.argmin(), dist.shape)
    res = {'gridi': i, 'gridj': j, 'data': []}

    ppt = nc.variables['p01m'][sidx:eidx, j, i]
    nc.close()

    for tx, pt in enumerate(ppt):
        valid = sts + datetime.timedelta(hours=tx)
        res['data'].append({
            'end_valid':
            valid.strftime("%Y-%m-%dT%H:00:00Z"),
            'precip_in':
            myrounder(datatypes.distance(pt, 'MM').value('IN'), 2)
        })

    return json.dumps(res)
Ejemplo n.º 23
0
def main():
    """Go Main Go"""
    fh = open("insert.sql", "w")

    req = requests.get("http://www.weather.gov/source/aprfc/nrcs_swe.json")
    j = req.json()
    for feature in j["features"]:
        nwsli = feature["properties"]["lid"]
        name = feature["properties"]["name"]
        elev = distance(float(feature["properties"]["elev"]), "FT").value("M")
        lon, lat = feature["geometry"]["coordinates"]
        country = nwsli2country.get(nwsli[3:])
        state = nwsli2state.get(nwsli[3:])
        network = "%s_DCP" % (state, )

        sql = """INSERT into stations(id, name, network, country, state,
        plot_name, elevation, online, metasite, geom) VALUES ('%s', '%s', '%s',
        '%s', '%s', '%s', %s, 't', 'f', 'SRID=4326;POINT(%s %s)');
        """ % (
            nwsli,
            name,
            network,
            country,
            state,
            name,
            float(elev),
            float(lon),
            float(lat),
        )
        fh.write(sql)
    fh.close()
Ejemplo n.º 24
0
def do_month(year, month, routes):
    """ Generate a MRMS plot for the month!"""

    sts = datetime.datetime(year, month, 1)
    ets = sts + datetime.timedelta(days=35)
    ets = ets.replace(day=1)

    today = datetime.datetime.now()
    if ets > today:
        ets = today

    idx0 = iemre.daily_offset(sts)
    idx1 = iemre.daily_offset(ets)

    nc = ncopen(iemre.get_daily_mrms_ncname(year), 'r')

    lats = nc.variables['lat'][:]
    lons = nc.variables['lon'][:]
    p01d = distance(np.sum(nc.variables['p01d'][idx0:idx1, :, :], 0),
                    'MM').value('IN')
    nc.close()

    mp = MapPlot(sector='iowa', title='MRMS %s - %s Total Precipitation' % (
                 sts.strftime("%-d %b"),
                 (ets - datetime.timedelta(days=1)).strftime("%-d %b %Y")),
                 subtitle='Data from NOAA MRMS Project')
    x, y = np.meshgrid(lons, lats)
    bins = [0.01, 0.1, 0.5, 1, 1.5, 2, 3, 4, 5, 6, 7, 8, 12, 16, 20]
    mp.pcolormesh(x, y, p01d, bins, units='inches')
    mp.drawcounties()
    currentfn = "summary/iowa_mrms_q3_month.png"
    archivefn = sts.strftime("%Y/%m/summary/iowa_mrms_q3_month.png")
    pqstr = "plot %s %s00 %s %s png" % (
                routes, sts.strftime("%Y%m%d%H"), currentfn, archivefn)
    mp.postprocess(pqstr=pqstr)
Ejemplo n.º 25
0
Archivo: p141.py Proyecto: akrherz/iem
def load(dirname, location, sdate):
    """ Read a file please """
    data = []
    idx = []
    mindoy = int(sdate.strftime("%j"))
    for line in open("%s/%s.met" % (dirname, location)):
        line = line.strip()
        if not line.startswith('19') and not line.startswith('20'):
            continue
        tokens = line.split()
        if int(tokens[1]) < mindoy:
            continue
        data.append(tokens)
        ts = (datetime.date(int(tokens[0]), 1, 1) +
              datetime.timedelta(days=int(tokens[1])-1))
        idx.append(ts)
    if len(data[0]) < 10:
        cols = ['year', 'doy', 'radn', 'maxt', 'mint', 'rain']
    else:
        cols = ['year', 'doy', 'radn', 'maxt', 'mint',
                'rain', 'gdd', 'st4', 'st12', 'st24',
                'st50', 'sm12', 'sm24', 'sm50']
    df = pd.DataFrame(data, index=idx,
                      columns=cols)
    for col in cols:
        df[col] = pd.to_numeric(df[col], errors='coerce')
    if len(data[0]) < 10:
        df['gdd'] = gdd(temperature(df['maxt'].values, 'C'),
                        temperature(df['mint'].values, 'C'))
    df['gddcum'] = df.groupby(['year'])['gdd'].apply(lambda x: x.cumsum())
    df['raincum'] = distance(
        df.groupby(['year'])['rain'].apply(lambda x: x.cumsum()),
        'MM').value('IN')
    return df
Ejemplo n.º 26
0
def grid_day(nc, ts):
    """
    I proctor the gridding of data on an hourly basis
    @param ts Timestamp of the analysis, we'll consider a 20 minute window
    """
    cursor = COOP.cursor(cursor_factory=psycopg2.extras.DictCursor)
    offset = iemre.daily_offset(ts)
    if ts.day == 29 and ts.month == 2:
        ts = datetime.datetime(2000, 3, 1)

    sql = """SELECT * from ncdc_climate71 WHERE valid = '%s' and
             substr(station,3,4) != '0000' and substr(station,3,1) != 'C'
             """ % (ts.strftime("%Y-%m-%d"), )
    cursor.execute(sql)
    if cursor.rowcount > 4:
        res = generic_gridder(nc, cursor, 'high')
        if res is not None:
            nc.variables['tmax'][offset] = datatypes.temperature(
                res, 'F').value('C')
        cursor.scroll(0, mode='absolute')
        res = generic_gridder(nc, cursor, 'low')
        if res is not None:
            nc.variables['tmin'][offset] = datatypes.temperature(
                res, 'F').value('C')
        cursor.scroll(0, mode='absolute')
        res = generic_gridder(nc, cursor, 'precip')
        if res is not None:
            nc.variables['ppt'][offset] = datatypes.distance(res,
                                                             'IN').value('MM')
    else:
        print(("%s has %02i entries, FAIL") %
              (ts.strftime("%Y-%m-%d"), cursor.rowcount))
Ejemplo n.º 27
0
def main():
    """Go Main"""
    nt = NetworkTable(['AWOS', 'IA_ASOS'])
    pgconn = psycopg2.connect(database='mos', host='localhost', user='******',
                              port=5555)
    df = read_sql("""
    select station, avg(pwater) from model_gridpoint where
    model = 'NAM' and extract(hour from runtime at time zone 'UTC') = 0
    and pwater > 0 and pwater < 100 and
    extract(month from runtime) between 4 and 9 and ftime = runtime
    GROUP by station ORDER by avg
    """, pgconn, index_col='station')
    df['lat'] = None
    df['lon'] = None
    df['pwater'] = distance(df['avg'].values, 'MM').value('IN')
    for station in df.index.values:
        df.at[station, 'lat'] = nt.sts[station[1:]]['lat']
        df.at[station, 'lon'] = nt.sts[station[1:]]['lon']

    mp = MapPlot(title=('00z Analysis NAM Warm-Season Average '
                        'Precipitable Water [in]'),
                 subtitle=("based on grid point samples "
                           "from 2004-2017 (April-September)"))
    cmap = plt.get_cmap("plasma_r")
    cmap.set_under('white')
    cmap.set_over('black')
    mp.contourf(df['lon'], df['lat'], df['pwater'],
                np.arange(0.94, 1.13, 0.03), cmap=cmap,
                units='inch')
    mp.drawcounties()
    mp.drawcities()
    mp.postprocess(filename='170901.png')
    mp.close()
Ejemplo n.º 28
0
def replace_forecast(df, location):
    """Replace dataframe data with forecast for this location"""
    pgconn = psycopg2.connect(database='coop', host='iemdb', user='******')
    cursor = pgconn.cursor()
    today = datetime.date.today()
    nextjan1 = datetime.date(today.year + 1, 1, 1)
    coop = XREF[location]['climodat']
    years = [int(y) for y in np.arange(df.index.values.min().year,
                                       df.index.values.max().year + 1)]
    cursor.execute("""
        SELECT day, high, low, precip from alldata_forecast WHERE
        modelid = (SELECT id from forecast_inventory WHERE model = 'NDFD'
        ORDER by modelts DESC LIMIT 1) and station = %s and day >= %s
    """, (coop, today))
    rcols = ['maxt', 'mint', 'rain']
    for row in cursor:
        valid = row[0]
        maxc = temperature(row[1], 'F').value('C')
        minc = temperature(row[2], 'F').value('C')
        rain = distance(row[3], 'IN').value('MM')
        for year in years:
            df.loc[valid.replace(year=year), rcols] = (maxc, minc, rain)

    # Need to get radiation from CFS
    cursor.execute("""
        SELECT day, srad from alldata_forecast WHERE
        modelid = (SELECT id from forecast_inventory WHERE model = 'CFS'
        ORDER by modelts DESC LIMIT 1) and station = %s and day >= %s
        and day < %s
    """, (coop, today, nextjan1))
    for row in cursor:
        valid = row[0]
        for year in years:
            df.loc[valid.replace(year=year), 'radn'] = row[1]
Ejemplo n.º 29
0
def daily_process(nwsli, maxts):
    """ Process the daily file """
    fn = "%s/%s_DailySI.dat" % (BASE, STATIONS[nwsli])
    df = common_df_logic(fn, maxts, nwsli, "sm_daily")
    if df is None:
        return 0
    LOG.debug("processing %s rows from %s", len(df.index), fn)
    processed = 0
    acursor = ACCESS.cursor()
    for _i, row in df.iterrows():
        # Need a timezone
        valid = datetime.datetime(
            row["valid"].year, row["valid"].month, row["valid"].day, 12, 0
        )
        valid = valid.replace(tzinfo=pytz.timezone("America/Chicago"))
        ob = Observation(nwsli, "ISUSM", valid)
        ob.data["max_tmpf"] = temperature(row["tair_c_max_qc"], "C").value("F")
        ob.data["min_tmpf"] = temperature(row["tair_c_min_qc"], "C").value("F")
        ob.data["pday"] = round(
            distance(row["rain_mm_tot_qc"], "MM").value("IN"), 2
        )
        if valid not in EVENTS["days"]:
            EVENTS["days"].append(valid)
        ob.data["et_inch"] = distance(row["dailyet_qc"], "MM").value("IN")
        ob.data["srad_mj"] = row["slrmj_tot_qc"]
        # Someday check if this is apples to apples here
        ob.data["vector_avg_drct"] = row["winddir_d1_wvt_qc"]
        if ob.data["max_tmpf"] is None:
            EVENTS["reprocess_temps"] = True
        if ob.data["srad_mj"] == 0 or np.isnan(ob.data["srad_mj"]):
            LOG.info(
                "soilm_ingest.py station: %s ts: %s has 0 solar",
                nwsli,
                valid.strftime("%Y-%m-%d"),
            )
            EVENTS["reprocess_solar"] = True
        if "ws_mps_max" in df.columns:
            ob.data["max_sknt"] = speed(row["ws_mps_max_qc"], "MPS").value(
                "KT"
            )
        ob.data["avg_sknt"] = speed(row["ws_mps_s_wvt_qc"], "MPS").value("KT")
        ob.save(acursor)

        processed += 1
    acursor.close()
    ACCESS.commit()
    return processed
Ejemplo n.º 30
0
def do_var(varname):
    """
    Run our estimator for a given variable
    """
    currentnc = None
    sql = """select day, station from alldata_%s WHERE %s is null
        and day >= '1893-01-01' ORDER by day ASC""" % (state.lower(), varname)
    ccursor.execute(sql)
    for row in ccursor:
        day = row[0]
        station = row[1]
        if station not in nt.sts:
            continue

        sql = """
            SELECT station, %s from alldata_%s WHERE %s is not NULL
            and station in %s and day = '%s'
            """ % (varname, state, varname, tuple(friends[station]), day)
        ccursor2.execute(sql)
        weight = []
        value = []
        for row2 in ccursor2:
            idx = friends[station].index(row2[0])
            weight.append(weights[station][idx])
            value.append(row2[1])

        if len(weight) < 3:
            # Nearest neighbors failed, so lets look at our grided analysis
            # and sample from it
            if currentnc is None or currentnc.title.find(str(day.year)) == -1:
                currentnc = netCDF4.Dataset(("/mesonet/data/iemre/"
                                             "%s_mw_daily.nc") % (day.year,))
            tidx = iemre.daily_offset(datetime.datetime(day.year, day.month,
                                                        day.day))
            iidx, jidx = iemre.find_ij(nt.sts[station]['lon'],
                                       nt.sts[station]['lat'])
            iemreval = currentnc.variables[vnameconv[varname]][tidx, jidx,
                                                               iidx]
            if varname in ('high', 'low'):
                interp = temperature(iemreval, 'K').value('F')
            else:
                interp = distance(iemreval, 'MM').value('IN')
            print '--> Neighbor failure, %s %s %s' % (station, day, varname)
        else:
            mass = sum(weight)
            interp = np.sum(np.array(weight) * np.array(value) / mass)

        dataformat = '%.2f'
        if varname in ['high', 'low']:
            dataformat = '%.0f'
        print(('Set station: %s day: %s varname: %s value: %s'
               ) % (station, day, varname, dataformat % (interp,)))
        sql = """
            UPDATE alldata_%s SET estimated = true, %s = %s WHERE
            station = '%s' and day = '%s'
            """ % (state.lower(), varname,
                   dataformat % (interp,), station, day)
        sql = sql.replace(' nan ', ' null ')
        ccursor2.execute(sql)
Ejemplo n.º 31
0
def doday(ts, realtime):
    """
    Create a plot of precipitation stage4 estimates for some day
    """
    lts = utc(ts.year, ts.month, ts.day, 12)
    lts = lts.astimezone(pytz.timezone("America/Chicago"))
    # make assumptions about the last valid MRMS data
    if realtime:
        # Up until :59 after of the last hour
        lts = (datetime.datetime.now() -
               datetime.timedelta(hours=1)).replace(minute=59)
    else:
        lts = lts.replace(year=ts.year,
                          month=ts.month,
                          day=ts.day,
                          hour=23,
                          minute=59)

    idx = iemre.daily_offset(ts)
    ncfn = iemre.get_daily_mrms_ncname(ts.year)
    if not os.path.isfile(ncfn):
        LOG.info("File %s missing, abort.", ncfn)
        return
    with ncopen(ncfn, timeout=300) as nc:
        precip = nc.variables['p01d'][idx, :, :]
        lats = nc.variables['lat'][:]
        lons = nc.variables['lon'][:]
    subtitle = "Total between 12:00 AM and %s" % (
        lts.strftime("%I:%M %p %Z"), )
    routes = 'ac'
    if not realtime:
        routes = 'a'

    # clevs = np.arange(0, 0.25, 0.05)
    # clevs = np.append(clevs, np.arange(0.25, 3., 0.25))
    # clevs = np.append(clevs, np.arange(3., 10.0, 1))
    clevs = [
        0.01, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10
    ]

    (xx, yy) = np.meshgrid(lons, lats)
    for sector in ['iowa', 'midwest']:
        pqstr = ("plot %s %s00 %s_q2_1d.png %s_q2_1d.png png") % (
            routes, ts.strftime("%Y%m%d%H"), sector, sector)
        mp = MapPlot(title=("%s NCEP MRMS Q3 Today's Precipitation") %
                     (ts.strftime("%-d %b %Y"), ),
                     subtitle=subtitle,
                     sector=sector)

        mp.pcolormesh(xx,
                      yy,
                      distance(precip, 'MM').value('IN'),
                      clevs,
                      cmap=nwsprecip(),
                      units='inch')
        if sector == 'iowa':
            mp.drawcounties()
        mp.postprocess(pqstr=pqstr, view=False)
        mp.close()
Ejemplo n.º 32
0
def plotter(fdict):
    """ Go """
    ctx = get_autoplot_context(fdict, get_description())
    csector = ctx['csector']
    date = ctx['date']
    z = ctx['z']
    period = ctx['f']
    scale = ctx['scale']
    valid = utc(date.year, date.month, date.day, int(z))
    gribfn = valid.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/model/wpc/"
                             "p" + period + "m_%Y%m%d%Hf" + period + ".grb"))
    if not os.path.isfile(gribfn):
        raise ValueError("gribfn %s missing" % (gribfn, ))

    grbs = pygrib.open(gribfn)
    grb = grbs[1]
    precip = distance(grb.values, 'MM').value('IN')
    lats, lons = grb.latlons()

    title = ("Weather Prediction Center %s Quantitative "
             "Precipitation Forecast") % (PDICT[period])
    subtitle = ("%sWPC Forcast %s UTC to %s UTC") % (
        ("US Drought Monitor Overlaid, " if ctx['opt'] == 'both' else ''),
        valid.strftime("%d %b %Y %H"),
        (valid +
         datetime.timedelta(hours=int(period))).strftime("%d %b %Y %H"))
    mp = MapPlot(sector=('state' if len(csector) == 2 else csector),
                 state=ctx['csector'],
                 title=title,
                 subtitle=subtitle,
                 continentalcolor='white',
                 titlefontsize=16)
    cmap = plt.get_cmap('gist_ncar')
    cmap.set_under('#EEEEEE')
    cmap.set_over('black')
    if scale == 'auto':
        levs = np.linspace(0, np.max(precip) * 1.1, 10)
        levs = [round(lev, 2) for lev in levs]
        levs[0] = 0.01
    elif scale == '10':
        levs = np.arange(0, 10.1, 1.)
        levs[0] = 0.01
    elif scale == '7':
        levs = np.arange(0, 7.1, 0.5)
        levs[0] = 0.01
    elif scale == '3.5':
        levs = np.arange(0, 3.6, 0.25)
        levs[0] = 0.01
    mp.pcolormesh(lons,
                  lats,
                  precip,
                  levs,
                  cmap=cmap,
                  units='inch',
                  clip_on=False)
    if ctx['opt'] == 'both':
        mp.draw_usdm(valid=valid, filled=False, hatched=True)

    return mp.fig
Ejemplo n.º 33
0
def replace_obs_iem(df, location):
    """Replace dataframe data with obs for this location

    Tricky part, if the baseline already provides data for this year, we should
    use it!
    """
    pgconn = psycopg2.connect(database='iem', host='iemdb', user='******')
    cursor = pgconn.cursor()
    station = XREF[location]['station']
    today = datetime.date.today()
    jan1 = today.replace(month=1, day=1)
    years = [int(y) for y in np.arange(df.index.values.min().year,
                                       df.index.values.max().year + 1)]

    table = "summary_%s" % (jan1.year,)
    cursor.execute("""
        select day, max_tmpf, min_tmpf, srad_mj, pday
        from """ + table + """ s JOIN stations t on (s.iemid = t.iemid)
        WHERE t.id = %s and max_tmpf is not null
        and day < 'TODAY' ORDER by day ASC
        """, (station,))
    rcols = ['maxt', 'mint', 'radn', 'gdd', 'rain']
    replaced = []
    for row in cursor:
        valid = row[0]
        # Does our df currently have data for this date?  If so, we shall do
        # no more
        dont_replace = not np.isnan(df.at[valid, 'mint'])
        if not dont_replace:
            replaced.append(valid)
        _gdd = gdd(temperature(row[1], 'F'), temperature(row[2], 'F'))
        for year in years:
            if valid.month == 2 and valid.day == 29 and year % 4 != 0:
                continue
            if dont_replace:
                df.loc[valid.replace(year=year),
                       rcols[3:-1]] = (_gdd,
                                       distance(row[4], 'in').value('mm'))
                continue
            df.loc[valid.replace(year=year), rcols] = (
                temperature(row[1], 'F').value('C'),
                temperature(row[2], 'F').value('C'), row[3],
                _gdd, distance(row[4], 'in').value('mm'))
    if len(replaced) > 0:
        print(("  used IEM Access %s from %s->%s"
               ) % (station, replaced[0], replaced[-1]))
Ejemplo n.º 34
0
def doday(ts, realtime):
    """
    Create a plot of precipitation stage4 estimates for some day

    We should total files from 1 AM to midnight local time
    """
    sts = ts.replace(hour=1)
    ets = sts + datetime.timedelta(hours=24)
    interval = datetime.timedelta(hours=1)
    now = sts
    total = None
    lts = None
    while now < ets:
        gmt = now.astimezone(pytz.timezone("UTC"))
        fn = gmt.strftime(("/mesonet/ARCHIVE/data/%Y/%m/%d/"
                           "stage4/ST4.%Y%m%d%H.01h.grib"))
        if os.path.isfile(fn):
            lts = now
            grbs = pygrib.open(fn)

            if total is None:
                g = grbs[1]
                total = g["values"]
                lats, lons = g.latlons()
            else:
                total += grbs[1]["values"]
            grbs.close()
        now += interval

    if lts is None and ts.hour > 1:
        print 'stage4_today_total.py found no data!'
    if lts is None:
        return
    lts = lts - datetime.timedelta(minutes=1)
    subtitle = "Total between 12:00 AM and %s" % (lts.strftime("%I:%M %p %Z"),)
    routes = 'ac'
    if not realtime:
        routes = 'a'
    for sector in ['iowa', 'midwest', 'conus']:
        pqstr = ("plot %s %s00 %s_stage4_1d.png %s_stage4_1d.png png"
                 ) % (routes,
                      ts.strftime("%Y%m%d%H"), sector, sector)

        m = MapPlot(sector=sector,
                    title="%s NCEP Stage IV Today's Precipitation" % (
                                                    ts.strftime("%-d %b %Y"),),
                    subtitle=subtitle)

        clevs = [0.01, 0.1, 0.3, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8, 10]
        m.pcolormesh(lons, lats, distance(total, 'MM').value('IN'), clevs,
                     cmap=nwsprecip(), units='inch')

        # map.drawstates(zorder=2)
        if sector == 'iowa':
            m.drawcounties()
        m.postprocess(pqstr=pqstr)
        m.close()
Ejemplo n.º 35
0
def do(ts, hours):
    """
    Create a plot of precipitation stage4 estimates for some day
    """
    ts = ts.replace(minute=0)
    sts = ts - datetime.timedelta(hours=hours)
    ets = ts
    interval = datetime.timedelta(hours=1)
    now = sts
    total = None
    lts = None
    while now < ets:
        fn = ("/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.01h.grib") % (
            now.strftime("%Y/%m/%d"), now.strftime("%Y%m%d%H"))

        if os.path.isfile(fn):
            lts = now
            grbs = pygrib.open(fn)

            if total is None:
                g = grbs[1]
                total = g["values"].filled(0)
                lats, lons = g.latlons()
            else:
                total += grbs[1]["values"].filled(0)
            grbs.close()
        now += interval

    if lts is None and ts.hour > 1:
        print 'Missing StageIV data!'
    if lts is None:
        return

    cmap = cm.get_cmap("jet")
    cmap.set_under('white')
    cmap.set_over('black')
    clevs = [0.01, 0.1, 0.25, 0.5, 1, 2, 3, 5, 8, 9.9]
    localtime = (ts - datetime.timedelta(minutes=1)).astimezone(
        pytz.timezone("America/Chicago"))

    for sector in ['iowa', 'midwest', 'conus']:
        m = MapPlot(sector=sector,
                    title='NCEP Stage IV %s Hour Precipitation' % (hours, ),
                    subtitle='Total up to %s' %
                    (localtime.strftime("%d %B %Y %I %p %Z"), ))
        m.pcolormesh(lons,
                     lats,
                     distance(total, 'MM').value('IN'),
                     clevs,
                     units='inch')
        pqstr = "plot %s %s00 %s_stage4_%sh.png %s_stage4_%sh_%s.png png" % (
            'ac', ts.strftime("%Y%m%d%H"), sector, hours, sector, hours,
            ts.strftime("%H"))
        if sector == 'iowa':
            m.drawcounties()
        m.postprocess(pqstr=pqstr)
        m.close()
Ejemplo n.º 36
0
def daily_process(nwsli, maxts):
    """ Process the daily file """
    # print '-------------- DAILY PROCESS ----------------'
    fn = "%s/%s_DailySI.dat" % (BASE, STATIONS[nwsli])
    df = common_df_logic(fn, maxts, nwsli, "sm_daily")
    if df is None:
        return 0
    LOG.debug("processing %s rows from %s", len(df.index), fn)
    processed = 0
    acursor = ACCESS.cursor()
    for _i, row in df.iterrows():
        # Need a timezone
        valid = datetime.datetime(row['valid'].year, row['valid'].month,
                                  row['valid'].day, 12, 0)
        valid = valid.replace(tzinfo=pytz.timezone("America/Chicago"))
        ob = Observation(nwsli, 'ISUSM', valid)
        ob.data['max_tmpf'] = temperature(row['tair_c_max_qc'], 'C').value('F')
        ob.data['min_tmpf'] = temperature(row['tair_c_min_qc'], 'C').value('F')
        ob.data['pday'] = round(distance(row['rain_mm_tot_qc'],
                                         'MM').value('IN'), 2)
        if valid not in EVENTS['days']:
            EVENTS['days'].append(valid)
        ob.data['et_inch'] = distance(row['dailyet_qc'], 'MM').value('IN')
        ob.data['srad_mj'] = row['slrmj_tot_qc']
        # Someday check if this is apples to apples here
        ob.data['vector_avg_drct'] = row['winddir_d1_wvt_qc']
        if ob.data['max_tmpf'] is None:
            EVENTS['reprocess_temps'] = True
        if ob.data['srad_mj'] == 0 or np.isnan(ob.data['srad_mj']):
            LOG.info(
                "soilm_ingest.py station: %s ts: %s has 0 solar",
                nwsli, valid.strftime("%Y-%m-%d")
            )
            EVENTS['reprocess_solar'] = True
        if 'ws_mps_max' in df.columns:
            ob.data['max_sknt'] = speed(row['ws_mps_max_qc'],
                                        'MPS').value('KT')
        ob.data['avg_sknt'] = speed(row['ws_mps_s_wvt_qc'], 'MPS').value('KT')
        ob.save(acursor)

        processed += 1
    acursor.close()
    ACCESS.commit()
    return processed
Ejemplo n.º 37
0
Archivo: p143.py Proyecto: akrherz/iem
def load(dirname, location, sdate):
    """ Read a file please """
    data = []
    idx = []
    for line in open("%s/%s.met" % (dirname, location)):
        line = line.strip()
        if not line.startswith("19") and not line.startswith("20"):
            continue
        tokens = line.split()
        data.append(tokens)
        ts = datetime.date(int(tokens[0]), 1, 1) + datetime.timedelta(days=int(tokens[1]) - 1)
        idx.append(ts)
    if len(data[0]) < 10:
        cols = ["year", "doy", "radn", "maxt", "mint", "rain"]
    else:
        cols = [
            "year",
            "doy",
            "radn",
            "maxt",
            "mint",
            "rain",
            "gdd",
            "st4",
            "st12",
            "st24",
            "st50",
            "sm12",
            "sm24",
            "sm50",
        ]
    df = pd.DataFrame(data, index=idx, columns=cols)
    for col in cols:
        df[col] = pd.to_numeric(df[col], errors="coerce")
    if len(data[0]) < 10:
        df["gdd"] = gdd(temperature(df["maxt"].values, "C"), temperature(df["mint"].values, "C"))
    bins = []
    today = datetime.date.today()
    for valid, _ in df.iterrows():
        if valid >= today:
            bins.append(0)
            continue
        if sdate == "nov1" and valid.month >= 11:
            bins.append(valid.year + 1)
            continue
        if valid.month < today.month:
            bins.append(valid.year)
            continue
        if valid.month == today.month and valid.day < today.day:
            bins.append(valid.year)
            continue
        bins.append(0)
    df["bin"] = bins
    df["rain"] = distance(df["rain"].values, "MM").value("IN")
    df["avgt"] = temperature((df["maxt"] + df["mint"]) / 2.0, "C").value("F")
    return df
Ejemplo n.º 38
0
def estimate_snow(ts):
    """Estimate the Snow based on COOP reports"""
    idx = iemre.daily_offset(ts)
    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_daily.nc" % (ts.year, ),
                         'r')
    nc.set_auto_mask(True)
    snowgrid12 = distance(nc.variables['snow_12z'][idx, :, :],
                          'MM').value('IN')
    snowdgrid12 = distance(nc.variables['snowd_12z'][idx, :, :],
                           'MM').value('IN')
    nc.close()

    for sid in nt.sts:
        val = snowgrid12[nt.sts[sid]['gridj'], nt.sts[sid]['gridi']]
        if val >= 0 and val < 100:
            nt.sts[sid]['snow'] = "%.1f" % (val, )
        val = snowdgrid12[nt.sts[sid]['gridj'], nt.sts[sid]['gridi']]
        if val >= 0 and val < 140:
            nt.sts[sid]['snowd'] = "%.1f" % (val, )
Ejemplo n.º 39
0
def test_distance_conv():
    """ Speed distance from M to MI to FT to SM to KM """
    mi = datatypes.distance(1.0, "mi")
    assert abs(1609.344 - mi.value("M")) < 0.1
    assert abs(5280.0 - mi.value("FT")) < 0.1
    assert abs(1.0 - mi.value("SM")) < 0.1
    assert abs(1.61 - mi.value("KM")) < 0.01
    assert abs(63360.00 - mi.value("IN")) < 0.01
    assert abs(1609344.00 - mi.value("mm")) < 0.01
    assert abs(160934.40 - mi.value("CM")) < 0.01
Ejemplo n.º 40
0
def test_distance_conv():
    """ Speed distance from M to MI to FT to SM to KM """
    mi = datatypes.distance(1.0, 'mi')
    assert abs(1609.344 - mi.value('M')) < 0.1
    assert abs(5280.0 - mi.value('FT')) < 0.1
    assert abs(1.0 - mi.value('SM')) < 0.1
    assert abs(1.61 - mi.value('KM')) < 0.01
    assert abs(63360.00 - mi.value('IN')) < 0.01
    assert abs(1609344.00 - mi.value('mm')) < 0.01
    assert abs(160934.40 - mi.value('CM')) < 0.01
Ejemplo n.º 41
0
 def test_distance_conv(self):
     """ Speed distance from M to MI to FT to SM to KM """
     mi = datatypes.distance(1.0, 'MI')
     self.assertAlmostEquals(1609.344, mi.value('M'), 1)
     self.assertAlmostEquals(5280.0, mi.value('FT'), 1)
     self.assertAlmostEquals(1.0, mi.value('SM'), 1)
     self.assertAlmostEquals(1.61, mi.value('KM'), 2)
     self.assertAlmostEquals(63360.00, mi.value('IN'), 2)
     self.assertAlmostEquals(1609344.00, mi.value('MM'), 2)
     self.assertAlmostEquals(160934.40, mi.value('CM'), 2)
Ejemplo n.º 42
0
def one():
    icursor.execute("""SELECT station, valid, rain_mm_tot, tair_c_max,
        tair_c_min from sm_daily WHERE tair_c_max is not null""")

    for row in icursor:
        high = temperature(row[3], 'C').value('F')
        low = temperature(row[4], 'C').value('F')

        iemcursor.execute("""
        SELECT pday, max_tmpf, min_tmpf from summary s JOIN stations t on
        (t.iemid = s.iemid)
        WHERE day = %s and t.id = %s and t.network = 'ISUSM'
        """, (row[1], row[0]))
        if iemcursor.rowcount == 0:
            print 'Adding summary_%s row %s %s' % (row[1].year, row[0], row[1])
            iemcursor.execute("""
            INSERT into summary_""" + str(row[1].year) + """
            (iemid, day, pday, max_tmpf, min_tmpf) VALUES (
                (SELECT iemid from stations where id = '%s' and
                network = 'ISUSM'), '%s', %s, %s, %s)
            """ % (row[0], row[1], distance(row[2], 'MM').value('IN'), high,
                   low))
        else:
            row2 = iemcursor.fetchone()
            if (row2[1] is None or row2[2] is None or
                    round(row2[0], 2) != round((distance(row[2],
                                                         'MM').value('IN')),
                                               2) or
                    round(high, 2) != round(row2[1], 2) or
                    round(low, 2) != round(row2[2], 2)):
                print(('Mismatch %s %s old: %s new: %s'
                       ) % (row[0], row[1], row2[0],
                            distance(row[2], 'MM').value('IN')))

                iemcursor.execute("""
                UPDATE summary SET pday = %s, max_tmpf = %s,
                min_tmpf = %s WHERE
                iemid = (select iemid from stations WHERE network = 'ISUSM' and
                id = %s) and day = %s
                """, (distance(row[2], 'MM').value('IN'), high, low, row[0],
                      row[1]))
Ejemplo n.º 43
0
 def compute_loc(self, loc, dist, bearing):
     """ Figure out the lon/lat for this location """
     lat = self.nwsli_provider[loc]['lat']
     lon = self.nwsli_provider[loc]['lon']
     # shortcut
     if dist == 0:
         return lon, lat
     meters = distance(float(dist), "MI").value("M")
     northing = meters * math.cos(math.radians(bearing)) / 111111.0
     easting = (meters * math.sin(math.radians(bearing)) /
                math.cos(math.radians(lat)) / 111111.0)
     return lon + easting, lat + northing
Ejemplo n.º 44
0
def main():
    """Go Main Go"""
    nc = ncopen("/mesonet/data/iemre/1979_narr.nc")

    data = np.sum(nc.variables['apcp'][:, :, :], axis=0)

    m = MapPlot(sector='conus', axisbg='tan',
                title=(""),
                subtitle='',
                titlefontsize=16)

    t = distance(data, 'MM').value('IN')
    m.pcolormesh(nc.variables['lon'][:], nc.variables['lat'][:], t,
                 np.arange(0, 60, 2), units='F')

    m.postprocess(filename='test.png')
Ejemplo n.º 45
0
def doday(ts, realtime):
    """
    Create a plot of precipitation stage4 estimates for some day
    """
    lts = datetime.datetime.utcnow().replace(
                tzinfo=pytz.timezone("UTC"))
    lts = lts.astimezone(pytz.timezone("America/Chicago"))
    # make assumptions about the last valid MRMS data
    if realtime:
        # Up until :59 after of the last hour
        lts = (lts - datetime.timedelta(hours=1)).replace(minute=59)
    else:
        lts = lts.replace(year=ts.year, month=ts.month, day=ts.day,
                          hour=23, minute=59)

    idx = iemre.daily_offset(ts)
    ncfn = "/mesonet/data/iemre/%s_mw_mrms_daily.nc" % (ts.year,)
    nc = netCDF4.Dataset(ncfn)
    precip = nc.variables['p01d'][idx, :, :]
    lats = nc.variables['lat'][:]
    lons = nc.variables['lon'][:]
    subtitle = "Total between 12:00 AM and %s" % (
                                            lts.strftime("%I:%M %p %Z"),)
    routes = 'ac'
    if not realtime:
        routes = 'a'

    # clevs = np.arange(0, 0.25, 0.05)
    # clevs = np.append(clevs, np.arange(0.25, 3., 0.25))
    # clevs = np.append(clevs, np.arange(3., 10.0, 1))
    clevs = [0.01, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8,
             10]

    sector = 'iowa'
    pqstr = ("plot %s %s00 %s_q2_1d.png %s_q2_1d.png png"
             ) % (routes, ts.strftime("%Y%m%d%H"), sector, sector)
    m = MapPlot(title=("%s NCEP MRMS Q3 Today's Precipitation"
                       ) % (ts.strftime("%-d %b %Y"),),
                subtitle=subtitle, sector=sector)

    (x, y) = np.meshgrid(lons, lats)

    m.pcolormesh(x, y, distance(precip, 'MM').value('IN'), clevs,
                 cmap=nwsprecip(), units='inch')
    m.drawcounties()
    m.postprocess(pqstr=pqstr, view=False)
    m.close()
Ejemplo n.º 46
0
def snowd(grids, valid, iarchive):
    """ Do the snowdepth grid"""
    pgconn = psycopg2.connect(database='iem', host='iemdb', user='******')
    df = read_sql("""
        SELECT ST_x(geom) as lon, ST_y(geom) as lat,
        max(snowd) as snow
        from summary s JOIN stations t on (s.iemid = t.iemid)
        WHERE s.day in (%s, %s) and
        t.network in ('IA_COOP', 'MN_COOP', 'WI_COOP', 'IL_COOP',
        'MO_COOP', 'NE_COOP', 'KS_COOP', 'SD_COOP') and snowd >= 0
        and snowd < 100 GROUP by lon, lat
        """, pgconn, params=(valid.date(),
                             (valid - datetime.timedelta(days=1)).date()),
                  index_col=None)

    nn = NearestNDInterpolator((df['lon'].values, df['lat'].values),
                               distance(df['snow'].values, 'IN').value('MM'))
    grids['snwd'] = nn(XI, YI)
Ejemplo n.º 47
0
Archivo: prism.py Proyecto: akrherz/iem
def do(form):
    """Do work!"""
    dates = compute_dates(form.getfirst('valid'))
    lat = float(form.getfirst("lat"))
    lon = float(form.getfirst("lon"))

    i, j = prism.find_ij(lon, lat)

    res = {'gridi': i, 'gridj': j, 'data': [],
           'disclaimer': ("PRISM Climate Group, Oregon State University, "
                          "http://prism.oregonstate.edu, created 4 Feb 2004.")}

    if i is None or j is None:
        sys.stdout.write(json.dumps({'error': 'Coordinates outside of domain'}
                                    ))
        return

    for dpair in dates:
        sts = dpair[0]
        ets = dpair[-1]
        sidx = prism.daily_offset(sts)
        eidx = prism.daily_offset(ets) + 1

        ncfn = "/mesonet/data/prism/%s_daily.nc" % (sts.year, )
        nc = netCDF4.Dataset(ncfn, 'r')

        tmax = nc.variables['tmax'][sidx:eidx, j, i]
        tmin = nc.variables['tmin'][sidx:eidx, j, i]
        ppt = nc.variables['ppt'][sidx:eidx, j, i]
        nc.close()

        for tx, (mt, nt, pt) in enumerate(zip(tmax, tmin, ppt)):
            valid = sts + datetime.timedelta(days=tx)
            res['data'].append({
                'valid': valid.strftime("%Y-%m-%dT12:00:00Z"),
                'high_f': myrounder(
                            datatypes.temperature(mt, 'C').value('F'), 1),
                'low_f': myrounder(
                            datatypes.temperature(nt, 'C').value('F'), 1),
                'precip_in': myrounder(
                            datatypes.distance(pt, 'MM').value('IN'), 2)
                })

    return json.dumps(res)
Ejemplo n.º 48
0
def do_work(form):
    """do great things"""
    pgconn = get_dbconn('sustainablecorn')
    stations = form.getlist('stations')
    if not stations:
        stations.append("XXX")
    sts, ets = get_cgi_dates(form)
    df = read_sql("""
    SELECT station as uniqueid, valid as day, extract(doy from valid) as doy,
    high, low, precip, sknt,
    srad_mj, drct from weather_data_daily WHERE station in %s
    and valid >= %s and valid <= %s
    ORDER by station ASC, valid ASC
    """, pgconn, params=(tuple(stations), sts, ets), index_col=None)
    df['highc'] = temperature(df['high'].values, 'F').value('C')
    df['lowc'] = temperature(df['low'].values, 'F').value('C')
    df['precipmm'] = distance(df['precip'].values, 'IN').value('MM')

    metarows = [{}, {}]
    cols = df.columns
    for i, colname in enumerate(cols):
        if i == 0:
            metarows[0][colname] = 'description'
            metarows[1][colname] = 'units'
            continue
        metarows[0][colname] = VARDF.get(colname, '')
        metarows[1][colname] = UVARDF.get(colname, '')
    df = pd.concat([pd.DataFrame(metarows), df], ignore_index=True)
    # re-establish the correct column sorting
    df = df.reindex_axis(cols, axis=1)

    writer = pd.ExcelWriter("/tmp/ss.xlsx", engine='xlsxwriter')
    df.to_excel(writer, 'Daily Weather', index=False)
    worksheet = writer.sheets['Daily Weather']
    worksheet.freeze_panes(3, 0)
    writer.close()

    fn = ",".join(stations)
    ssw("Content-type: application/vnd.ms-excel\n")
    ssw(("Content-Disposition: attachment;Filename=wx_%s.xls\n\n"
         ) % (fn, ))
    ssw(open('/tmp/ss.xlsx', 'rb').read())
    os.unlink('/tmp/ss.xlsx')
Ejemplo n.º 49
0
def replace_cfs(df, location):
    """Replace the CFS data for this year!"""
    pgconn = psycopg2.connect(database='coop', host='iemdb', user='******')
    cursor = pgconn.cursor()
    coop = XREF[location]['climodat']
    today = datetime.date.today() + datetime.timedelta(days=3)
    dec31 = today.replace(day=31, month=12)
    cursor.execute("""
        SELECT day, high, low, precip, srad from alldata_forecast WHERE
        modelid = (SELECT id from forecast_inventory WHERE model = 'CFS'
        ORDER by modelts DESC LIMIT 1) and station = %s and day >= %s
        and day <= %s
    """, (coop, today, dec31))
    rcols = ['maxt', 'mint', 'rain', 'radn']
    for row in cursor:
        maxt = temperature(row[1], 'F').value('C')
        mint = temperature(row[2], 'F').value('C')
        rain = distance(row[3], 'IN').value('MM')
        radn = row[4]
        df.loc[row[0], rcols] = [maxt, mint, rain, radn]
Ejemplo n.º 50
0
def doday(ts, realtime):
    """
    Create a plot of precipitation stage4 estimates for some day
    """
    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_ifc_daily.nc" % (ts.year,))
    idx = daily_offset(ts)
    xaxis = nc.variables['lon'][:]
    yaxis = nc.variables['lat'][:]
    total = nc.variables['p01d'][idx, :, :]
    nc.close()
    lastts = datetime.datetime(ts.year, ts.month, ts.day, 23, 59)
    if realtime:
        now = datetime.datetime.now() - datetime.timedelta(minutes=60)
        lastts = now.replace(minute=59)
    subtitle = "Total between 12:00 AM and %s" % (
                                                  lastts.strftime("%I:%M %p"),)
    routes = 'ac'
    if not realtime:
        routes = 'a'

    clevs = [0.01, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, 3, 3.5, 4, 5, 6, 8,
             10]

    pqstr = ("plot %s %s00 iowa_ifc_1d.png iowa_ifc_1d.png png"
             ) % (routes, ts.strftime("%Y%m%d%H"))
    m = MapPlot(title=("%s Iowa Flood Center Today's Precipitation"
                       ) % (ts.strftime("%-d %b %Y"),),
                subtitle=subtitle, sector='custom',
                west=xaxis[0], east=xaxis[-1],
                south=yaxis[0], north=yaxis[-1])

    (x, y) = np.meshgrid(xaxis, yaxis)

    m.pcolormesh(x, y, distance(total, 'MM').value("IN"), clevs,
                 cmap=nwsprecip(), units='inch')
    m.drawcounties()
    m.postprocess(pqstr=pqstr, view=False)
    m.close()
Ejemplo n.º 51
0
def main():
    """Go Main Go"""
    nc = netCDF4.Dataset('/mesonet/data/stage4/2018_stage4_hourly.nc')
    precip = nc.variables['p01m']
    # Compute needed grid bounds
    #     y,  x
    # NW 678, 412
    # SW 313, 417
    # NE 764, 787
    # SE 432, 941
    # (west, east, south, north) = compute_bounds(nc)
    south = 313
    north = 678
    west = 412
    east = 900
    lats = nc.variables['lat'][south:north, west:east]
    lons = nc.variables['lon'][south:north, west:east]
    pts = []
    for lon, lat in zip(np.ravel(lons), np.ravel(lats)):
        pts.append(Point(lon, lat))
    df = gpd.GeoDataFrame({'geometry': pts})
    # iterate over days
    now = datetime.date(2018, 7, 1)
    ets = datetime.date(2019, 1, 1)
    while now < ets:
        valid = util.utc(now.year, now.month, now.day, 5)
        tidx0 = iemre.hourly_offset(valid)
        now += datetime.timedelta(days=1)
        valid = util.utc(now.year, now.month, now.day, 5)
        tidx1 = iemre.hourly_offset(valid)
        data = precip[tidx0:tidx1, south:north, west:east]
        total = np.sum(data, axis=0)
        print("Processing %s max precip: %.2f" % (
            now - datetime.timedelta(days=1), np.max(total)))
        df[(now - datetime.timedelta(days=1)).strftime("%b%d")
           ] = distance(np.ravel(total), 'MM').value("IN")
    df.to_file('combined.shp')
Ejemplo n.º 52
0
def do_output():
    pgconn = psycopg2.connect(database='isuag', host='iemdb', user='******')
    cursor = pgconn.cursor(cursor_factory=psycopg2.extras.DictCursor)
    cursor.execute("""
    with data as (
        select rank() OVER (ORDER by valid DESC), * from sm_hourly
        where valid > now() - '48 hours'::interval)
    SELECT *, valid at time zone 'UTC' as utc_valid from data
    where rank = 1 ORDER by station ASC""")

    sys.stdout.write(("station_id,LAT [deg N],LON [deg E],date_time,ELEV [m],"
                      "depth [m]#SOILT [C],depth [m]#SOILM [kg/kg]"
                      "\n"))

    nt = NetworkTable("ISUSM")
    for row in cursor:
        sid = row['station']
        sys.stdout.write(
            ("%s,%.4f,%.4f,%s,%.1f,"
             "%.3f;%.3f;%.3f;%.3f#%s;%s;%s;%s,"
             "%.3f;%.3f;%.3f#%s;%s;%s,"
             "\n"
             ) % (sid, nt.sts[sid]['lat'],
                  nt.sts[sid]['lon'],
                  row['utc_valid'].strftime("%Y-%m-%dT%H:%M:%SZ"),
                  nt.sts[sid]['elevation'],
                  distance(4, 'IN').value("M"), distance(12, 'IN').value("M"),
                  distance(24, 'IN').value("M"), distance(50, 'IN').value("M"),
                  p(row['tsoil_c_avg'], 3, -90, 90),
                  p(row['t12_c_avg'], 3, -90, 90),
                  p(row['t24_c_avg'], 3, -90, 90),
                  p(row['t50_c_avg'], 3, -90, 90),
                  distance(12, 'IN').value("M"),
                  distance(24, 'IN').value("M"), distance(50, 'IN').value("M"),
                  p(row['vwc_12_avg'], 1, 0, 100),
                  p(row['vwc_24_avg'], 1, 0, 100),
                  p(row['vwc_50_avg'], 1, 0, 100),
                  ))
    sys.stdout.write(".EOO\n")
Ejemplo n.º 53
0
def do_month(year, month, routes):
    """ Generate a MRMS plot for the month!"""

    sts = datetime.datetime(year,month,1)
    ets = sts + datetime.timedelta(days=35)
    ets = ets.replace(day=1)

    today = datetime.datetime.now()
    if ets > today:
        ets = today

    idx0 = iemre.daily_offset(sts)
    idx1 = iemre.daily_offset(ets)

    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_mrms_daily.nc" % (year,),
                          'r')

    lats = nc.variables['lat'][:]
    lons = nc.variables['lon'][:]
    p01d = distance(np.sum(nc.variables['p01d'][idx0:idx1,:,:],0),
                    'MM').value('IN')
    nc.close()

    m = MapPlot(sector='iowa', title='MRMS %s - %s Total Precipitation' % (
            sts.strftime("%-d %b"), 
            (ets - datetime.timedelta(days=1)).strftime("%-d %b %Y")),
            subtitle='Data from NOAA MRMS Project')
    x,y = np.meshgrid(lons, lats)
    bins = [0.01, 0.1, 0.5, 1, 1.5, 2, 3, 4, 5, 6, 7, 8, 12, 16, 20]
    m.pcolormesh(x, y, p01d, bins, units='inches')
    m.drawcounties()
    currentfn = "summary/iowa_mrms_q3_month.png"
    archivefn = sts.strftime("%Y/%m/summary/iowa_mrms_q3_month.png")
    pqstr = "plot %s %s00 %s %s png" % (
                routes, sts.strftime("%Y%m%d%H"), currentfn, archivefn)
    m.postprocess(pqstr=pqstr)
Ejemplo n.º 54
0
def process(sheets):

    resdf = pd.DataFrame({
            'precip': sheets['RainOut']['Rain_mm_Tot'],
            'tmpf': sheets['TempRHVPOut']['AirT_C_Avg'],
            'rh': sheets['TempRHVPOut']['RH'],
            'drct': sheets['WindOut']['WindDir_D1_WVT'],
            'sknt': sheets['WindOut']['WS_ms_S_WVT'],
            'srad': sheets['SolarRad1Out']['Slr_kW_Avg']})
    # Do unit conversion
    resdf['srad'] = resdf['srad'] * 1000.
    resdf['precip'] = distance(resdf['precip'], 'MM').value('IN')
    resdf['tmpf'] = temperature(resdf['tmpf'], 'C').value('F')
    resdf['dwpf'] = dewpoint(temperature(resdf['tmpf'], 'F'),
                             humidity(resdf['rh'], '%')).value('F')
    resdf['sknt'] = speed(resdf['sknt'], 'MPS').value('KT')
    print(resdf.describe())
    minval = resdf.index.min()
    maxval = resdf.index.max()
    cursor = pgconn.cursor()
    cursor.execute("""DELETE from weather_data_obs WHERE
    valid between '%s-06' and '%s-06' and station = 'HICKS.P'
    """ % (minval.strftime("%Y-%m-%d %H:%M"),
           maxval.strftime("%Y-%m-%d %H:%M")))
    print("DELETED %s rows between %s and %s" % (cursor.rowcount, minval,
                                                 maxval))
    for valid, row in resdf.iterrows():
        if pd.isnull(valid):
            continue
        cursor.execute("""INSERT into weather_data_obs
        (station, valid, tmpf, dwpf, drct, precip, srad, sknt) VALUES
        ('HICKS.P', %s, %s, %s, %s, %s, %s, %s)
        """, (valid.strftime("%Y-%m-%d %H:%M-06"), row['tmpf'], row['dwpf'],
              row['drct'], row['precip'], row['srad'], row['sknt']))
    cursor.close()
    pgconn.commit()
Ejemplo n.º 55
0
def do_daily(fn):
    df = pd.read_table(fn, sep=' ')
    df['sknt'] = speed(df['WINDSPEED'], 'MPS').value('KT')
    df['high'] = temperature(df['TMAX'], 'C').value('F')
    df['low'] = temperature(df['TMIN'], 'C').value('F')
    df['pday'] = distance(df['PRECIP'], 'MM').value('IN')
    df['date'] = df[['YEAR', 'MONTH', 'DAY']].apply(lambda x:
                                                    datetime.date(x[0], x[1],
                                                                  x[2]),
                                                    axis=1)
    print("fn: %s valid: %s - %s" % (fn, df['date'].min(), df['date'].max()))
    cursor = pgconn.cursor()
    cursor.execute("""DELETE from weather_data_daily where station = 'DPAC'
    and valid >= %s and valid <= %s""", (df['date'].min(), df['date'].max()))
    if cursor.rowcount > 0:
        print("Deleted %s rows" % (cursor.rowcount, ))
    for i, row in df.iterrows():
        cursor.execute("""INSERT into weather_data_daily
        (station, valid, high, low, precip, sknt) VALUES ('DPAC',
        %s, %s, %s, %s, %s)""", (row['date'], row['high'], row['low'],
                                 row['pday'], row['sknt']))
    print("Inserted %s rows..." % (i + 1, ))
    cursor.close()
    pgconn.commit()
Ejemplo n.º 56
0
def do_hourly(fn):
    df = pd.read_table(fn, sep=' ')
    df['sknt'] = speed(df['WINDSPEED'], 'MPS').value('KT')
    df['tmpf'] = temperature(df['TAIR'], 'C').value('F')
    df['precip'] = distance(df['PREC'], 'MM').value('IN')
    df['valid'] = df[['YEAR', 'MONTH', 'DAY', 'HOUR']].apply(lambda x:
                                                             d(*x),
                                                             axis=1)
    print("fn: %s valid: %s - %s" % (fn, df['valid'].min(),
                                     df['valid'].max()))
    cursor = pgconn.cursor()
    cursor.execute("""DELETE from weather_data_obs where station = 'DPAC'
    and valid >= %s and valid <= %s""", (df['valid'].min(), df['valid'].max()))
    if cursor.rowcount > 0:
        print("Deleted %s rows" % (cursor.rowcount, ))
    for i, row in df.iterrows():
        cursor.execute("""INSERT into weather_data_obs
        (station, valid, tmpf, sknt, precip, srad) VALUES ('DPAC',
        %s, %s, %s, %s, %s)""", (row['valid'].strftime("%Y-%m-%d %H:%M-05"),
                                 row['tmpf'], row['sknt'],
                                 row['precip'], row['RADIATION']))
    print("Inserted %s rows..." % (i + 1, ))
    cursor.close()
    pgconn.commit()
Ejemplo n.º 57
0
def process(station, metadata):
    ''' Lets process something, stat

    ['TMAX', 'TMIN', 'TOBS', 'PRCP', 'SNOW', 'SNWD', 'EVAP', 'MNPN', 'MXPN',
     'WDMV', 'DAEV', 'MDEV', 'DAWM', 'MDWM', 'WT05', 'SN01', 'SN02', 'SN03', 
     'SX01', 'SX02', 'SX03', 'MDPR', 'MDSF', 'SN51', 'SN52', 'SN53', 'SX51', 
     'SX52', 'SX53', 'WT01', 'SN31', 'SN32', 'SN33', 'SX31', 'SX32', 'SX33']

    '''
    fp = get_file(station)
    if fp is None:
        return#
    nc = create_netcdf(station, metadata)
    if nc is None:
        return
    data = {}
    reclength = len(nc.dimensions['time'])
    for line in fp:
        m = DATARE.match(line)
        d = m.groupdict()
        if d['element'] not in ['TMAX', 'TMIN', 'PRCP', 'SNOW', 'SNWD']:
            continue
        day = datetime.date( int(d['year']), int(d['month']), 1)
        days = get_days_for_month(day)
        for i in range(1, days+1):
            day = day.replace(day=i)
            # We don't want data in the future!
            if day >= TODAY:
                continue
            if not data.has_key(day):
                data[day] = {}
            #if d['flag%s' % (i,)] != "   ":
            #    print d['flag%s' % (i,)]
            data[day][d['element']+"mflag"] = d['flag%s' % (i,)][0]
            data[day][d['element']+"qflag"] = d['flag%s' % (i,)][1]
            data[day][d['element']+"sflag"] = d['flag%s' % (i,)][2]
            v = varconv(d['value%s' % (i,)], d['element'])
            if v is not None:
                data[day][d['element']] = v

    table = "alldata_%s" % (station[:2],)
    keys = data.keys()
    keys.sort()
    
    obs = {}
    cursor.execute("""SELECT day, high, low, precip, snow, snowd from
            """+table+""" where station = %s""", (station,))
    for row in cursor:
        obs[row[0]] = row[1:]
    print 'loadvars'
    pr_mflag = nc.variables['pr_mflag'][:]
    pr_sflag = nc.variables['pr_sflag'][:]
    pr_qflag = nc.variables['pr_qflag'][:]
    pr = nc.variables['pr'][:]

    prsn_mflag = nc.variables['prsn_mflag'][:]
    prsn_qflag = nc.variables['prsn_qflag'][:]
    prsn_sflag = nc.variables['prsn_sflag'][:]
    prsn = nc.variables['prsn'][:]

    snowdepth_mflag = nc.variables['snowdepth_mflag'][:]
    snowdepth_qflag = nc.variables['snowdepth_qflag'][:]
    snowdepth_sflag = nc.variables['snowdepth_sflag'][:]
    snowdepth = nc.variables['snowdepth'][:]

    tmax_mflag= nc.variables['tmax_mflag'][:]
    tmax_qflag= nc.variables['tmax_qflag'][:]
    tmax_sflag= nc.variables['tmax_sflag'][:]
    tmax= nc.variables['tmax'][:]

    tmin_mflag= nc.variables['tmin_mflag'][:]
    tmin_qflag= nc.variables['tmin_qflag'][:]
    tmin_sflag= nc.variables['tmin_sflag'][:]
    tmin= nc.variables['tmin'][:]
    #print 'loaded'
    for d in keys:
        #row = obs.get(d)
        offset = (d - BASE).days - 1
        pr_mflag[offset] = ord(data[d].get('PRCPmflag', ' '))
        pr_qflag[offset] = ord(data[d].get('PRCPqflag', ' '))
        pr_sflag[offset] = ord(data[d].get('PRCPsflag', ' '))
        if data[d].get('PRCP') is not None:
            pr[offset] = distance(data[d].get('PRCP'), 
                                                  'IN').value("MM") / 86400.
        prsn_mflag[offset] = ord(data[d].get('SNOWmflag', ' '))
        prsn_qflag[offset] = ord(data[d].get('SNOWqflag', ' '))
        prsn_sflag[offset] = ord(data[d].get('SNOWsflag', ' '))
        if data[d].get('SNOW') is not None:
            prsn[offset] = distance(data[d].get('SNOW'), 
                                                  'IN').value("MM") / 86400.
        snowdepth_mflag[offset] = ord(data[d].get('SNWDmflag', ' '))
        snowdepth_qflag[offset] = ord(data[d].get('SNWDqflag', ' '))
        snowdepth_sflag[offset] = ord(data[d].get('SNWDsflag', ' '))
        if data[d].get('SNWD') is not None:
            snowdepth[offset] = distance(data[d].get('SNWD'), 
                                                  'IN').value("MM")
        tmax_mflag[offset] = ord(data[d].get('TMAXmflag', ' '))
        tmax_qflag[offset] = ord(data[d].get('TMAXqflag', ' '))
        tmax_sflag[offset] = ord(data[d].get('TMAXsflag', ' '))
        if data[d].get('TMAX') is not None:
            tmax[offset] = temperature(data[d].get('TMAX'),
                                                   'F').value('K')
        tmin_mflag[offset] = ord(data[d].get('TMINmflag', ' '))
        tmin_qflag[offset] = ord(data[d].get('TMINqflag', ' '))
        tmin_sflag[offset] = ord(data[d].get('TMINsflag', ' '))
        if data[d].get('TMIN') is not None:
            tmin[offset] = temperature(data[d].get('TMIN'),
                                                   'F').value('K')
        if row is None:
            print 'No data for %s %s' % (station, d)
            cursor.execute("""INSERT into %s(station, day, sday,
            year, month) VALUES ('%s', '%s', '%s', %s, %s)""" % (
                                            table, station, d, 
                            "%02i%02i" % (d.month, d.day), d.year, 
                                             d.month))
            row = [None, None, None, None, None]
        s = ""
        if (data[d].get('TMAX') is not None and 
            (row[0] is None or row[0] != data[d]['TMAX'])):
            print 'Update %s High   %5s -> %5s' % (d, row[0], 
                                                      data[d]['TMAX'])
            s += "high = %.0f," % (data[d]['TMAX'],)
            
        if (data[d].get('TMIN') is not None and 
            (row[1] is None or row[1] != data[d]['TMIN'])):
            print 'Update %s Low    %5s -> %5s' % (d, row[1], 
                                                     data[d]['TMIN'])
            s += "low = %.0f," % (data[d]['TMIN'],)

        if (data[d].get('PRCP') is not None and 
            (row[2] is None or row[2] != data[d]['PRCP'])):
            print 'Update %s Precip %5s -> %5s' % (d, row[2], 
                                                        data[d]['PRCP'])
            s += "precip = %.2f," % (data[d]['PRCP'],)

        if (data[d].get('SNOW') is not None and 
            (row[3] is None or row[3] != data[d]['SNOW'])):
            print 'Update %s Snow   %5s -> %5s' % (d, row[3], 
                                                      data[d]['SNOW'])
            s += "snow = %.1f," % (data[d]['SNOW'],)

        if (data[d].get('SNWD') is not None and 
            (row[4] is None or row[4] != data[d]['SNWD'])):
            print 'Update %s Snowd  %5s -> %5s' % (d, row[4], 
                                                       data[d]['SNWD'])
            s += "snowd = %.1f," % (data[d]['SNWD'],)

        if s != "":
            sql = """UPDATE %s SET %s WHERE day = '%s' and 
            station = '%s'""" % (table, s[:-1],          
                             d, station )
            cursor.execute(sql)

    
    #print 'Write pr netcdf'
    nc.variables['pr'][:] = pr
    #print 'Write pr_mflag netcdf'
    nc.variables['pr_mflag'][:] = pr_mflag
    #print 'Write pr_qflag netcdf'
    nc.variables['pr_qflag'][:] = pr_qflag
    #print 'Write pr_sflag netcdf'
    nc.variables['pr_sflag'][:] = pr_sflag
    
    #print 'Write prsn netcdf'
    nc.variables['prsn'][:] = prsn
    nc.variables['prsn_mflag'][:] = prsn_mflag
    nc.variables['prsn_qflag'][:] = prsn_qflag
    nc.variables['prsn_sflag'][:] = prsn_sflag

    #print 'Write snowdepth netcdf'
    nc.variables['snowdepth'][:] = snowdepth
    nc.variables['snowdepth_mflag'][:] = snowdepth_mflag
    nc.variables['snowdepth_qflag'][:] = snowdepth_qflag
    nc.variables['snowdepth_sflag'][:] = snowdepth_sflag

    #print 'Write tmax netcdf'
    nc.variables['tmax'][:] = tmax
    nc.variables['tmax_mflag'][:] = tmax_mflag
    nc.variables['tmax_qflag'][:] = tmax_qflag
    nc.variables['tmax_sflag'][:] = tmax_sflag

    #print 'Write tmin netcdf'
    nc.variables['tmin'][:] = tmin
    nc.variables['tmin_mflag'][:] = tmin_mflag
    nc.variables['tmin_qflag'][:] = tmin_qflag
    nc.variables['tmin_sflag'][:] = tmin_sflag
    #print 'Done writing'
    nc.close()
Ejemplo n.º 58
0
    basefn = compute_fn(info['lon'], info['lat'])
    # print basefn, info['lon'], info['lat']
    # load precip
    fn = "../arrm/swatfiles_%s/%s.pcp" % (model, basefn)
    if not os.path.isfile(fn):
        continue
    obs = {}
    for line in open(fn):
        if not line.startswith('20'):
            continue
        if line.find('-99.0') > 0:
            line = line.replace('-99.0', '  0.0')
        valid = datetime.datetime.strptime(line[:7], '%Y%j')
        obs[valid] = {'valid': valid,
                      'precip_in':
                      distance(float(line[7:12]), 'MM').value('IN')}
    fn = "../arrm/swatfiles_%s/%s.tmp" % (model, basefn)
    for line in open(fn):
        if not line.startswith('20'):
            continue
        if line.find('-99.0') > 0:
            continue
        valid = datetime.datetime.strptime(line[:7], '%Y%j')
        obs[valid]['high_f'] = temperature(float(line[7:12]), 'C').value('F')
        obs[valid]['low_f'] = temperature(float(line[12:17]), 'C').value('F')

    res = {'fips': fips}
    rows = []
    for key, row in obs.iteritems():
        rows.append(row)
    df = pd.DataFrame(rows)
Ejemplo n.º 59
0
import pygrib
from pyiem.plot import MapPlot
from pyiem import reference
import numpy as np
from pyiem.datatypes import distance

g = pygrib.open('p168m_2016090600f168.grb')

grb = g[1]
lats, lons = grb.latlons()

m = MapPlot(sector='custom', project='aea',
            south=reference.MW_SOUTH, north=reference.MW_NORTH,
            east=reference.MW_EAST, west=reference.MW_WEST,
            axisbg='tan',
            title=("Weather Prediction Center (WPC) "
                   "7 Day Forecasted Precipitation"),
            subtitle='7 PM 5 September thru 7 PM 12 September 2016')

m.contourf(lons, lats, distance(grb['values'], 'MM').value('IN'),
           np.arange(0, 4.6, 0.25), units='inches')

m.postprocess(filename='test.png')