コード例 #1
0
def do_precip12(ts):
    """Compute the 24 Hour precip at 12 UTC, we do some more tricks though"""
    offset = iemre.daily_offset(ts)
    ets = utc(ts.year, ts.month, ts.day, 12)
    sts = ets - datetime.timedelta(hours=24)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    if ts.month == 1 and ts.day == 1:
        if sts.year >= 1900:
            print(("p01d_12z  for %s [idx:%s] %s(%s)->%s(%s) SPECIAL") %
                  (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
                   ets.strftime("%Y%m%d%H"), offset2))
        ncfn = iemre.get_hourly_ncname(ets.year)
        if not os.path.isfile(ncfn):
            print("Missing %s" % (ncfn, ))
            return
        hnc = ncopen(ncfn, timeout=600)
        phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0)
        hnc.close()
        hnc = ncopen(iemre.get_hourly_ncname(sts.year), timeout=600)
        phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0)
        hnc.close()
    else:
        ncfn = iemre.get_hourly_ncname(ts.year)
        if not os.path.isfile(ncfn):
            print("Missing %s" % (ncfn, ))
            return
        hnc = ncopen(ncfn, timeout=600)
        phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0)
        hnc.close()
    write_grid(ts, 'p01d_12z', np.where(phour < 0, 0, phour))
コード例 #2
0
ファイル: daily_analysis.py プロジェクト: akrherz/iem
def do_precip12(nc, ts):
    """Compute the 24 Hour precip at 12 UTC, we do some more tricks though"""
    offset = iemre.daily_offset(ts)
    ets = datetime.datetime(ts.year, ts.month, ts.day, 12)
    ets = ets.replace(tzinfo=pytz.timezone("UTC"))
    sts = ets - datetime.timedelta(hours=24)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    if ts.month == 1 and ts.day == 1:
        print(("p01d_12z  for %s [idx:%s] %s(%s)->%s(%s) SPECIAL"
               ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
                    ets.strftime("%Y%m%d%H"), offset2))
        ncfn = "/mesonet/data/iemre/%s_mw_hourly.nc" % (ets.year,)
        if not os.path.isfile(ncfn):
            print("Missing %s" % (ncfn,))
            return
        hnc = netCDF4.Dataset(ncfn)
        phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0)
        hnc.close()
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (
            sts.year,))
        phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0)
        hnc.close()
    else:
        print(("p01d_12z  for %s [idx:%s] %s(%s)->%s(%s)"
               ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
                    ets.strftime("%Y%m%d%H"), offset2))
        ncfn = "/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year,)
        if not os.path.isfile(ncfn):
            print("Missing %s" % (ncfn,))
            return
        hnc = netCDF4.Dataset(ncfn)
        phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0)
        hnc.close()
    nc.variables['p01d_12z'][offset] = phour
コード例 #3
0
def dowork(form):
    """Do work!"""
    date = datetime.datetime.strptime(form.getfirst('valid'), '%Y-%m-%d')
    lat = float(form.getfirst("lat"))
    lon = float(form.getfirst("lon"))

    # We want data for the UTC date and timestamps are in the rears, so from
    # 1z through 1z
    sts = datetime.datetime(date.year, date.month, date.day, 1)
    sts = sts.replace(tzinfo=pytz.utc)
    ets = sts + datetime.timedelta(hours=24)
    sidx = iemre.hourly_offset(sts)
    eidx = iemre.hourly_offset(ets)

    ncfn = "/mesonet/data/stage4/%s_stage4_hourly.nc" % (date.year, )
    nc = netCDF4.Dataset(ncfn, 'r')

    dist = ((nc.variables['lon'][:] - lon)**2 +
            (nc.variables['lat'][:] - lat)**2)**0.5
    (j, i) = np.unravel_index(dist.argmin(), dist.shape)
    res = {'gridi': i, 'gridj': j, 'data': []}

    ppt = nc.variables['p01m'][sidx:eidx, j, i]
    nc.close()

    for tx, pt in enumerate(ppt):
        valid = sts + datetime.timedelta(hours=tx)
        res['data'].append({
            'end_valid':
            valid.strftime("%Y-%m-%dT%H:00:00Z"),
            'precip_in':
            myrounder(datatypes.distance(pt, 'MM').value('IN'), 2)
        })

    return json.dumps(res)
コード例 #4
0
ファイル: daily_analysis.py プロジェクト: smartparrot/iem
def do_precip12(ts, ds):
    """Compute the 24 Hour precip at 12 UTC, we do some more tricks though"""
    offset = iemre.daily_offset(ts)
    ets = utc(ts.year, ts.month, ts.day, 12)
    sts = ets - datetime.timedelta(hours=24)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    if ts.month == 1 and ts.day == 1:
        if sts.year >= 1900:
            LOG.warning(
                "p01d_12z  for %s [idx:%s] %s(%s)->%s(%s) SPECIAL",
                ts,
                offset,
                sts.strftime("%Y%m%d%H"),
                offset1,
                ets.strftime("%Y%m%d%H"),
                offset2,
            )
        ncfn = iemre.get_hourly_ncname(ets.year)
        if not os.path.isfile(ncfn):
            LOG.warning("Missing %s", ncfn)
            return
        with ncopen(ncfn, timeout=600) as hnc:
            phour = np.sum(hnc.variables["p01m"][:offset2, :, :], 0)
        with ncopen(iemre.get_hourly_ncname(sts.year), timeout=600) as hnc:
            phour += np.sum(hnc.variables["p01m"][offset1:, :, :], 0)
    else:
        ncfn = iemre.get_hourly_ncname(ts.year)
        if not os.path.isfile(ncfn):
            LOG.warning("Missing %s", ncfn)
            return
        with ncopen(ncfn, timeout=600) as hnc:
            phour = np.sum(hnc.variables["p01m"][offset1:offset2, :, :], 0)
    ds["p01d_12z"].values = np.where(phour < 0, 0, phour)
コード例 #5
0
ファイル: daily_analysis.py プロジェクト: nbackas/iem
def do_precip(nc, ts):
    """Compute the 6 UTC to 6 UTC precip

    We need to be careful here as the timestamp sent to this app is today,
    we are actually creating the analysis for yesterday
    """
    sts = datetime.datetime(ts.year, ts.month, ts.day, 6)
    sts = sts.replace(tzinfo=pytz.timezone("UTC"))
    ets = sts + datetime.timedelta(hours=24)
    offset = iemre.daily_offset(ts)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    if ts.month == 12 and ts.day == 31:
        print(("p01d      for %s [idx:%s] %s(%s)->%s(%s) SPECIAL") %
              (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
               ets.strftime("%Y%m%d%H"), offset2))
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" %
                              (ets.year, ))
        phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0)
        hnc.close()
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" %
                              (sts.year, ))
        phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0)
        hnc.close()
    else:
        print(("p01d      for %s [idx:%s] %s(%s)->%s(%s)") %
              (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
               ets.strftime("%Y%m%d%H"), offset2))
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" %
                              (sts.year, ))
        phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0)
        hnc.close()
    nc.variables['p01d'][offset] = phour
コード例 #6
0
ファイル: daily_analysis.py プロジェクト: muthulatha/iem
def do_precip(nc, ts):
    """Compute the 6 UTC to 6 UTC precip

    We need to be careful here as the timestamp sent to this app is today,
    we are actually creating the analysis for yesterday
    """
    sts = datetime.datetime(ts.year, ts.month, ts.day, 6)
    sts = sts.replace(tzinfo=pytz.timezone("UTC"))
    ets = sts + datetime.timedelta(hours=24)
    offset = iemre.daily_offset(ts)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    if ts.month == 12 and ts.day == 31:
        print(("p01d      for %s [idx:%s] %s(%s)->%s(%s) SPECIAL"
               ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
                    ets.strftime("%Y%m%d%H"), offset2))
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (
            ets.year,))
        phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0)
        hnc.close()
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (
            sts.year,))
        phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0)
        hnc.close()
    else:
        print(("p01d      for %s [idx:%s] %s(%s)->%s(%s)"
               ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
                    ets.strftime("%Y%m%d%H"), offset2))
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (
            sts.year,))
        phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0)
        hnc.close()
    nc.variables['p01d'][offset] = phour
コード例 #7
0
ファイル: daily_analysis.py プロジェクト: nbackas/iem
def do_precip12(nc, ts):
    """Compute the 24 Hour precip at 12 UTC, we do some more tricks though"""
    offset = iemre.daily_offset(ts)
    ets = datetime.datetime(ts.year, ts.month, ts.day, 12)
    ets = ets.replace(tzinfo=pytz.timezone("UTC"))
    sts = ets - datetime.timedelta(hours=24)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    if ts.month == 1 and ts.day == 1:
        print(("p01d_12z  for %s [idx:%s] %s(%s)->%s(%s) SPECIAL") %
              (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
               ets.strftime("%Y%m%d%H"), offset2))
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" %
                              (ets.year, ))
        phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0)
        hnc.close()
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" %
                              (sts.year, ))
        phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0)
        hnc.close()
    else:
        print(("p01d_12z  for %s [idx:%s] %s(%s)->%s(%s)") %
              (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
               ets.strftime("%Y%m%d%H"), offset2))
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" %
                              (ts.year, ))
        phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0)
        hnc.close()
    nc.variables['p01d_12z'][offset] = phour
コード例 #8
0
ファイル: stage4.py プロジェクト: jamayfieldjr/iem
def dowork(form):
    """Do work!"""
    date = datetime.datetime.strptime(form.getfirst('valid'), '%Y-%m-%d')
    lat = float(form.getfirst("lat"))
    lon = float(form.getfirst("lon"))

    # We want data for the UTC date and timestamps are in the rears, so from
    # 1z through 1z
    sts = utc(date.year, date.month, date.day, 1)
    ets = sts + datetime.timedelta(hours=24)
    sidx = iemre.hourly_offset(sts)
    eidx = iemre.hourly_offset(ets)

    ncfn = "/mesonet/data/stage4/%s_stage4_hourly.nc" % (date.year, )
    res = {'gridi': -1, 'gridj': -1, 'data': []}
    if not os.path.isfile(ncfn):
        return json.dumps(res)
    with ncopen(ncfn) as nc:
        dist = ((nc.variables['lon'][:] - lon)**2 +
                (nc.variables['lat'][:] - lat)**2)**0.5
        (j, i) = np.unravel_index(dist.argmin(), dist.shape)
        res['gridi'] = int(i)
        res['gridj'] = int(j)

        ppt = nc.variables['p01m'][sidx:eidx, j, i]

    for tx, pt in enumerate(ppt):
        valid = sts + datetime.timedelta(hours=tx)
        res['data'].append({
            'end_valid': valid.strftime("%Y-%m-%dT%H:00:00Z"),
            'precip_in': myrounder(
                        datatypes.distance(pt, 'MM').value('IN'), 2)
            })

    return json.dumps(res)
コード例 #9
0
def workflow(valid):
    """Our workflow"""
    if valid.month == 1 and valid.day == 1:
        print("prism_adjust_stage4, sorry Jan 1 processing is a TODO!")
        return
    # read prism
    tidx = daily_offset(valid)
    nc = netCDF4.Dataset("/mesonet/data/prism/%s_daily.nc" % (valid.year, ),
                         'r')
    ppt = nc.variables['ppt'][tidx, :, :]
    # missing as zero
    ppt = np.where(ppt.mask, 0, ppt)
    lons = nc.variables['lon'][:]
    lats = nc.variables['lat'][:]
    nc.close()
    (lons, lats) = np.meshgrid(lons, lats)

    # Interpolate this onto the stage4 grid
    nc = netCDF4.Dataset(
        ("/mesonet/data/stage4/%s_stage4_hourly.nc") % (valid.year, ), 'a')
    p01m = nc.variables['p01m']
    p01m_status = nc.variables['p01m_status']
    s4lons = nc.variables['lon'][:]
    s4lats = nc.variables['lat'][:]
    # Values are in the hourly arrears, so start at -23 and thru current hour
    sts_tidx = hourly_offset(valid - datetime.timedelta(hours=23))
    ets_tidx = hourly_offset(valid + datetime.timedelta(hours=1))
    s4total = np.sum(p01m[sts_tidx:ets_tidx, :, :], axis=0)
    # make sure the s4total does not have zeros
    s4total = np.where(s4total < 0.001, 0.001, s4total)

    nn = NearestNDInterpolator((lons.flat, lats.flat), ppt.flat)
    prism_on_s4grid = nn(s4lons, s4lats)
    multiplier = prism_on_s4grid / s4total

    # Do the work now, we should not have to worry about the scale factor
    for tidx in range(sts_tidx, ets_tidx):
        newval = p01m[tidx, :, :] * multiplier
        p01m[tidx, :, :] = newval
        # make sure have data
        if np.ma.max(newval) > 0:
            p01m_status[tidx] = 2
        else:
            print(("prism_adjust_stage4 NOOP for time %s[idx:%s]") %
                  ((datetime.datetime(valid.year, 1, 1, 0) +
                    datetime.timedelta(hours=tidx)).strftime("%Y-%m-%dT%H"),
                   tidx))
    """
    s4total_v2 = np.sum(p01m[sts_tidx:ets_tidx, :, :], axis=0)

    from pyiem.plot.geoplot import MapPlot
    import matplotlib.pyplot as plt
    mp = MapPlot(sector='iowa')
    mp.pcolormesh(s4lons, s4lats, s4total_v2,
                  np.arange(-10, 11, 1), cmap=plt.get_cmap("BrBG"))
    mp.postprocess(filename='test.png')
    mp.close()
    """
    nc.close()
コード例 #10
0
ファイル: daily_clifile_editor.py プロジェクト: akrherz/idep
def load_stage4():
    """ It sucks, but we need to load the stage IV data to give us something
    to benchmark the MRMS data against, to account for two things:
    1) Wind Farms
    2) Over-estimates
    """
    printt("load_stage4() called...")
    # The stage4 files store precip in the rears, so compute 1 AM
    one_am = datetime.datetime(VALID.year, VALID.month, VALID.day, 12, 0)
    one_am = one_am.replace(tzinfo=pytz.timezone("UTC"))
    one_am = one_am.astimezone(pytz.timezone("America/Chicago"))
    # stage4 data is stored in the rears, so 1am
    one_am = one_am.replace(hour=1, minute=0, second=0)
    # clever hack for CST/CDT
    tomorrow = one_am + datetime.timedelta(hours=36)
    tomorrow = tomorrow.replace(hour=1)

    sts_tidx = iemre.hourly_offset(one_am)
    ets_tidx = iemre.hourly_offset(tomorrow)
    printt(("stage4 sts_tidx:%s[%s] ets_tidx:%s[%s]"
            ) % (sts_tidx, one_am, ets_tidx, tomorrow))
    nc = ncopen("/mesonet/data/stage4/%s_stage4_hourly.nc" % (VALID.year, ))
    p01m = nc.variables['p01m']

    lats = nc.variables['lat'][:]
    lons = nc.variables['lon'][:]
    # crossing jan 1
    if ets_tidx < sts_tidx:
        printt("Exercise special stageIV logic for jan1!")
        totals = np.sum(p01m[sts_tidx:, :, :], axis=0)
        nc.close()
        nc = ncopen(
            "/mesonet/data/stage4/%s_stage4_hourly.nc" % (tomorrow.year, )
        )
        p01m = nc.variables['p01m']
        totals += np.sum(p01m[:ets_tidx, :, :], axis=0)
    else:
        totals = np.sum(p01m[sts_tidx:ets_tidx, :, :], axis=0)
    nc.close()

    if np.ma.max(totals) > 0:
        pass
    else:
        print('No StageIV data found, aborting...')
        sys.exit()
    # set a small non-zero number to keep things non-zero
    totals = np.where(totals > 0.001, totals, 0.001)

    xaxis = np.arange(MYWEST, MYEAST, 0.01)
    yaxis = np.arange(MYSOUTH, MYNORTH, 0.01)
    xi, yi = np.meshgrid(xaxis, yaxis)
    nn = NearestNDInterpolator((lons.flatten(), lats.flatten()),
                               totals.flatten())
    STAGE4[:] = nn(xi, yi)
    write_grid(STAGE4, 'stage4')
    printt("load_stage4() finished...")
コード例 #11
0
def merge(ts):
    """
    Process an hour's worth of stage4 data into the hourly RE
    """

    # Load up the 12z 24h total, this is what we base our deltas on
    fn = ("/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.24h.grib") % (
        ts.strftime("%Y/%m/%d"),
        ts.strftime("%Y%m%d%H"),
    )
    if not os.path.isfile(fn):
        LOG.info("stage4_12z_adjust %s is missing", fn)
        return False

    grbs = pygrib.open(fn)
    grb = grbs[1]
    val = grb.values
    lats, lons = grb.latlons()
    # can save a bit of memory as we don't need all data
    stride = slice(None, None, 3)
    lats = np.ravel(lats[stride, stride])
    lons = np.ravel(lons[stride, stride])
    vals = np.ravel(val[stride, stride])
    # Clip large values
    vals = np.where(vals > 250.0, 0, vals)
    nn = NearestNDInterpolator((lons, lats), vals)
    xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS)
    stage4 = nn(xi, yi)
    # Prevent Large numbers, negative numbers
    stage4 = np.where(stage4 < 10000.0, stage4, 0.0)
    stage4 = np.where(stage4 < 0.0, 0.0, stage4)

    ts0 = ts - datetime.timedelta(days=1)
    offset0 = iemre.hourly_offset(ts0)
    offset1 = iemre.hourly_offset(ts)
    # Running at 12 UTC 1 Jan
    if offset0 > offset1:
        offset0 = 0
    # Open up our RE file
    with ncopen(iemre.get_hourly_ncname(ts.year), "a", timeout=300) as nc:
        iemre_total = np.sum(nc.variables["p01m"][offset0:offset1, :, :],
                             axis=0)
        iemre_total = np.where(iemre_total > 0.0, iemre_total, 0.00024)
        iemre_total = np.where(iemre_total < 10000.0, iemre_total, 0.00024)
        multiplier = stage4 / iemre_total
        for offset in range(offset0, offset1):
            # Get the unmasked dadta
            data = nc.variables["p01m"][offset, :, :]

            # Keep data within reason
            data = np.where(data > 10000.0, 0.0, data)
            # 0.00024 / 24
            adjust = np.where(data > 0, data, 0.00001) * multiplier
            adjust = np.where(adjust > 250.0, 0, adjust)
            nc.variables["p01m"][offset, :, :] = np.where(
                adjust < 0.01, 0, adjust)
コード例 #12
0
def load_stage4():
    """ It sucks, but we need to load the stage IV data to give us something
    to benchmark the MRMS data against, to account for two things:
    1) Wind Farms
    2) Over-estimates
    """
    printt("load_stage4() called...")
    # The stage4 files store precip in the rears, so compute 1 AM
    one_am = datetime.datetime(VALID.year, VALID.month, VALID.day, 12, 0)
    one_am = one_am.replace(tzinfo=pytz.timezone("UTC"))
    one_am = one_am.astimezone(pytz.timezone("America/Chicago"))
    # stage4 data is stored in the rears, so 1am
    one_am = one_am.replace(hour=1, minute=0, second=0)
    # clever hack for CST/CDT
    tomorrow = one_am + datetime.timedelta(hours=36)
    tomorrow = tomorrow.replace(hour=1)

    sts_tidx = iemre.hourly_offset(one_am)
    ets_tidx = iemre.hourly_offset(tomorrow)
    printt(("stage4 sts_tidx:%s[%s] ets_tidx:%s[%s]") %
           (sts_tidx, one_am, ets_tidx, tomorrow))
    nc = ncopen("/mesonet/data/stage4/%s_stage4_hourly.nc" % (VALID.year, ))
    p01m = nc.variables["p01m"]

    lats = nc.variables["lat"][:]
    lons = nc.variables["lon"][:]
    # crossing jan 1
    if ets_tidx < sts_tidx:
        printt("Exercise special stageIV logic for jan1!")
        totals = np.sum(p01m[sts_tidx:, :, :], axis=0)
        nc.close()
        nc = ncopen("/mesonet/data/stage4/%s_stage4_hourly.nc" %
                    (tomorrow.year, ))
        p01m = nc.variables["p01m"]
        totals += np.sum(p01m[:ets_tidx, :, :], axis=0)
    else:
        totals = np.sum(p01m[sts_tidx:ets_tidx, :, :], axis=0)
    nc.close()

    if np.ma.max(totals) > 0:
        pass
    else:
        print("No StageIV data found, aborting...")
        sys.exit()
    # set a small non-zero number to keep things non-zero
    totals = np.where(totals > 0.001, totals, 0.001)

    xaxis = np.arange(MYWEST, MYEAST, 0.01)
    yaxis = np.arange(MYSOUTH, MYNORTH, 0.01)
    xi, yi = np.meshgrid(xaxis, yaxis)
    nn = NearestNDInterpolator((lons.flatten(), lats.flatten()),
                               totals.flatten())
    STAGE4[:] = nn(xi, yi)
    write_grid(STAGE4, "stage4")
    printt("load_stage4() finished...")
コード例 #13
0
    def test_hourly_offset(self):
        """ Compute the offsets """
        ts = datetime.datetime(2013, 1, 1, 0, 0)
        ts = ts.replace(tzinfo=pytz.timezone("UTC"))
        offset = iemre.hourly_offset(ts)
        self.assertEqual(offset, 0)

        ts = datetime.datetime(2013, 1, 5, 12, 0)
        ts = ts.replace(tzinfo=pytz.timezone("UTC"))
        offset = iemre.hourly_offset(ts)
        self.assertEqual(offset, 4*24 + 12)
コード例 #14
0
ファイル: daily_analysis.py プロジェクト: danhreitz/iem
def do_precip(nc, ts):
    """Compute the precip totals based on the hourly analysis totals"""
    offset = iemre.daily_offset(ts)
    ets = ts.replace(hour=12, tzinfo=pytz.timezone("UTC"))
    sts = ets - datetime.timedelta(hours=24)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year,))
    phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0)
    nc.variables['p01d'][offset] = phour
    hnc.close()
コード例 #15
0
ファイル: stage4_12z_adjust.py プロジェクト: nbackas/iem
def merge(ts):
    """
    Process an hour's worth of stage4 data into the hourly RE
    """

    # Load up the 12z 24h total, this is what we base our deltas on
    fn = ("/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.24h.grib"
          ) % (ts.strftime("%Y/%m/%d"), ts.strftime("%Y%m%d%H"))

    grbs = pygrib.open(fn)
    grb = grbs[1]
    val = grb.values
    lats, lons = grb.latlons()
    # Rough subsample, since the whole enchillata is too much
    lats = np.ravel(lats[200:-100:5, 300:900:5])
    lons = np.ravel(lons[200:-100:5, 300:900:5])
    vals = np.ravel(val[200:-100:5, 300:900:5])
    # Clip large values
    vals = np.where(vals > 250., 0, vals)
    nn = NearestNDInterpolator((lons, lats), vals)
    xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS)
    stage4 = nn(xi, yi)
    # Prevent Large numbers, negative numbers
    stage4 = np.where(stage4 < 10000., stage4, 0.)
    stage4 = np.where(stage4 < 0., 0., stage4)

    # Open up our RE file
    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year,),
                         'a')
    ts0 = ts - datetime.timedelta(days=1)
    offset0 = iemre.hourly_offset(ts0)
    offset1 = iemre.hourly_offset(ts)
    # Running at 12 UTC 1 Jan
    if offset0 > offset1:
        offset0 = 0
    iemre_total = np.sum(nc.variables["p01m"][offset0:offset1, :, :], axis=0)
    iemre_total = np.where(iemre_total > 0., iemre_total, 0.00024)
    iemre_total = np.where(iemre_total < 10000., iemre_total, 0.00024)
    multiplier = stage4 / iemre_total
    for offset in range(offset0, offset1):
        # Get the unmasked dadta
        data = nc.variables["p01m"][offset, :, :]

        # Keep data within reason
        data = np.where(data > 10000., 0., data)
        # 0.00024 / 24
        adjust = np.where(data > 0, data, 0.00001) * multiplier
        adjust = np.where(adjust > 250.0, 0, adjust)
        nc.variables["p01m"][offset, :, :] = np.where(adjust < 0.01, 0, adjust)
        ts = ts0 + datetime.timedelta(hours=offset-offset0)
    nc.sync()
    nc.close()
コード例 #16
0
ファイル: test_iemre.py プロジェクト: akrherz/pyIEM
def test_hourly_offset():
    """ Compute the offsets """
    ts = utc(2013, 1, 1, 0, 0)
    offset = iemre.hourly_offset(ts)
    assert offset == 0

    ts = utc(2013, 1, 1, 6, 0)
    ts = ts.astimezone(pytz.timezone("America/Chicago"))
    offset = iemre.hourly_offset(ts)
    assert offset == 6

    ts = utc(2013, 1, 5, 12, 0)
    offset = iemre.hourly_offset(ts)
    assert offset == 4*24 + 12
コード例 #17
0
def test_hourly_offset():
    """ Compute the offsets """
    ts = utc(2013, 1, 1, 0, 0)
    offset = iemre.hourly_offset(ts)
    assert offset == 0

    ts = utc(2013, 1, 1, 6, 0)
    ts = ts.astimezone(pytz.timezone("America/Chicago"))
    offset = iemre.hourly_offset(ts)
    assert offset == 6

    ts = utc(2013, 1, 5, 12, 0)
    offset = iemre.hourly_offset(ts)
    assert offset == 4 * 24 + 12
コード例 #18
0
ファイル: daily_analysis.py プロジェクト: raprasad/iem
def do_precip12(nc, ts):
    """Compute the 24 Hour precip at 12 UTC, we do some more tricks though"""
    offset = iemre.daily_offset(ts)
    ets = datetime.datetime(ts.year, ts.month, ts.day, 12)
    ets = ets.replace(tzinfo=pytz.timezone("UTC"))
    sts = ets - datetime.timedelta(hours=24)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    print(("p01d_12z  for %s [idx:%s] %s(%s)->%s(%s)"
           ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
                ets.strftime("%Y%m%d%H"), offset2))
    hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year,))
    phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0)
    nc.variables['p01d_12z'][offset] = phour
    hnc.close()
コード例 #19
0
ファイル: stage4_hourlyre.py プロジェクト: josempolm/iem
def merge(valid):
    """
    Process an hour's worth of stage4 data into the hourly RE
    """
    nc = netCDF4.Dataset(
        ("/mesonet/data/stage4/%s_stage4_hourly.nc") % (valid.year, ), 'r')
    tidx = iemre.hourly_offset(valid)
    val = nc.variables['p01m'][tidx, :, :]
    # print("stage4 mean: %.2f max: %.2f" % (np.mean(val), np.max(val)))
    lats = nc.variables['lat'][:]
    lons = nc.variables['lon'][:]

    # Rough subsample, since the whole enchillata is too much
    lats = np.ravel(lats[200:-100:5, 300:900:5])
    lons = np.ravel(lons[200:-100:5, 300:900:5])
    vals = np.ravel(val[200:-100:5, 300:900:5])
    nn = NearestNDInterpolator((lons, lats), vals)
    xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS)
    res = nn(xi, yi)

    # Lets clip bad data
    # 10 inches per hour is bad data
    res = np.where(np.logical_or(res < 0, res > 250), 0., res)
    # print("Resulting mean: %.2f max: %.2f" % (np.mean(res), np.max(res)))

    # Open up our RE file
    nc = netCDF4.Dataset(
        "/mesonet/data/iemre/%s_mw_hourly.nc" % (valid.year, ), 'a')
    nc.variables["p01m"][tidx, :, :] = res
    # print(("Readback mean: %.2f max: %.2f"
    #       ) % (np.mean(nc.variables["p01m"][tidx, :, :]),
    #            np.max(nc.variables["p01m"][tidx, :, :])))
    nc.close()
コード例 #20
0
ファイル: stage4_hourlyre.py プロジェクト: josempolm/iem
def to_netcdf(valid):
    """Persist this 1 hour precip information to the netcdf storage

    Recall that this timestep has data for the previous hour"""
    fn = ("/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.01h.grib") % (
        valid.strftime("%Y/%m/%d"), valid.strftime("%Y%m%d%H"))
    if not os.path.isfile(fn):
        print("stage4_hourlyre: missing stageIV hourly file %s" % (fn, ))
        return False
    gribs = pygrib.open(fn)
    grb = gribs[1]
    val = grb.values
    # values over 10 inches are bad
    val = np.where(val > 250., 0, val)

    nc = netCDF4.Dataset(
        ("/mesonet/data/stage4/%s_stage4_hourly.nc") % (valid.year, ), 'a')
    tidx = iemre.hourly_offset(valid)
    if nc.variables['p01m_status'][tidx] > 1:
        print("Skipping stage4_hourlyre write as variable status is >1")
        nc.close()
        return True
    p01m = nc.variables['p01m']
    p01m[tidx, :, :] = val
    nc.variables['p01m_status'][tidx] = 1
    nc.close()

    return True
コード例 #21
0
ファイル: qc_summarize.py プロジェクト: akrherz/idep
def compute_stage4(lon, lat, year):
    """Build a daily dataframe for the stage4 data"""
    nc = netCDF4.Dataset("/mesonet/data/stage4/%s_stage4_hourly.nc" % (year,))
    lons = nc.variables['lon'][:]
    lats = nc.variables['lat'][:]
    dist = ((lons - lon)**2 + (lats - lat)**2)**0.5
    (yidx, xidx) = np.unravel_index(dist.argmin(), dist.shape)
    print(("Computed stage4 nclon:%.2f nclat:%.2f yidx:%s xidx:%s "
           ) % (lons[yidx, xidx], lats[yidx, xidx], yidx, xidx))
    p01i = distance(nc.variables['p01m'][:, yidx, xidx], 'MM').value('IN')
    nc.close()
    df = pd.DataFrame({'precip': 0.0},
                      index=pd.date_range('%s-01-01' % (year, ),
                                          '%s-12-31' % (year, ),
                                          tz='America/Chicago'))
    for date in df.index.values:
        date2 = datetime.datetime.utcfromtimestamp(date.tolist()/1e9)
        ts = datetime.datetime(date2.year, date2.month, date2.day, 6)
        ts = ts.replace(tzinfo=pytz.utc)
        ts = ts.astimezone(pytz.timezone("America/Chicago"))
        ts = ts.replace(hour=0)
        ts = ts.astimezone(pytz.utc)
        tidx = hourly_offset(ts)
        # values are in the rears
        val = np.ma.sum(p01i[tidx+1:tidx+25])
        if val > 0:
            df.at[date, 'precip'] = val  # close enough
    return df
コード例 #22
0
def to_netcdf(valid):
    """Persist this 1 hour precip information to the netcdf storage

    Recall that this timestep has data for the previous hour"""
    fn = ("/mesonet/ARCHIVE/data/%s/model/NARR/apcp_%s.grib") % (
        valid.strftime("%Y/%m/%d"),
        valid.strftime("%Y%m%d%H%M"),
    )
    if not os.path.isfile(fn):
        print("merge_narr: missing file %s" % (fn,))
        return False
    gribs = pygrib.open(fn)
    grb = gribs[1]
    val = grb.values

    nc = ncopen(
        ("/mesonet/data/iemre/%s_narr.nc") % (valid.year,), "a", timeout=300
    )
    tidx = int((iemre.hourly_offset(valid) + 1) / 3)
    print("%s np.min: %s np.max: %s" % (tidx, np.min(val), np.max(val)))
    apcp = nc.variables["apcp"]
    apcp[tidx, :, :] = val
    nc.close()

    return True
コード例 #23
0
ファイル: stage4_hourlyre.py プロジェクト: skmichaeldavis/iem
def merge(valid):
    """
    Process an hour's worth of stage4 data into the hourly RE
    """
    nc = ncopen(("/mesonet/data/stage4/%s_stage4_hourly.nc"
                 ) % (valid.year, ), 'r')
    tidx = iemre.hourly_offset(valid)
    val = nc.variables['p01m'][tidx, :, :]
    # print("stage4 mean: %.2f max: %.2f" % (np.mean(val), np.max(val)))
    lats = nc.variables['lat'][:]
    lons = nc.variables['lon'][:]

    # Our data is 4km, iemre is 0.125deg, so we stride some to cut down on mem
    stride = slice(None, None, 3)
    lats = np.ravel(lats[stride, stride])
    lons = np.ravel(lons[stride, stride])
    vals = np.ravel(val[stride, stride])
    nn = NearestNDInterpolator((lons, lats), vals)
    xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS)
    res = nn(xi, yi)

    # Lets clip bad data
    # 10 inches per hour is bad data
    res = np.where(np.logical_or(res < 0, res > 250), 0., res)
    # print("Resulting mean: %.2f max: %.2f" % (np.mean(res), np.max(res)))

    # Open up our RE file
    nc = ncopen(iemre.get_hourly_ncname(valid.year), 'a', timeout=300)
    nc.variables["p01m"][tidx, :, :] = res
    # print(("Readback mean: %.2f max: %.2f"
    #      ) % (np.mean(nc.variables["p01m"][tidx, :, :]),
    #           np.max(nc.variables["p01m"][tidx, :, :])))
    nc.close()
コード例 #24
0
def compute_stage4(lon, lat, year):
    """Build a daily dataframe for the stage4 data"""
    nc = netCDF4.Dataset("/mesonet/data/stage4/%s_stage4_hourly.nc" % (year, ))
    lons = nc.variables['lon'][:]
    lats = nc.variables['lat'][:]
    dist = ((lons - lon)**2 + (lats - lat)**2)**0.5
    (yidx, xidx) = np.unravel_index(dist.argmin(), dist.shape)
    print(("Computed stage4 nclon:%.2f nclat:%.2f yidx:%s xidx:%s ") %
          (lons[yidx, xidx], lats[yidx, xidx], yidx, xidx))
    p01i = distance(nc.variables['p01m'][:, yidx, xidx], 'MM').value('IN')
    nc.close()
    df = pd.DataFrame({'precip': 0.0},
                      index=pd.date_range('%s-01-01' % (year, ),
                                          '%s-12-31' % (year, ),
                                          tz='America/Chicago'))
    for date in df.index.values:
        date2 = datetime.datetime.utcfromtimestamp(date.tolist() / 1e9)
        ts = datetime.datetime(date2.year, date2.month, date2.day, 6)
        ts = ts.replace(tzinfo=pytz.utc)
        ts = ts.astimezone(pytz.timezone("America/Chicago"))
        ts = ts.replace(hour=0)
        ts = ts.astimezone(pytz.utc)
        tidx = hourly_offset(ts)
        # values are in the rears
        val = np.ma.sum(p01i[tidx + 1:tidx + 25])
        if val > 0:
            df.at[date, 'precip'] = val  # close enough
    return df
コード例 #25
0
ファイル: precip_ingest.py プロジェクト: jamayfieldjr/iem
def copy_to_iemre(valid):
    """verbatim copy over to IEMRE."""
    tidx = iemre.hourly_offset(valid)
    nc = ncopen(("/mesonet/data/stage4/%s_stage4_hourly.nc") % (valid.year, ),
                'a',
                timeout=300)
    lats = nc.variables['lat'][:]
    lons = nc.variables['lon'][:]
    val = nc.variables['p01m'][tidx]
    nc.close()

    # Our data is 4km, iemre is 0.125deg, so we stride some to cut down on mem
    stride = slice(None, None, 3)
    lats = np.ravel(lats[stride, stride])
    lons = np.ravel(lons[stride, stride])
    vals = np.ravel(val[stride, stride])
    nn = NearestNDInterpolator((lons, lats), vals)
    xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS)
    res = nn(xi, yi)

    # Lets clip bad data
    # 10 inches per hour is bad data
    res = np.where(np.logical_or(res < 0, res > 250), 0., res)

    # Open up our RE file
    nc = ncopen(iemre.get_hourly_ncname(valid.year), 'a', timeout=300)
    nc.variables["p01m"][tidx, :, :] = res
    LOG.debug("wrote data to hourly IEMRE min: %.2f avg: %.2f max: %.2f",
              np.min(res), np.mean(res), np.max(res))
    nc.close()
コード例 #26
0
ファイル: precip_ingest.py プロジェクト: jamayfieldjr/iem
def ingest_hourly_grib(valid):
    """Copy the hourly grib data into the netcdf storage.

    Returns:
      int value of the new p01m_status
    """
    tidx = iemre.hourly_offset(valid)
    fn = valid.strftime(
        "/mesonet/ARCHIVE/data/%Y/%m/%d/stage4/ST4.%Y%m%d%H.01h.grib")
    if not os.path.isfile(fn):
        LOG.info("stage4_ingest: missing file %s", fn)
        return 0
    gribs = pygrib.open(fn)
    grb = gribs[1]
    val = grb.values
    # values over 10 inches are bad
    val = np.where(val > 250., 0, val)

    nc = ncopen(("/mesonet/data/stage4/%s_stage4_hourly.nc") % (valid.year, ),
                'a',
                timeout=300)
    p01m = nc.variables['p01m']
    # account for legacy grid prior to 2002
    if val.shape == (880, 1160):
        p01m[tidx, 1:, :] = val[:, 39:]
    else:
        p01m[tidx, :, :] = val
    nc.variables['p01m_status'][tidx] = 1
    LOG.debug("write p01m to stage4 netcdf min: %.2f avg: %.2f max: %.2f",
              np.min(val), np.mean(val), np.max(val))
    nc.close()
    return 1
コード例 #27
0
    def test_hourly_offset(self):
        """ Compute the offsets """
        ts = datetime.datetime(2013, 1, 1, 0, 0)
        ts = ts.replace(tzinfo=pytz.timezone("UTC"))
        offset = iemre.hourly_offset(ts)
        self.assertEqual(offset, 0)

        ts = datetime.datetime(2013, 1, 1, 6, 0)
        ts = ts.replace(tzinfo=pytz.timezone("UTC"))
        ts = ts.astimezone(pytz.timezone("America/Chicago"))
        offset = iemre.hourly_offset(ts)
        self.assertEqual(offset, 6)

        ts = datetime.datetime(2013, 1, 5, 12, 0)
        ts = ts.replace(tzinfo=pytz.timezone("UTC"))
        offset = iemre.hourly_offset(ts)
        self.assertEqual(offset, 4 * 24 + 12)
コード例 #28
0
ファイル: hourly_analysis.py プロジェクト: stormchas4/iem
def write_grid(valid, vname, grid):
    """Atomic write of data to our netcdf storage

    This is isolated so that we don't 'lock' up our file while intensive
    work is done
    """
    offset = iemre.hourly_offset(valid)
    with ncopen(iemre.get_hourly_ncname(valid.year), "a", timeout=300) as nc:
        nc.variables[vname][offset] = grid
コード例 #29
0
ファイル: daily_analysis.py プロジェクト: smartparrot/iem
def do_precip(ts, ds):
    """Compute the 6 UTC to 6 UTC precip

    We need to be careful here as the timestamp sent to this app is today,
    we are actually creating the analysis for yesterday
    """
    sts = utc(ts.year, ts.month, ts.day, 6)
    ets = sts + datetime.timedelta(hours=24)
    offset = iemre.daily_offset(ts)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    if ts.month == 12 and ts.day == 31:
        LOG.warning(
            "p01d      for %s [idx:%s] %s(%s)->%s(%s) SPECIAL",
            ts,
            offset,
            sts.strftime("%Y%m%d%H"),
            offset1,
            ets.strftime("%Y%m%d%H"),
            offset2,
        )
        ncfn = iemre.get_hourly_ncname(ets.year)
        if not os.path.isfile(ncfn):
            LOG.warning("Missing %s", ncfn)
            return
        with ncopen(ncfn, timeout=600) as hnc:
            phour = np.sum(hnc.variables["p01m"][:offset2, :, :], 0)
        ncfn = iemre.get_hourly_ncname(sts.year)
        if os.path.isfile(ncfn):
            with ncopen(ncfn, timeout=600) as hnc:
                phour += np.sum(hnc.variables["p01m"][offset1:, :, :], 0)
    else:
        ncfn = iemre.get_hourly_ncname(sts.year)
        if not os.path.isfile(ncfn):
            LOG.warning("Missing %s", ncfn)
            return
        with ncopen(ncfn, timeout=600) as hnc:
            # for offset in range(offset1, offset2):
            #    LOG.info(
            #        "offset: %s min: %s max: %s",
            #        offset, np.ma.min(hnc.variables['p01m'][offset, :, :]),
            #        np.max(hnc.variables['p01m'][offset, :, :]))
            phour = np.sum(hnc.variables["p01m"][offset1:offset2, :, :], 0)
    ds["p01d"].values = np.where(phour < 0, 0, phour)
コード例 #30
0
ファイル: precip_ingest.py プロジェクト: jamayfieldjr/iem
def get_p01m_status(valid):
    """Figure out what our current status is of this hour."""
    nc = ncopen(("/mesonet/data/stage4/%s_stage4_hourly.nc") % (valid.year, ),
                timeout=300)
    tidx = iemre.hourly_offset(valid)
    # 2 prism_adjust_stage4 ran
    # 1 copied hourly data in
    # 0 nothing happened
    p01m_status = nc.variables['p01m_status'][tidx]
    nc.close()
    LOG.debug("p01m_status is %s for valid %s", p01m_status, valid)
    return p01m_status
コード例 #31
0
ファイル: daily_analysis.py プロジェクト: claudemp/iem
def do_precip(nc, ts):
    """Compute the 6 UTC to 6 UTC precip

    We need to be careful here as the timestamp sent to this app is today,
    we are actually creating the analysis for yesterday
    """
    sts = utc(ts.year, ts.month, ts.day, 6)
    ets = sts + datetime.timedelta(hours=24)
    offset = iemre.daily_offset(ts)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    if ts.month == 12 and ts.day == 31:
        print(("p01d      for %s [idx:%s] %s(%s)->%s(%s) SPECIAL"
               ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
                    ets.strftime("%Y%m%d%H"), offset2))
        ncfn = "/mesonet/data/iemre/%s_mw_hourly.nc" % (ets.year,)
        if not os.path.isfile(ncfn):
            print("Missing %s" % (ncfn,))
            return
        hnc = netCDF4.Dataset(ncfn)
        phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0)
        hnc.close()
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (
            sts.year,))
        phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0)
        hnc.close()
    else:
        if sts.year >= 1900:
            print(("p01d      for %s [idx:%s] %s(%s)->%s(%s)"
                   ) % (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
                        ets.strftime("%Y%m%d%H"), offset2))
        ncfn = "/mesonet/data/iemre/%s_mw_hourly.nc" % (sts.year,)
        if not os.path.isfile(ncfn):
            print("Missing %s" % (ncfn,))
            return
        hnc = netCDF4.Dataset(ncfn)
        phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0)
        hnc.close()
    nc.variables['p01d'][offset] = np.where(phour < 0, 0, phour)
コード例 #32
0
ファイル: daily_stage4_shapefile.py プロジェクト: akrherz/DEV
def main():
    """Go Main Go"""
    nc = netCDF4.Dataset('/mesonet/data/stage4/2018_stage4_hourly.nc')
    precip = nc.variables['p01m']
    # Compute needed grid bounds
    #     y,  x
    # NW 678, 412
    # SW 313, 417
    # NE 764, 787
    # SE 432, 941
    # (west, east, south, north) = compute_bounds(nc)
    south = 313
    north = 678
    west = 412
    east = 900
    lats = nc.variables['lat'][south:north, west:east]
    lons = nc.variables['lon'][south:north, west:east]
    pts = []
    for lon, lat in zip(np.ravel(lons), np.ravel(lats)):
        pts.append(Point(lon, lat))
    df = gpd.GeoDataFrame({'geometry': pts})
    # iterate over days
    now = datetime.date(2018, 7, 1)
    ets = datetime.date(2019, 1, 1)
    while now < ets:
        valid = util.utc(now.year, now.month, now.day, 5)
        tidx0 = iemre.hourly_offset(valid)
        now += datetime.timedelta(days=1)
        valid = util.utc(now.year, now.month, now.day, 5)
        tidx1 = iemre.hourly_offset(valid)
        data = precip[tidx0:tidx1, south:north, west:east]
        total = np.sum(data, axis=0)
        print("Processing %s max precip: %.2f" % (
            now - datetime.timedelta(days=1), np.max(total)))
        df[(now - datetime.timedelta(days=1)).strftime("%b%d")
           ] = distance(np.ravel(total), 'MM').value("IN")
    df.to_file('combined.shp')
コード例 #33
0
ファイル: interval_between_rain.py プロジェクト: akrherz/DEV
def main():
    """Go Main Go"""
    sts = datetime.datetime(2018, 4, 20, 0)
    sts = sts.replace(tzinfo=pytz.utc)
    ets = datetime.datetime(2018, 5, 11, 0)
    ets = ets.replace(tzinfo=pytz.utc)

    nc = ncopen(iemre.get_hourly_ncname(sts.year))
    lons = nc.variables['lon'][:]
    lats = nc.variables['lat'][:]
    running = np.zeros((len(nc.dimensions['lat']), len(nc.dimensions['lon'])))
    maxval = np.zeros((len(nc.dimensions['lat']), len(nc.dimensions['lon'])))
    interval = datetime.timedelta(hours=1)
    now = sts
    i, j = iemre.find_ij(-93.61, 41.99)
    while now < ets:
        offset = iemre.hourly_offset(now)
        p01m = np.sum(nc.variables['p01m'][offset - 24:offset], axis=0)
        # 0.05in is 1.27 mm
        this = np.where(p01m > THRESHOLD, 1, 0)
        running = np.where(this == 1, 0, running + 1)
        maxval = np.where(running > maxval, running, maxval)
        print("%s %s %s" % (now, running[j, i], maxval[j, i]))

        now += interval

    # maxval = numpy.where(domain == 1, maxval, 1.e20)

    m = plot.MapPlot(sector='midwest',
                     title=('Max Period '
                            'between 24 Hour 0.25+ inch Total Precipitation'),
                     subtitle=('Period of 20 Apr - 11 May 2018, '
                               'based on NCEP Stage IV data'))

    extra = lons[-1] + (lons[-1] - lons[-2])
    lons[-1] = extra
    # lons = np.concatenate([lons, [extra, ]])

    extra = lats[-1] + (lats[-1] - lats[-2])
    lats[-1] = extra
    # lats = np.concatenate([lats, [extra, ]])

    lons, lats = np.meshgrid(lons, lats)
    # m.pcolormesh(x, y, maxval / 24.0, numpy.arange(0,25,1), units='days')
    maxval = np.where(maxval > 800, 73., maxval)
    cmap = plt.get_cmap('terrain')
    m.contourf(lons, lats, maxval / 24.0, np.arange(1, 11.1, 1), cmap=cmap,
               units='days', clip_on=False)
    m.postprocess(filename='test.png')
コード例 #34
0
def dowork(fields):
    """Do work!"""
    date = datetime.datetime.strptime(fields.get("valid"), "%Y-%m-%d")
    lat = float(fields.get("lat"))
    lon = float(fields.get("lon"))

    # We want data for the UTC date and timestamps are in the rears, so from
    # 1z through 1z
    sts = utc(date.year, date.month, date.day, 1)
    ets = sts + datetime.timedelta(hours=24)
    sidx = iemre.hourly_offset(sts)
    eidx = iemre.hourly_offset(ets)

    ncfn = "/mesonet/data/stage4/%s_stage4_hourly.nc" % (date.year, )
    res = {"gridi": -1, "gridj": -1, "data": []}
    if not os.path.isfile(ncfn):
        return json.dumps(res)
    with ncopen(ncfn) as nc:
        dist = ((nc.variables["lon"][:] - lon)**2 +
                (nc.variables["lat"][:] - lat)**2)**0.5
        (j, i) = np.unravel_index(dist.argmin(), dist.shape)
        res["gridi"] = int(i)
        res["gridj"] = int(j)

        ppt = nc.variables["p01m"][sidx:eidx, j, i]

    for tx, pt in enumerate(ppt):
        valid = sts + datetime.timedelta(hours=tx)
        res["data"].append({
            "end_valid":
            valid.strftime("%Y-%m-%dT%H:00:00Z"),
            "precip_in":
            myrounder(datatypes.distance(pt, "MM").value("IN"), 2),
        })

    return json.dumps(res)
コード例 #35
0
def do_precip(ts):
    """Compute the 6 UTC to 6 UTC precip

    We need to be careful here as the timestamp sent to this app is today,
    we are actually creating the analysis for yesterday
    """
    sts = utc(ts.year, ts.month, ts.day, 6)
    ets = sts + datetime.timedelta(hours=24)
    offset = iemre.daily_offset(ts)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    if ts.month == 12 and ts.day == 31:
        print(("p01d      for %s [idx:%s] %s(%s)->%s(%s) SPECIAL") %
              (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
               ets.strftime("%Y%m%d%H"), offset2))
        ncfn = iemre.get_hourly_ncname(ets.year)
        if not os.path.isfile(ncfn):
            print("Missing %s" % (ncfn, ))
            return
        hnc = ncopen(ncfn, timeout=600)
        phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0)
        hnc.close()
        ncfn = iemre.get_hourly_ncname(sts.year)
        if os.path.isfile(ncfn):
            hnc = ncopen(ncfn, timeout=600)
            phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0)
            hnc.close()
    else:
        ncfn = iemre.get_hourly_ncname(sts.year)
        if not os.path.isfile(ncfn):
            print("Missing %s" % (ncfn, ))
            return
        hnc = ncopen(ncfn, timeout=600)
        phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0)
        hnc.close()
    write_grid(ts, 'p01d', np.where(phour < 0, 0, phour))
コード例 #36
0
ファイル: daily_analysis.py プロジェクト: josempolm/iem
def do_precip12(nc, ts):
    """Compute the 24 Hour precip at 12 UTC, we do some more tricks though"""
    offset = iemre.daily_offset(ts)
    ets = utc(ts.year, ts.month, ts.day, 12)
    sts = ets - datetime.timedelta(hours=24)
    offset1 = iemre.hourly_offset(sts)
    offset2 = iemre.hourly_offset(ets)
    if ts.month == 1 and ts.day == 1:
        if sts.year >= 1900:
            print(("p01d_12z  for %s [idx:%s] %s(%s)->%s(%s) SPECIAL") %
                  (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
                   ets.strftime("%Y%m%d%H"), offset2))
        ncfn = "/mesonet/data/iemre/%s_mw_hourly.nc" % (ets.year, )
        if not os.path.isfile(ncfn):
            print("Missing %s" % (ncfn, ))
            return
        hnc = netCDF4.Dataset(ncfn)
        phour = np.sum(hnc.variables['p01m'][:offset2, :, :], 0)
        hnc.close()
        hnc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" %
                              (sts.year, ))
        phour += np.sum(hnc.variables['p01m'][offset1:, :, :], 0)
        hnc.close()
    else:
        if sts.year >= 1900:
            print(("p01d_12z  for %s [idx:%s] %s(%s)->%s(%s)") %
                  (ts, offset, sts.strftime("%Y%m%d%H"), offset1,
                   ets.strftime("%Y%m%d%H"), offset2))
        ncfn = "/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year, )
        if not os.path.isfile(ncfn):
            print("Missing %s" % (ncfn, ))
            return
        hnc = netCDF4.Dataset(ncfn)
        phour = np.sum(hnc.variables['p01m'][offset1:offset2, :, :], 0)
        hnc.close()
    nc.variables['p01d_12z'][offset] = phour
コード例 #37
0
def workflow(sts, ets, i, j):
    """Return a dict of our data."""
    res = []

    # BUG here for Dec 31.
    fn = iemre.get_hourly_ncname(sts.year)

    if not os.path.isfile(fn):
        return res

    if i is None or j is None:
        return {"error": "Coordinates outside of domain"}

    UTC = ZoneInfo("UTC")
    with ncopen(fn) as nc:
        now = sts
        while now <= ets:
            offset = iemre.hourly_offset(now)
            res.append(
                {
                    "valid_utc": now.astimezone(UTC).strftime(ISO),
                    "valid_local": now.strftime(ISO[:-1]),
                    "skyc_%": myrounder(nc.variables["skyc"][offset, j, i], 1),
                    "air_temp_f": myrounder(
                        convert_value(
                            nc.variables["tmpk"][offset, j, i], "degK", "degF"
                        ),
                        1,
                    ),
                    "dew_point_f": myrounder(
                        convert_value(
                            nc.variables["dwpk"][offset, j, i], "degK", "degF"
                        ),
                        1,
                    ),
                    "uwnd_mps": myrounder(
                        nc.variables["uwnd"][offset, j, i], 2
                    ),
                    "vwnd_mps": myrounder(
                        nc.variables["vwnd"][offset, j, i], 2
                    ),
                    "hourly_precip_in": myrounder(
                        mm2inch(nc.variables["p01m"][offset, j, i]), 2
                    ),
                }
            )
            now += datetime.timedelta(hours=1)
    return pd.DataFrame(res)
コード例 #38
0
def workflow(sts, ets, i, j):
    """Return a dict of our data."""
    res = {"data": [], "generated_at": utc().strftime(ISO)}

    # BUG here for Dec 31.
    fn = iemre.get_hourly_ncname(sts.year)

    if not os.path.isfile(fn):
        return res

    if i is None or j is None:
        return {"error": "Coordinates outside of domain"}

    with ncopen(fn) as nc:
        now = sts
        while now <= ets:
            offset = iemre.hourly_offset(now)
            res["data"].append(
                {
                    "valid_utc": now.astimezone(pytz.UTC).strftime(ISO),
                    "valid_local": now.strftime(ISO),
                    "skyc_%": myrounder(nc.variables["skyc"][offset, j, i], 1),
                    "air_temp_f": myrounder(
                        datatypes.temperature(
                            nc.variables["tmpk"][offset, j, i], "K"
                        ).value("F"),
                        1,
                    ),
                    "dew_point_f": myrounder(
                        datatypes.temperature(
                            nc.variables["dwpk"][offset, j, i], "K"
                        ).value("F"),
                        1,
                    ),
                    "uwnd_mps": myrounder(
                        nc.variables["uwnd"][offset, j, i], 2
                    ),
                    "vwnd_mps": myrounder(
                        nc.variables["vwnd"][offset, j, i], 2
                    ),
                    "hourly_precip_in": myrounder(
                        nc.variables["p01m"][offset, j, i] / 25.4, 2
                    ),
                }
            )
            now += datetime.timedelta(hours=1)
    return res
コード例 #39
0
def main(argv):
    """Go Main Go."""
    log = logger()
    if len(argv) == 6:
        valid = utc(int(argv[1]), int(argv[2]), int(argv[3]), int(argv[4]))
        ncfn = iemre.get_hourly_ncname(valid.year)
        idx = iemre.hourly_offset(valid)
    else:
        valid = datetime.date(int(argv[1]), int(argv[2]), int(argv[3]))
        ncfn = iemre.get_daily_ncname(valid.year)
        idx = iemre.daily_offset(valid)
    ds = iemre.get_grids(valid)
    with ncopen(ncfn, 'a', timeout=600) as nc:
        for vname in ds:
            if vname not in nc.variables:
                continue
            log.debug("copying database var %s to netcdf", vname)
            nc.variables[vname][idx, :, :] = ds[vname].values
コード例 #40
0
ファイル: stage4_hourlyre.py プロジェクト: nbackas/iem
def merge(ts):
    """
    Process an hour's worth of stage4 data into the hourly RE
    """

    fn = ("/mesonet/ARCHIVE/data/%s/stage4/ST4.%s.01h.grib"
          ) % (ts.strftime("%Y/%m/%d"), ts.strftime("%Y%m%d%H"))
    if os.path.isfile(fn):
        gribs = pygrib.open(fn)
        grb = gribs[1]
        val = grb.values
        lats, lons = grb.latlons()
        # Rough subsample, since the whole enchillata is too much
        lats = np.ravel(lats[200:-100:5, 300:900:5])
        lons = np.ravel(lons[200:-100:5, 300:900:5])
        vals = np.ravel(val[200:-100:5, 300:900:5])
        # Clip large values
        vals = np.where(vals > 250., 0, vals)
        nn = NearestNDInterpolator((lons, lats), vals)
        xi, yi = np.meshgrid(iemre.XAXIS, iemre.YAXIS)
        res = nn(xi, yi)
    else:
        print 'Missing stage4 %s' % (fn,)
        res = np.zeros((iemre.NX, iemre.NY))

    # Lets clip bad data
    res = np.where(res < 0, 0., res)
    # 10 inches per hour is bad data
    res = np.where(res > 250., 0., res)

    # Open up our RE file
    nc = netCDF4.Dataset("/mesonet/data/iemre/%s_mw_hourly.nc" % (ts.year,),
                         'a')
    offset = iemre.hourly_offset(ts)
    nc.variables["p01m"][offset, :, :] = res

    nc.close()
    del nc
コード例 #41
0
ファイル: hourly_analysis.py プロジェクト: bthoover/iem
def grid_hour(nc, ts):
    """
    I proctor the gridding of data on an hourly basis
    @param ts Timestamp of the analysis, we'll consider a 20 minute window
    """
    ts0 = ts - datetime.timedelta(minutes=10)
    ts1 = ts + datetime.timedelta(minutes=10)
    offset = iemre.hourly_offset(ts)
    utcnow = datetime.datetime.utcnow()
    utcnow = utcnow.replace(tzinfo=pytz.timezone("UTC")) - datetime.timedelta(hours=36)

    # If we are near realtime, look in IEMAccess instead of ASOS database
    if utcnow < ts:
        dbconn = psycopg2.connect(database="iem", host="iemdb", user="******")
        pcursor = dbconn.cursor(cursor_factory=psycopg2.extras.DictCursor)
        table = "current_log"
        pcolumn = "(phour * 25.4)"
        sql = """SELECT t.id as station,
 max(case when tmpf > -60 and tmpf < 130 THEN tmpf else null end) as max_tmpf,
 max(case when sknt > 0 and sknt < 100 then sknt else 0 end) as max_sknt,
 max(getskyc(skyc1)) as max_skyc1,
 max(getskyc(skyc2)) as max_skyc2,
 max(getskyc(skyc3)) as max_skyc3,
 max(case when %s > 0 and %s < 1000 then %s else 0 end) as max_p01m,
 max(case when dwpf > -60 and dwpf < 100 THEN dwpf else null end) as max_dwpf,
 max(case when sknt >= 0 then sknt else 0 end) as sknt,
 max(case when sknt >= 0 then drct else 0 end) as drct from %s s, stations t
 WHERE t.id in %s and t.iemid = s.iemid and
 valid >= '%s' and valid < '%s' GROUP by station
         """ % (
            pcolumn,
            pcolumn,
            pcolumn,
            table,
            ids,
            ts0,
            ts1,
        )
    else:
        dbconn = psycopg2.connect(database="asos", host="iemdb", user="******")
        pcursor = dbconn.cursor(cursor_factory=psycopg2.extras.DictCursor)
        table = "t%s" % (ts.year,)
        pcolumn = "p01i"
        sql = """SELECT station,
 max(case when tmpf > -60 and tmpf < 130 THEN tmpf else null end) as max_tmpf,
 max(case when sknt > 0 and sknt < 100 then sknt else 0 end) as max_sknt,
 max(getskyc(skyc1)) as max_skyc1,
 max(getskyc(skyc2)) as max_skyc2,
 max(getskyc(skyc3)) as max_skyc3,
 max(case when %s > 0 and %s < 1000 then %s else 0 end) as max_p01m,
 max(case when dwpf > -60 and dwpf < 100 THEN dwpf else null end) as max_dwpf,
 max(case when sknt >= 0 then sknt else 0 end) as sknt,
 max(case when sknt >= 0 then drct else 0 end) as drct from %s
 WHERE station in %s and
 valid >= '%s' and valid < '%s' GROUP by station
         """ % (
            pcolumn,
            pcolumn,
            pcolumn,
            table,
            ids,
            ts0,
            ts1,
        )

    pcursor.execute(sql)

    if pcursor.rowcount > 4:
        rs = []
        for row in pcursor:
            rs.append(row)
        ures, vres = grid_wind(rs)
        if ures is not None:
            nc.variables["uwnd"][offset] = ures
            nc.variables["vwnd"][offset] = vres

        res = generic_gridder(rs, "max_tmpf")
        if res is not None:
            nc.variables["tmpk"][offset] = dt.temperature(res, "F").value("K")

        res = grid_skyc(rs)
        if res is not None:
            nc.variables["skyc"][offset] = res
    else:
        print "%s has %02i entries, FAIL" % (ts.strftime("%Y-%m-%d %H:%M"), pcursor.rowcount)
コード例 #42
0
sts = datetime.datetime(2013,5,1, 0)
sts = sts.replace(tzinfo=pytz.timezone("UTC"))
ets = datetime.datetime(2013,6,10, 0)
ets = ets.replace(tzinfo=pytz.timezone("UTC"))


nc = netCDF4.Dataset('/mesonet/data/iemre/2013_mw_hourly.nc')
lons = nc.variables['lon'][:]
lats = nc.variables['lat'][:]
running = numpy.zeros( (len(nc.dimensions['lat']), len(nc.dimensions['lon'])))
maxval = numpy.zeros( (len(nc.dimensions['lat']), len(nc.dimensions['lon'])))
interval = datetime.timedelta(hours=1)
now = sts
i,j = iemre.find_ij(-93.61, 41.99)
while now < ets:
    offset = iemre.hourly_offset(now)
    p01m = nc.variables['p01m'][offset]
    # 0.05in is 1.27 mm
    this = numpy.where(p01m > 1.27, 1, 0)
    running = numpy.where(this == 1, 0, running + 1)
    maxval = numpy.where(running > maxval, running, maxval)
    print now, running[j,i], maxval[j,i]
    
    now += interval

nc2 = netCDF4.Dataset("/mesonet/data/iemre/state_weights.nc")
domain = nc2.variables['domain'][:]
nc2.close()
maxval = numpy.where(domain == 1, maxval, 1.e20)

m = plot.MapPlot(sector='midwest',