def test_dewpoint(self):
""" test out computation of dew point """
for t0, r0, a0 in [[80, 80, 73.42], [80, 20, 35.87]]:
t = datatypes.temperature(t0, 'F')
rh = datatypes.humidity(r0, '%')
dwpk = meteorology.dewpoint(t, rh)
self.assertAlmostEqual(dwpk.value("F"), a0, 2)
def test_dewpoint():
""" test out computation of dew point """
for t0, r0, a0 in [[80, 80, 73.42], [80, 20, 35.87]]:
t = datatypes.temperature(t0, 'F')
rh = datatypes.humidity(r0, '%')
dwpk = meteorology.dewpoint(t, rh)
assert abs(dwpk.value("F") - a0) < 0.01
def test_dewpoint():
""" test out computation of dew point """
for t0, r0, a0 in [[80, 80, 73.42], [80, 20, 35.87]]:
t = datatypes.temperature(t0, "F")
rh = datatypes.humidity(r0, "%")
dwpk = meteorology.dewpoint(t, rh)
assert abs(dwpk.value("F") - a0) < 0.01
def test_dewpoint(self):
""" test out computation of dew point """
for t0,r0,a0 in [[80,80,73.42], [80,20,35.87]]:
t = datatypes.temperature(t0, 'F')
rh = datatypes.humidity(r0, '%')
dwpk = meteorology.dewpoint(t, rh)
self.assertAlmostEqual( dwpk.value("F"), a0, 2)
def hourly_process(nwsli, maxts):
""" Process the hourly file """
fn = "%s/%s_HrlySI.dat" % (BASE, STATIONS[nwsli])
df = common_df_logic(fn, maxts, nwsli, "sm_hourly")
if df is None:
return 0
processed = 0
LOG.debug("processing %s rows from %s", len(df.index), fn)
acursor = ACCESS.cursor()
for _i, row in df.iterrows():
# Update IEMAccess
ob = Observation(nwsli, "ISUSM", row["valid"])
tmpc = temperature(row["tair_c_avg_qc"], "C")
if tmpc.value("F") > -50 and tmpc.value("F") < 140:
ob.data["tmpf"] = tmpc.value("F")
relh = humidity(row["rh_qc"], "%")
ob.data["relh"] = relh.value("%")
ob.data["dwpf"] = met.dewpoint(tmpc, relh).value("F")
ob.data["srad"] = row["slrkw_avg_qc"]
ob.data["phour"] = round(
distance(row["rain_mm_tot_qc"], "MM").value("IN"), 2
)
ob.data["sknt"] = speed(row["ws_mps_s_wvt_qc"], "MPS").value("KT")
if "ws_mph_max" in df.columns:
ob.data["gust"] = speed(row["ws_mph_max_qc"], "MPH").value("KT")
ob.data["max_gust_ts"] = row["ws_mph_tmx"]
ob.data["drct"] = row["winddir_d1_wvt_qc"]
if "tsoil_c_avg" in df.columns:
ob.data["c1tmpf"] = temperature(row["tsoil_c_avg_qc"], "C").value(
"F"
)
if "t12_c_avg_qc" in df.columns:
ob.data["c2tmpf"] = temperature(row["t12_c_avg_qc"], "C").value(
"F"
)
if "t24_c_avg_qc" in df.columns:
ob.data["c3tmpf"] = temperature(row["t24_c_avg_qc"], "C").value(
"F"
)
if "t50_c_avg" in df.columns:
ob.data["c4tmpf"] = temperature(row["t50_c_avg_qc"], "C").value(
"F"
)
if "calc_vwc_12_avg" in df.columns:
ob.data["c2smv"] = row["calc_vwc_12_avg_qc"] * 100.0
if "calc_vwc_24_avg" in df.columns:
ob.data["c3smv"] = row["calc_vwc_24_avg_qc"] * 100.0
if "calc_vwc_50_avg" in df.columns:
ob.data["c4smv"] = row["calc_vwc_50_avg_qc"] * 100.0
ob.save(acursor)
processed += 1
acursor.close()
ACCESS.commit()
return processed
def processfile(icursor, filename):
"""Process this file"""
lines = open("/mesonet/data/incoming/rwis/%s" % (filename, ),
'r').readlines()
if len(lines) < 2:
return
heading = lines[0].split(",")
cols = lines[1].split(",")
data = {}
if len(cols) < len(heading):
return
for i in range(len(heading)):
if cols[i].strip() != "/":
data[heading[i].strip()] = cols[i].strip()
nwsli = LOOKUP[filename]
if filename in ['portableExportP1.csv', 'miniExportIFB.csv']:
ts = datetime.datetime.strptime(data['date_time'][:16],
'%Y-%m-%d %H:%M')
else:
ts = datetime.datetime.strptime(data['date_time'][:-6],
'%Y-%m-%d %H:%M')
ts = ts.replace(tzinfo=pytz.timezone("UTC"))
iem = Observation(nwsli, 'IA_RWIS', ts)
if ts.year < 2010:
print(("rwis/mini_portable.py file: %s bad timestamp: %s"
"") % (filename, data['date_time']))
return
iem.load(icursor)
# IEM Tracker stuff is missing
if data.get('wind_speed', '') != '':
iem.data['sknt'] = float(data['wind_speed']) * 1.17
if data.get('sub', '') != '':
iem.data['rwis_subf'] = float(data['sub'])
if data.get('air_temp', '') != '':
iem.data['tmpf'] = float(data['air_temp'])
if data.get('pave_temp', '') != '':
iem.data['tsf0'] = float(data['pave_temp'])
if data.get('pave_temp2', '') != '':
iem.data['tsf1'] = float(data['pave_temp2'])
if data.get('press', '') != '':
iem.data['alti'] = float(data['press'])
if data.get('RH', '') != '':
if float(data['RH']) > 1:
t = temperature(iem.data['tmpf'], 'F')
rh = humidity(float(data['RH']), '%')
iem.data['dwpf'] = meteorology.dewpoint(t, rh).value('F')
if data.get('wind_dir', '') != '':
iem.data['drct'] = float(data['wind_dir'])
iem.save(icursor)
def processfile(icursor, filename):
"""Process this file"""
lines = open("/mesonet/data/incoming/rwis/%s" % (filename, ),
"r").readlines()
if len(lines) < 2:
return
heading = lines[0].split(",")
cols = lines[1].split(",")
data = {}
if len(cols) < len(heading):
return
for i, h in enumerate(heading):
if cols[i].strip() != "/":
data[h.strip()] = cols[i].strip()
nwsli = LOOKUP[filename]
if filename in ["portableExportP1.csv", "miniExportIFB.csv"]:
ts = datetime.datetime.strptime(data["date_time"][:16],
"%Y-%m-%d %H:%M")
else:
ts = datetime.datetime.strptime(data["date_time"][:-6],
"%Y-%m-%d %H:%M")
ts = ts.replace(tzinfo=pytz.UTC)
iem = Observation(nwsli, "IA_RWIS", ts)
if ts.year < 2010:
print(("rwis/mini_portable.py file: %s bad timestamp: %s"
"") % (filename, data["date_time"]))
return
iem.load(icursor)
# IEM Tracker stuff is missing
if data.get("wind_speed", "") != "":
iem.data["sknt"] = float(data["wind_speed"]) * 1.17
if data.get("sub", "") != "":
iem.data["rwis_subf"] = float(data["sub"])
if data.get("air_temp", "") != "":
iem.data["tmpf"] = float(data["air_temp"])
if data.get("pave_temp", "") != "":
iem.data["tsf0"] = float(data["pave_temp"])
if data.get("pave_temp2", "") != "":
iem.data["tsf1"] = float(data["pave_temp2"])
if data.get("press", "") != "":
iem.data["alti"] = float(data["press"])
if data.get("RH", "") != "":
if float(data["RH"]) > 1:
t = temperature(iem.data["tmpf"], "F")
rh = humidity(float(data["RH"]), "%")
iem.data["dwpf"] = meteorology.dewpoint(t, rh).value("F")
if data.get("wind_dir", "") != "":
iem.data["drct"] = float(data["wind_dir"])
iem.save(icursor)
def hourly_process(nwsli, maxts):
""" Process the hourly file """
fn = "%s/%s_HrlySI.dat" % (BASE, STATIONS[nwsli])
df = common_df_logic(fn, maxts, nwsli, "sm_hourly")
if df is None:
return 0
processed = 0
LOG.debug("processing %s rows from %s", len(df.index), fn)
acursor = ACCESS.cursor()
for _i, row in df.iterrows():
# Update IEMAccess
# print nwsli, valid
ob = Observation(nwsli, 'ISUSM', row['valid'])
tmpc = temperature(row['tair_c_avg_qc'], 'C')
if tmpc.value('F') > -50 and tmpc.value('F') < 140:
ob.data['tmpf'] = tmpc.value('F')
relh = humidity(row['rh_qc'], '%')
ob.data['relh'] = relh.value('%')
ob.data['dwpf'] = met.dewpoint(tmpc, relh).value('F')
ob.data['srad'] = row['slrkw_avg_qc']
ob.data['phour'] = round(distance(row['rain_mm_tot_qc'],
'MM').value('IN'), 2)
ob.data['sknt'] = speed(row['ws_mps_s_wvt_qc'], 'MPS').value("KT")
if 'ws_mph_max' in df.columns:
ob.data['gust'] = speed(row['ws_mph_max_qc'], 'MPH').value('KT')
ob.data['max_gust_ts'] = row['ws_mph_tmx']
ob.data['drct'] = row['winddir_d1_wvt_qc']
if 'tsoil_c_avg' in df.columns:
ob.data['c1tmpf'] = temperature(row['tsoil_c_avg_qc'],
'C').value('F')
if 't12_c_avg_qc' in df.columns:
ob.data['c2tmpf'] = temperature(
row['t12_c_avg_qc'], 'C').value('F')
if 't24_c_avg_qc' in df.columns:
ob.data['c3tmpf'] = temperature(
row['t24_c_avg_qc'], 'C').value('F')
if 't50_c_avg' in df.columns:
ob.data['c4tmpf'] = temperature(row['t50_c_avg_qc'],
'C').value('F')
if 'calc_vwc_12_avg' in df.columns:
ob.data['c2smv'] = row['calc_vwc_12_avg_qc'] * 100.0
if 'calc_vwc_24_avg' in df.columns:
ob.data['c3smv'] = row['calc_vwc_24_avg_qc'] * 100.0
if 'calc_vwc_50_avg' in df.columns:
ob.data['c4smv'] = row['calc_vwc_50_avg_qc'] * 100.0
ob.save(acursor)
# print 'soilm_ingest.py station: %s ts: %s hrly updated no data?' % (
# nwsli, valid)
processed += 1
acursor.close()
ACCESS.commit()
return processed
def processfile( fp ):
o = open("/mesonet/data/incoming/rwis/%s" % (fp,), 'r').readlines()
if len(o) < 2:
return
heading = o[0].split(",")
cols = o[1].split(",")
data = {}
if len(cols) < len(heading):
return
for i in range(len(heading)):
if cols[i].strip() != "/":
data[heading[i].strip()] = cols[i].strip()
nwsli = lkp[fp]
if fp in ['portableExportP1.csv', 'miniExportIFB.csv']:
ts = datetime.datetime.strptime(data['date_time'][:16],
'%Y-%m-%d %H:%M')
else:
ts = datetime.datetime.strptime(data['date_time'][:-6],
'%Y-%m-%d %H:%M')
ts = ts.replace(tzinfo=pytz.timezone("UTC"))
iem = Observation(nwsli, 'IA_RWIS', ts)
if ts.year < 2010:
print(("rwis/mini_portable.py file: %s bad timestamp: %s"
"") % (fp, data['date_time']))
return
iem.load(icursor)
# IEM Tracker stuff is missing
if data.get('wind_speed', '') != '':
iem.data['sknt'] = float(data['wind_speed']) * 1.17
if data.get('sub', '') != '':
iem.data['rwis_subf'] = float(data['sub'])
if data.get('air_temp', '') != '':
iem.data['tmpf'] = float(data['air_temp'])
if data.get('pave_temp', '') != '':
iem.data['tsf0'] = float(data['pave_temp'])
if data.get('pave_temp2', '') != '':
iem.data['tsf1'] = float(data['pave_temp2'])
if data.get('press', '') != '':
iem.data['alti'] = float(data['press'])
if data.get('RH', '') != '':
if float(data['RH']) > 1:
t = temperature(iem.data['tmpf'], 'F')
rh = humidity(float(data['RH']), '%')
iem.data['dwpf'] = meteorology.dewpoint(t, rh).value('F')
if data.get('wind_dir', '') != '':
iem.data['drct'] = float(data['wind_dir'])
iem.save(icursor)
def main():
"""Go Main Go"""
pgconn = get_dbconn("iem")
icursor = pgconn.cursor()
now = datetime.datetime.now().replace(
tzinfo=pytz.timezone("America/Chicago"))
fn = ("/mesonet/ARCHIVE/data/%s/text/ot/ot0006.dat"
"") % (now.strftime("%Y/%m/%d"), )
if not os.path.isfile(fn):
return
lines = open(fn, 'r').readlines()
line = lines[-1]
# January 17, 2017 02:57 PM
# 35.1 35.8 33.4 92 3 351 14 2:13PM 30.03 0.00 1.12 1.12 68.6 36
tokens = line.split(" ")
if len(tokens) != 19:
sys.exit(0)
ts = datetime.datetime.strptime(" ".join(tokens[:5]), "%B %d, %Y %I:%M %p")
ts = now.replace(year=ts.year,
month=ts.month,
day=ts.day,
hour=ts.hour,
minute=ts.minute)
iemob = Observation("OT0006", "OT", ts)
iemob.data['tmpf'] = float(tokens[5])
iemob.data['relh'] = float(tokens[8])
tmpf = temperature(iemob.data['tmpf'], 'F')
relh = humidity(iemob.data['relh'], '%')
iemob.data['dwpf'] = meteorology.dewpoint(tmpf, relh).value('F')
iemob.data['sknt'] = speed(float(tokens[9]), 'MPH').value('KT')
iemob.data['drct'] = tokens[10]
iemob.data['alti'] = float(tokens[13])
iemob.data['pday'] = float(tokens[14])
iemob.save(icursor)
icursor.close()
pgconn.commit()
pgconn.close()
def main():
"""Go Main Go"""
pgconn = get_dbconn("iem")
icursor = pgconn.cursor()
now = datetime.datetime.now()
fn = ("/mesonet/ARCHIVE/data/%s/text/ot/ot0007.dat"
"") % (now.strftime("%Y/%m/%d"), )
if not os.path.isfile(fn):
sys.exit(0)
lines = open(fn, "r").readlines()
line = lines[-1]
# 114,2006,240,1530,18.17,64.70, 88.9,2.675,21.91,1014.3,0.000
tokens = line.split(",")
if len(tokens) != 11:
sys.exit(0)
hhmm = "%04i" % (int(tokens[3]), )
ts = now.replace(
hour=int(hhmm[:2]),
minute=int(hhmm[2:]),
second=0,
microsecond=0,
tzinfo=pytz.timezone("America/Chicago"),
)
iemob = Observation("OT0007", "OT", ts)
iemob.data["tmpf"] = float(tokens[5])
iemob.data["relh"] = float(tokens[6])
tmpf = temperature(iemob.data["tmpf"], "F")
relh = humidity(iemob.data["relh"], "%")
iemob.data["dwpf"] = meteorology.dewpoint(tmpf, relh).value("F")
iemob.data["sknt"] = float(tokens[7]) * 1.94
iemob.data["drct"] = tokens[8]
iemob.data["pres"] = float(tokens[9]) / 960 * 28.36
iemob.save(icursor)
icursor.close()
pgconn.commit()
pgconn.close()
def main():
"""Go Main Go"""
iemaccess = get_dbconn('iem')
cursor = iemaccess.cursor()
valid = datetime.datetime.utcnow()
valid = valid.replace(tzinfo=pytz.utc)
valid = valid.astimezone(pytz.timezone("America/Chicago"))
fn = valid.strftime("/mesonet/ARCHIVE/data/%Y/%m/%d/text/ot/ot0010.dat")
if not os.path.isfile(fn):
sys.exit(0)
lines = open(fn, "r").readlines()
lastline = lines[-1].strip()
tokens = re.split(r"[\s+]+", lastline)
if len(tokens) != 20:
return
tparts = re.split(":", tokens[3])
valid = valid.replace(hour=int(tparts[0]),
minute=int(tparts[1]),
second=0,
microsecond=0)
iem = Observation("OT0010", "OT", valid)
iem.data['tmpf'] = float(tokens[4])
iem.data['max_tmpf'] = float(tokens[5])
iem.data['min_tmpf'] = float(tokens[6])
iem.data['relh'] = int(tokens[7])
iem.data['dwpf'] = dewpoint(temperature(iem.data['tmpf'], 'F'),
humidity(iem.data['relh'], '%')).value("F")
iem.data['sknt'] = speed(float(tokens[8]), 'mph').value('KT')
iem.data['drct'] = int(tokens[9])
iem.data['max_sknt'] = speed(float(tokens[10]), 'mph').value('KT')
iem.data['alti'] = float(tokens[12])
iem.data['pday'] = float(tokens[13])
iem.data['srad'] = float(tokens[18])
iem.save(cursor)
cursor.close()
iemaccess.commit()
def minute_iemaccess(df):
"""Process dataframe into iemaccess."""
pgconn = get_dbconn("iem")
cursor = pgconn.cursor()
for _i, row in df.iterrows():
# Update IEMAccess
# print nwsli, valid
ob = Observation(row["station"], "ISUSM", row["valid"])
tmpc = temperature(row["tair_c_avg_qc"], "C")
if tmpc.value("F") > -50 and tmpc.value("F") < 140:
ob.data["tmpf"] = tmpc.value("F")
relh = humidity(row["rh_avg_qc"], "%")
ob.data["relh"] = relh.value("%")
ob.data["dwpf"] = met.dewpoint(tmpc, relh).value("F")
# database srad is W/ms2
ob.data["srad"] = row["slrkj_tot_qc"] / 60.0 * 1000.0
ob.data["pcounter"] = row["rain_in_tot_qc"]
ob.data["sknt"] = speed(row["ws_mph_s_wvt_qc"], "MPH").value("KT")
if "ws_mph_max" in df.columns:
ob.data["gust"] = speed(row["ws_mph_max_qc"], "MPH").value("KT")
ob.data["drct"] = row["winddir_d1_wvt_qc"]
if "tsoil_c_avg" in df.columns:
ob.data["c1tmpf"] = temperature(row["tsoil_c_avg_qc"], "C").value(
"F"
)
ob.data["c2tmpf"] = temperature(row["t12_c_avg_qc"], "C").value("F")
ob.data["c3tmpf"] = temperature(row["t24_c_avg_qc"], "C").value("F")
if "t50_c_avg" in df.columns:
ob.data["c4tmpf"] = temperature(row["t50_c_avg_qc"], "C").value(
"F"
)
if "calcvwc12_avg" in df.columns:
ob.data["c2smv"] = row["calcvwc12_avg_qc"] * 100.0
if "calcvwc24_avg" in df.columns:
ob.data["c3smv"] = row["calcvwc24_avg_qc"] * 100.0
if "calcvwc50_avg" in df.columns:
ob.data["c4smv"] = row["calcvwc50_avg_qc"] * 100.0
ob.save(cursor)
cursor.close()
pgconn.commit()
def process(sheets):
resdf = pd.DataFrame({
'precip': sheets['RainOut']['Rain_mm_Tot'],
'tmpf': sheets['TempRHVPOut']['AirT_C_Avg'],
'rh': sheets['TempRHVPOut']['RH'],
'drct': sheets['WindOut']['WindDir_D1_WVT'],
'sknt': sheets['WindOut']['WS_ms_S_WVT'],
'srad': sheets['SolarRad1Out']['Slr_kW_Avg']
})
# Do unit conversion
resdf['srad'] = resdf['srad'] * 1000.
resdf['precip'] = distance(resdf['precip'], 'MM').value('IN')
resdf['tmpf'] = temperature(resdf['tmpf'], 'C').value('F')
resdf['dwpf'] = dewpoint(temperature(resdf['tmpf'], 'F'),
humidity(resdf['rh'], '%')).value('F')
resdf['sknt'] = speed(resdf['sknt'], 'MPS').value('KT')
print(resdf.describe())
minval = resdf.index.min()
maxval = resdf.index.max()
cursor = pgconn.cursor()
cursor.execute(
"""DELETE from weather_data_obs WHERE
valid between '%s-06' and '%s-06' and station = 'HICKS.P'
""" %
(minval.strftime("%Y-%m-%d %H:%M"), maxval.strftime("%Y-%m-%d %H:%M")))
print("DELETED %s rows between %s and %s" %
(cursor.rowcount, minval, maxval))
for valid, row in resdf.iterrows():
if pd.isnull(valid):
continue
cursor.execute(
"""INSERT into weather_data_obs
(station, valid, tmpf, dwpf, drct, precip, srad, sknt) VALUES
('HICKS.P', %s, %s, %s, %s, %s, %s, %s)
""", (valid.strftime("%Y-%m-%d %H:%M-06"), row['tmpf'], row['dwpf'],
row['drct'], row['precip'], row['srad'], row['sknt']))
cursor.close()
pgconn.commit()
def minute_iemaccess(df):
"""Process dataframe into iemaccess."""
pgconn = get_dbconn('iem')
cursor = pgconn.cursor()
for _i, row in df.iterrows():
# Update IEMAccess
# print nwsli, valid
ob = Observation(row['station'], 'ISUSM', row['valid'])
tmpc = temperature(row['tair_c_avg_qc'], 'C')
if tmpc.value('F') > -50 and tmpc.value('F') < 140:
ob.data['tmpf'] = tmpc.value('F')
relh = humidity(row['rh_avg_qc'], '%')
ob.data['relh'] = relh.value('%')
ob.data['dwpf'] = met.dewpoint(tmpc, relh).value('F')
# database srad is W/ms2
ob.data['srad'] = row['slrkj_tot_qc'] / 60. * 1000.
ob.data['pcounter'] = row['rain_in_tot_qc']
ob.data['sknt'] = speed(row['ws_mph_s_wvt_qc'], 'MPH').value("KT")
if 'ws_mph_max' in df.columns:
ob.data['gust'] = speed(row['ws_mph_max_qc'], 'MPH').value('KT')
ob.data['drct'] = row['winddir_d1_wvt_qc']
if 'tsoil_c_avg' in df.columns:
ob.data['c1tmpf'] = temperature(row['tsoil_c_avg_qc'],
'C').value('F')
ob.data['c2tmpf'] = temperature(row['t12_c_avg_qc'], 'C').value('F')
ob.data['c3tmpf'] = temperature(row['t24_c_avg_qc'], 'C').value('F')
if 't50_c_avg' in df.columns:
ob.data['c4tmpf'] = temperature(row['t50_c_avg_qc'],
'C').value('F')
if 'calcvwc12_avg' in df.columns:
ob.data['c2smv'] = row['calcvwc12_avg_qc'] * 100.0
if 'calcvwc24_avg' in df.columns:
ob.data['c3smv'] = row['calcvwc24_avg_qc'] * 100.0
if 'calcvwc50_avg' in df.columns:
ob.data['c4smv'] = row['calcvwc50_avg_qc'] * 100.0
ob.save(cursor)
cursor.close()
pgconn.commit()
def process(sheets):
resdf = pd.DataFrame({
'precip': sheets['RainOut']['Rain_mm_Tot'],
'tmpf': sheets['TempRHVPOut']['AirT_C_Avg'],
'rh': sheets['TempRHVPOut']['RH'],
'drct': sheets['WindOut']['WindDir_D1_WVT'],
'sknt': sheets['WindOut']['WS_ms_S_WVT'],
'srad': sheets['SolarRad1Out']['Slr_kW_Avg']})
# Do unit conversion
resdf['srad'] = resdf['srad'] * 1000.
resdf['precip'] = distance(resdf['precip'], 'MM').value('IN')
resdf['tmpf'] = temperature(resdf['tmpf'], 'C').value('F')
resdf['dwpf'] = dewpoint(temperature(resdf['tmpf'], 'F'),
humidity(resdf['rh'], '%')).value('F')
resdf['sknt'] = speed(resdf['sknt'], 'MPS').value('KT')
print(resdf.describe())
minval = resdf.index.min()
maxval = resdf.index.max()
cursor = pgconn.cursor()
cursor.execute("""DELETE from weather_data_obs WHERE
valid between '%s-06' and '%s-06' and station = 'HICKS.P'
""" % (minval.strftime("%Y-%m-%d %H:%M"),
maxval.strftime("%Y-%m-%d %H:%M")))
print("DELETED %s rows between %s and %s" % (cursor.rowcount, minval,
maxval))
for valid, row in resdf.iterrows():
if pd.isnull(valid):
continue
cursor.execute("""INSERT into weather_data_obs
(station, valid, tmpf, dwpf, drct, precip, srad, sknt) VALUES
('HICKS.P', %s, %s, %s, %s, %s, %s, %s)
""", (valid.strftime("%Y-%m-%d %H:%M-06"), row['tmpf'], row['dwpf'],
row['drct'], row['precip'], row['srad'], row['sknt']))
cursor.close()
pgconn.commit()
def write_grids(fp, valid, fhour):
"""Do the write to disk"""
gribfn = "%s/%sF%03i.grib2" % (TMP, valid.strftime("%Y%m%d%H%M"),
fhour)
if not os.path.isfile(gribfn):
print("Skipping write_grids because of missing fn: %s" % (gribfn,))
return
gribs = pygrib.open(gribfn)
grids = dict()
for grib in gribs:
grids[grib.name] = grib
d = dict()
if '2 metre temperature' in grids:
g = grids['2 metre temperature']
if G['LATS'] is None:
G['LATS'], G['LONS'] = g.latlons()
vals = temperature(g.values, 'K').value('C')
nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()),
vals.flatten())
d['tmpc'] = nn(XI, YI)
if 'Relative humidity' in grids:
g = grids['Relative humidity']
vals = g.values
nn = NearestNDInterpolator((G['LONS'].flatten(),
G['LATS'].flatten()),
vals.flatten())
rh = nn(XI, YI)
d['dwpc'] = dewpoint(temperature(d['tmpc'], 'C'),
humidity(rh, '%')).value('C')
if ('10 metre U wind component' in grids and
'10 metre V wind component' in grids):
u = grids['10 metre U wind component'].values
v = grids['10 metre V wind component'].values
vals = ((u ** 2) + (v ** 2)) ** 0.5
nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()),
vals.flatten())
d['smps'] = nn(XI, YI)
vals = drct(speed(u, 'MPS'), speed(v, 'MPS')).value('deg')
nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()),
vals.flatten())
d['drct'] = nn(XI, YI)
if 'Total Precipitation' in grids:
vals = grids['Total Precipitation'].values
nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()),
vals.flatten())
d['pcpn'] = nn(XI, YI)
if 'Visibility' in grids:
vals = grids['Visibility'].values / 1000. # km
nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()),
vals.flatten())
d['vsby'] = nn(XI, YI)
fp.write("""{"forecast_hour": "%03i",
"gids": [
""" % (fhour,))
fmt = ('{"gid": %s, "tmpc": %s, "dwpc": %s, '
'"smps": %s, "drct": %s, "vsby": %s, "pcpn": %s}')
i = 1
ar = []
def f(label, row, col, fmt):
if label not in d:
return 'null'
return fmt % d[label][row, col]
for row in range(len(YAXIS)):
for col in range(len(XAXIS)):
ar.append(fmt % (i, f('tmpc', row, col, '%.2f'),
f('dwpc', row, col, '%.2f'),
f('smps', row, col, '%.1f'),
f('drct', row, col, '%i'),
f('vsby', row, col, '%.3f'),
f('pcpn', row, col, '%.2f')))
i += 1
fp.write(",\n".join(ar))
fp.write("]}%s\n" % ("," if fhour != 84 else '',))
def m15_process(nwsli, maxts):
""" Process the 15minute file """
fn = "%s/%s_Min15SI.dat" % (BASE, STATIONS[nwsli])
if not os.path.isfile(fn):
return
lines = open(fn).readlines()
if len(lines) < 5:
return
# Read header....
headers = []
for col in lines[1].strip().replace('"', '').split(","):
headers.append(VARCONV.get(col.lower(), col.lower()))
# Read data
processed = 0
for i in range(len(lines) - 1, 3, -1):
tokens = lines[i].strip().replace('"', '').split(",")
if len(tokens) != len(headers):
continue
valid = make_time(tokens[headers.index('timestamp')])
if valid <= maxts:
break
gust_valid = make_time(tokens[headers.index('ws_mph_tmx')])
# print valid, tokens[ headers.index('timestamp')]
# We are ready for dbinserting, we duplicate the data for the _qc
# column
dbcols = ("station,valid,%s,%s") % (",".join(headers[2:]), ",".join(
["%s_qc" % (h, ) for h in headers[2:]]))
dbvals = "'%s','%s-06'," % (nwsli, valid.strftime("%Y-%m-%d %H:%M:%S"))
for v in tokens[2:]:
dbvals += "%s," % (formatter(v), )
for v in tokens[2:]:
dbvals += "%s," % (formatter(v), )
sql = "INSERT into sm_15minute (%s) values (%s)" % (dbcols,
dbvals[:-1])
icursor.execute(sql)
# Update IEMAccess
# print nwsli, valid
ob = Observation(nwsli, 'ISUSM',
valid.astimezone(pytz.timezone("America/Chicago")))
tmpc = temperature(float(tokens[headers.index('tair_c_avg')]), 'C')
if tmpc.value('F') > -50 and tmpc.value('F') < 140:
ob.data['tmpf'] = tmpc.value('F')
relh = humidity(float(tokens[headers.index('rh')]), '%')
ob.data['relh'] = relh.value('%')
ob.data['dwpf'] = met.dewpoint(tmpc, relh).value('F')
ob.data['srad'] = tokens[headers.index('slrkw_avg')]
ob.data['phour'] = round(
distance(float(tokens[headers.index('rain_mm_tot')]),
'MM').value('IN'), 2)
ob.data['sknt'] = float(tokens[headers.index('ws_mps_s_wvt')]) * 1.94
ob.data['gust'] = float(tokens[headers.index('ws_mph_max')]) / 1.15
ob.data['max_gust_ts'] = "%s-06" % (
gust_valid.strftime("%Y-%m-%d %H:%M:%S"), )
ob.data['drct'] = float(tokens[headers.index('winddir_d1_wvt')])
ob.data['c1tmpf'] = temperature(
float(tokens[headers.index('tsoil_c_avg')]), 'C').value('F')
ob.data['c2tmpf'] = temperature(
float(tokens[headers.index('t12_c_avg')]), 'C').value('F')
ob.data['c3tmpf'] = temperature(
float(tokens[headers.index('t24_c_avg')]), 'C').value('F')
ob.data['c4tmpf'] = temperature(
float(tokens[headers.index('t50_c_avg')]), 'C').value('F')
ob.data['c2smv'] = float(tokens[headers.index('vwc_12_avg')]) * 100.0
ob.data['c3smv'] = float(tokens[headers.index('vwc_24_avg')]) * 100.0
ob.data['c4smv'] = float(tokens[headers.index('vwc_50_avg')]) * 100.0
ob.save(accesstxn, force_current_log=True)
# print 'soilm_ingest.py station: %s ts: %s hrly updated no data?' % (
# nwsli, valid)
processed += 1
return processed
if not os.path.isfile(fn):
sys.exit(0)
lines = open(fn, 'r').readlines()
line = lines[-1]
# 114,2006,240,1530,18.17,64.70, 88.9,2.675,21.91,1014.3,0.000
tokens = line.split(",")
if (len(tokens) != 11):
sys.exit(0)
hhmm = "%04i" % (int(tokens[3]),)
ts = now.replace(hour=int(hhmm[:2]), minute=int(hhmm[2:]), second=0,
microsecond=0, tzinfo=pytz.timezone("America/Chicago"))
iemob = Observation("OT0007", "OT", ts)
iemob.data['tmpf'] = float(tokens[5])
iemob.data['relh'] = float(tokens[6])
tmpf = temperature(iemob.data['tmpf'], 'F')
relh = humidity(iemob.data['relh'], '%')
iemob.data['dwpf'] = meteorology.dewpoint(tmpf, relh).value('F')
iemob.data['sknt'] = float(tokens[7]) * 1.94
iemob.data['drct'] = tokens[8]
iemob.data['pres'] = float(tokens[9]) / 960 * 28.36
iemob.save(icursor)
icursor.close()
IEM.commit()
IEM.close()
def m15_process(nwsli, maxts):
""" Process the 15minute file """
fn = "%s/%s_Min15SI.dat" % (BASE, STATIONS[nwsli])
if not os.path.isfile(fn):
return
lines = open(fn).readlines()
if len(lines) < 5:
return
# Read header....
headers = []
for col in lines[1].strip().replace('"', '').split(","):
headers.append(VARCONV.get(col.lower(), col.lower()))
# Read data
processed = 0
for i in range(len(lines)-1, 3, -1):
tokens = lines[i].strip().replace('"', '').split(",")
if len(tokens) != len(headers):
continue
valid = make_time(tokens[headers.index('timestamp')])
if valid <= maxts:
break
gust_valid = make_time(tokens[headers.index('ws_mph_tmx')])
# print valid, tokens[ headers.index('timestamp')]
# We are ready for dbinserting, we duplicate the data for the _qc
# column
dbcols = ("station,valid,%s,%s"
) % (",".join(headers[2:]),
",".join(["%s_qc" % (h,) for h in headers[2:]]))
dbvals = "'%s','%s-06'," % (nwsli, valid.strftime("%Y-%m-%d %H:%M:%S"))
for v in tokens[2:]:
dbvals += "%s," % (formatter(v),)
for v in tokens[2:]:
dbvals += "%s," % (formatter(v),)
sql = "INSERT into sm_15minute (%s) values (%s)" % (dbcols,
dbvals[:-1])
icursor.execute(sql)
# Update IEMAccess
# print nwsli, valid
ob = Observation(nwsli, 'ISUSM',
valid.astimezone(pytz.timezone("America/Chicago")))
tmpc = temperature(float(tokens[headers.index('tair_c_avg')]), 'C')
if tmpc.value('F') > -50 and tmpc.value('F') < 140:
ob.data['tmpf'] = tmpc.value('F')
relh = humidity(float(tokens[headers.index('rh')]), '%')
ob.data['relh'] = relh.value('%')
ob.data['dwpf'] = met.dewpoint(tmpc, relh).value('F')
ob.data['srad'] = tokens[headers.index('slrkw_avg')]
ob.data['phour'] = round(
distance(
float(tokens[headers.index('rain_mm_tot')]),
'MM').value('IN'), 2)
ob.data['sknt'] = float(tokens[headers.index('ws_mps_s_wvt')]) * 1.94
ob.data['gust'] = float(tokens[headers.index('ws_mph_max')]) / 1.15
ob.data['max_gust_ts'] = "%s-06" % (
gust_valid.strftime("%Y-%m-%d %H:%M:%S"),)
ob.data['drct'] = float(tokens[headers.index('winddir_d1_wvt')])
ob.data['c1tmpf'] = temperature(
float(tokens[headers.index('tsoil_c_avg')]), 'C').value('F')
ob.data['c2tmpf'] = temperature(
float(tokens[headers.index('t12_c_avg')]), 'C').value('F')
ob.data['c3tmpf'] = temperature(
float(tokens[headers.index('t24_c_avg')]), 'C').value('F')
ob.data['c4tmpf'] = temperature(
float(tokens[headers.index('t50_c_avg')]), 'C').value('F')
ob.data['c2smv'] = float(tokens[headers.index('vwc_12_avg')]) * 100.0
ob.data['c3smv'] = float(tokens[headers.index('vwc_24_avg')]) * 100.0
ob.data['c4smv'] = float(tokens[headers.index('vwc_50_avg')]) * 100.0
ob.save(accesstxn, force_current_log=True)
# print 'soilm_ingest.py station: %s ts: %s hrly updated no data?' % (
# nwsli, valid)
processed += 1
return processed
lines = open(fn, "r").readlines()
lastLine = lines[-1].strip()
tokens = re.split("[\s+]+", lastLine)
if len(tokens) != 20:
sys.exit(0)
tparts = re.split(":", tokens[3])
valid = valid.replace(hour=int(tparts[0]),
minute=int(tparts[1]), second=0, microsecond=0)
iem = Observation("OT0010", "OT", valid)
iem.data['tmpf'] = float(tokens[4])
iem.data['max_tmpf'] = float(tokens[5])
iem.data['min_tmpf'] = float(tokens[6])
iem.data['relh'] = int(tokens[7])
iem.data['dwpf'] = dewpoint(temperature(iem.data['tmpf'], 'F'),
humidity(iem.data['relh'], '%')).value("F")
iem.data['sknt'] = speed(float(tokens[8]), 'mph').value('KT')
iem.data['drct'] = int(tokens[9])
iem.data['max_sknt'] = speed(float(tokens[10]), 'mph').value('KT')
iem.data['alti'] = float(tokens[12])
iem.data['pday'] = float(tokens[13])
iem.data['srad'] = float(tokens[18])
iem.save(cursor)
cursor.close()
iemaccess.commit()
lines = open(fn, 'r').readlines()
line = lines[-1]
# 114,2006,240,1530,18.17,64.70, 88.9,2.675,21.91,1014.3,0.000
tokens = line.split(",")
if (len(tokens) != 11):
sys.exit(0)
hhmm = "%04i" % (int(tokens[3]), )
ts = now.replace(hour=int(hhmm[:2]),
minute=int(hhmm[2:]),
second=0,
microsecond=0,
tzinfo=pytz.timezone("America/Chicago"))
iemob = Observation("OT0007", "OT", ts)
iemob.data['tmpf'] = float(tokens[5])
iemob.data['relh'] = float(tokens[6])
tmpf = temperature(iemob.data['tmpf'], 'F')
relh = humidity(iemob.data['relh'], '%')
iemob.data['dwpf'] = meteorology.dewpoint(tmpf, relh).value('F')
iemob.data['sknt'] = float(tokens[7]) * 1.94
iemob.data['drct'] = tokens[8]
iemob.data['pres'] = float(tokens[9]) / 960 * 28.36
iemob.save(icursor)
icursor.close()
IEM.commit()
IEM.close()
def hourly_process(nwsli, maxts):
""" Process the hourly file """
#print '-------------- HOURLY PROCESS ---------------'
fn = "%s%s" % (BASE, STATIONS[nwsli]['hourly'].split("/")[1])
if not os.path.isfile(fn):
return
lines = open(fn).readlines()
if len(lines) < 5:
return
# Read header....
headers = []
for col in lines[1].strip().replace('"', '').split(","):
headers.append(VARCONV.get(col.lower(), col.lower()))
# Read data
processed = 0
for i in range(len(lines)-1, 3, -1):
tokens = lines[i].strip().replace('"','').split(",")
if len(tokens) != len(headers):
continue
valid = datetime.datetime.strptime(tokens[ headers.index('timestamp')],
'%Y-%m-%d %H:%M:%S')
valid = valid.replace(tzinfo=pytz.FixedOffset(-360))
if valid <= maxts:
break
#print valid, tokens[ headers.index('timestamp')]
# We are ready for dbinserting!
dbcols = "station,valid," + ",".join(headers[2:])
dbvals = "'%s','%s-06'," % (nwsli, valid.strftime("%Y-%m-%d %H:%M:%S"))
for v in tokens[2:]:
dbvals += "%s," % (formatter(v),)
sql = "INSERT into sm_hourly (%s) values (%s)" % (dbcols, dbvals[:-1])
icursor.execute(sql)
# Update IEMAccess
#print nwsli, valid
ob = Observation(nwsli, 'ISUSM',
valid.astimezone(pytz.timezone("America/Chicago")))
tmpc = temperature(float(tokens[headers.index('tair_c_avg')]), 'C')
if tmpc.value('F') > -50 and tmpc.value('F') < 140:
ob.data['tmpf'] = tmpc.value('F')
relh = humidity(float(tokens[headers.index('rh')]), '%')
ob.data['relh'] = relh.value('%')
ob.data['dwpf'] = met.dewpoint(tmpc, relh).value('F')
ob.data['srad'] = tokens[headers.index('slrkw_avg')]
ob.data['phour'] = round(
float(tokens[headers.index('rain_mm_tot')]) / 24.5, 2)
ob.data['sknt'] = float(tokens[headers.index('ws_mps_s_wvt')]) * 1.94
ob.data['gust'] = float(tokens[headers.index('ws_mph_max')]) / 1.15
ob.data['drct'] = float(tokens[headers.index('winddir_d1_wvt')])
ob.data['c1tmpf'] = temperature(
float(tokens[headers.index('tsoil_c_avg')]), 'C').value('F')
ob.data['c2tmpf'] = temperature(
float(tokens[headers.index('t12_c_avg')]), 'C').value('F')
ob.data['c3tmpf'] = temperature(
float(tokens[headers.index('t24_c_avg')]), 'C').value('F')
ob.data['c4tmpf'] = temperature(
float(tokens[headers.index('t50_c_avg')]), 'C').value('F')
ob.data['c2smv'] = float(tokens[headers.index('vwc_12_avg')]) * 100.0
ob.data['c3smv'] = float(tokens[headers.index('vwc_24_avg')]) * 100.0
ob.data['c4smv'] = float(tokens[headers.index('vwc_50_avg')]) * 100.0
ob.save(accesstxn)
# print 'soilm_ingest.py station: %s ts: %s hrly updated no data?' % (
# nwsli, valid)
processed += 1
return processed
def write_grids(fp, valid, fhour):
"""Do the write to disk"""
gribfn = "%s/%sF%03i.grib2" % (TMP, valid.strftime("%Y%m%d%H%M"), fhour)
if not os.path.isfile(gribfn):
print("Skipping write_grids because of missing fn: %s" % (gribfn, ))
return
gribs = pygrib.open(gribfn)
grids = dict()
for grib in gribs:
grids[grib.name] = grib
d = dict()
if '2 metre temperature' in grids:
g = grids['2 metre temperature']
if G['LATS'] is None:
G['LATS'], G['LONS'] = g.latlons()
vals = temperature(g.values, 'K').value('C')
nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()),
vals.flatten())
d['tmpc'] = nn(XI, YI)
if 'Relative humidity' in grids:
g = grids['Relative humidity']
vals = g.values
nn = NearestNDInterpolator(
(G['LONS'].flatten(), G['LATS'].flatten()), vals.flatten())
rh = nn(XI, YI)
d['dwpc'] = dewpoint(temperature(d['tmpc'], 'C'),
humidity(rh, '%')).value('C')
if ('10 metre U wind component' in grids
and '10 metre V wind component' in grids):
u = grids['10 metre U wind component'].values
v = grids['10 metre V wind component'].values
vals = ((u**2) + (v**2))**0.5
nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()),
vals.flatten())
d['smps'] = nn(XI, YI)
vals = drct(speed(u, 'MPS'), speed(v, 'MPS')).value('deg')
nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()),
vals.flatten())
d['drct'] = nn(XI, YI)
if 'Total Precipitation' in grids:
vals = grids['Total Precipitation'].values
nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()),
vals.flatten())
d['pcpn'] = nn(XI, YI)
if 'Visibility' in grids:
vals = grids['Visibility'].values / 1000. # km
nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()),
vals.flatten())
d['vsby'] = nn(XI, YI)
fp.write("""{"forecast_hour": "%03i",
"gids": [
""" % (fhour, ))
fmt = ('{"gid": %s, "tmpc": %s, "dwpc": %s, '
'"smps": %s, "drct": %s, "vsby": %s, "pcpn": %s}')
i = 1
ar = []
def f(label, row, col, fmt):
if label not in d:
return 'null'
return fmt % d[label][row, col]
for row in range(len(YAXIS)):
for col in range(len(XAXIS)):
ar.append(
fmt %
(i, f('tmpc', row, col, '%.2f'), f('dwpc', row, col, '%.2f'),
f('smps', row, col, '%.1f'), f('drct', row, col, '%i'),
f('vsby', row, col, '%.3f'), f('pcpn', row, col, '%.2f')))
i += 1
fp.write(",\n".join(ar))
fp.write("]}%s\n" % ("," if fhour != 84 else '', ))
def main():
"""Go Main Go"""
valid = []
pres = []
sped = []
drct = []
tmpf = []
dwpf = []
pday = []
lines = []
now = datetime.datetime(2018, 4, 13, 0, 0)
for line in open('jefferson.txt'):
if line.find("Min") > 0 or line.find("Max") > 0:
continue
ts = datetime.datetime.strptime(line[7:21], '%H:%M %m/%d/%y')
if len(lines) == 0:
now = ts
if ts == now:
lines.append(line)
continue
# Spread obs
now = ts
delta = 60.0 / len(lines)
for line in lines:
ts += datetime.timedelta(seconds=delta)
t = float(line[42:45])
rhval = float(line[47:50])
tmpf.append(t)
dwpf.append(dewpoint(temperature(t, 'F'),
humidity(rhval, '%')).value("F"))
pres.append(float(line[52:57]) * 33.86375)
sped.append(float(line[26:28]))
pday.append(float(line[59:64]))
drct.append(float(txt2drct[line[22:25].strip()]) - 20.0)
# Clock was not in CDT
valid.append(ts + datetime.timedelta(minutes=60))
lines = []
sts = datetime.datetime(2018, 4, 13, 17, 0)
ets = datetime.datetime(2018, 4, 13, 18, 0)
interval = datetime.timedelta(minutes=15)
xticks = []
xticklabels = []
now = sts
while now < ets:
fmt = ":%M"
if now.minute == 0 or not xticks:
fmt = "%-I:%M %p"
xticks.append(now)
xticklabels.append(now.strftime(fmt))
now += interval
fig, ax = plt.subplots(3, 1, sharex=True)
"""
fancybox = mpatches.FancyBboxPatch(
[mx.DateTime.DateTime(2012,5,3,8,6).ticks(),
40], 60*18, 50,
boxstyle='round', alpha=0.2, facecolor='#7EE5DE', edgecolor='#EEEEEE',
zorder=1)
ax[0].add_patch( fancybox)
"""
ax[0].plot(valid, np.array(tmpf), label='Air')
# print("%s %s" % (tmpf, max(tmpf)))
ax[0].plot(valid, np.array(dwpf), color='g', label='Dew Point')
# ax[0].set_ylim(40, 92)
ax[1].scatter(valid, drct, zorder=2)
ax2 = ax[1].twinx()
ax2.plot(valid, sped, 'r')
ax2.set_ylabel("Wind Speed [mph]")
ax[2].plot(valid, pres, 'b', label="Pressure [mb]")
ax[2].set_ylim(1006, 1014)
#ax3 = ax[2].twinx()
#ax3.plot(valid, pday, 'r')
#ax3.set_ylabel("Daily Precip [inch]")
ax[0].set_xticks(xticks)
ax[0].set_xticklabels(xticklabels)
ax[0].set_xlim(sts, ets)
prop = matplotlib.font_manager.FontProperties(size=12)
ax[0].legend(loc=2, prop=prop, ncol=2)
ax[2].legend(loc=1, prop=prop)
ax[0].grid(True)
ax[1].grid(True)
ax[2].grid(True)
ax[0].set_title("Jefferson SchoolNet8 Site (~ $\Delta$6 second data)")
ax[2].set_xlabel("Afternoon of 13 Apr 2018")
#ax[2].set_ylabel("Pressure [mb]")
#[t.set_color('red') for t in ax3.yaxis.get_ticklines()]
#[t.set_color('red') for t in ax3.yaxis.get_ticklabels()]
[t.set_color('red') for t in ax2.yaxis.get_ticklines()]
[t.set_color('red') for t in ax2.yaxis.get_ticklabels()]
[t.set_color('b') for t in ax[2].yaxis.get_ticklines()]
[t.set_color('b') for t in ax[2].yaxis.get_ticklabels()]
#[t.set_color('b') for t in ax[0].yaxis.get_ticklines()]
#[t.set_color('b') for t in ax[0].yaxis.get_ticklabels()]
[t.set_color('b') for t in ax[1].yaxis.get_ticklines()]
[t.set_color('b') for t in ax[1].yaxis.get_ticklabels()]
ax[1].set_ylabel("Wind Direction")
ax[0].set_ylabel("Temperature $^{\circ}\mathrm{F}$")
ax[1].set_yticks(np.arange(0, 361, 90))
ax[1].set_ylim(0, 361)
ax[1].set_yticklabels(['N', 'E', 'S', 'W', 'N'])
ax[2].set_ylim(min(pres) - 2, max(pres) + 4)
fig.savefig('test.png')