def main(network, nwsli):
"""Do the query and work
Args:
network (str): IEM network identifier
nwsli (str): NWSLI ID
"""
payload = {
"sid": nwsli,
"sdate": "1850-01-01",
"edate": "2012-01-01",
"elems": "obst,maxt,mint,pcpn,snow,snwd",
}
req = requests.post(SERVICE, json=payload)
j = req.json()
pgconn = get_dbconn("iem")
cursor = pgconn.cursor()
for row in tqdm(j["data"], disable=not sys.stdout.isatty()):
date = datetime.datetime.strptime(row[0], "%Y-%m-%d")
(obst, high, low, precip, snow, snowd) = map(safe, row[1:])
if all([a is None for a in (obst, high, low, precip, snow, snowd)]):
continue
ob = Observation(nwsli, network,
utc(date.year, date.month, date.day, 12))
ob.data["max_tmpf"] = high
ob.data["min_tmpf"] = low
ob.data["coop_tmpf"] = obst
ob.data["pday"] = precip
ob.data["snow"] = snow
ob.data["snowd"] = snowd
ob.save(cursor, skip_current=True)
cursor.close()
pgconn.commit()
def daily_process(nwsli, maxts):
""" Process the daily file """
# print '-------------- DAILY PROCESS ----------------'
fn = "%s%s" % (BASE, STATIONS[nwsli]['daily'].split("/")[1])
if not os.path.isfile(fn):
return 0
lines = open(fn).readlines()
if len(lines) < 5:
return 0
# Read header....
headers = []
for col in lines[1].strip().replace('"', '').split(","):
headers.append(VARCONV.get(col.lower(), col.lower()))
# Read data
processed = 0
for i in range(len(lines) - 1, 3, -1):
tokens = lines[i].strip().replace('"', '').split(",")
if len(tokens) != len(headers):
continue
valid = datetime.datetime.strptime(
tokens[headers.index('timestamp')][:10], '%Y-%m-%d')
valid = valid.date() - datetime.timedelta(days=1)
if valid < maxts:
break
if valid == maxts: # Reprocess
icursor.execute("""DELETE from sm_daily WHERE valid = '%s' and
station = '%s' """ % (valid.strftime("%Y-%m-%d"), nwsli))
# We are ready for dbinserting!
dbcols = "station,valid," + ",".join(headers[2:])
dbvals = "'%s','%s'," % (nwsli, valid.strftime("%Y-%m-%d"))
for v in tokens[2:]:
dbvals += "%s," % (formatter(v), )
sql = "INSERT into sm_daily (%s) values (%s)" % (dbcols, dbvals[:-1])
icursor.execute(sql)
# Need a timezone
valid = datetime.datetime(valid.year, valid.month, valid.day, 12, 0)
valid = valid.replace(tzinfo=pytz.timezone("America/Chicago"))
ob = Observation(nwsli, 'ISUSM', valid)
ob.data['max_tmpf'] = temperature(
float(tokens[headers.index('tair_c_max')]), 'C').value('F')
ob.data['min_tmpf'] = temperature(
float(tokens[headers.index('tair_c_min')]), 'C').value('F')
ob.data['pday'] = round(
float(tokens[headers.index('rain_mm_tot')]) / 24.5, 2)
ob.data['et_inch'] = float(tokens[headers.index('dailyet')]) / 24.5
ob.data['srad_mj'] = float(tokens[headers.index('slrmj_tot')])
ob.data['max_sknt'] = float(tokens[headers.index('ws_mps_max')]) * 1.94
ob.save(accesstxn)
# print 'soilm_ingest.py station: %s ts: %s daily updated no data?' % (
# nwsli, valid.strftime("%Y-%m-%d"))
processed += 1
return processed
def daily_process(nwsli, maxts):
""" Process the daily file """
# print '-------------- DAILY PROCESS ----------------'
fn = "%s%s" % (BASE, STATIONS[nwsli]['daily'].split("/")[1])
if not os.path.isfile(fn):
return 0
lines = open(fn).readlines()
if len(lines) < 5:
return 0
# Read header....
headers = []
for col in lines[1].strip().replace('"', '').split(","):
headers.append(VARCONV.get(col.lower(), col.lower()))
# Read data
processed = 0
for i in range(len(lines)-1, 3, -1):
tokens = lines[i].strip().replace('"','').split(",")
if len(tokens) != len(headers):
continue
valid = datetime.datetime.strptime(tokens[ headers.index('timestamp')][:10],
'%Y-%m-%d')
valid = valid.date() - datetime.timedelta(days=1)
if valid < maxts:
break
if valid == maxts: # Reprocess
icursor.execute("""DELETE from sm_daily WHERE valid = '%s' and
station = '%s' """ % (valid.strftime("%Y-%m-%d"), nwsli))
# We are ready for dbinserting!
dbcols = "station,valid," + ",".join(headers[2:])
dbvals = "'%s','%s'," % (nwsli, valid.strftime("%Y-%m-%d"))
for v in tokens[2:]:
dbvals += "%s," % (formatter(v),)
sql = "INSERT into sm_daily (%s) values (%s)" % (dbcols, dbvals[:-1])
icursor.execute(sql)
# Need a timezone
valid = datetime.datetime(valid.year, valid.month, valid.day, 12, 0)
valid = valid.replace(tzinfo=pytz.timezone("America/Chicago"))
ob = Observation(nwsli, 'ISUSM', valid)
ob.data['max_tmpf'] = temperature(
float(tokens[headers.index('tair_c_max')]), 'C').value('F')
ob.data['min_tmpf'] = temperature(
float(tokens[headers.index('tair_c_min')]), 'C').value('F')
ob.data['pday'] = round(
float(tokens[headers.index('rain_mm_tot')]) / 24.5, 2)
ob.data['et_inch'] = float(tokens[headers.index('dailyet')]) / 24.5
ob.data['srad_mj'] = float(tokens[headers.index('slrmj_tot')])
ob.data['max_sknt'] = float(tokens[headers.index('ws_mps_max')]) * 1.94
ob.save(accesstxn)
# print 'soilm_ingest.py station: %s ts: %s daily updated no data?' % (
# nwsli, valid.strftime("%Y-%m-%d"))
processed += 1
return processed
def handler(
key, time, tmpf, max_tmpf, min_tmpf, dwpf, relh, sknt, pday, alti, drct
):
"""Handle the request, return dict"""
# sys.stderr.write(repr(fields))
if not PROPS:
PROPS.update(get_properties())
lookup = {}
for sid in ["OT0013", "OT0014", "OT0015", "OT0016"]:
lookup[PROPS.get("meteobridge.key." + sid)] = sid
if key not in lookup:
raise HTTPException(status_code=404, detail="BAD_KEY")
sid = lookup[key]
if len(time) == 14:
_t = time
now = utc(
int(_t[:4]),
int(_t[4:6]),
int(_t[6:8]),
int(_t[8:10]),
int(_t[10:12]),
int(_t[12:14]),
)
else:
now = datetime.datetime.utcnow()
now = now.replace(tzinfo=pytz.UTC)
ob = Observation(sid, "OT", now)
for fname in [
"tmpf",
"max_tmpf",
"min_tmpf",
"dwpf",
"relh",
"sknt",
"pday",
"alti",
"drct",
]:
if vars()[fname] == "M":
continue
ob.data[fname] = float(vars()[fname])
pgconn = get_dbconn("iem")
cursor = pgconn.cursor()
ob.save(cursor)
cursor.close()
pgconn.commit()
return "OK"
def main():
"""Go Main Go"""
iemaccess = get_dbconn('iem')
cursor = iemaccess.cursor()
valid = datetime.datetime.utcnow()
valid = valid.replace(tzinfo=pytz.utc)
valid = valid.astimezone(pytz.timezone("America/Chicago"))
fn = valid.strftime("/mesonet/ARCHIVE/data/%Y/%m/%d/text/ot/ot0002.dat")
if not os.path.isfile(fn):
sys.exit(0)
lines = open(fn, "r").readlines()
lastline = lines[-1]
tokens = re.split(r"[\s+]+", lastline)
tparts = re.split(":", tokens[4])
valid = valid.replace(hour=int(tparts[0]),
minute=int(tparts[1]),
second=int(tparts[2]))
iem = Observation("OT0002", "OT", valid)
sknt = speed(float(tokens[8]), 'MPH').value('KT')
iem.data['sknt'] = sknt
iem.data['drct'] = tokens[9]
iem.data['tmpf'] = tokens[7]
iem.save(cursor)
cursor.close()
iemaccess.commit()
def update_iemaccess(obs):
"""Update the IEMAccess database"""
icursor = IEM.cursor()
for sid in obs:
ob = obs[sid]
iemob = Observation(sid, "IA_RWIS", ob['valid'])
for varname in [
'tmpf', 'dwpf', 'drct', 'sknt', 'gust', 'vsby', 'pday', 'tsf0',
'tsf1', 'tsf2', 'tsf3', 'scond0', 'scond1', 'scond2', 'scond3',
'relh'
]:
# Don't insert NaN values into iemaccess
thisval = ob.get(varname)
if thisval is None:
continue
# strings fail the isnan check
if isinstance(thisval, str):
iemob.data[varname] = ob.get(varname)
elif not np.isnan(thisval):
iemob.data[varname] = ob.get(varname)
for varname in ['tsub0', 'tsub1', 'tsub2', 'tsub3']:
if ob.get(varname) is not None:
iemob.data['rwis_subf'] = ob.get(varname)
break
iemob.save(icursor)
icursor.close()
IEM.commit()
def main():
"""Go Main Go"""
pgconn = get_dbconn("iem")
icursor = pgconn.cursor()
now = datetime.datetime.now().replace(
tzinfo=pytz.timezone("America/Chicago"))
fn = ("/mesonet/ARCHIVE/data/%s/text/ot/ot0006.dat"
"") % (now.strftime("%Y/%m/%d"), )
if not os.path.isfile(fn):
return
lines = open(fn, "r").readlines()
line = lines[-1]
# January 17, 2017 02:57 PM
# 35.1 35.8 33.4 92 3 351 14 2:13PM 30.03 0.00 1.12 1.12 68.6 36
tokens = line.split(" ")
if len(tokens) != 19:
sys.exit(0)
ts = datetime.datetime.strptime(" ".join(tokens[:5]), "%B %d, %Y %I:%M %p")
ts = now.replace(
year=ts.year,
month=ts.month,
day=ts.day,
hour=ts.hour,
minute=ts.minute,
)
iemob = Observation("OT0006", "OT", ts)
iemob.data["tmpf"] = float(tokens[5])
iemob.data["relh"] = float(tokens[8])
iemob.data["sknt"] = speed(float(tokens[9]), "MPH").value("KT")
iemob.data["drct"] = tokens[10]
iemob.data["alti"] = float(tokens[13])
iemob.data["pday"] = float(tokens[14])
iemob.save(icursor)
icursor.close()
pgconn.commit()
pgconn.close()
def main():
"""Go Main Go"""
pgconn = get_dbconn("iem")
icursor = pgconn.cursor()
now = datetime.datetime.now()
fn = ("/mesonet/ARCHIVE/data/%s/text/ot/ot0007.dat"
"") % (now.strftime("%Y/%m/%d"), )
if not os.path.isfile(fn):
sys.exit(0)
lines = open(fn, "r").readlines()
line = lines[-1]
# 114,2006,240,1530,18.17,64.70, 88.9,2.675,21.91,1014.3,0.000
tokens = line.split(",")
if len(tokens) != 11:
sys.exit(0)
hhmm = "%04i" % (int(tokens[3]), )
ts = now.replace(
hour=int(hhmm[:2]),
minute=int(hhmm[2:]),
second=0,
microsecond=0,
tzinfo=pytz.timezone("America/Chicago"),
)
iemob = Observation("OT0007", "OT", ts)
iemob.data["tmpf"] = float(tokens[5])
iemob.data["relh"] = float(tokens[6])
tmpf = temperature(iemob.data["tmpf"], "F")
relh = humidity(iemob.data["relh"], "%")
iemob.data["dwpf"] = meteorology.dewpoint(tmpf, relh).value("F")
iemob.data["sknt"] = float(tokens[7]) * 1.94
iemob.data["drct"] = tokens[8]
iemob.data["pres"] = float(tokens[9]) / 960 * 28.36
iemob.save(icursor)
icursor.close()
pgconn.commit()
pgconn.close()
def main():
"""Go Main Go"""
dbconn = get_dbconn("iem")
cursor = dbconn.cursor()
now = datetime.datetime.now() - datetime.timedelta(hours=3)
lts = datetime.datetime.utcnow()
lts = lts.replace(tzinfo=pytz.utc)
lts = lts.astimezone(pytz.timezone("America/Chicago"))
state = sys.argv[1]
url = ("http://data.cocorahs.org/Cocorahs/export/exportreports.aspx"
"?ReportType=Daily&dtf=1&Format=CSV&State=%s&"
"ReportDateType=timestamp&Date=%s&TimesInGMT=False") % (
state, now.strftime("%m/%d/%Y%%20%H:00%%20%P"))
data = requests.get(url, timeout=30).content.decode("ascii").split("\r\n")
# Process Header
header = {}
h = data[0].split(",")
for i, _h in enumerate(h):
header[_h] = i
if "StationNumber" not in header:
return
for row in data[1:]:
cols = row.split(",")
if len(cols) < 4:
continue
sid = cols[header["StationNumber"]].strip()
t = "%s %s" % (
cols[header["ObservationDate"]],
cols[header["ObservationTime"]].strip(),
)
ts = datetime.datetime.strptime(t, "%Y-%m-%d %I:%M %p")
lts = lts.replace(
year=ts.year,
month=ts.month,
day=ts.day,
hour=ts.hour,
minute=ts.minute,
)
iem = Observation(sid, "%sCOCORAHS" % (state, ), lts)
iem.data["pday"] = safeP(cols[header["TotalPrecipAmt"]])
if cols[header["NewSnowDepth"]].strip() != "NA":
iem.data["snow"] = safeP(cols[header["NewSnowDepth"]])
if cols[header["TotalSnowDepth"]].strip() != "NA":
iem.data["snowd"] = safeP(cols[header["TotalSnowDepth"]])
iem.save(cursor)
del iem
cursor.close()
dbconn.commit()
def update_iemaccess(obs):
"""Update the IEMAccess database"""
icursor = IEM.cursor()
for sid in obs:
ob = obs[sid]
iemob = Observation(sid, "IA_RWIS", ob['valid'])
for varname in ['tmpf', 'dwpf', 'drct', 'sknt', 'gust', 'vsby',
'pday', 'tsf0', 'tsf1', 'tsf2', 'tsf3', 'scond0',
'scond1', 'scond2', 'scond3']:
# Don't insert NaN values into iemaccess
thisval = ob.get(varname)
if thisval is None:
continue
# strings fail the isnan check
if isinstance(thisval, str):
iemob.data[varname] = ob.get(varname)
elif not np.isnan(thisval):
iemob.data[varname] = ob.get(varname)
for varname in ['tsub0', 'tsub1', 'tsub2', 'tsub3']:
if ob.get(varname) is not None:
iemob.data['rwis_subf'] = ob.get(varname)
break
iemob.save(icursor)
icursor.close()
IEM.commit()
def main():
"""Go Main Go"""
dbconn = get_dbconn('iem')
cursor = dbconn.cursor()
now = datetime.datetime.now() - datetime.timedelta(hours=3)
lts = datetime.datetime.utcnow()
lts = lts.replace(tzinfo=pytz.utc)
lts = lts.astimezone(pytz.timezone("America/Chicago"))
state = sys.argv[1]
url = ("http://data.cocorahs.org/Cocorahs/export/exportreports.aspx"
"?ReportType=Daily&dtf=1&Format=CSV&State=%s&"
"ReportDateType=timestamp&Date=%s&TimesInGMT=False") % (
state, now.strftime("%m/%d/%Y%%20%H:00%%20%P"))
req = urllib2.Request(url)
data = urllib2.urlopen(req, timeout=30).readlines()
# Process Header
header = {}
h = data[0].split(",")
for i in range(len(h)):
header[h[i]] = i
if 'StationNumber' not in header:
return
for row in data[1:]:
cols = row.split(",")
sid = cols[header["StationNumber"]].strip()
t = "%s %s" % (cols[header["ObservationDate"]],
cols[header["ObservationTime"]].strip())
ts = datetime.datetime.strptime(t, "%Y-%m-%d %I:%M %p")
lts = lts.replace(year=ts.year,
month=ts.month,
day=ts.day,
hour=ts.hour,
minute=ts.minute)
iem = Observation(sid, "%sCOCORAHS" % (state, ), lts)
iem.data['pday'] = safeP(cols[header["TotalPrecipAmt"]])
if cols[header["NewSnowDepth"]].strip() != "NA":
iem.data['snow'] = safeP(cols[header["NewSnowDepth"]])
if cols[header["TotalSnowDepth"]].strip() != "NA":
iem.data['snowd'] = safeP(cols[header["TotalSnowDepth"]])
iem.save(cursor)
del iem
cursor.close()
dbconn.commit()
def database(lastob, ddf, hdf, force_currentlog):
"""Do the tricky database work"""
# This should be okay as we are always going to CST
maxts = (hdf["TIMESTAMP"].max().replace(
tzinfo=pytz.timezone("America/Chicago")))
if lastob is not None and maxts <= lastob:
# print("maxts: %s lastob: %s" % (maxts, lastob))
return
iemdb = get_dbconn("iem")
icursor = iemdb.cursor()
if lastob is None:
df2 = hdf
else:
df2 = hdf[hdf["valid"] > lastob]
for _, row in df2.iterrows():
localts = row["valid"].tz_convert(pytz.timezone("America/Chicago"))
# Find, if it exists, the summary table entry here
daily = ddf[ddf["date"] == localts.date()]
ob = Observation(SID, "OT", localts)
if len(daily.index) == 1:
for key, value in DAILYCONV.items():
if value is None:
continue
# print("D: %s -> %s" % (key, value))
ob.data[value] = clean(key, daily.iloc[0][key])
# print("date: %s srad_mj: %s" % (localts.date(), ob.data['srad_mj']))
if ob.data["srad_mj"] is None:
ob.data["srad_mj"] = sum_hourly(hdf, localts.date(), "SlrMJ_Tot")
if ob.data["pday"] is None:
ob.data["pday"] = sum_hourly(hdf, localts.date(), "Rain_in_Tot")
for key, value in HOURLYCONV.items():
if value is None:
continue
# print("H: %s -> %s" % (key, value))
ob.data[value] = clean(key, row[key])
ob.save(
icursor,
force_current_log=force_currentlog,
skip_current=force_currentlog,
)
icursor.close()
iemdb.commit()
def update_iemaccess(obs):
"""Update the IEMAccess database"""
icursor = IEM.cursor()
for sid in obs:
ob = obs[sid]
iemob = Observation(sid, "IA_RWIS", ob["valid"])
for varname in [
"tmpf",
"dwpf",
"drct",
"sknt",
"gust",
"vsby",
"pday",
"tsf0",
"tsf1",
"tsf2",
"tsf3",
"scond0",
"scond1",
"scond2",
"scond3",
"relh",
]:
# Don't insert NaN values into iemaccess
thisval = ob.get(varname)
if thisval is None:
continue
# strings fail the isnan check
if isinstance(thisval, str):
iemob.data[varname] = ob.get(varname)
elif not np.isnan(thisval):
iemob.data[varname] = ob.get(varname)
for varname in ["tsub0", "tsub1", "tsub2", "tsub3"]:
if ob.get(varname) is not None:
iemob.data["rwis_subf"] = ob.get(varname)
break
iemob.save(icursor)
icursor.close()
IEM.commit()
def savedata(data, maxts):
"""
Save away our data into IEM Access
"""
if 'Time' in data:
tstr = "%s %s" % (data['Date'], data['Time'])
elif 'Time (CST)' in data:
tstr = "%s %s" % (data['Date'], data['Time (CST)'])
else:
tstr = "%s %s" % (data['Date'], data['Time (CDT)'])
ts = datetime.datetime.strptime(tstr, '%Y-%m-%d %H:%M')
utc = datetime.datetime.utcnow()
utc = utc.replace(tzinfo=pytz.timezone("UTC"))
localts = utc.astimezone(pytz.timezone("America/Chicago"))
ts = localts.replace(year=ts.year, month=ts.month, day=ts.day,
hour=ts.hour, minute=ts.minute, second=0,
microsecond=0)
sid = "S%s" % (data['Site Id'],)
if sid in maxts and maxts[sid] >= ts:
return
iem = Observation(sid, 'SCAN', ts)
iem.data['ts'] = ts
iem.data['year'] = ts.astimezone(pytz.timezone("UTC")).year
for key in data.keys():
if (key in mapping and mapping[key]['iemvar'] != "" and
key != 'Site Id'):
iem.data[mapping[key]['iemvar']] = float(data[key].strip())
iem.data['valid'] = ts
iem.data['tmpf'] = temperature(float(iem.data.get('tmpc')), 'C').value('F')
iem.data['dwpf'] = temperature(float(iem.data.get('dwpc')), 'C').value('F')
iem.data['c1tmpf'] = temperature(float(iem.data.get('c1tmpc')),
'C').value('F')
iem.data['c2tmpf'] = temperature(float(iem.data.get('c2tmpc')),
'C').value('F')
iem.data['c3tmpf'] = temperature(float(iem.data.get('c3tmpc')),
'C').value('F')
iem.data['c4tmpf'] = temperature(float(iem.data.get('c4tmpc')),
'C').value('F')
iem.data['c5tmpf'] = temperature(float(iem.data.get('c5tmpc')),
'C').value('F')
iem.data['c1smv'] = float(iem.data.get('c1smv'))
iem.data['c2smv'] = float(iem.data.get('c2smv'))
iem.data['c3smv'] = float(iem.data.get('c3smv'))
iem.data['c4smv'] = float(iem.data.get('c4smv'))
iem.data['c5smv'] = float(iem.data.get('c5smv'))
iem.data['phour'] = float(iem.data.get('phour'))
if not iem.save(icursor):
print(('scan_ingest.py iemaccess for sid: %s ts: %s updated no rows'
) % (sid, ts))
sql = """INSERT into t%(year)s_hourly (station, valid, tmpf,
dwpf, srad,
sknt, drct, relh, pres, c1tmpf, c2tmpf, c3tmpf, c4tmpf,
c5tmpf,
c1smv, c2smv, c3smv, c4smv, c5smv, phour)
VALUES
('%(station)s', '%(valid)s', %(tmpf)s, %(dwpf)s,
%(srad)s,%(sknt)s,
%(drct)s, %(relh)s, %(pres)s, %(c1tmpf)s,
%(c2tmpf)s,
%(c3tmpf)s, %(c4tmpf)s, %(c5tmpf)s, %(c1smv)s,
%(c2smv)s,
%(c3smv)s, %(c4smv)s, %(c5smv)s, %(phour)s)
""" % iem.data
scursor.execute(sql.replace("None", "null"))
def main():
"""Go Main Go"""
pgconn = get_dbconn("other")
ipgconn = get_dbconn("iem")
cursor = pgconn.cursor()
# Figure out max valid times
maxts = {}
cursor.execute(
"SELECT station, max(valid) from flux_data GROUP by station")
for row in cursor:
maxts[row[0]] = row[1]
processed = 0
for station, fns in FILENAMES.items():
if station not in maxts:
LOG.info("%s has no prior db archive", station)
maxts[station] = datetime.datetime(1980, 1,
1).replace(tzinfo=pytz.utc)
dfs = []
for fn in fns:
myfn = "%s%s" % (DIR, fn)
if not os.path.isfile(myfn):
LOG.info("missing file: %s", myfn)
continue
df = pd.read_csv(myfn,
skiprows=[0, 2, 3],
index_col=0,
na_values=["NAN"])
df.drop("RECORD", axis=1, inplace=True)
if df.empty:
LOG.info("file: %s has no data", fn)
continue
dfs.append(df)
if not dfs:
LOG.info("no data for: %s", station)
continue
df = dfs[0]
if len(dfs) > 1:
df = df.join(dfs[1]).copy()
# get index back into a column
df.reset_index(inplace=True)
# lowercase all column names
df.columns = [x.lower() for x in df.columns]
df["timestamp"] = df["timestamp"].apply(make_time)
df = df[df["timestamp"] > maxts[station]].copy()
if df.empty:
continue
df.rename(columns=CONVERT, inplace=True)
# We need a UTC year to allow for the database insert below to work
df["utcyear"] = df["valid"].dt.tz_convert(pytz.utc).dt.year
df["station"] = station
for year, gdf in df.groupby("utcyear"):
exclude = []
for colname in gdf.columns:
if colname not in DBCOLS:
exclude.append(colname)
if colname not in DROPCOLS:
LOG.info("%s has more cols: %s", station, exclude)
gdf2 = gdf[gdf.columns.difference(exclude)]
processed += len(gdf2.index)
output = StringIO()
gdf2.to_csv(output, sep="\t", header=False, index=False)
cursor = pgconn.cursor()
output.seek(0)
cursor.copy_from(output,
"flux%s" % (year, ),
columns=gdf2.columns,
null="")
cursor.close()
pgconn.commit()
icursor = ipgconn.cursor()
for _i, row in df.iterrows():
iemob = Observation(station, "NSTLFLUX", row["valid"])
if "t_hmp_avg" in df.columns:
iemob.data["tmpf"] = temperature(row["t_hmp_avg"],
"C").value("F")
if "wnd_spd" in df.columns:
iemob.data["sknt"] = speed(row["wnd_spd"], "MPS").value("KT")
if "press_avg" in df.columns:
iemob.data["pres"] = pressure(row["press_avg"] * 1000.0,
"PA").value("MB")
for cvar, ivar in zip(
["solarrad_w_avg", "rh_hmp_avg", "wnd_dir_compass"],
["srad", "rh", "drct"],
):
if cvar in df.columns:
iemob.data[ivar] = row[cvar]
iemob.save(icursor)
icursor.close()
ipgconn.commit()
def to_iemaccess(self, txn, force_current_log=False, skip_current=False):
"""Persist parsed data to IEMAccess Database.
Args:
txn (psycopg2.cursor): database cursor / transaction
force_current_log (boolean): should this ob always go to current_log
skip_current (boolean): should this ob always skip current table
"""
gts = self.time.replace(tzinfo=pytz.utc)
iem = Observation(self.iemid, self.network, gts)
# Load the observation from the database, if the same time exists!
iem.load(txn)
# Need to figure out if we have a duplicate ob, if so, check
# the length of the raw data, if greater, take the temps
if (iem.data['raw'] is not None and
len(iem.data['raw']) >= len(self.code)):
pass
else:
if self.temp:
val = self.temp.value("F")
# Place reasonable bounds on the temperature before saving it!
if val > -90 and val < 150:
iem.data['tmpf'] = round(val, 1)
if self.dewpt:
val = self.dewpt.value("F")
# Place reasonable bounds on the temperature before saving it!
if val > -150 and val < 100:
iem.data['dwpf'] = round(val, 1)
# Daabase only allows len 254
iem.data['raw'] = self.code[:254]
wind_logic(iem, self)
if self.max_temp_6hr:
iem.data['max_tmpf_6hr'] = round(self.max_temp_6hr.value("F"), 1)
if self.min_temp_6hr:
iem.data['min_tmpf_6hr'] = round(self.min_temp_6hr.value("F"), 1)
if self.max_temp_24hr:
iem.data['max_tmpf_24hr'] = round(self.max_temp_24hr.value("F"), 1)
if self.min_temp_24hr:
iem.data['min_tmpf_24hr'] = round(self.min_temp_24hr.value("F"), 1)
if self.precip_3hr:
iem.data['p03i'] = trace(self.precip_3hr)
if self.precip_6hr:
iem.data['p06i'] = trace(self.precip_6hr)
if self.precip_24hr:
iem.data['p24i'] = trace(self.precip_24hr)
# We assume the value is zero, sad!
iem.data['phour'] = 0
if self.precip_1hr:
iem.data['phour'] = trace(self.precip_1hr)
if self.snowdepth:
iem.data['snowd'] = self.snowdepth.value("IN")
if self.vis:
iem.data['vsby'] = self.vis.value("SM")
if self.press:
iem.data['alti'] = self.press.value("IN")
if self.press_sea_level:
iem.data['mslp'] = self.press_sea_level.value("MB")
if self.press_sea_level and self.press:
alti = self.press.value("MB")
mslp = self.press_sea_level.value("MB")
if abs(alti - mslp) > 25:
print(("PRESSURE ERROR %s %s ALTI: %s MSLP: %s"
) % (iem.data['station'], iem.data['valid'],
alti, mslp))
if alti > mslp:
iem.data['mslp'] += 100.
else:
iem.data['mslp'] -= 100.
# Do something with sky coverage
for i in range(len(self.sky)):
(cov, hgh, _) = self.sky[i]
iem.data['skyc%s' % (i+1)] = cov
if hgh is not None:
iem.data['skyl%s' % (i+1)] = hgh.value("FT")
# Presentwx
if self.weather:
pwx = []
for wx in self.weather:
val = "".join([a for a in wx if a is not None])
if val == "" or val == len(val) * "/":
continue
pwx.append(val)
iem.data['wxcodes'] = pwx
# Ice Accretion
for hr in [1, 3, 6]:
key = 'ice_accretion_%shr' % (hr, )
iem.data[key] = trace(getattr(self, key))
return iem, iem.save(txn, force_current_log, skip_current)
db[thisStation]['pres'] = sanityCheck(pressure[recnum], 0, 1000000, -99)
db[thisStation]['tmpk'] = sanityCheck(tmpk[recnum], 0, 500, -99)
db[thisStation]['dwpk'] = sanityCheck(dwpk[recnum], 0, 500, -99)
db[thisStation]['tmpk_dd'] = tmpk_dd[recnum]
db[thisStation]['drct'] = sanityCheck(drct[recnum], -1, 361, -99)
db[thisStation]['smps'] = sanityCheck(smps[recnum], -1, 200, -99)
db[thisStation]['gmps'] = sanityCheck(gmps[recnum], -1, 200, -99)
db[thisStation]['rtk1'] = sanityCheck(rtk1[recnum], 0, 500, -99)
db[thisStation]['rtk2'] = sanityCheck(rtk2[recnum], 0, 500, -99)
db[thisStation]['rtk3'] = sanityCheck(rtk3[recnum], 0, 500, -99)
db[thisStation]['rtk4'] = sanityCheck(rtk4[recnum], 0, 500, -99)
db[thisStation]['subk'] = sanityCheck(subk1[recnum], 0, 500, -99)
db[thisStation]['pday'] = sanityCheck(pcpn[recnum], -1, 5000, -99)
for sid in db.keys():
iem = Observation(sid, db[sid]['network'], db[sid]['ts'])
# if not iem.load(icursor):
# print 'Missing fp: %s network: %s station: %s' % (fp,
# db[sid]['network'],
# sid)
# subprocess.call("python sync_stations.py %s" % (fp,), shell=True)
# os.chdir("../../dbutil")
# subprocess.call("sh SYNC_STATIONS.sh", shell=True)
# os.chdir("../ingestors/madis")
iem.data['tmpf'] = temperature(db[sid]['tmpk'], 'K').value('F')
iem.data['dwpf'] = temperature(db[sid]['dwpk'], 'K').value('F')
if db[sid]['drct'] >= 0:
iem.data['drct'] = db[sid]['drct']
if db[sid]['smps'] >= 0:
iem.data['sknt'] = speed(db[sid]['smps'], 'MPS').value('KT')
if db[sid]['gmps'] >= 0:
df['sknt'] = df['wia'] / 10.15 # x10
df['gust'] = df['wih'] / 10.15
df['pres'] = df['bia'] / 100.0
df['max_gust'] = df['wdh'] / 10.15
sdf = df.sort_values(by=['utc'], ascending=[True])
for _, row in sdf.iterrows():
if pd.isnull(row['utc']):
continue
utc = datetime.datetime.strptime(row['utc'], '%Y-%m-%d %H:%M:%S')
utc = utc.replace(tzinfo=pytz.timezone("UTC"))
if lastts is not None and utc < lastts:
continue
# print nwsli, utc
iem = Observation(nwsli, 'IA_RWIS', utc)
for iemvar in conv:
iem.data[iemvar] = row[conv[iemvar]]
iem.save(icursor)
if __name__ == '__main__':
data = get_last_obs()
for nwsli in assoc:
process(nwsli, data.get(nwsli, None))
icursor.close()
IEM.commit()
IEM.close()
def hourly_process(nwsli, maxts):
""" Process the hourly file """
#print '-------------- HOURLY PROCESS ---------------'
fn = "%s%s" % (BASE, STATIONS[nwsli]['hourly'].split("/")[1])
if not os.path.isfile(fn):
return
lines = open(fn).readlines()
if len(lines) < 5:
return
# Read header....
headers = []
for col in lines[1].strip().replace('"', '').split(","):
headers.append(VARCONV.get(col.lower(), col.lower()))
# Read data
processed = 0
for i in range(len(lines)-1, 3, -1):
tokens = lines[i].strip().replace('"','').split(",")
if len(tokens) != len(headers):
continue
valid = datetime.datetime.strptime(tokens[ headers.index('timestamp')],
'%Y-%m-%d %H:%M:%S')
valid = valid.replace(tzinfo=pytz.FixedOffset(-360))
if valid <= maxts:
break
#print valid, tokens[ headers.index('timestamp')]
# We are ready for dbinserting!
dbcols = "station,valid," + ",".join(headers[2:])
dbvals = "'%s','%s-06'," % (nwsli, valid.strftime("%Y-%m-%d %H:%M:%S"))
for v in tokens[2:]:
dbvals += "%s," % (formatter(v),)
sql = "INSERT into sm_hourly (%s) values (%s)" % (dbcols, dbvals[:-1])
icursor.execute(sql)
# Update IEMAccess
#print nwsli, valid
ob = Observation(nwsli, 'ISUSM',
valid.astimezone(pytz.timezone("America/Chicago")))
tmpc = temperature(float(tokens[headers.index('tair_c_avg')]), 'C')
if tmpc.value('F') > -50 and tmpc.value('F') < 140:
ob.data['tmpf'] = tmpc.value('F')
relh = humidity(float(tokens[headers.index('rh')]), '%')
ob.data['relh'] = relh.value('%')
ob.data['dwpf'] = met.dewpoint(tmpc, relh).value('F')
ob.data['srad'] = tokens[headers.index('slrkw_avg')]
ob.data['phour'] = round(
float(tokens[headers.index('rain_mm_tot')]) / 24.5, 2)
ob.data['sknt'] = float(tokens[headers.index('ws_mps_s_wvt')]) * 1.94
ob.data['gust'] = float(tokens[headers.index('ws_mph_max')]) / 1.15
ob.data['drct'] = float(tokens[headers.index('winddir_d1_wvt')])
ob.data['c1tmpf'] = temperature(
float(tokens[headers.index('tsoil_c_avg')]), 'C').value('F')
ob.data['c2tmpf'] = temperature(
float(tokens[headers.index('t12_c_avg')]), 'C').value('F')
ob.data['c3tmpf'] = temperature(
float(tokens[headers.index('t24_c_avg')]), 'C').value('F')
ob.data['c4tmpf'] = temperature(
float(tokens[headers.index('t50_c_avg')]), 'C').value('F')
ob.data['c2smv'] = float(tokens[headers.index('vwc_12_avg')]) * 100.0
ob.data['c3smv'] = float(tokens[headers.index('vwc_24_avg')]) * 100.0
ob.data['c4smv'] = float(tokens[headers.index('vwc_50_avg')]) * 100.0
ob.save(accesstxn)
# print 'soilm_ingest.py station: %s ts: %s hrly updated no data?' % (
# nwsli, valid)
processed += 1
return processed
df['min_tmpf'] = df['tdl'] / 10.0
df['tsf0'] = df['t1ia'] / 10.0
df['tsf1'] = df['t2ia'] / 10.0
df['sknt'] = df['wia'] / 10.15 # x10
df['gust'] = df['wih'] / 10.15
df['pres'] = df['bia'] / 100.0
df['max_gust'] = df['wdh'] / 10.15
sdf = df.sort_values(by=['utc'], ascending=[True])
for _, row in sdf.iterrows():
utc = datetime.datetime.strptime(row['utc'], '%Y-%m-%d %H:%M:%S')
utc = utc.replace(tzinfo=pytz.timezone("UTC"))
if lastts is not None and utc < lastts:
continue
# print nwsli, utc
iem = Observation(nwsli, 'IA_RWIS', utc)
for iemvar in conv:
iem.data[iemvar] = row[conv[iemvar]]
iem.save(icursor)
if __name__ == '__main__':
data = get_last_obs()
for nwsli in assoc:
process(nwsli, data.get(nwsli, None))
icursor.close()
IEM.commit()
IEM.close()
def daily_process(nwsli, maxts):
""" Process the daily file """
fn = "%s/%s_DailySI.dat" % (BASE, STATIONS[nwsli])
df = common_df_logic(fn, maxts, nwsli, "sm_daily")
if df is None:
return 0
LOG.debug("processing %s rows from %s", len(df.index), fn)
processed = 0
acursor = ACCESS.cursor()
for _i, row in df.iterrows():
# Need a timezone
valid = datetime.datetime(row["valid"].year, row["valid"].month,
row["valid"].day, 12, 0)
valid = valid.replace(tzinfo=pytz.timezone("America/Chicago"))
ob = Observation(nwsli, "ISUSM", valid)
ob.data["max_tmpf"] = temperature(row["tair_c_max_qc"], "C").value("F")
ob.data["min_tmpf"] = temperature(row["tair_c_min_qc"], "C").value("F")
ob.data["pday"] = round(
distance(row["rain_mm_tot_qc"], "MM").value("IN"), 2)
if valid not in EVENTS["days"]:
EVENTS["days"].append(valid)
ob.data["et_inch"] = distance(row["dailyet_qc"], "MM").value("IN")
ob.data["srad_mj"] = row["slrmj_tot_qc"]
# Someday check if this is apples to apples here
ob.data["vector_avg_drct"] = row["winddir_d1_wvt_qc"]
if ob.data["max_tmpf"] is None:
EVENTS["reprocess_temps"] = True
if ob.data["srad_mj"] == 0 or np.isnan(ob.data["srad_mj"]):
LOG.info(
"soilm_ingest.py station: %s ts: %s has 0 solar",
nwsli,
valid.strftime("%Y-%m-%d"),
)
EVENTS["reprocess_solar"] = True
if "ws_mps_max" in df.columns:
ob.data["max_sknt"] = speed(row["ws_mps_max_qc"],
"MPS").value("KT")
ob.data["avg_sknt"] = speed(row["ws_mps_s_wvt_qc"], "MPS").value("KT")
ob.save(acursor)
processed += 1
acursor.close()
ACCESS.commit()
return processed
def daily_process(nwsli, maxts):
""" Process the daily file """
acursor = ACCESS.cursor()
# print '-------------- DAILY PROCESS ----------------'
fn = "%s/%s_DailySI.dat" % (BASE, STATIONS[nwsli])
df = common_df_logic(fn, maxts, nwsli, "sm_daily")
if df is None:
return 0
processed = 0
for _i, row in df.iterrows():
# Need a timezone
valid = datetime.datetime(row['valid'].year, row['valid'].month,
row['valid'].day, 12, 0)
valid = valid.replace(tzinfo=pytz.timezone("America/Chicago"))
ob = Observation(nwsli, 'ISUSM', valid)
ob.data['max_tmpf'] = temperature(row['tair_c_max_qc'], 'C').value('F')
ob.data['min_tmpf'] = temperature(row['tair_c_min_qc'], 'C').value('F')
ob.data['pday'] = round(
distance(row['rain_mm_tot_qc'], 'MM').value('IN'), 2)
if valid not in EVENTS['days']:
EVENTS['days'].append(valid)
ob.data['et_inch'] = distance(row['dailyet_qc'], 'MM').value('IN')
ob.data['srad_mj'] = row['slrmj_tot_qc']
# Someday check if this is apples to apples here
ob.data['vector_avg_drct'] = row['winddir_d1_wvt_qc']
if ob.data['max_tmpf'] is None:
EVENTS['reprocess_temps'] = True
if ob.data['srad_mj'] == 0 or np.isnan(ob.data['srad_mj']):
print(("soilm_ingest.py station: %s ts: %s has 0 solar") %
(nwsli, valid.strftime("%Y-%m-%d")))
EVENTS['reprocess_solar'] = True
if 'ws_mps_max' in df.columns:
ob.data['max_sknt'] = speed(row['ws_mps_max_qc'],
'MPS').value('KT')
ob.data['avg_sknt'] = speed(row['ws_mps_s_wvt_qc'], 'MPS').value('KT')
ob.save(acursor)
processed += 1
acursor.close()
ACCESS.commit()
return processed
def process(ncfn):
"""Process this file """
IEM = psycopg2.connect(database="iem", host="iemdb")
icursor = IEM.cursor()
xref = {}
icursor.execute(
"""SELECT id, network from stations where
network ~* 'ASOS' or network = 'AWOS' and country = 'US'"""
)
for row in icursor:
xref[row[0]] = row[1]
icursor.close()
nc = netCDF4.Dataset(ncfn)
data = {}
for vname in [
"stationId",
"observationTime",
"temperature",
"dewpoint",
"altimeter", # Pa
"windDir",
"windSpeed", # mps
"windGust",
"visibility", # m
"precipAccum",
"presWx",
"skyCvr",
"skyCovLayerBase",
"autoRemark",
"operatorRemark",
]:
data[vname] = nc.variables[vname][:]
vname += "QCR"
if vname in nc.variables:
data[vname] = nc.variables[vname][:]
for vname in ["temperature", "dewpoint"]:
data[vname + "C"] = temperature(data[vname], "K").value("C")
data[vname] = temperature(data[vname], "K").value("F")
for vname in ["windSpeed", "windGust"]:
data[vname] = speed(data[vname], "MPS").value("KT")
data["altimeter"] = pressure(data["altimeter"], "PA").value("IN")
data["skyCovLayerBase"] = distance(data["skyCovLayerBase"], "M").value("FT")
data["visibility"] = distance(data["visibility"], "M").value("MI")
data["precipAccum"] = distance(data["precipAccum"], "MM").value("IN")
for i in range(len(data["stationId"])):
sid = tostring(data["stationId"][i])
sid3 = sid[1:] if sid[0] == "K" else sid
ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=data["observationTime"][i])
ts = ts.replace(tzinfo=pytz.timezone("UTC"))
mtr = "%s %sZ AUTO " % (sid, ts.strftime("%d%H%M"))
network = xref.get(sid3, "ASOS")
iem = Observation(sid3, network, ts.astimezone(TIMEZONES[LOC2TZ.get(sid3, None)]))
# 06019G23KT
if data["windDirQCR"][i] == 0 and data["windDir"][i] is not np.ma.masked:
iem.data["drct"] = int(data["windDir"][i])
mtr += "%03i" % (iem.data["drct"],)
else:
mtr += "///"
if data["windSpeedQCR"][i] == 0 and data["windSpeed"][i] is not np.ma.masked:
iem.data["sknt"] = int(data["windSpeed"][i])
mtr += "%02i" % (iem.data["sknt"],)
else:
mtr += "//"
if data["windGustQCR"][i] == 0 and data["windGust"][i] is not np.ma.masked and data["windGust"][i] > 0:
iem.data["gust"] = int(data["windGust"][i])
mtr += "G%02i" % (iem.data["gust"],)
mtr += "KT "
if data["visibilityQCR"][i] == 0 and data["visibility"][i] is not np.ma.masked:
iem.data["vsby"] = float(data["visibility"][i])
mtr += "%sSM " % (vsbyfmt(iem.data["vsby"]),)
presentwx = tostring(data["presWx"][i])
if presentwx != "":
iem.data["presentwx"] = presentwx
mtr += "%s " % (presentwx,)
for _i, (_c, _l) in enumerate(zip(data["skyCvr"][i], data["skyCovLayerBase"][i])):
if tostring(_c) != "":
skyc = tostring(_c)
iem.data["skyc%s" % (_i + 1,)] = skyc
if skyc != "CLR":
iem.data["skyl%s" % (_i + 1,)] = int(_l)
mtr += "%s%03i " % (tostring(_c), int(_l) / 100)
else:
mtr += "CLR "
t = ""
tgroup = "T"
if data["temperatureQCR"][i] == 0 and data["temperature"][i] is not np.ma.masked:
# iem.data['tmpf'] = float(data['temperature'][i])
tmpc = float(data["temperatureC"][i])
t = "%s%02i/" % ("M" if tmpc < 0 else "", tmpc if tmpc > 0 else (0 - tmpc))
tgroup += "%s%03i" % ("1" if tmpc < 0 else "0", (tmpc if tmpc > 0 else (0 - tmpc)) * 10.0)
if data["dewpointQCR"][i] == 0 and data["dewpoint"][i] is not np.ma.masked:
# iem.data['dwpf'] = float(data['dewpoint'][i])
tmpc = float(data["dewpointC"][i])
if t != "":
t = "%s%s%02i " % (t, "M" if tmpc < 0 else "", tmpc if tmpc > 0 else 0 - tmpc)
tgroup += "%s%03i" % ("1" if tmpc < 0 else "0", (tmpc if tmpc > 0 else (0 - tmpc)) * 10.0)
if len(t) > 4:
mtr += t
if data["altimeterQCR"][i] == 0 and data["altimeter"][i] is not np.ma.masked:
iem.data["alti"] = round(data["altimeter"][i], 2)
mtr += "A%4i " % (iem.data["alti"] * 100.0,)
mtr += "RMK "
if data["precipAccumQCR"][i] == 0 and data["precipAccum"][i] is not np.ma.masked:
if data["precipAccum"][i] >= 0.01:
iem.data["phour"] = round(data["precipAccum"][i], 2)
mtr += "P%04i " % (iem.data["phour"] * 100.0,)
elif data["precipAccum"][i] < 0.01 and data["precipAccum"][i] > 0:
# Trace
mtr += "P0000 "
iem.data["phour"] = 0.0001
if tgroup != "T":
mtr += "%s " % (tgroup,)
autoremark = tostring(data["autoRemark"][i])
opremark = tostring(data["operatorRemark"][i])
if autoremark != "" or opremark != "":
mtr += "%s %s " % (autoremark, opremark)
mtr += "MADISHF"
# Eat our own dogfood
try:
Metar(mtr)
iem.data["raw"] = mtr
except:
pass
icursor = IEM.cursor()
if not iem.save(icursor, force_current_log=True, skip_current=True):
print(("extract_hfmetar: unknown station? %s %s %s\n%s") % (sid3, network, ts, mtr))
pass
icursor.close()
IEM.commit()
def process(ncfn):
"""Process this file """
pgconn = get_dbconn('iem')
icursor = pgconn.cursor()
xref = {}
icursor.execute("""SELECT id, network from stations where
network ~* 'ASOS' or network = 'AWOS' and country = 'US'""")
for row in icursor:
xref[row[0]] = row[1]
icursor.close()
nc = netCDF4.Dataset(ncfn)
data = {}
for vname in [
'stationId',
'observationTime',
'temperature',
'dewpoint',
'altimeter', # Pa
'windDir',
'windSpeed', # mps
'windGust',
'visibility', # m
'precipAccum',
'presWx',
'skyCvr',
'skyCovLayerBase',
'autoRemark',
'operatorRemark'
]:
data[vname] = nc.variables[vname][:]
vname += "QCR"
if vname in nc.variables:
data[vname] = nc.variables[vname][:]
for vname in ['temperature', 'dewpoint']:
data[vname + "C"] = temperature(data[vname], 'K').value('C')
data[vname] = temperature(data[vname], 'K').value('F')
for vname in ['windSpeed', 'windGust']:
data[vname] = speed(data[vname], 'MPS').value('KT')
data['altimeter'] = pressure(data['altimeter'], 'PA').value("IN")
data['skyCovLayerBase'] = distance(data['skyCovLayerBase'],
'M').value("FT")
data['visibility'] = distance(data['visibility'], 'M').value("MI")
data['precipAccum'] = distance(data['precipAccum'], 'MM').value("IN")
for i in range(len(data['stationId'])):
sid = tostring(data['stationId'][i])
sid3 = sid[1:] if sid[0] == 'K' else sid
ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(
seconds=data['observationTime'][i])
ts = ts.replace(tzinfo=pytz.timezone("UTC"))
mtr = "%s %sZ AUTO " % (sid, ts.strftime("%d%H%M"))
network = xref.get(sid3, 'ASOS')
iem = Observation(sid3, network, ts)
# 06019G23KT
if (data['windDirQCR'][i] == 0
and data['windDir'][i] is not np.ma.masked):
iem.data['drct'] = int(data['windDir'][i])
mtr += "%03i" % (iem.data['drct'], )
else:
mtr += "///"
if (data['windSpeedQCR'][i] == 0
and data['windSpeed'][i] is not np.ma.masked):
iem.data['sknt'] = int(data['windSpeed'][i])
mtr += "%02i" % (iem.data['sknt'], )
else:
mtr += "//"
if (data['windGustQCR'][i] == 0
and data['windGust'][i] is not np.ma.masked
and data['windGust'][i] > 0):
iem.data['gust'] = int(data['windGust'][i])
mtr += "G%02i" % (iem.data['gust'], )
mtr += "KT "
if (data['visibilityQCR'][i] == 0
and data['visibility'][i] is not np.ma.masked):
iem.data['vsby'] = float(data['visibility'][i])
mtr += "%sSM " % (vsbyfmt(iem.data['vsby']), )
presentwx = tostring(data['presWx'][i])
if presentwx != '':
# database storage is comma delimited
iem.data['wxcodes'] = presentwx.split(" ")
mtr += "%s " % (presentwx, )
for _i, (_c, _l) in enumerate(
zip(data['skyCvr'][i], data['skyCovLayerBase'][i])):
if tostring(_c) != '':
skyc = tostring(_c)
iem.data['skyc%s' % (_i + 1, )] = skyc
if skyc != 'CLR':
iem.data['skyl%s' % (_i + 1, )] = int(_l)
mtr += "%s%03i " % (tostring(_c), int(_l) / 100)
else:
mtr += "CLR "
t = ""
tgroup = "T"
if (data['temperatureQCR'][i] == 0
and data['temperature'][i] is not np.ma.masked):
# iem.data['tmpf'] = float(data['temperature'][i])
tmpc = float(data['temperatureC'][i])
t = "%s%02i/" % ("M" if tmpc < 0 else "", tmpc if tmpc > 0 else
(0 - tmpc))
tgroup += "%s%03i" % ("1" if tmpc < 0 else "0",
(tmpc if tmpc > 0 else (0 - tmpc)) * 10.)
if (data['dewpointQCR'][i] == 0
and data['dewpoint'][i] is not np.ma.masked):
# iem.data['dwpf'] = float(data['dewpoint'][i])
tmpc = float(data['dewpointC'][i])
if t != "":
t = "%s%s%02i " % (t, "M" if tmpc < 0 else "",
tmpc if tmpc > 0 else 0 - tmpc)
tgroup += "%s%03i" % ("1" if tmpc < 0 else "0",
(tmpc if tmpc > 0 else (0 - tmpc)) * 10.)
if len(t) > 4:
mtr += t
if (data['altimeterQCR'][i] == 0
and data['altimeter'][i] is not np.ma.masked):
iem.data['alti'] = round(data['altimeter'][i], 2)
mtr += "A%4i " % (iem.data['alti'] * 100., )
mtr += "RMK "
if (data['precipAccumQCR'][i] == 0
and data['precipAccum'][i] is not np.ma.masked):
if data['precipAccum'][i] >= 0.01:
iem.data['phour'] = round(data['precipAccum'][i], 2)
mtr += "P%04i " % (iem.data['phour'] * 100., )
elif data['precipAccum'][i] < 0.01 and data['precipAccum'][i] > 0:
# Trace
mtr += "P0000 "
iem.data['phour'] = TRACE_VALUE
if tgroup != "T":
mtr += "%s " % (tgroup, )
autoremark = tostring(data['autoRemark'][i])
opremark = tostring(data['operatorRemark'][i])
if autoremark != '' or opremark != '':
mtr += "%s %s " % (autoremark, opremark)
mtr += "MADISHF"
# Eat our own dogfood
try:
Metar.Metar(mtr)
iem.data['raw'] = mtr
except Exception as exp:
pass
icursor = pgconn.cursor()
if not iem.save(icursor, force_current_log=True, skip_current=True):
print(("extract_hfmetar: unknown station? %s %s %s\n%s") %
(sid3, network, ts, mtr))
icursor.close()
pgconn.commit()
def processfile( fp ):
o = open("/mesonet/data/incoming/rwis/%s" % (fp,), 'r').readlines()
if len(o) < 2:
return
heading = o[0].split(",")
cols = o[1].split(",")
data = {}
if len(cols) < len(heading):
return
for i in range(len(heading)):
if cols[i].strip() != "/":
data[heading[i].strip()] = cols[i].strip()
nwsli = lkp[fp]
if fp in ['portableExportP1.csv', 'miniExportIFB.csv']:
ts = datetime.datetime.strptime(data['date_time'][:16],
'%Y-%m-%d %H:%M')
else:
ts = datetime.datetime.strptime(data['date_time'][:-6],
'%Y-%m-%d %H:%M')
ts = ts.replace(tzinfo=pytz.timezone("UTC"))
iem = Observation(nwsli, 'IA_RWIS', ts)
if ts.year < 2010:
print(("rwis/mini_portable.py file: %s bad timestamp: %s"
"") % (fp, data['date_time']))
return
iem.load(icursor)
# IEM Tracker stuff is missing
if data.get('wind_speed', '') != '':
iem.data['sknt'] = float(data['wind_speed']) * 1.17
if data.get('sub', '') != '':
iem.data['rwis_subf'] = float(data['sub'])
if data.get('air_temp', '') != '':
iem.data['tmpf'] = float(data['air_temp'])
if data.get('pave_temp', '') != '':
iem.data['tsf0'] = float(data['pave_temp'])
if data.get('pave_temp2', '') != '':
iem.data['tsf1'] = float(data['pave_temp2'])
if data.get('press', '') != '':
iem.data['alti'] = float(data['press'])
if data.get('RH', '') != '':
if float(data['RH']) > 1:
t = temperature(iem.data['tmpf'], 'F')
rh = humidity(float(data['RH']), '%')
iem.data['dwpf'] = meteorology.dewpoint(t, rh).value('F')
if data.get('wind_dir', '') != '':
iem.data['drct'] = float(data['wind_dir'])
iem.save(icursor)
db[thisStation]["pres"] = sanityCheck(pressure[recnum], 0, 1000000, -99)
db[thisStation]["tmpk"] = sanityCheck(tmpk[recnum], 0, 500, -99)
db[thisStation]["dwpk"] = sanityCheck(dwpk[recnum], 0, 500, -99)
db[thisStation]["tmpk_dd"] = tmpk_dd[recnum]
db[thisStation]["drct"] = sanityCheck(drct[recnum], -1, 361, -99)
db[thisStation]["smps"] = sanityCheck(smps[recnum], -1, 200, -99)
db[thisStation]["gmps"] = sanityCheck(gmps[recnum], -1, 200, -99)
db[thisStation]["rtk1"] = sanityCheck(rtk1[recnum], 0, 500, -99)
db[thisStation]["rtk2"] = sanityCheck(rtk2[recnum], 0, 500, -99)
db[thisStation]["rtk3"] = sanityCheck(rtk3[recnum], 0, 500, -99)
db[thisStation]["rtk4"] = sanityCheck(rtk4[recnum], 0, 500, -99)
db[thisStation]["subk"] = sanityCheck(subk1[recnum], 0, 500, -99)
db[thisStation]["pday"] = sanityCheck(pcpn[recnum], -1, 5000, -99)
for sid in db.keys():
iem = Observation(sid, db[sid]["network"], db[sid]["ts"])
# if not iem.load(icursor):
# print 'Missing fp: %s network: %s station: %s' % (fp,
# db[sid]['network'],
# sid)
# subprocess.call("python sync_stations.py %s" % (fp,), shell=True)
# os.chdir("../../dbutil")
# subprocess.call("sh SYNC_STATIONS.sh", shell=True)
# os.chdir("../ingestors/madis")
iem.data["tmpf"] = temperature(db[sid]["tmpk"], "K").value("F")
iem.data["dwpf"] = temperature(db[sid]["dwpk"], "K").value("F")
if db[sid]["drct"] >= 0:
iem.data["drct"] = db[sid]["drct"]
if db[sid]["smps"] >= 0:
iem.data["sknt"] = db[sid]["smps"] * (1 / 0.5148)
if db[sid]["gmps"] >= 0:
sys.exit(0)
if len(data) < 2:
sys.exit(0)
keys = data[0].strip().replace('"', '').split(',')
vals = data[1].strip().replace('"', '').split(',')
d = {}
for i in range(len(vals)):
d[keys[i]] = vals[i]
# Ob times are always CDT
ts1 = datetime.datetime.strptime(d['TIMESTAMP'], '%Y-%m-%d %H:%M:%S')
gts1 = ts1 + datetime.timedelta(hours=5)
gts1 = gts1.replace(tzinfo=pytz.timezone("UTC"))
lts = gts1.astimezone(pytz.timezone("America/Chicago"))
iem = Observation('RSAI4', "OT", lts)
drct = d['WindDir']
iem.data['drct'] = drct
sknt = float(d['WS_mph_S_WVT']) / 1.15
iem.data['sknt'] = sknt
gust = float(d['WS_mph_Max']) / 1.15
iem.data['gust'] = gust
iem.save(icursor)
csv.write("%s,%s,%s,%.1f,%.1f\n" % ('RSAI4',
gts1.strftime("%Y/%m/%d %H:%M:%S"),
drct, sknt, gust))
# Red Rock
try:
req = urllib2.Request(("ftp://%s:%[email protected]/Red Rock_Table3"
if not os.path.isfile(fn):
sys.exit(0)
lines = open(fn, 'r').readlines()
line = lines[-1]
# 114,2006,240,1530,18.17,64.70, 88.9,2.675,21.91,1014.3,0.000
tokens = line.split(",")
if (len(tokens) != 11):
sys.exit(0)
hhmm = "%04i" % (int(tokens[3]),)
ts = now.replace(hour=int(hhmm[:2]), minute=int(hhmm[2:]), second=0,
microsecond=0, tzinfo=pytz.timezone("America/Chicago"))
iemob = Observation("OT0007", "OT", ts)
iemob.data['tmpf'] = float(tokens[5])
iemob.data['relh'] = float(tokens[6])
tmpf = temperature(iemob.data['tmpf'], 'F')
relh = humidity(iemob.data['relh'], '%')
iemob.data['dwpf'] = meteorology.dewpoint(tmpf, relh).value('F')
iemob.data['sknt'] = float(tokens[7]) * 1.94
iemob.data['drct'] = tokens[8]
iemob.data['pres'] = float(tokens[9]) / 960 * 28.36
iemob.save(icursor)
icursor.close()
IEM.commit()
IEM.close()
def saveData(txn):
"""
Save data to the IEM database
"""
timer_start = mx.DateTime.now()
updates = 0
skips = 0
# need to call keys() as DB is voliatle
for key in DB.keys():
val = DB[key]
if not val.updated or val.nwsli is None:
skips += 1
continue
val.sanityCheck()
iem = Observation(val.nwsli, val.network, val.ts)
val.avgWinds()
if val.tmpf is None or val.tmpf == 460:
iem.data['tmpf'] = None
iem.data['relh'] = None
iem.data['dwpf'] = None
else:
iem.data['tmpf'] = val.tmpf
iem.data['relh'] = val.humid
iem.data['dwpf'] = val.dwpf
iem.data['drct'] = val.avg_drct
iem.data['sknt'] = val.avg_sknt
iem.data['srad'] = val.rad
iem.data['max_srad'] = val.xsrad
iem.data['pres'] = val.pres
iem.data['pday'] = float(val.pDay)
iem.data['pmonth'] = float(val.pMonth)
iem.data['gust'] = float(val.xsped) * 0.86897
iem.data['max_drct'] = val.xdrct
iem.save(txn)
val.updated = False
updates += 1
val.xsped = 0
del(iem)
timer_end = mx.DateTime.now()
print "Time: %.2fs Updates: %s Skips: %s" % (timer_end - timer_start,
updates, skips)
def saveData(txn):
"""
Save data to the IEM database
"""
timer_start = datetime.datetime.now()
updates = 0
skips = 0
# need to call keys() as DB is voliatle
for key in DB.keys():
val = DB[key]
if not val.updated or val.nwsli is None:
skips += 1
continue
val.sanityCheck()
iem = Observation(val.nwsli, val.network, val.ts)
val.avgWinds()
if val.tmpf is None or val.tmpf == 460:
iem.data["tmpf"] = None
iem.data["relh"] = None
iem.data["dwpf"] = None
else:
iem.data["tmpf"] = val.tmpf
iem.data["relh"] = val.humid
iem.data["dwpf"] = val.dwpf
iem.data["drct"] = val.avg_drct
iem.data["sknt"] = val.avg_sknt
iem.data["srad"] = val.rad
iem.data["max_srad"] = val.xsrad
iem.data["pres"] = val.pres
iem.data["pday"] = float(val.pDay)
iem.data["pmonth"] = float(val.pMonth)
iem.data["gust"] = float(val.xsped) * 0.86897
iem.data["max_drct"] = val.xdrct
iem.save(txn)
val.updated = False
updates += 1
val.xsped = 0
del (iem)
timer_end = datetime.datetime.now()
print(("Time: %.2fs Updates: %s Skips: %s") % ((timer_end - timer_start).total_seconds(), updates, skips))
def main():
"""Go Main Go"""
iemaccess = get_dbconn('iem')
cursor = iemaccess.cursor()
valid = datetime.datetime.utcnow()
valid = valid.replace(tzinfo=pytz.utc)
valid = valid.astimezone(pytz.timezone("America/Chicago"))
fn = valid.strftime("/mesonet/ARCHIVE/data/%Y/%m/%d/text/ot/ot0010.dat")
if not os.path.isfile(fn):
sys.exit(0)
lines = open(fn, "r").readlines()
lastline = lines[-1].strip()
tokens = re.split(r"[\s+]+", lastline)
if len(tokens) != 20:
return
tparts = re.split(":", tokens[3])
valid = valid.replace(hour=int(tparts[0]),
minute=int(tparts[1]),
second=0,
microsecond=0)
iem = Observation("OT0010", "OT", valid)
iem.data['tmpf'] = float(tokens[4])
iem.data['max_tmpf'] = float(tokens[5])
iem.data['min_tmpf'] = float(tokens[6])
iem.data['relh'] = int(tokens[7])
iem.data['dwpf'] = dewpoint(temperature(iem.data['tmpf'], 'F'),
humidity(iem.data['relh'], '%')).value("F")
iem.data['sknt'] = speed(float(tokens[8]), 'mph').value('KT')
iem.data['drct'] = int(tokens[9])
iem.data['max_sknt'] = speed(float(tokens[10]), 'mph').value('KT')
iem.data['alti'] = float(tokens[12])
iem.data['pday'] = float(tokens[13])
iem.data['srad'] = float(tokens[18])
iem.save(cursor)
cursor.close()
iemaccess.commit()
def saveData(txn):
"""
Save data to the IEM database
"""
timer_start = datetime.datetime.now()
updates = 0
skips = 0
# need to call keys() as DB is voliatle
for key in DB.keys():
val = DB[key]
if not val.updated or val.nwsli is None:
skips += 1
continue
val.sanityCheck()
iem = Observation(val.nwsli, val.network, val.ts)
val.avgWinds()
if val.tmpf is None or val.tmpf == 460:
iem.data['tmpf'] = None
iem.data['relh'] = None
iem.data['dwpf'] = None
else:
iem.data['tmpf'] = val.tmpf
iem.data['relh'] = val.humid
iem.data['dwpf'] = val.dwpf
iem.data['drct'] = val.avg_drct
iem.data['sknt'] = val.avg_sknt
iem.data['srad'] = val.rad
iem.data['max_srad'] = val.xsrad
iem.data['pres'] = val.pres
iem.data['pday'] = float(val.pDay)
iem.data['pmonth'] = float(val.pMonth)
iem.data['gust'] = float(val.xsped) * 0.86897
iem.data['max_drct'] = val.xdrct
iem.save(txn)
val.updated = False
updates += 1
val.xsped = 0
del (iem)
timer_end = datetime.datetime.now()
print(("Time: %.2fs Updates: %s Skips: %s") %
((timer_end - timer_start).total_seconds(), updates, skips))
cursor = iemaccess.cursor()
valid = datetime.datetime.utcnow()
valid = valid.replace(tzinfo=pytz.timezone("UTC"))
valid = valid.astimezone(pytz.timezone("America/Chicago"))
fn = valid.strftime("/mesonet/ARCHIVE/data/%Y/%m/%d/text/ot/ot0002.dat")
if not os.path.isfile(fn):
sys.exit(0)
lines = open(fn, "r").readlines()
lastLine = lines[-1]
tokens = re.split("[\s+]+", lastLine)
tparts = re.split(":", tokens[4])
valid = valid.replace(hour=int(tparts[0]),
minute=int(tparts[1]), second=int(tparts[2]))
iem = Observation("OT0002", "OT", valid)
sknt = speed(float(tokens[8]), 'MPH').value('KT')
iem.data['sknt'] = sknt
iem.data['drct'] = tokens[9]
iem.data['tmpf'] = tokens[7]
iem.save(cursor)
cursor.close()
iemaccess.commit()
def hourly_process(nwsli, maxts):
""" Process the hourly file """
fn = "%s/%s_HrlySI.dat" % (BASE, STATIONS[nwsli])
df = common_df_logic(fn, maxts, nwsli, "sm_hourly")
if df is None:
return 0
processed = 0
LOG.debug("processing %s rows from %s", len(df.index), fn)
acursor = ACCESS.cursor()
for _i, row in df.iterrows():
# Update IEMAccess
ob = Observation(nwsli, "ISUSM", row["valid"])
tmpc = temperature(row["tair_c_avg_qc"], "C")
if tmpc.value("F") > -50 and tmpc.value("F") < 140:
ob.data["tmpf"] = tmpc.value("F")
relh = humidity(row["rh_qc"], "%")
ob.data["relh"] = relh.value("%")
ob.data["dwpf"] = met.dewpoint(tmpc, relh).value("F")
# SIC the units of slrkw are actually W/m2
ob.data["srad"] = row["slrkw_avg_qc"]
ob.data["phour"] = round(
distance(row["rain_mm_tot_qc"], "MM").value("IN"), 2)
ob.data["sknt"] = speed(row["ws_mps_s_wvt_qc"], "MPS").value("KT")
if "ws_mph_max" in df.columns:
ob.data["gust"] = speed(row["ws_mph_max_qc"], "MPH").value("KT")
ob.data["max_gust_ts"] = row["ws_mph_tmx"]
ob.data["drct"] = row["winddir_d1_wvt_qc"]
if "tsoil_c_avg" in df.columns:
ob.data["c1tmpf"] = temperature(row["tsoil_c_avg_qc"],
"C").value("F")
if "t12_c_avg_qc" in df.columns:
ob.data["c2tmpf"] = temperature(row["t12_c_avg_qc"],
"C").value("F")
if "t24_c_avg_qc" in df.columns:
ob.data["c3tmpf"] = temperature(row["t24_c_avg_qc"],
"C").value("F")
if "t50_c_avg" in df.columns:
ob.data["c4tmpf"] = temperature(row["t50_c_avg_qc"],
"C").value("F")
if "calc_vwc_12_avg" in df.columns:
ob.data["c2smv"] = row["calc_vwc_12_avg_qc"] * 100.0
if "calc_vwc_24_avg" in df.columns:
ob.data["c3smv"] = row["calc_vwc_24_avg_qc"] * 100.0
if "calc_vwc_50_avg" in df.columns:
ob.data["c4smv"] = row["calc_vwc_50_avg_qc"] * 100.0
ob.save(acursor)
processed += 1
acursor.close()
ACCESS.commit()
return processed
db[thisStation]['pres'] = sanityCheck(pressure[recnum], 0, 1000000, -99)
db[thisStation]['tmpk'] = sanityCheck(tmpk[recnum], 0, 500, -99)
db[thisStation]['dwpk'] = sanityCheck(dwpk[recnum], 0, 500, -99)
db[thisStation]['tmpk_dd'] = tmpk_dd[recnum]
db[thisStation]['drct'] = sanityCheck(drct[recnum], -1, 361, -99)
db[thisStation]['smps'] = sanityCheck(smps[recnum], -1, 200, -99)
db[thisStation]['gmps'] = sanityCheck(gmps[recnum], -1, 200, -99)
db[thisStation]['rtk1'] = sanityCheck(rtk1[recnum], 0, 500, -99)
db[thisStation]['rtk2'] = sanityCheck(rtk2[recnum], 0, 500, -99)
db[thisStation]['rtk3'] = sanityCheck(rtk3[recnum], 0, 500, -99)
db[thisStation]['rtk4'] = sanityCheck(rtk4[recnum], 0, 500, -99)
db[thisStation]['subk'] = sanityCheck(subk1[recnum], 0, 500, -99)
db[thisStation]['pday'] = sanityCheck(pcpn[recnum], -1, 5000, -99)
for sid in db.keys():
iem = Observation(sid, db[sid]['network'], db[sid]['ts'])
# if not iem.load(icursor):
# print 'Missing fp: %s network: %s station: %s' % (fp,
# db[sid]['network'],
# sid)
# subprocess.call("python sync_stations.py %s" % (fp,), shell=True)
# os.chdir("../../dbutil")
# subprocess.call("sh SYNC_STATIONS.sh", shell=True)
# os.chdir("../ingestors/madis")
iem.data['tmpf'] = temperature(db[sid]['tmpk'], 'K').value('F')
iem.data['dwpf'] = temperature(db[sid]['dwpk'], 'K').value('F')
if db[sid]['drct'] >= 0:
iem.data['drct'] = db[sid]['drct']
if db[sid]['smps'] >= 0:
iem.data['sknt'] = speed(db[sid]['smps'], 'MPS').value('KT')
if db[sid]['gmps'] >= 0:
def hourly_process(nwsli, maxts):
""" Process the hourly file """
# print '-------------- HOURLY PROCESS ---------------'
acursor = ACCESS.cursor()
fn = "%s/%s_HrlySI.dat" % (BASE, STATIONS[nwsli])
df = common_df_logic(fn, maxts, nwsli, "sm_hourly")
if df is None:
return 0
processed = 0
for _i, row in df.iterrows():
# Update IEMAccess
# print nwsli, valid
ob = Observation(nwsli, 'ISUSM', row['valid'])
tmpc = temperature(row['tair_c_avg_qc'], 'C')
if tmpc.value('F') > -50 and tmpc.value('F') < 140:
ob.data['tmpf'] = tmpc.value('F')
relh = humidity(row['rh_qc'], '%')
ob.data['relh'] = relh.value('%')
ob.data['dwpf'] = met.dewpoint(tmpc, relh).value('F')
ob.data['srad'] = row['slrkw_avg_qc']
ob.data['phour'] = round(
distance(row['rain_mm_tot_qc'], 'MM').value('IN'), 2)
ob.data['sknt'] = speed(row['ws_mps_s_wvt_qc'], 'MPS').value("KT")
if 'ws_mph_max' in df.columns:
ob.data['gust'] = speed(row['ws_mph_max_qc'], 'MPH').value('KT')
ob.data['max_gust_ts'] = row['ws_mph_tmx']
ob.data['drct'] = row['winddir_d1_wvt_qc']
if 'tsoil_c_avg' in df.columns:
ob.data['c1tmpf'] = temperature(row['tsoil_c_avg_qc'],
'C').value('F')
ob.data['c2tmpf'] = temperature(row['t12_c_avg_qc'], 'C').value('F')
ob.data['c3tmpf'] = temperature(row['t24_c_avg_qc'], 'C').value('F')
if 't50_c_avg' in df.columns:
ob.data['c4tmpf'] = temperature(row['t50_c_avg_qc'],
'C').value('F')
if 'calc_vwc_12_avg' in df.columns:
ob.data['c2smv'] = row['calc_vwc_12_avg_qc'] * 100.0
if 'calc_vwc_24_avg' in df.columns:
ob.data['c3smv'] = row['calc_vwc_24_avg_qc'] * 100.0
if 'calc_vwc_50_avg' in df.columns:
ob.data['c4smv'] = row['calc_vwc_50_avg_qc'] * 100.0
ob.save(acursor)
# print 'soilm_ingest.py station: %s ts: %s hrly updated no data?' % (
# nwsli, valid)
processed += 1
acursor.close()
ACCESS.commit()
return processed
def m15_process(nwsli, maxts):
""" Process the 15minute file """
fn = "%s/%s_Min15SI.dat" % (BASE, STATIONS[nwsli])
if not os.path.isfile(fn):
return
lines = open(fn).readlines()
if len(lines) < 5:
return
# Read header....
headers = []
for col in lines[1].strip().replace('"', '').split(","):
headers.append(VARCONV.get(col.lower(), col.lower()))
# Read data
processed = 0
for i in range(len(lines) - 1, 3, -1):
tokens = lines[i].strip().replace('"', '').split(",")
if len(tokens) != len(headers):
continue
valid = make_time(tokens[headers.index('timestamp')])
if valid <= maxts:
break
gust_valid = make_time(tokens[headers.index('ws_mph_tmx')])
# print valid, tokens[ headers.index('timestamp')]
# We are ready for dbinserting, we duplicate the data for the _qc
# column
dbcols = ("station,valid,%s,%s") % (",".join(headers[2:]), ",".join(
["%s_qc" % (h, ) for h in headers[2:]]))
dbvals = "'%s','%s-06'," % (nwsli, valid.strftime("%Y-%m-%d %H:%M:%S"))
for v in tokens[2:]:
dbvals += "%s," % (formatter(v), )
for v in tokens[2:]:
dbvals += "%s," % (formatter(v), )
sql = "INSERT into sm_15minute (%s) values (%s)" % (dbcols,
dbvals[:-1])
icursor.execute(sql)
# Update IEMAccess
# print nwsli, valid
ob = Observation(nwsli, 'ISUSM',
valid.astimezone(pytz.timezone("America/Chicago")))
tmpc = temperature(float(tokens[headers.index('tair_c_avg')]), 'C')
if tmpc.value('F') > -50 and tmpc.value('F') < 140:
ob.data['tmpf'] = tmpc.value('F')
relh = humidity(float(tokens[headers.index('rh')]), '%')
ob.data['relh'] = relh.value('%')
ob.data['dwpf'] = met.dewpoint(tmpc, relh).value('F')
ob.data['srad'] = tokens[headers.index('slrkw_avg')]
ob.data['phour'] = round(
distance(float(tokens[headers.index('rain_mm_tot')]),
'MM').value('IN'), 2)
ob.data['sknt'] = float(tokens[headers.index('ws_mps_s_wvt')]) * 1.94
ob.data['gust'] = float(tokens[headers.index('ws_mph_max')]) / 1.15
ob.data['max_gust_ts'] = "%s-06" % (
gust_valid.strftime("%Y-%m-%d %H:%M:%S"), )
ob.data['drct'] = float(tokens[headers.index('winddir_d1_wvt')])
ob.data['c1tmpf'] = temperature(
float(tokens[headers.index('tsoil_c_avg')]), 'C').value('F')
ob.data['c2tmpf'] = temperature(
float(tokens[headers.index('t12_c_avg')]), 'C').value('F')
ob.data['c3tmpf'] = temperature(
float(tokens[headers.index('t24_c_avg')]), 'C').value('F')
ob.data['c4tmpf'] = temperature(
float(tokens[headers.index('t50_c_avg')]), 'C').value('F')
ob.data['c2smv'] = float(tokens[headers.index('vwc_12_avg')]) * 100.0
ob.data['c3smv'] = float(tokens[headers.index('vwc_24_avg')]) * 100.0
ob.data['c4smv'] = float(tokens[headers.index('vwc_50_avg')]) * 100.0
ob.save(accesstxn, force_current_log=True)
# print 'soilm_ingest.py station: %s ts: %s hrly updated no data?' % (
# nwsli, valid)
processed += 1
return processed
def daily_process(nwsli, maxts):
""" Process the daily file """
icursor = ISUAG.cursor()
acursor = ACCESS.cursor()
# print '-------------- DAILY PROCESS ----------------'
fn = "%s/%s_DailySI.dat" % (BASE, STATIONS[nwsli])
if not os.path.isfile(fn):
return 0
lines = open(fn).readlines()
if len(lines) < 5:
return 0
# Read header....
headers = []
for col in lines[1].strip().replace('"', '').split(","):
headers.append(VARCONV.get(col.lower(), col.lower()))
# Read data
processed = 0
for i in range(len(lines) - 1, 3, -1):
tokens = lines[i].strip().replace('"', '').split(",")
if len(tokens) != len(headers):
continue
valid = datetime.datetime.strptime(
tokens[headers.index('timestamp')][:10], '%Y-%m-%d')
valid = valid.date() - datetime.timedelta(days=1)
if valid <= maxts:
break
# if valid == maxts: # Reprocess
# icursor.execute("""DELETE from sm_daily WHERE valid = '%s' and
# station = '%s' """ % (valid.strftime("%Y-%m-%d"), nwsli))
# We are ready for dbinserting!
dbcols = ("station,valid,%s,%s") % (",".join(headers[2:]), ",".join(
["%s_qc" % (h, ) for h in headers[2:]]))
dbvals = "'%s','%s-06'," % (nwsli, valid.strftime("%Y-%m-%d %H:%M:%S"))
for v in tokens[2:]:
dbvals += "%s," % (formatter(v), )
for v in tokens[2:]:
dbvals += "%s," % (formatter(v), )
sql = "INSERT into sm_daily (%s) values (%s)" % (dbcols, dbvals[:-1])
icursor.execute(sql)
# Need a timezone
valid = datetime.datetime(valid.year, valid.month, valid.day, 12, 0)
valid = valid.replace(tzinfo=pytz.timezone("America/Chicago"))
ob = Observation(nwsli, 'ISUSM', valid)
ob.data['max_tmpf'] = temperature(
float(tokens[headers.index('tair_c_max')]), 'C').value('F')
ob.data['min_tmpf'] = temperature(
float(tokens[headers.index('tair_c_min')]), 'C').value('F')
ob.data['pday'] = round(
distance(float(tokens[headers.index('rain_mm_tot')]),
'MM').value('IN'), 2)
if valid not in EVENTS['days']:
EVENTS['days'].append(valid)
ob.data['et_inch'] = distance(float(tokens[headers.index('dailyet')]),
'MM').value('IN')
ob.data['srad_mj'] = float(tokens[headers.index('slrmj_tot')])
if ob.data['srad_mj'] == 0 or np.isnan(ob.data['srad_mj']):
print(("soilm_ingest.py station: %s ts: %s has 0 solar") %
(nwsli, valid.strftime("%Y-%m-%d")))
EVENTS['reprocess_solar'] = True
ob.data['max_sknt'] = speed(float(tokens[headers.index('ws_mps_max')]),
'MPS').value('KT')
ob.data['avg_sknt'] = speed(
float(tokens[headers.index('ws_mps_s_wvt')]), 'MPS').value('KT')
# TODO: assumption this is done right by the campbell logger
ob.data['vector_avg_drct'] = float(
tokens[headers.index('winddir_d1_wvt')])
ob.save(acursor)
processed += 1
icursor.close()
acursor.close()
ISUAG.commit()
ACCESS.commit()
return processed
def m15_process(nwsli, maxts):
""" Process the 15minute file """
fn = "%s/%s_Min15SI.dat" % (BASE, STATIONS[nwsli])
if not os.path.isfile(fn):
return
lines = open(fn).readlines()
if len(lines) < 5:
return
# Read header....
headers = []
for col in lines[1].strip().replace('"', '').split(","):
headers.append(VARCONV.get(col.lower(), col.lower()))
# Read data
processed = 0
for i in range(len(lines)-1, 3, -1):
tokens = lines[i].strip().replace('"', '').split(",")
if len(tokens) != len(headers):
continue
valid = make_time(tokens[headers.index('timestamp')])
if valid <= maxts:
break
gust_valid = make_time(tokens[headers.index('ws_mph_tmx')])
# print valid, tokens[ headers.index('timestamp')]
# We are ready for dbinserting, we duplicate the data for the _qc
# column
dbcols = ("station,valid,%s,%s"
) % (",".join(headers[2:]),
",".join(["%s_qc" % (h,) for h in headers[2:]]))
dbvals = "'%s','%s-06'," % (nwsli, valid.strftime("%Y-%m-%d %H:%M:%S"))
for v in tokens[2:]:
dbvals += "%s," % (formatter(v),)
for v in tokens[2:]:
dbvals += "%s," % (formatter(v),)
sql = "INSERT into sm_15minute (%s) values (%s)" % (dbcols,
dbvals[:-1])
icursor.execute(sql)
# Update IEMAccess
# print nwsli, valid
ob = Observation(nwsli, 'ISUSM',
valid.astimezone(pytz.timezone("America/Chicago")))
tmpc = temperature(float(tokens[headers.index('tair_c_avg')]), 'C')
if tmpc.value('F') > -50 and tmpc.value('F') < 140:
ob.data['tmpf'] = tmpc.value('F')
relh = humidity(float(tokens[headers.index('rh')]), '%')
ob.data['relh'] = relh.value('%')
ob.data['dwpf'] = met.dewpoint(tmpc, relh).value('F')
ob.data['srad'] = tokens[headers.index('slrkw_avg')]
ob.data['phour'] = round(
distance(
float(tokens[headers.index('rain_mm_tot')]),
'MM').value('IN'), 2)
ob.data['sknt'] = float(tokens[headers.index('ws_mps_s_wvt')]) * 1.94
ob.data['gust'] = float(tokens[headers.index('ws_mph_max')]) / 1.15
ob.data['max_gust_ts'] = "%s-06" % (
gust_valid.strftime("%Y-%m-%d %H:%M:%S"),)
ob.data['drct'] = float(tokens[headers.index('winddir_d1_wvt')])
ob.data['c1tmpf'] = temperature(
float(tokens[headers.index('tsoil_c_avg')]), 'C').value('F')
ob.data['c2tmpf'] = temperature(
float(tokens[headers.index('t12_c_avg')]), 'C').value('F')
ob.data['c3tmpf'] = temperature(
float(tokens[headers.index('t24_c_avg')]), 'C').value('F')
ob.data['c4tmpf'] = temperature(
float(tokens[headers.index('t50_c_avg')]), 'C').value('F')
ob.data['c2smv'] = float(tokens[headers.index('vwc_12_avg')]) * 100.0
ob.data['c3smv'] = float(tokens[headers.index('vwc_24_avg')]) * 100.0
ob.data['c4smv'] = float(tokens[headers.index('vwc_50_avg')]) * 100.0
ob.save(accesstxn, force_current_log=True)
# print 'soilm_ingest.py station: %s ts: %s hrly updated no data?' % (
# nwsli, valid)
processed += 1
return processed
def process_site(prod, sid, ts, data):
"""
I process a dictionary of data for a particular site
"""
localts = get_localtime(sid, ts)
# Insert data into database regardless
for varname in data:
value = data[varname]
deffer = HADSDB.runOperation(
"""INSERT into raw_inbound
(station, valid, key, value)
VALUES(%s,%s, %s, %s)
""",
(sid, ts.strftime("%Y-%m-%d %H:%M+00"), varname, value),
)
deffer.addErrback(common.email_error, prod.text)
deffer.addErrback(log.err)
key = "%s|%s" % (sid, varname)
cur = CURRENT_QUEUE.setdefault(key,
dict(valid=ts, value=value, dirty=True))
if ts > cur["valid"]:
cur["valid"] = ts
cur["value"] = value
cur["dirty"] = True
# Don't bother with stranger locations
if len(sid) == 8 and sid[0] == "X":
return
# Don't bother with unknown sites
if sid in UNKNOWN:
return
network = get_network(prod, sid, ts, data)
if network in ["KCCI", "KIMT", "KELO", "ISUSM", None]:
return
# Do not send DCP sites with old data to IEMAccess
if network.find("_DCP") > 0 and localts < LOCS.get(sid, dict()).get(
"valid", localts):
return
metadata = LOCS.setdefault(sid,
{network: dict(valid=localts, tzname=None)})
metadata[network]["valid"] = localts
# Okay, time for a hack, if our observation is at midnight!
if localts.hour == 0 and localts.minute == 0:
localts -= datetime.timedelta(minutes=1)
# log.msg("Shifting %s [%s] back one minute: %s" % (sid, network,
# localts))
iemob = Observation(sid, network, localts)
iscoop = network.find("COOP") > 0
hasdata = False
for var in data:
# shefit uses -9999 as a missing sentinel
val = None if data[var] < -9998 else data[var]
iemvar = MAPPING[var]
if iemvar == "":
continue
if val is None:
# Behold, glorious hack here to force nulls into the summary
# database that uses coerce
iemob.data["null_%s" % (iemvar, )] = None
else:
val *= MULTIPLIER.get(var[:2], 1.0)
hasdata = True
iemob.data[iemvar] = val
if MAPPING[var] in ["pday", "snow", "snowd"]:
# Convert 0.001 to 0.0001 for Trace values
if val is not None and val == 0.001:
iemob.data[iemvar] = TRACE_VALUE
# Prevent negative numbers
elif val is not None and val < 0:
iemob.data[iemvar] = 0
if iscoop:
# Save COOP 'at-ob' temperature into summary table
if iemvar == "tmpf":
iemob.data["coop_tmpf"] = val
# Save observation time into the summary table
if iemvar in [
"tmpf",
"max_tmpf",
"min_tmpf",
"pday",
"snow",
"snowd",
]:
iemob.data["coop_valid"] = iemob.data["valid"]
if hasdata:
iemob.data["raw"] = prod.product_id
deffer = ACCESSDB.runInteraction(iemob.save)
deffer.addCallback(got_results, prod.product_id, sid, network, localts)
deffer.addErrback(common.email_error, prod.text)
deffer.addErrback(log.err)
fn = valid.strftime("/mesonet/ARCHIVE/data/%Y/%m/%d/text/ot/ot0010.dat")
if not os.path.isfile(fn):
sys.exit(0)
lines = open(fn, "r").readlines()
lastLine = lines[-1].strip()
tokens = re.split("[\s+]+", lastLine)
if len(tokens) != 20:
sys.exit(0)
tparts = re.split(":", tokens[3])
valid = valid.replace(hour=int(tparts[0]),
minute=int(tparts[1]), second=0, microsecond=0)
iem = Observation("OT0010", "OT", valid)
iem.data['tmpf'] = float(tokens[4])
iem.data['max_tmpf'] = float(tokens[5])
iem.data['min_tmpf'] = float(tokens[6])
iem.data['relh'] = int(tokens[7])
iem.data['dwpf'] = dewpoint(temperature(iem.data['tmpf'], 'F'),
humidity(iem.data['relh'], '%')).value("F")
iem.data['sknt'] = speed(float(tokens[8]), 'mph').value('KT')
iem.data['drct'] = int(tokens[9])
iem.data['max_sknt'] = speed(float(tokens[10]), 'mph').value('KT')
iem.data['alti'] = float(tokens[12])
iem.data['pday'] = float(tokens[13])
iem.data['srad'] = float(tokens[18])
db[thisStation]['pres'] = sanityCheck(pressure[recnum], 0, 1000000, -99)
db[thisStation]['tmpk'] = sanityCheck(tmpk[recnum], 0, 500, -99)
db[thisStation]['dwpk'] = sanityCheck(dwpk[recnum], 0, 500, -99)
db[thisStation]['tmpk_dd'] = tmpk_dd[recnum]
db[thisStation]['drct'] = sanityCheck(drct[recnum], -1, 361, -99)
db[thisStation]['smps'] = sanityCheck(smps[recnum], -1, 200, -99)
db[thisStation]['gmps'] = sanityCheck(gmps[recnum], -1, 200, -99)
db[thisStation]['rtk1'] = sanityCheck(rtk1[recnum], 0, 500, -99)
db[thisStation]['rtk2'] = sanityCheck(rtk2[recnum], 0, 500, -99)
db[thisStation]['rtk3'] = sanityCheck(rtk3[recnum], 0, 500, -99)
db[thisStation]['rtk4'] = sanityCheck(rtk4[recnum], 0, 500, -99)
db[thisStation]['subk'] = sanityCheck(subk1[recnum], 0, 500, -99)
db[thisStation]['pday'] = sanityCheck(pcpn[recnum], -1, 5000, -99)
for sid in db.keys():
iem = Observation(sid, db[sid]['network'], db[sid]['ts'])
# if not iem.load(icursor):
# print 'Missing fp: %s network: %s station: %s' % (fp,
# db[sid]['network'],
# sid)
# subprocess.call("python sync_stations.py %s" % (fp,), shell=True)
# os.chdir("../../dbutil")
# subprocess.call("sh SYNC_STATIONS.sh", shell=True)
# os.chdir("../ingestors/madis")
iem.data['tmpf'] = temperature(db[sid]['tmpk'], 'K').value('F')
iem.data['dwpf'] = temperature(db[sid]['dwpk'], 'K').value('F')
if db[sid]['drct'] >= 0:
iem.data['drct'] = db[sid]['drct']
if db[sid]['smps'] >= 0:
iem.data['sknt'] = db[sid]['smps'] * (1/0.5148)
if db[sid]['gmps'] >= 0:
def to_iemaccess(self, txn):
"""Persist this data object to IEMAccess"""
gts = self.time.replace(tzinfo=pytz.timezone("UTC"))
iem = Observation(self.iemid, self.network, gts)
# Load the observation from the database, if the same time exists!
iem.load(txn)
# Need to figure out if we have a duplicate ob, if so, check
# the length of the raw data, if greater, take the temps
if (iem.data['raw'] is not None and
len(iem.data['raw']) >= len(self.code)):
pass
else:
if self.temp:
val = self.temp.value("F")
# Place reasonable bounds on the temperature before saving it!
if val > -90 and val < 150:
iem.data['tmpf'] = round(val, 1)
if self.dewpt:
iem.data['dwpf'] = round(self.dewpt.value("F"), 1)
# Daabase only allows len 254
iem.data['raw'] = self.code[:254]
if self.wind_speed:
iem.data['sknt'] = self.wind_speed.value("KT")
if self.wind_gust:
iem.data['gust'] = self.wind_gust.value("KT")
if self.wind_dir:
if self.wind_dir.value() == 'VRB':
iem.data['drct'] = 0
else:
iem.data['drct'] = float(self.wind_dir.value())
if not self.wind_speed_peak:
old_max_wind = max([iem.data.get('max_sknt', 0) or 0,
iem.data.get('max_gust', 0) or 0])
new_max_wind = max([iem.data.get('sknt', 0),
iem.data.get('gust', 0)])
if new_max_wind > old_max_wind:
# print 'Setting max_drct manually: %s' % (clean_metar,)
iem.data['max_drct'] = iem.data.get('drct', 0)
if self.wind_speed_peak:
iem.data['max_gust'] = self.wind_speed_peak.value("KT")
if self.wind_dir_peak:
iem.data['max_drct'] = self.wind_dir_peak.value()
if self.peak_wind_time:
iem.data['max_gust_ts'] = self.peak_wind_time.replace(
tzinfo=pytz.timezone("UTC"))
if self.max_temp_6hr:
iem.data['max_tmpf_6hr'] = round(self.max_temp_6hr.value("F"), 1)
if iem.data['valid'].hour >= 6:
iem.data['max_tmpf'] = round(self.max_temp_6hr.value("F"), 1)
if self.min_temp_6hr:
iem.data['min_tmpf_6hr'] = round(self.min_temp_6hr.value("F"), 1)
if iem.data['valid'].hour >= 6:
iem.data['min_tmpf'] = round(self.min_temp_6hr.value("F"), 1)
if self.max_temp_24hr:
iem.data['max_tmpf_24hr'] = round(self.max_temp_24hr.value("F"), 1)
if self.min_temp_24hr:
iem.data['min_tmpf_24hr'] = round(self.min_temp_24hr.value("F"), 1)
if self.precip_3hr:
iem.data['p03i'] = trace(self.precip_3hr)
if self.precip_6hr:
iem.data['p06i'] = trace(self.precip_6hr)
if self.precip_24hr:
iem.data['p24i'] = trace(self.precip_24hr)
# We assume the value is zero, sad!
iem.data['phour'] = 0
if self.precip_1hr:
iem.data['phour'] = trace(self.precip_1hr)
if self.snowdepth:
iem.data['snowd'] = self.snowdepth.value("IN")
if self.vis:
iem.data['vsby'] = self.vis.value("SM")
if self.press:
iem.data['alti'] = self.press.value("IN")
if self.press_sea_level:
iem.data['mslp'] = self.press_sea_level.value("MB")
if self.press_sea_level and self.press:
alti = self.press.value("MB")
mslp = self.press_sea_level.value("MB")
if abs(alti - mslp) > 25:
print(("PRESSURE ERROR %s %s ALTI: %s MSLP: %s"
) % (iem.data['station'], iem.data['valid'],
alti, mslp))
if alti > mslp:
iem.data['mslp'] += 100.
else:
iem.data['mslp'] -= 100.
# Do something with sky coverage
for i in range(len(self.sky)):
(cov, hgh, _) = self.sky[i]
iem.data['skyc%s' % (i+1)] = cov
if hgh is not None:
iem.data['skyl%s' % (i+1)] = hgh.value("FT")
# Presentwx
if self.weather:
pwx = []
for wx in self.weather:
pwx.append(("").join([a for a in wx if a is not None]))
iem.data['presentwx'] = (",".join(pwx))[:24]
return iem, iem.save(txn)
def process_site(prod, sid, ts, data):
"""
I process a dictionary of data for a particular site
"""
localts = get_localtime(sid, ts)
# Insert data into database regardless
for varname in data:
value = data[varname]
deffer = HADSDB.runOperation("""INSERT into raw_inbound
(station, valid, key, value)
VALUES(%s,%s, %s, %s)
""", (sid, ts.strftime("%Y-%m-%d %H:%M+00"), varname, value))
deffer.addErrback(common.email_error, prod.text)
deffer.addErrback(log.err)
key = "%s|%s" % (sid, varname)
cur = CURRENT_QUEUE.setdefault(key, dict(valid=ts, value=value,
dirty=True))
if ts > cur['valid']:
cur['valid'] = ts
cur['value'] = value
cur['dirty'] = True
# Don't bother with stranger locations
if len(sid) == 8 and sid[0] == 'X':
return
# Don't bother with unknown sites
if sid in UNKNOWN:
return
network = get_network(prod, sid, ts, data)
if network in ['KCCI', 'KIMT', 'KELO', 'ISUSM', None]:
return
# Do not send DCP sites with old data to IEMAccess
if network.find("_DCP") > 0 and localts < LOCS.get(
sid, dict()).get('valid', localts):
return
metadata = LOCS.setdefault(sid,
{network: dict(valid=localts, tzname=None)})
metadata[network]['valid'] = localts
# Okay, time for a hack, if our observation is at midnight!
if localts.hour == 0 and localts.minute == 0:
localts -= datetime.timedelta(minutes=1)
# log.msg("Shifting %s [%s] back one minute: %s" % (sid, network,
# localts))
iemob = Observation(sid, network, localts)
iscoop = (network.find('COOP') > 0)
hasdata = False
for var in data:
if data[var] == -9999:
continue
iemvar = MAPPING[var]
if iemvar == '':
continue
hasdata = True
myval = data[var] * MULTIPLIER.get(var[:2], 1.0)
iemob.data[MAPPING[var]] = myval
# Convert 0.001 to 0.0001 for Trace values
if myval == 0.001 and MAPPING[var] in ['pday', 'snow', 'snowd']:
iemob.data[MAPPING[var]] = TRACE_VALUE
if iscoop:
# Save COOP 'at-ob' temperature into summary table
if MAPPING[var] == 'tmpf':
iemob.data['coop_tmpf'] = myval
# Save observation time into the summary table
if MAPPING[var] in ['tmpf', 'max_tmpf', 'min_tmpf', 'pday',
'snow', 'snowd']:
iemob.data['coop_valid'] = iemob.data['valid']
if hasdata:
iemob.data['raw'] = prod.product_id
deffer = ACCESSDB.runInteraction(iemob.save)
deffer.addCallback(got_results, prod.product_id, sid, network,
localts)
deffer.addErrback(common.email_error, prod.text)
deffer.addErrback(log.err)
def main():
"""Do Something"""
pgconn = get_dbconn("iem")
icursor = pgconn.cursor(cursor_factory=psycopg2.extras.DictCursor)
fn = find_file()
nc = ncopen(fn, timeout=300)
stations = chartostring(nc.variables["stationId"][:])
providers = chartostring(nc.variables["dataProvider"][:])
names = chartostring(nc.variables["stationName"][:])
tmpk = nc.variables["temperature"][:]
dwpk = nc.variables["dewpoint"][:]
relh = nc.variables["relHumidity"][:]
# Set some data bounds to keep mcalc from complaining
tmpk = np.where(
np.ma.logical_or(np.ma.less(tmpk, 200), np.ma.greater(tmpk, 320)),
np.nan,
tmpk,
)
dwpk = np.where(
np.ma.logical_or(np.ma.less(dwpk, 200), np.ma.greater(dwpk, 320)),
np.nan,
dwpk,
)
relh = np.where(
np.ma.logical_and(np.ma.less(relh, 100.1), np.ma.greater(relh, 0)),
relh,
np.nan,
)
obtime = nc.variables["observationTime"][:]
pressure = nc.variables["stationPressure"][:]
# altimeter = nc.variables["altimeter"][:]
# slp = nc.variables["seaLevelPressure"][:]
drct = nc.variables["windDir"][:]
smps = nc.variables["windSpeed"][:]
gmps = nc.variables["windGust"][:]
# gmps_drct = nc.variables["windDirMax"][:]
pcpn = nc.variables["precipAccum"][:]
rtk1 = nc.variables["roadTemperature1"][:]
rtk2 = nc.variables["roadTemperature2"][:]
rtk3 = nc.variables["roadTemperature3"][:]
rtk4 = nc.variables["roadTemperature4"][:]
subk1 = nc.variables["roadSubsurfaceTemp1"][:]
nc.close()
db = {}
for recnum, provider in enumerate(providers):
name = names[recnum]
network = provider2network(provider, name)
if network is None:
continue
LOG.debug("provider: %s name: %s network: %s", provider, name, network)
this_station = stations[recnum]
db[this_station] = {}
ticks = obtime[recnum]
ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=ticks)
db[this_station]["ts"] = ts.replace(tzinfo=pytz.utc)
db[this_station]["network"] = network
db[this_station]["pres"] = sanity_check(pressure[recnum], 0, 1000000)
db[this_station]["tmpk"] = sanity_check(tmpk[recnum], 200, 330)
db[this_station]["dwpk"] = sanity_check(dwpk[recnum], 200, 330)
db[this_station]["relh"] = sanity_check(relh[recnum], 0, 100.1)
db[this_station]["drct"] = sanity_check(drct[recnum], -1, 361)
db[this_station]["smps"] = sanity_check(smps[recnum], -1, 200)
db[this_station]["gmps"] = sanity_check(gmps[recnum], -1, 200)
db[this_station]["rtk1"] = sanity_check(rtk1[recnum], 0, 500)
db[this_station]["rtk2"] = sanity_check(rtk2[recnum], 0, 500)
db[this_station]["rtk3"] = sanity_check(rtk3[recnum], 0, 500)
db[this_station]["rtk4"] = sanity_check(rtk4[recnum], 0, 500)
db[this_station]["subk"] = sanity_check(subk1[recnum], 0, 500)
db[this_station]["pday"] = sanity_check(pcpn[recnum], -1, 5000)
for sid in db:
iem = Observation(sid, db[sid]["network"], db[sid]["ts"])
if db[sid]["tmpk"] is not None:
iem.data["tmpf"] = temperature(db[sid]["tmpk"], "K").value("F")
if db[sid]["dwpk"] is not None:
iem.data["dwpf"] = temperature(db[sid]["dwpk"], "K").value("F")
if db[sid]["relh"] is not None and db[sid]["relh"] is not np.ma.masked:
iem.data["relh"] = float(db[sid]["relh"])
if db[sid]["drct"] is not None:
iem.data["drct"] = db[sid]["drct"]
if db[sid]["smps"] is not None:
iem.data["sknt"] = speed(db[sid]["smps"], "MPS").value("KT")
if db[sid]["gmps"] is not None:
iem.data["gust"] = speed(db[sid]["gmps"], "MPS").value("KT")
if db[sid]["pres"] is not None:
iem.data["pres"] = (float(db[sid]["pres"]) / 100.00) * 0.02952
if db[sid]["rtk1"] is not None:
iem.data["tsf0"] = temperature(db[sid]["rtk1"], "K").value("F")
if db[sid]["rtk2"] is not None:
iem.data["tsf1"] = temperature(db[sid]["rtk2"], "K").value("F")
if db[sid]["rtk3"] is not None:
iem.data["tsf2"] = temperature(db[sid]["rtk3"], "K").value("F")
if db[sid]["rtk4"] is not None:
iem.data["tsf3"] = temperature(db[sid]["rtk4"], "K").value("F")
if db[sid]["subk"] is not None:
iem.data["rwis_subf"] = temperature(db[sid]["subk"],
"K").value("F")
if db[sid]["pday"] is not None:
iem.data["pday"] = round(
distance(db[sid]["pday"], "MM").value("IN"), 2)
if not iem.save(icursor):
LOG.info(
"MADIS Extract: %s found new station: %s network: %s"
"",
fn.split("/")[-1],
sid,
db[sid]["network"],
)
subprocess.call("python sync_stations.py %s" % (fn, ), shell=True)
os.chdir("../../dbutil")
subprocess.call("sh SYNC_STATIONS.sh", shell=True)
os.chdir("../ingestors/madis")
LOG.info("...done with sync.")
del iem
icursor.close()
pgconn.commit()
pgconn.close()
def process(ncfn):
"""Process this file """
pgconn = get_dbconn("iem")
icursor = pgconn.cursor()
xref = {}
icursor.execute("SELECT id, network from stations where "
"network ~* 'ASOS' or network = 'AWOS' and country = 'US'")
for row in icursor:
xref[row[0]] = row[1]
icursor.close()
nc = ncopen(ncfn)
data = {}
for vname in [
"stationId",
"observationTime",
"temperature",
"dewpoint",
"altimeter", # Pa
"windDir",
"windSpeed", # mps
"windGust", # mps
"visibility", # m
"precipAccum",
"presWx",
"skyCvr",
"skyCovLayerBase",
"autoRemark",
"operatorRemark",
]:
data[vname] = nc.variables[vname][:]
for qc in ["QCR", "QCD"]:
vname2 = vname + qc
if vname2 in nc.variables:
data[vname2] = nc.variables[vname2][:]
for vname in ["temperature", "dewpoint"]:
data[vname + "C"] = temperature(data[vname], "K").value("C")
data[vname] = temperature(data[vname], "K").value("F")
for vname in ["windSpeed", "windGust"]:
data[vname] = (masked_array(data[vname], units("meter / second")).to(
units("knots")).magnitude)
data["altimeter"] = pressure(data["altimeter"], "PA").value("IN")
data["skyCovLayerBase"] = distance(data["skyCovLayerBase"],
"M").value("FT")
data["visibility"] = distance(data["visibility"], "M").value("MI")
data["precipAccum"] = distance(data["precipAccum"], "MM").value("IN")
stations = chartostring(data["stationId"][:])
presentwxs = chartostring(data["presWx"][:])
skycs = chartostring(data["skyCvr"][:])
autoremarks = chartostring(data["autoRemark"][:])
opremarks = chartostring(data["operatorRemark"][:])
def decision(i, fieldname, tolerance):
"""Our decision if we are going to take a HFMETAR value or not"""
if data[fieldname][i] is np.ma.masked:
return None
if data["%sQCR" % (fieldname, )][i] == 0:
return data[fieldname][i]
# Now we have work to do
departure = np.ma.max(np.ma.abs(data["%sQCD" % (fieldname, )][i, :]))
# print("departure: %s tolerance: %s" % (departure, tolerance))
if departure <= tolerance:
return data[fieldname][i]
return None
for i, sid in tqdm(
enumerate(stations),
total=len(stations),
disable=(not sys.stdout.isatty()),
):
if len(sid) < 3:
continue
sid3 = sid[1:] if sid.startswith("K") else sid
ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(
seconds=data["observationTime"][i])
ts = ts.replace(tzinfo=pytz.UTC)
mtr = "%s %sZ AUTO " % (sid, ts.strftime("%d%H%M"))
network = xref.get(sid3, "ASOS")
iem = Observation(sid3, network, ts)
# 06019G23KT
val = decision(i, "windDir", 15)
if val is not None:
iem.data["drct"] = int(val)
mtr += "%03i" % (iem.data["drct"], )
else:
mtr += "///"
val = decision(i, "windSpeed", 10)
if val is not None:
iem.data["sknt"] = int(val)
mtr += "%02i" % (iem.data["sknt"], )
else:
mtr += "//"
val = decision(i, "windGust", 10)
if val is not None and val > 0:
iem.data["gust"] = int(val)
mtr += "G%02i" % (iem.data["gust"], )
mtr += "KT "
val = decision(i, "visibility", 4)
if val is not None:
iem.data["vsby"] = float(val)
mtr += "%sSM " % (vsbyfmt(iem.data["vsby"]), )
presentwx = presentwxs[i]
if presentwx != "":
# database storage is comma delimited
iem.data["wxcodes"] = presentwx.split(" ")
mtr += "%s " % (presentwx, )
for _i, (skyc,
_l) in enumerate(zip(skycs[i], data["skyCovLayerBase"][i])):
if skyc != "":
iem.data["skyc%s" % (_i + 1, )] = skyc
if skyc != "CLR":
iem.data["skyl%s" % (_i + 1, )] = int(_l)
mtr += "%s%03i " % (skyc, int(_l) / 100)
else:
mtr += "CLR "
t = ""
tgroup = "T"
val = decision(i, "temperature", 10)
if val is not None:
# Recall the pain enabling this
# iem.data['tmpf'] = float(data['temperature'][i])
tmpc = float(data["temperatureC"][i])
t = "%s%02i/" % (
"M" if tmpc < 0 else "",
tmpc if tmpc > 0 else (0 - tmpc),
)
tgroup += "%s%03i" % (
"1" if tmpc < 0 else "0",
(tmpc if tmpc > 0 else (0 - tmpc)) * 10.0,
)
val = decision(i, "dewpoint", 10)
if val is not None:
# iem.data['dwpf'] = float(data['dewpoint'][i])
tmpc = float(data["dewpointC"][i])
if t != "":
t = "%s%s%02i " % (
t,
"M" if tmpc < 0 else "",
tmpc if tmpc > 0 else 0 - tmpc,
)
tgroup += "%s%03i" % (
"1" if tmpc < 0 else "0",
(tmpc if tmpc > 0 else (0 - tmpc)) * 10.0,
)
if len(t) > 4:
mtr += t
val = decision(i, "altimeter", 20)
if val is not None:
iem.data["alti"] = float(round(val, 2))
mtr += "A%4i " % (iem.data["alti"] * 100.0, )
mtr += "RMK "
val = decision(i, "precipAccum", 25)
if val is not None:
if val > 0.009:
iem.data["phour"] = float(round(val, 2))
mtr += "P%04i " % (iem.data["phour"] * 100.0, )
elif val > 0:
# Trace
mtr += "P0000 "
iem.data["phour"] = TRACE_VALUE
if tgroup != "T":
mtr += "%s " % (tgroup, )
if autoremarks[i] != "" or opremarks[i] != "":
mtr += "%s %s " % (autoremarks[i], opremarks[i])
mtr += "MADISHF"
# Eat our own dogfood
try:
Metar.Metar(mtr)
iem.data["raw"] = mtr
except Exception as exp:
print("dogfooding extract_hfmetar %s resulted in %s" % (mtr, exp))
continue
for key in iem.data:
if isinstance(iem.data[key], np.float32):
print("key: %s type: %s" % (key, type(iem.data[key])))
icursor = pgconn.cursor()
if not iem.save(icursor, force_current_log=True, skip_current=True):
print(("extract_hfmetar: unknown station? %s %s %s\n%s") %
(sid3, network, ts, mtr))
icursor.close()
pgconn.commit()
def safeP(v):
v = v.strip()
if v == "T":
return 0.0001
if v == "NA":
return -99
return float(v)
for row in data[1:]:
cols = row.split(",")
sid = cols[header["StationNumber"]].strip()
t = "%s %s" % (cols[header["ObservationDate"]],
cols[header["ObservationTime"]].strip())
ts = datetime.datetime.strptime(t, "%Y-%m-%d %I:%M %p")
lts = lts.replace(year=ts.year, month=ts.month, day=ts.day,
hour=ts.hour, minute=ts.minute)
iem = Observation(sid, "%sCOCORAHS" % (state,), lts)
iem.data['pday'] = safeP(cols[header["TotalPrecipAmt"]])
if cols[header["NewSnowDepth"]].strip() != "NA":
iem.data['snow'] = safeP(cols[header["NewSnowDepth"]])
if cols[header["TotalSnowDepth"]].strip() != "NA":
iem.data['snowd'] = safeP(cols[header["TotalSnowDepth"]])
iem.save(cursor)
del iem
cursor.close()
dbconn.commit()
def process(ncfn):
"""Process this file """
pgconn = get_dbconn('iem')
icursor = pgconn.cursor()
xref = {}
icursor.execute("""SELECT id, network from stations where
network ~* 'ASOS' or network = 'AWOS' and country = 'US'""")
for row in icursor:
xref[row[0]] = row[1]
icursor.close()
nc = ncopen(ncfn)
data = {}
for vname in [
'stationId',
'observationTime',
'temperature',
'dewpoint',
'altimeter', # Pa
'windDir',
'windSpeed', # mps
'windGust',
'visibility', # m
'precipAccum',
'presWx',
'skyCvr',
'skyCovLayerBase',
'autoRemark',
'operatorRemark'
]:
data[vname] = nc.variables[vname][:]
for qc in ['QCR', 'QCD']:
vname2 = vname + qc
if vname2 in nc.variables:
data[vname2] = nc.variables[vname2][:]
for vname in ['temperature', 'dewpoint']:
data[vname + "C"] = temperature(data[vname], 'K').value('C')
data[vname] = temperature(data[vname], 'K').value('F')
for vname in ['windSpeed', 'windGust']:
data[vname] = speed(data[vname], 'MPS').value('KT')
data['altimeter'] = pressure(data['altimeter'], 'PA').value("IN")
data['skyCovLayerBase'] = distance(data['skyCovLayerBase'],
'M').value("FT")
data['visibility'] = distance(data['visibility'], 'M').value("MI")
data['precipAccum'] = distance(data['precipAccum'], 'MM').value("IN")
stations = chartostring(data['stationId'][:])
presentwxs = chartostring(data['presWx'][:])
skycs = chartostring(data['skyCvr'][:])
autoremarks = chartostring(data['autoRemark'][:])
opremarks = chartostring(data['operatorRemark'][:])
def decision(i, fieldname, tolerance):
"""Our decision if we are going to take a HFMETAR value or not"""
if data[fieldname][i] is np.ma.masked:
return None
if data["%sQCR" % (fieldname, )][i] == 0:
return data[fieldname][i]
# Now we have work to do
departure = np.ma.max(np.ma.abs(data['%sQCD' % (fieldname, )][i, :]))
# print("departure: %s tolerance: %s" % (departure, tolerance))
if departure <= tolerance:
return data[fieldname][i]
return None
for i, sid in tqdm(enumerate(stations),
total=len(stations),
disable=(not sys.stdout.isatty())):
sid3 = sid[1:] if sid[0] == 'K' else sid
ts = datetime.datetime(1970, 1, 1) + datetime.timedelta(
seconds=data['observationTime'][i])
ts = ts.replace(tzinfo=pytz.utc)
mtr = "%s %sZ AUTO " % (sid, ts.strftime("%d%H%M"))
network = xref.get(sid3, 'ASOS')
iem = Observation(sid3, network, ts)
# 06019G23KT
val = decision(i, 'windDir', 15)
if val is not None:
iem.data['drct'] = int(val)
mtr += "%03i" % (iem.data['drct'], )
else:
mtr += "///"
val = decision(i, 'windSpeed', 10)
if val is not None:
iem.data['sknt'] = int(val)
mtr += "%02i" % (iem.data['sknt'], )
else:
mtr += "//"
val = decision(i, 'windGust', 10)
if val is not None and val > 0:
iem.data['gust'] = int(val)
mtr += "G%02i" % (iem.data['gust'], )
mtr += "KT "
val = decision(i, 'visibility', 4)
if val is not None:
iem.data['vsby'] = float(val)
mtr += "%sSM " % (vsbyfmt(iem.data['vsby']), )
presentwx = presentwxs[i]
if presentwx != '':
# database storage is comma delimited
iem.data['wxcodes'] = presentwx.split(" ")
mtr += "%s " % (presentwx, )
for _i, (skyc,
_l) in enumerate(zip(skycs[i], data['skyCovLayerBase'][i])):
if skyc != '':
iem.data['skyc%s' % (_i + 1, )] = skyc
if skyc != 'CLR':
iem.data['skyl%s' % (_i + 1, )] = int(_l)
mtr += "%s%03i " % (skyc, int(_l) / 100)
else:
mtr += "CLR "
t = ""
tgroup = "T"
val = decision(i, 'temperature', 10)
if val is not None:
# Recall the pain enabling this
# iem.data['tmpf'] = float(data['temperature'][i])
tmpc = float(data['temperatureC'][i])
t = "%s%02i/" % ("M" if tmpc < 0 else "", tmpc if tmpc > 0 else
(0 - tmpc))
tgroup += "%s%03i" % ("1" if tmpc < 0 else "0",
(tmpc if tmpc > 0 else (0 - tmpc)) * 10.)
val = decision(i, 'dewpoint', 10)
if val is not None:
# iem.data['dwpf'] = float(data['dewpoint'][i])
tmpc = float(data['dewpointC'][i])
if t != "":
t = "%s%s%02i " % (t, "M" if tmpc < 0 else "",
tmpc if tmpc > 0 else 0 - tmpc)
tgroup += "%s%03i" % ("1" if tmpc < 0 else "0",
(tmpc if tmpc > 0 else (0 - tmpc)) * 10.)
if len(t) > 4:
mtr += t
val = decision(i, 'altimeter', 20)
if val is not None:
iem.data['alti'] = float(round(val, 2))
mtr += "A%4i " % (iem.data['alti'] * 100., )
mtr += "RMK "
val = decision(i, 'precipAccum', 25)
if val is not None:
if val >= 0.01:
iem.data['phour'] = float(round(val, 2))
mtr += "P%04i " % (iem.data['phour'] * 100., )
elif val < 0.01 and val > 0:
# Trace
mtr += "P0000 "
iem.data['phour'] = TRACE_VALUE
if tgroup != "T":
mtr += "%s " % (tgroup, )
if autoremarks[i] != '' or opremarks[i] != '':
mtr += "%s %s " % (autoremarks[i], opremarks[i])
mtr += "MADISHF"
# Eat our own dogfood
try:
Metar.Metar(mtr)
iem.data['raw'] = mtr
except Exception as exp:
print("dogfooding extract_hfmetar %s resulted in %s" % (mtr, exp))
continue
for key in iem.data:
if isinstance(iem.data[key], np.float32):
print("key: %s type: %s" % (key, type(iem.data[key])))
icursor = pgconn.cursor()
if not iem.save(icursor, force_current_log=True, skip_current=True):
print(("extract_hfmetar: unknown station? %s %s %s\n%s") %
(sid3, network, ts, mtr))
icursor.close()
pgconn.commit()
fp = "/mesonet/ARCHIVE/data/%s/text/ot/ot0007.dat" % (now.strftime("%Y/%m/%d"),)
if (not os.path.isfile(fp)):
sys.exit(0)
lines = open(fp, 'r').readlines()
line = lines[-1]
# 114,2006,240,1530,18.17,64.70, 88.9,2.675,21.91,1014.3,0.000
tokens = line.split(",")
if (len(tokens) != 11):
sys.exit(0)
hhmm = "%04i" % (int(tokens[3]),)
ts = now.replace(hour= int(hhmm[:2]), minute= int(hhmm[2:]), second = 0,
microsecond=0 , tzinfo=pytz.timezone("America/Chicago"))
iemob = Observation("OT0007", "OT", ts)
iemob.data['tmpf'] = tokens[5]
iemob.data['relh'] = tokens[6]
iemob.data['dwpf'] = mesonet.dwpf(float(iemob.data['tmpf']), float(iemob.data['relh']) )
iemob.data['sknt'] = float(tokens[7]) * 1.94
iemob.data['drct'] = tokens[8]
iemob.data['pres'] = float(tokens[9]) / 960 * 28.36
iemob.save(icursor)
icursor.close()
IEM.commit()
IEM.close()