def main(argv):
"""Go"""
pgconn = get_dbconn('asos')
airforce = int(argv[1])
wban = int(argv[2])
faa = argv[3]
year = int(argv[4])
year2 = int(argv[5])
failedyears = []
msgs = []
for year in tqdm.tqdm(range(year, year2)):
sts = utc(year, 1, 1)
ets = sts.replace(year=year + 1)
cursor = pgconn.cursor()
lfn = "%06i-%05i-%s" % (airforce, wban, year)
if not os.path.isfile(lfn):
fn = "ftp://ftp.ncdc.noaa.gov/pub/data/noaa/%s/%s.gz" % (year, lfn)
subprocess.call("wget -q -O %s.gz %s" % (lfn, fn), shell=True)
if (not os.path.isfile(lfn + ".gz")
or os.path.getsize(lfn + ".gz") == 0):
failedyears.append(year)
continue
subprocess.call("gunzip %s.gz" % (lfn, ),
shell=True,
stderr=subprocess.PIPE)
added = 0
bad = 0
removed = 0
skipped = 0
# ignore any bad bytes, sigh
for line in open(lfn, errors='ignore'):
data = ds3505.parser(line.strip(), faa, add_metar=True)
if data is None:
bad += 1
continue
if added == 0:
dbid = faa if len(faa) == 4 and faa[0] != 'K' else faa[1:]
cursor.execute(
"""
DELETE from alldata where station = %s
and valid >= %s and valid < %s
""", (dbid, sts, ets))
if cursor.rowcount > 0:
print("deleted %s rows for %s" % (cursor.rowcount, dbid))
removed = cursor.rowcount
res = ds3505.sql(cursor, faa, data)
if res is None:
skipped += 1
else:
added += 1
msgs.append(
(" %s: %s added: %s removed: %s bad: %s"
" skipped: %s") % (year, faa, added, removed, bad, skipped))
cursor.close()
pgconn.commit()
year += 1
print(" failed years: %s" % (failedyears, ))
print("\n".join(msgs))
def main(argv):
"""Go"""
pgconn = psycopg2.connect(database='asos', host='iemdb')
airforce = int(argv[1])
wban = int(argv[2])
faa = argv[3]
year = int(argv[4])
year2 = int(argv[5])
failedyears = []
msgs = []
for year in tqdm.tqdm(range(year, year2)):
sts = datetime.datetime(year, 1, 1).replace(tzinfo=pytz.utc)
ets = sts.replace(year=year+1)
cursor = pgconn.cursor()
lfn = "%06i-%05i-%s" % (airforce, wban, year)
if not os.path.isfile(lfn):
fn = "ftp://ftp.ncdc.noaa.gov/pub/data/noaa/%s/%s.gz" % (year, lfn)
subprocess.call("wget -q -O %s.gz %s" % (lfn, fn),
shell=True)
if (not os.path.isfile(lfn+".gz") or
os.path.getsize(lfn+".gz") == 0):
failedyears.append(year)
continue
subprocess.call("gunzip %s.gz" % (lfn, ), shell=True,
stderr=subprocess.PIPE)
added = 0
bad = 0
removed = 0
skipped = 0
for line in open(lfn):
data = ds3505.parser(line.strip(), faa, add_metar=True)
if data is None:
bad += 1
continue
if added == 0:
cursor.execute("""
DELETE from alldata where station = %s
and valid >= %s and valid < %s
""", (faa, sts, ets))
removed = cursor.rowcount
res = ds3505.sql(cursor, faa, data)
if res is None:
skipped += 1
else:
added += 1
msgs.append((" %s: %s added: %s removed: %s bad: %s"
" skipped: %s") % (year, faa, added, removed, bad,
skipped))
cursor.close()
pgconn.commit()
year += 1
print(" failed years: %s" % (failedyears, ))
print("\n".join(msgs))
def test_6hour_temp(dbcursor):
"""6 hour high/low"""
# 2016-08-12 23:53:00
# KAMW 122353Z AUTO 35014G23KT 10SM CLR 25/21 A2983 RMK AO2 SLP092
# 60000 T02500211 10272 20250 55001
msg = ("0271725472949892016081223537+41991-093619FM-15+0291KAMW V03035"
"05N007252200059N0160935N5+02505+02115100925ADDAA101000095AA206"
"000021GA1005+999999999GD10991+9999999GF1009919999999999999999"
"99KA1060M+02721KA2060N+02501MA1101025097575MD1590019+9999"
"OC101185REMMET11708/12/16 17:53:02 METAR KAMW 122353Z "
"35014G23KT 10SM CLR 25/21 A2983 RMK AO2 SLP092 60000 "
"T02500211 10272 20250 55001")
data = ds3505.parser(msg, "KAMW", add_metar=True)
# db schema for testing only goes to 2015
data["valid"] = utc(2011, 1, 12, 23, 53)
ans = ("KAMW 122353Z AUTO 35014G23KT 10SM CLR 25/21 A2983 "
"RMK 60000 SLP092 T02500211 10272 20250 55001 IEM_DS3505")
assert data["metar"] == ans
assert ds3505.sql(dbcursor, "AMW", data) == 1
def test_6hour_temp(dbcursor):
"""6 hour high/low"""
# 2016-08-12 23:53:00
# KAMW 122353Z AUTO 35014G23KT 10SM CLR 25/21 A2983 RMK AO2 SLP092
# 60000 T02500211 10272 20250 55001
msg = (
"0271725472949892016081223537+41991-093619FM-15+0291KAMW V03035"
"05N007252200059N0160935N5+02505+02115100925ADDAA101000095AA206"
"000021GA1005+999999999GD10991+9999999GF1009919999999999999999"
"99KA1060M+02721KA2060N+02501MA1101025097575MD1590019+9999"
"OC101185REMMET11708/12/16 17:53:02 METAR KAMW 122353Z "
"35014G23KT 10SM CLR 25/21 A2983 RMK AO2 SLP092 60000 "
"T02500211 10272 20250 55001"
)
data = parser(msg, 'KAMW', add_metar=True)
# db schema for testing only goes to 2015
data['valid'] = utc(2011, 1, 12, 23, 53)
ans = (
"KAMW 122353Z AUTO 35014G23KT 10SM CLR 25/21 A2983 "
"RMK 60000 SLP092 T02500211 10272 20250 55001 IEM_DS3505")
assert data['metar'] == ans
assert sql(dbcursor, 'AMW', data) == 1
def main(argv):
"""Go"""
pgconn = get_dbconn("asos")
airforce = int(argv[1])
wban = int(argv[2])
faa = argv[3]
year = int(argv[4])
year2 = int(argv[5])
failedyears = []
msgs = []
dbid = faa if len(faa) == 4 and faa[0] != "K" else faa[1:]
for year in tqdm.tqdm(range(year, year2)):
sts = utc(year, 1, 1)
ets = sts.replace(year=year + 1)
cursor = pgconn.cursor()
lfn = "%06i-%05i-%s" % (airforce, wban, year)
if not os.path.isfile("%s/%s" % (TMPDIR, lfn)):
uri = "https://www1.ncdc.noaa.gov/pub/data/noaa/%s/%s.gz" % (
year,
lfn,
)
req = exponential_backoff(requests.get, uri, timeout=30)
if req is None or req.status_code != 200:
LOG.info("Failed to fetch %s", uri)
failedyears.append(year)
continue
with open("%s/%s.gz" % (TMPDIR, lfn), "wb") as fh:
fh.write(req.content)
subprocess.call(
"gunzip %s/%s.gz" % (TMPDIR, lfn),
shell=True,
stderr=subprocess.PIPE,
)
added = 0
bad = 0
removed = 0
skipped = 0
current = []
if ADD_ONLY:
# build out our current obs
cursor.execute(
"""
SELECT valid at time zone 'UTC'
from alldata where station = %s and
valid >= %s and valid < %s ORDER by valid ASC
""",
(dbid, sts, ets),
)
for row in cursor:
current.append(row[0].strftime("%Y%m%d%H%M"))
# ignore any bad bytes, sigh
for line in open("%s/%s" % (TMPDIR, lfn), errors="ignore"):
data = ds3505.parser(line.strip(), faa, add_metar=True)
if data is None:
bad += 1
continue
if added == 0 and not ADD_ONLY:
cursor.execute(
"""
DELETE from alldata where station = %s
and valid >= %s and valid < %s
""",
(dbid, sts, ets),
)
if cursor.rowcount > 0:
print("deleted %s rows for %s" % (cursor.rowcount, dbid))
removed = cursor.rowcount
if ADD_ONLY and data["valid"].strftime("%Y%m%d%H%M") in current:
skipped += 1
continue
res = ds3505.sql(cursor, faa, data)
if res is None:
skipped += 1
else:
added += 1
msgs.append(
(" %s: %s added: %s removed: %s bad: %s"
" skipped: %s") % (year, faa, added, removed, bad, skipped))
cursor.close()
pgconn.commit()
os.unlink("%s/%s" % (TMPDIR, lfn))
LOG.info(" failed years: %s", failedyears)
LOG.info("\n".join(msgs))