def test_dewpoint(self): """ test out computation of dew point """ for t0, r0, a0 in [[80, 80, 73.42], [80, 20, 35.87]]: t = datatypes.temperature(t0, 'F') rh = datatypes.humidity(r0, '%') dwpk = meteorology.dewpoint(t, rh) self.assertAlmostEqual(dwpk.value("F"), a0, 2)
def test_dewpoint(): """ test out computation of dew point """ for t0, r0, a0 in [[80, 80, 73.42], [80, 20, 35.87]]: t = datatypes.temperature(t0, 'F') rh = datatypes.humidity(r0, '%') dwpk = meteorology.dewpoint(t, rh) assert abs(dwpk.value("F") - a0) < 0.01
def test_dewpoint(): """ test out computation of dew point """ for t0, r0, a0 in [[80, 80, 73.42], [80, 20, 35.87]]: t = datatypes.temperature(t0, "F") rh = datatypes.humidity(r0, "%") dwpk = meteorology.dewpoint(t, rh) assert abs(dwpk.value("F") - a0) < 0.01
def test_dewpoint(self): """ test out computation of dew point """ for t0,r0,a0 in [[80,80,73.42], [80,20,35.87]]: t = datatypes.temperature(t0, 'F') rh = datatypes.humidity(r0, '%') dwpk = meteorology.dewpoint(t, rh) self.assertAlmostEqual( dwpk.value("F"), a0, 2)
def hourly_process(nwsli, maxts): """ Process the hourly file """ fn = "%s/%s_HrlySI.dat" % (BASE, STATIONS[nwsli]) df = common_df_logic(fn, maxts, nwsli, "sm_hourly") if df is None: return 0 processed = 0 LOG.debug("processing %s rows from %s", len(df.index), fn) acursor = ACCESS.cursor() for _i, row in df.iterrows(): # Update IEMAccess ob = Observation(nwsli, "ISUSM", row["valid"]) tmpc = temperature(row["tair_c_avg_qc"], "C") if tmpc.value("F") > -50 and tmpc.value("F") < 140: ob.data["tmpf"] = tmpc.value("F") relh = humidity(row["rh_qc"], "%") ob.data["relh"] = relh.value("%") ob.data["dwpf"] = met.dewpoint(tmpc, relh).value("F") ob.data["srad"] = row["slrkw_avg_qc"] ob.data["phour"] = round( distance(row["rain_mm_tot_qc"], "MM").value("IN"), 2 ) ob.data["sknt"] = speed(row["ws_mps_s_wvt_qc"], "MPS").value("KT") if "ws_mph_max" in df.columns: ob.data["gust"] = speed(row["ws_mph_max_qc"], "MPH").value("KT") ob.data["max_gust_ts"] = row["ws_mph_tmx"] ob.data["drct"] = row["winddir_d1_wvt_qc"] if "tsoil_c_avg" in df.columns: ob.data["c1tmpf"] = temperature(row["tsoil_c_avg_qc"], "C").value( "F" ) if "t12_c_avg_qc" in df.columns: ob.data["c2tmpf"] = temperature(row["t12_c_avg_qc"], "C").value( "F" ) if "t24_c_avg_qc" in df.columns: ob.data["c3tmpf"] = temperature(row["t24_c_avg_qc"], "C").value( "F" ) if "t50_c_avg" in df.columns: ob.data["c4tmpf"] = temperature(row["t50_c_avg_qc"], "C").value( "F" ) if "calc_vwc_12_avg" in df.columns: ob.data["c2smv"] = row["calc_vwc_12_avg_qc"] * 100.0 if "calc_vwc_24_avg" in df.columns: ob.data["c3smv"] = row["calc_vwc_24_avg_qc"] * 100.0 if "calc_vwc_50_avg" in df.columns: ob.data["c4smv"] = row["calc_vwc_50_avg_qc"] * 100.0 ob.save(acursor) processed += 1 acursor.close() ACCESS.commit() return processed
def relh(temperature, _dewpoint): """ Compute Relative Humidity based on a temperature and dew point """ # Get temperature in Celsius tmpc = temperature.value("C") dwpc = _dewpoint.value("C") e = 6.112 * np.exp((17.67 * dwpc) / (dwpc + 243.5)) es = 6.112 * np.exp((17.67 * tmpc) / (tmpc + 243.5)) _relh = (e / es) * 100.00 return dt.humidity(_relh, "%")
def relh(temperature, dewpoint): """ Compute Relative Humidity based on a temperature and dew point """ # Get temperature in Celsius tmpc = temperature.value("C") dwpc = dewpoint.value("C") e = 6.112 * np.exp((17.67 * dwpc) / (dwpc + 243.5)) es = 6.112 * np.exp((17.67 * tmpc) / (tmpc + 243.5)) relh = (e / es) * 100.00 return dt.humidity(relh, '%')
def hourly_process(nwsli, maxts): """ Process the hourly file """ fn = "%s/%s_HrlySI.dat" % (BASE, STATIONS[nwsli]) df = common_df_logic(fn, maxts, nwsli, "sm_hourly") if df is None: return 0 processed = 0 LOG.debug("processing %s rows from %s", len(df.index), fn) acursor = ACCESS.cursor() for _i, row in df.iterrows(): # Update IEMAccess # print nwsli, valid ob = Observation(nwsli, 'ISUSM', row['valid']) tmpc = temperature(row['tair_c_avg_qc'], 'C') if tmpc.value('F') > -50 and tmpc.value('F') < 140: ob.data['tmpf'] = tmpc.value('F') relh = humidity(row['rh_qc'], '%') ob.data['relh'] = relh.value('%') ob.data['dwpf'] = met.dewpoint(tmpc, relh).value('F') ob.data['srad'] = row['slrkw_avg_qc'] ob.data['phour'] = round(distance(row['rain_mm_tot_qc'], 'MM').value('IN'), 2) ob.data['sknt'] = speed(row['ws_mps_s_wvt_qc'], 'MPS').value("KT") if 'ws_mph_max' in df.columns: ob.data['gust'] = speed(row['ws_mph_max_qc'], 'MPH').value('KT') ob.data['max_gust_ts'] = row['ws_mph_tmx'] ob.data['drct'] = row['winddir_d1_wvt_qc'] if 'tsoil_c_avg' in df.columns: ob.data['c1tmpf'] = temperature(row['tsoil_c_avg_qc'], 'C').value('F') if 't12_c_avg_qc' in df.columns: ob.data['c2tmpf'] = temperature( row['t12_c_avg_qc'], 'C').value('F') if 't24_c_avg_qc' in df.columns: ob.data['c3tmpf'] = temperature( row['t24_c_avg_qc'], 'C').value('F') if 't50_c_avg' in df.columns: ob.data['c4tmpf'] = temperature(row['t50_c_avg_qc'], 'C').value('F') if 'calc_vwc_12_avg' in df.columns: ob.data['c2smv'] = row['calc_vwc_12_avg_qc'] * 100.0 if 'calc_vwc_24_avg' in df.columns: ob.data['c3smv'] = row['calc_vwc_24_avg_qc'] * 100.0 if 'calc_vwc_50_avg' in df.columns: ob.data['c4smv'] = row['calc_vwc_50_avg_qc'] * 100.0 ob.save(acursor) # print 'soilm_ingest.py station: %s ts: %s hrly updated no data?' % ( # nwsli, valid) processed += 1 acursor.close() ACCESS.commit() return processed
def processfile(icursor, filename): """Process this file""" lines = open("/mesonet/data/incoming/rwis/%s" % (filename, ), "r").readlines() if len(lines) < 2: return heading = lines[0].split(",") cols = lines[1].split(",") data = {} if len(cols) < len(heading): return for i, h in enumerate(heading): if cols[i].strip() != "/": data[h.strip()] = cols[i].strip() nwsli = LOOKUP[filename] if filename in ["portableExportP1.csv", "miniExportIFB.csv"]: ts = datetime.datetime.strptime(data["date_time"][:16], "%Y-%m-%d %H:%M") else: ts = datetime.datetime.strptime(data["date_time"][:-6], "%Y-%m-%d %H:%M") ts = ts.replace(tzinfo=pytz.UTC) iem = Observation(nwsli, "IA_RWIS", ts) if ts.year < 2010: print(("rwis/mini_portable.py file: %s bad timestamp: %s" "") % (filename, data["date_time"])) return iem.load(icursor) # IEM Tracker stuff is missing if data.get("wind_speed", "") != "": iem.data["sknt"] = float(data["wind_speed"]) * 1.17 if data.get("sub", "") != "": iem.data["rwis_subf"] = float(data["sub"]) if data.get("air_temp", "") != "": iem.data["tmpf"] = float(data["air_temp"]) if data.get("pave_temp", "") != "": iem.data["tsf0"] = float(data["pave_temp"]) if data.get("pave_temp2", "") != "": iem.data["tsf1"] = float(data["pave_temp2"]) if data.get("press", "") != "": iem.data["alti"] = float(data["press"]) if data.get("RH", "") != "": if float(data["RH"]) > 1: t = temperature(iem.data["tmpf"], "F") rh = humidity(float(data["RH"]), "%") iem.data["dwpf"] = meteorology.dewpoint(t, rh).value("F") if data.get("wind_dir", "") != "": iem.data["drct"] = float(data["wind_dir"]) iem.save(icursor)
def processfile(icursor, filename): """Process this file""" lines = open("/mesonet/data/incoming/rwis/%s" % (filename, ), 'r').readlines() if len(lines) < 2: return heading = lines[0].split(",") cols = lines[1].split(",") data = {} if len(cols) < len(heading): return for i in range(len(heading)): if cols[i].strip() != "/": data[heading[i].strip()] = cols[i].strip() nwsli = LOOKUP[filename] if filename in ['portableExportP1.csv', 'miniExportIFB.csv']: ts = datetime.datetime.strptime(data['date_time'][:16], '%Y-%m-%d %H:%M') else: ts = datetime.datetime.strptime(data['date_time'][:-6], '%Y-%m-%d %H:%M') ts = ts.replace(tzinfo=pytz.timezone("UTC")) iem = Observation(nwsli, 'IA_RWIS', ts) if ts.year < 2010: print(("rwis/mini_portable.py file: %s bad timestamp: %s" "") % (filename, data['date_time'])) return iem.load(icursor) # IEM Tracker stuff is missing if data.get('wind_speed', '') != '': iem.data['sknt'] = float(data['wind_speed']) * 1.17 if data.get('sub', '') != '': iem.data['rwis_subf'] = float(data['sub']) if data.get('air_temp', '') != '': iem.data['tmpf'] = float(data['air_temp']) if data.get('pave_temp', '') != '': iem.data['tsf0'] = float(data['pave_temp']) if data.get('pave_temp2', '') != '': iem.data['tsf1'] = float(data['pave_temp2']) if data.get('press', '') != '': iem.data['alti'] = float(data['press']) if data.get('RH', '') != '': if float(data['RH']) > 1: t = temperature(iem.data['tmpf'], 'F') rh = humidity(float(data['RH']), '%') iem.data['dwpf'] = meteorology.dewpoint(t, rh).value('F') if data.get('wind_dir', '') != '': iem.data['drct'] = float(data['wind_dir']) iem.save(icursor)
def processfile( fp ): o = open("/mesonet/data/incoming/rwis/%s" % (fp,), 'r').readlines() if len(o) < 2: return heading = o[0].split(",") cols = o[1].split(",") data = {} if len(cols) < len(heading): return for i in range(len(heading)): if cols[i].strip() != "/": data[heading[i].strip()] = cols[i].strip() nwsli = lkp[fp] if fp in ['portableExportP1.csv', 'miniExportIFB.csv']: ts = datetime.datetime.strptime(data['date_time'][:16], '%Y-%m-%d %H:%M') else: ts = datetime.datetime.strptime(data['date_time'][:-6], '%Y-%m-%d %H:%M') ts = ts.replace(tzinfo=pytz.timezone("UTC")) iem = Observation(nwsli, 'IA_RWIS', ts) if ts.year < 2010: print(("rwis/mini_portable.py file: %s bad timestamp: %s" "") % (fp, data['date_time'])) return iem.load(icursor) # IEM Tracker stuff is missing if data.get('wind_speed', '') != '': iem.data['sknt'] = float(data['wind_speed']) * 1.17 if data.get('sub', '') != '': iem.data['rwis_subf'] = float(data['sub']) if data.get('air_temp', '') != '': iem.data['tmpf'] = float(data['air_temp']) if data.get('pave_temp', '') != '': iem.data['tsf0'] = float(data['pave_temp']) if data.get('pave_temp2', '') != '': iem.data['tsf1'] = float(data['pave_temp2']) if data.get('press', '') != '': iem.data['alti'] = float(data['press']) if data.get('RH', '') != '': if float(data['RH']) > 1: t = temperature(iem.data['tmpf'], 'F') rh = humidity(float(data['RH']), '%') iem.data['dwpf'] = meteorology.dewpoint(t, rh).value('F') if data.get('wind_dir', '') != '': iem.data['drct'] = float(data['wind_dir']) iem.save(icursor)
def main(): """Go Main Go""" pgconn = get_dbconn("iem") icursor = pgconn.cursor() now = datetime.datetime.now().replace( tzinfo=pytz.timezone("America/Chicago")) fn = ("/mesonet/ARCHIVE/data/%s/text/ot/ot0006.dat" "") % (now.strftime("%Y/%m/%d"), ) if not os.path.isfile(fn): return lines = open(fn, 'r').readlines() line = lines[-1] # January 17, 2017 02:57 PM # 35.1 35.8 33.4 92 3 351 14 2:13PM 30.03 0.00 1.12 1.12 68.6 36 tokens = line.split(" ") if len(tokens) != 19: sys.exit(0) ts = datetime.datetime.strptime(" ".join(tokens[:5]), "%B %d, %Y %I:%M %p") ts = now.replace(year=ts.year, month=ts.month, day=ts.day, hour=ts.hour, minute=ts.minute) iemob = Observation("OT0006", "OT", ts) iemob.data['tmpf'] = float(tokens[5]) iemob.data['relh'] = float(tokens[8]) tmpf = temperature(iemob.data['tmpf'], 'F') relh = humidity(iemob.data['relh'], '%') iemob.data['dwpf'] = meteorology.dewpoint(tmpf, relh).value('F') iemob.data['sknt'] = speed(float(tokens[9]), 'MPH').value('KT') iemob.data['drct'] = tokens[10] iemob.data['alti'] = float(tokens[13]) iemob.data['pday'] = float(tokens[14]) iemob.save(icursor) icursor.close() pgconn.commit() pgconn.close()
def main(): """Go Main Go""" pgconn = get_dbconn("iem") icursor = pgconn.cursor() now = datetime.datetime.now() fn = ("/mesonet/ARCHIVE/data/%s/text/ot/ot0007.dat" "") % (now.strftime("%Y/%m/%d"), ) if not os.path.isfile(fn): sys.exit(0) lines = open(fn, "r").readlines() line = lines[-1] # 114,2006,240,1530,18.17,64.70, 88.9,2.675,21.91,1014.3,0.000 tokens = line.split(",") if len(tokens) != 11: sys.exit(0) hhmm = "%04i" % (int(tokens[3]), ) ts = now.replace( hour=int(hhmm[:2]), minute=int(hhmm[2:]), second=0, microsecond=0, tzinfo=pytz.timezone("America/Chicago"), ) iemob = Observation("OT0007", "OT", ts) iemob.data["tmpf"] = float(tokens[5]) iemob.data["relh"] = float(tokens[6]) tmpf = temperature(iemob.data["tmpf"], "F") relh = humidity(iemob.data["relh"], "%") iemob.data["dwpf"] = meteorology.dewpoint(tmpf, relh).value("F") iemob.data["sknt"] = float(tokens[7]) * 1.94 iemob.data["drct"] = tokens[8] iemob.data["pres"] = float(tokens[9]) / 960 * 28.36 iemob.save(icursor) icursor.close() pgconn.commit() pgconn.close()
def main(): """Go Main Go""" iemaccess = get_dbconn('iem') cursor = iemaccess.cursor() valid = datetime.datetime.utcnow() valid = valid.replace(tzinfo=pytz.utc) valid = valid.astimezone(pytz.timezone("America/Chicago")) fn = valid.strftime("/mesonet/ARCHIVE/data/%Y/%m/%d/text/ot/ot0010.dat") if not os.path.isfile(fn): sys.exit(0) lines = open(fn, "r").readlines() lastline = lines[-1].strip() tokens = re.split(r"[\s+]+", lastline) if len(tokens) != 20: return tparts = re.split(":", tokens[3]) valid = valid.replace(hour=int(tparts[0]), minute=int(tparts[1]), second=0, microsecond=0) iem = Observation("OT0010", "OT", valid) iem.data['tmpf'] = float(tokens[4]) iem.data['max_tmpf'] = float(tokens[5]) iem.data['min_tmpf'] = float(tokens[6]) iem.data['relh'] = int(tokens[7]) iem.data['dwpf'] = dewpoint(temperature(iem.data['tmpf'], 'F'), humidity(iem.data['relh'], '%')).value("F") iem.data['sknt'] = speed(float(tokens[8]), 'mph').value('KT') iem.data['drct'] = int(tokens[9]) iem.data['max_sknt'] = speed(float(tokens[10]), 'mph').value('KT') iem.data['alti'] = float(tokens[12]) iem.data['pday'] = float(tokens[13]) iem.data['srad'] = float(tokens[18]) iem.save(cursor) cursor.close() iemaccess.commit()
def minute_iemaccess(df): """Process dataframe into iemaccess.""" pgconn = get_dbconn("iem") cursor = pgconn.cursor() for _i, row in df.iterrows(): # Update IEMAccess # print nwsli, valid ob = Observation(row["station"], "ISUSM", row["valid"]) tmpc = temperature(row["tair_c_avg_qc"], "C") if tmpc.value("F") > -50 and tmpc.value("F") < 140: ob.data["tmpf"] = tmpc.value("F") relh = humidity(row["rh_avg_qc"], "%") ob.data["relh"] = relh.value("%") ob.data["dwpf"] = met.dewpoint(tmpc, relh).value("F") # database srad is W/ms2 ob.data["srad"] = row["slrkj_tot_qc"] / 60.0 * 1000.0 ob.data["pcounter"] = row["rain_in_tot_qc"] ob.data["sknt"] = speed(row["ws_mph_s_wvt_qc"], "MPH").value("KT") if "ws_mph_max" in df.columns: ob.data["gust"] = speed(row["ws_mph_max_qc"], "MPH").value("KT") ob.data["drct"] = row["winddir_d1_wvt_qc"] if "tsoil_c_avg" in df.columns: ob.data["c1tmpf"] = temperature(row["tsoil_c_avg_qc"], "C").value( "F" ) ob.data["c2tmpf"] = temperature(row["t12_c_avg_qc"], "C").value("F") ob.data["c3tmpf"] = temperature(row["t24_c_avg_qc"], "C").value("F") if "t50_c_avg" in df.columns: ob.data["c4tmpf"] = temperature(row["t50_c_avg_qc"], "C").value( "F" ) if "calcvwc12_avg" in df.columns: ob.data["c2smv"] = row["calcvwc12_avg_qc"] * 100.0 if "calcvwc24_avg" in df.columns: ob.data["c3smv"] = row["calcvwc24_avg_qc"] * 100.0 if "calcvwc50_avg" in df.columns: ob.data["c4smv"] = row["calcvwc50_avg_qc"] * 100.0 ob.save(cursor) cursor.close() pgconn.commit()
def process(sheets): resdf = pd.DataFrame({ 'precip': sheets['RainOut']['Rain_mm_Tot'], 'tmpf': sheets['TempRHVPOut']['AirT_C_Avg'], 'rh': sheets['TempRHVPOut']['RH'], 'drct': sheets['WindOut']['WindDir_D1_WVT'], 'sknt': sheets['WindOut']['WS_ms_S_WVT'], 'srad': sheets['SolarRad1Out']['Slr_kW_Avg'] }) # Do unit conversion resdf['srad'] = resdf['srad'] * 1000. resdf['precip'] = distance(resdf['precip'], 'MM').value('IN') resdf['tmpf'] = temperature(resdf['tmpf'], 'C').value('F') resdf['dwpf'] = dewpoint(temperature(resdf['tmpf'], 'F'), humidity(resdf['rh'], '%')).value('F') resdf['sknt'] = speed(resdf['sknt'], 'MPS').value('KT') print(resdf.describe()) minval = resdf.index.min() maxval = resdf.index.max() cursor = pgconn.cursor() cursor.execute( """DELETE from weather_data_obs WHERE valid between '%s-06' and '%s-06' and station = 'HICKS.P' """ % (minval.strftime("%Y-%m-%d %H:%M"), maxval.strftime("%Y-%m-%d %H:%M"))) print("DELETED %s rows between %s and %s" % (cursor.rowcount, minval, maxval)) for valid, row in resdf.iterrows(): if pd.isnull(valid): continue cursor.execute( """INSERT into weather_data_obs (station, valid, tmpf, dwpf, drct, precip, srad, sknt) VALUES ('HICKS.P', %s, %s, %s, %s, %s, %s, %s) """, (valid.strftime("%Y-%m-%d %H:%M-06"), row['tmpf'], row['dwpf'], row['drct'], row['precip'], row['srad'], row['sknt'])) cursor.close() pgconn.commit()
def minute_iemaccess(df): """Process dataframe into iemaccess.""" pgconn = get_dbconn('iem') cursor = pgconn.cursor() for _i, row in df.iterrows(): # Update IEMAccess # print nwsli, valid ob = Observation(row['station'], 'ISUSM', row['valid']) tmpc = temperature(row['tair_c_avg_qc'], 'C') if tmpc.value('F') > -50 and tmpc.value('F') < 140: ob.data['tmpf'] = tmpc.value('F') relh = humidity(row['rh_avg_qc'], '%') ob.data['relh'] = relh.value('%') ob.data['dwpf'] = met.dewpoint(tmpc, relh).value('F') # database srad is W/ms2 ob.data['srad'] = row['slrkj_tot_qc'] / 60. * 1000. ob.data['pcounter'] = row['rain_in_tot_qc'] ob.data['sknt'] = speed(row['ws_mph_s_wvt_qc'], 'MPH').value("KT") if 'ws_mph_max' in df.columns: ob.data['gust'] = speed(row['ws_mph_max_qc'], 'MPH').value('KT') ob.data['drct'] = row['winddir_d1_wvt_qc'] if 'tsoil_c_avg' in df.columns: ob.data['c1tmpf'] = temperature(row['tsoil_c_avg_qc'], 'C').value('F') ob.data['c2tmpf'] = temperature(row['t12_c_avg_qc'], 'C').value('F') ob.data['c3tmpf'] = temperature(row['t24_c_avg_qc'], 'C').value('F') if 't50_c_avg' in df.columns: ob.data['c4tmpf'] = temperature(row['t50_c_avg_qc'], 'C').value('F') if 'calcvwc12_avg' in df.columns: ob.data['c2smv'] = row['calcvwc12_avg_qc'] * 100.0 if 'calcvwc24_avg' in df.columns: ob.data['c3smv'] = row['calcvwc24_avg_qc'] * 100.0 if 'calcvwc50_avg' in df.columns: ob.data['c4smv'] = row['calcvwc50_avg_qc'] * 100.0 ob.save(cursor) cursor.close() pgconn.commit()
def process(sheets): resdf = pd.DataFrame({ 'precip': sheets['RainOut']['Rain_mm_Tot'], 'tmpf': sheets['TempRHVPOut']['AirT_C_Avg'], 'rh': sheets['TempRHVPOut']['RH'], 'drct': sheets['WindOut']['WindDir_D1_WVT'], 'sknt': sheets['WindOut']['WS_ms_S_WVT'], 'srad': sheets['SolarRad1Out']['Slr_kW_Avg']}) # Do unit conversion resdf['srad'] = resdf['srad'] * 1000. resdf['precip'] = distance(resdf['precip'], 'MM').value('IN') resdf['tmpf'] = temperature(resdf['tmpf'], 'C').value('F') resdf['dwpf'] = dewpoint(temperature(resdf['tmpf'], 'F'), humidity(resdf['rh'], '%')).value('F') resdf['sknt'] = speed(resdf['sknt'], 'MPS').value('KT') print(resdf.describe()) minval = resdf.index.min() maxval = resdf.index.max() cursor = pgconn.cursor() cursor.execute("""DELETE from weather_data_obs WHERE valid between '%s-06' and '%s-06' and station = 'HICKS.P' """ % (minval.strftime("%Y-%m-%d %H:%M"), maxval.strftime("%Y-%m-%d %H:%M"))) print("DELETED %s rows between %s and %s" % (cursor.rowcount, minval, maxval)) for valid, row in resdf.iterrows(): if pd.isnull(valid): continue cursor.execute("""INSERT into weather_data_obs (station, valid, tmpf, dwpf, drct, precip, srad, sknt) VALUES ('HICKS.P', %s, %s, %s, %s, %s, %s, %s) """, (valid.strftime("%Y-%m-%d %H:%M-06"), row['tmpf'], row['dwpf'], row['drct'], row['precip'], row['srad'], row['sknt'])) cursor.close() pgconn.commit()
def write_grids(fp, valid, fhour): """Do the write to disk""" gribfn = "%s/%sF%03i.grib2" % (TMP, valid.strftime("%Y%m%d%H%M"), fhour) if not os.path.isfile(gribfn): print("Skipping write_grids because of missing fn: %s" % (gribfn,)) return gribs = pygrib.open(gribfn) grids = dict() for grib in gribs: grids[grib.name] = grib d = dict() if '2 metre temperature' in grids: g = grids['2 metre temperature'] if G['LATS'] is None: G['LATS'], G['LONS'] = g.latlons() vals = temperature(g.values, 'K').value('C') nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()), vals.flatten()) d['tmpc'] = nn(XI, YI) if 'Relative humidity' in grids: g = grids['Relative humidity'] vals = g.values nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()), vals.flatten()) rh = nn(XI, YI) d['dwpc'] = dewpoint(temperature(d['tmpc'], 'C'), humidity(rh, '%')).value('C') if ('10 metre U wind component' in grids and '10 metre V wind component' in grids): u = grids['10 metre U wind component'].values v = grids['10 metre V wind component'].values vals = ((u ** 2) + (v ** 2)) ** 0.5 nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()), vals.flatten()) d['smps'] = nn(XI, YI) vals = drct(speed(u, 'MPS'), speed(v, 'MPS')).value('deg') nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()), vals.flatten()) d['drct'] = nn(XI, YI) if 'Total Precipitation' in grids: vals = grids['Total Precipitation'].values nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()), vals.flatten()) d['pcpn'] = nn(XI, YI) if 'Visibility' in grids: vals = grids['Visibility'].values / 1000. # km nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()), vals.flatten()) d['vsby'] = nn(XI, YI) fp.write("""{"forecast_hour": "%03i", "gids": [ """ % (fhour,)) fmt = ('{"gid": %s, "tmpc": %s, "dwpc": %s, ' '"smps": %s, "drct": %s, "vsby": %s, "pcpn": %s}') i = 1 ar = [] def f(label, row, col, fmt): if label not in d: return 'null' return fmt % d[label][row, col] for row in range(len(YAXIS)): for col in range(len(XAXIS)): ar.append(fmt % (i, f('tmpc', row, col, '%.2f'), f('dwpc', row, col, '%.2f'), f('smps', row, col, '%.1f'), f('drct', row, col, '%i'), f('vsby', row, col, '%.3f'), f('pcpn', row, col, '%.2f'))) i += 1 fp.write(",\n".join(ar)) fp.write("]}%s\n" % ("," if fhour != 84 else '',))
def m15_process(nwsli, maxts): """ Process the 15minute file """ fn = "%s/%s_Min15SI.dat" % (BASE, STATIONS[nwsli]) if not os.path.isfile(fn): return lines = open(fn).readlines() if len(lines) < 5: return # Read header.... headers = [] for col in lines[1].strip().replace('"', '').split(","): headers.append(VARCONV.get(col.lower(), col.lower())) # Read data processed = 0 for i in range(len(lines) - 1, 3, -1): tokens = lines[i].strip().replace('"', '').split(",") if len(tokens) != len(headers): continue valid = make_time(tokens[headers.index('timestamp')]) if valid <= maxts: break gust_valid = make_time(tokens[headers.index('ws_mph_tmx')]) # print valid, tokens[ headers.index('timestamp')] # We are ready for dbinserting, we duplicate the data for the _qc # column dbcols = ("station,valid,%s,%s") % (",".join(headers[2:]), ",".join( ["%s_qc" % (h, ) for h in headers[2:]])) dbvals = "'%s','%s-06'," % (nwsli, valid.strftime("%Y-%m-%d %H:%M:%S")) for v in tokens[2:]: dbvals += "%s," % (formatter(v), ) for v in tokens[2:]: dbvals += "%s," % (formatter(v), ) sql = "INSERT into sm_15minute (%s) values (%s)" % (dbcols, dbvals[:-1]) icursor.execute(sql) # Update IEMAccess # print nwsli, valid ob = Observation(nwsli, 'ISUSM', valid.astimezone(pytz.timezone("America/Chicago"))) tmpc = temperature(float(tokens[headers.index('tair_c_avg')]), 'C') if tmpc.value('F') > -50 and tmpc.value('F') < 140: ob.data['tmpf'] = tmpc.value('F') relh = humidity(float(tokens[headers.index('rh')]), '%') ob.data['relh'] = relh.value('%') ob.data['dwpf'] = met.dewpoint(tmpc, relh).value('F') ob.data['srad'] = tokens[headers.index('slrkw_avg')] ob.data['phour'] = round( distance(float(tokens[headers.index('rain_mm_tot')]), 'MM').value('IN'), 2) ob.data['sknt'] = float(tokens[headers.index('ws_mps_s_wvt')]) * 1.94 ob.data['gust'] = float(tokens[headers.index('ws_mph_max')]) / 1.15 ob.data['max_gust_ts'] = "%s-06" % ( gust_valid.strftime("%Y-%m-%d %H:%M:%S"), ) ob.data['drct'] = float(tokens[headers.index('winddir_d1_wvt')]) ob.data['c1tmpf'] = temperature( float(tokens[headers.index('tsoil_c_avg')]), 'C').value('F') ob.data['c2tmpf'] = temperature( float(tokens[headers.index('t12_c_avg')]), 'C').value('F') ob.data['c3tmpf'] = temperature( float(tokens[headers.index('t24_c_avg')]), 'C').value('F') ob.data['c4tmpf'] = temperature( float(tokens[headers.index('t50_c_avg')]), 'C').value('F') ob.data['c2smv'] = float(tokens[headers.index('vwc_12_avg')]) * 100.0 ob.data['c3smv'] = float(tokens[headers.index('vwc_24_avg')]) * 100.0 ob.data['c4smv'] = float(tokens[headers.index('vwc_50_avg')]) * 100.0 ob.save(accesstxn, force_current_log=True) # print 'soilm_ingest.py station: %s ts: %s hrly updated no data?' % ( # nwsli, valid) processed += 1 return processed
if not os.path.isfile(fn): sys.exit(0) lines = open(fn, 'r').readlines() line = lines[-1] # 114,2006,240,1530,18.17,64.70, 88.9,2.675,21.91,1014.3,0.000 tokens = line.split(",") if (len(tokens) != 11): sys.exit(0) hhmm = "%04i" % (int(tokens[3]),) ts = now.replace(hour=int(hhmm[:2]), minute=int(hhmm[2:]), second=0, microsecond=0, tzinfo=pytz.timezone("America/Chicago")) iemob = Observation("OT0007", "OT", ts) iemob.data['tmpf'] = float(tokens[5]) iemob.data['relh'] = float(tokens[6]) tmpf = temperature(iemob.data['tmpf'], 'F') relh = humidity(iemob.data['relh'], '%') iemob.data['dwpf'] = meteorology.dewpoint(tmpf, relh).value('F') iemob.data['sknt'] = float(tokens[7]) * 1.94 iemob.data['drct'] = tokens[8] iemob.data['pres'] = float(tokens[9]) / 960 * 28.36 iemob.save(icursor) icursor.close() IEM.commit() IEM.close()
def m15_process(nwsli, maxts): """ Process the 15minute file """ fn = "%s/%s_Min15SI.dat" % (BASE, STATIONS[nwsli]) if not os.path.isfile(fn): return lines = open(fn).readlines() if len(lines) < 5: return # Read header.... headers = [] for col in lines[1].strip().replace('"', '').split(","): headers.append(VARCONV.get(col.lower(), col.lower())) # Read data processed = 0 for i in range(len(lines)-1, 3, -1): tokens = lines[i].strip().replace('"', '').split(",") if len(tokens) != len(headers): continue valid = make_time(tokens[headers.index('timestamp')]) if valid <= maxts: break gust_valid = make_time(tokens[headers.index('ws_mph_tmx')]) # print valid, tokens[ headers.index('timestamp')] # We are ready for dbinserting, we duplicate the data for the _qc # column dbcols = ("station,valid,%s,%s" ) % (",".join(headers[2:]), ",".join(["%s_qc" % (h,) for h in headers[2:]])) dbvals = "'%s','%s-06'," % (nwsli, valid.strftime("%Y-%m-%d %H:%M:%S")) for v in tokens[2:]: dbvals += "%s," % (formatter(v),) for v in tokens[2:]: dbvals += "%s," % (formatter(v),) sql = "INSERT into sm_15minute (%s) values (%s)" % (dbcols, dbvals[:-1]) icursor.execute(sql) # Update IEMAccess # print nwsli, valid ob = Observation(nwsli, 'ISUSM', valid.astimezone(pytz.timezone("America/Chicago"))) tmpc = temperature(float(tokens[headers.index('tair_c_avg')]), 'C') if tmpc.value('F') > -50 and tmpc.value('F') < 140: ob.data['tmpf'] = tmpc.value('F') relh = humidity(float(tokens[headers.index('rh')]), '%') ob.data['relh'] = relh.value('%') ob.data['dwpf'] = met.dewpoint(tmpc, relh).value('F') ob.data['srad'] = tokens[headers.index('slrkw_avg')] ob.data['phour'] = round( distance( float(tokens[headers.index('rain_mm_tot')]), 'MM').value('IN'), 2) ob.data['sknt'] = float(tokens[headers.index('ws_mps_s_wvt')]) * 1.94 ob.data['gust'] = float(tokens[headers.index('ws_mph_max')]) / 1.15 ob.data['max_gust_ts'] = "%s-06" % ( gust_valid.strftime("%Y-%m-%d %H:%M:%S"),) ob.data['drct'] = float(tokens[headers.index('winddir_d1_wvt')]) ob.data['c1tmpf'] = temperature( float(tokens[headers.index('tsoil_c_avg')]), 'C').value('F') ob.data['c2tmpf'] = temperature( float(tokens[headers.index('t12_c_avg')]), 'C').value('F') ob.data['c3tmpf'] = temperature( float(tokens[headers.index('t24_c_avg')]), 'C').value('F') ob.data['c4tmpf'] = temperature( float(tokens[headers.index('t50_c_avg')]), 'C').value('F') ob.data['c2smv'] = float(tokens[headers.index('vwc_12_avg')]) * 100.0 ob.data['c3smv'] = float(tokens[headers.index('vwc_24_avg')]) * 100.0 ob.data['c4smv'] = float(tokens[headers.index('vwc_50_avg')]) * 100.0 ob.save(accesstxn, force_current_log=True) # print 'soilm_ingest.py station: %s ts: %s hrly updated no data?' % ( # nwsli, valid) processed += 1 return processed
lines = open(fn, "r").readlines() lastLine = lines[-1].strip() tokens = re.split("[\s+]+", lastLine) if len(tokens) != 20: sys.exit(0) tparts = re.split(":", tokens[3]) valid = valid.replace(hour=int(tparts[0]), minute=int(tparts[1]), second=0, microsecond=0) iem = Observation("OT0010", "OT", valid) iem.data['tmpf'] = float(tokens[4]) iem.data['max_tmpf'] = float(tokens[5]) iem.data['min_tmpf'] = float(tokens[6]) iem.data['relh'] = int(tokens[7]) iem.data['dwpf'] = dewpoint(temperature(iem.data['tmpf'], 'F'), humidity(iem.data['relh'], '%')).value("F") iem.data['sknt'] = speed(float(tokens[8]), 'mph').value('KT') iem.data['drct'] = int(tokens[9]) iem.data['max_sknt'] = speed(float(tokens[10]), 'mph').value('KT') iem.data['alti'] = float(tokens[12]) iem.data['pday'] = float(tokens[13]) iem.data['srad'] = float(tokens[18]) iem.save(cursor) cursor.close() iemaccess.commit()
lines = open(fn, 'r').readlines() line = lines[-1] # 114,2006,240,1530,18.17,64.70, 88.9,2.675,21.91,1014.3,0.000 tokens = line.split(",") if (len(tokens) != 11): sys.exit(0) hhmm = "%04i" % (int(tokens[3]), ) ts = now.replace(hour=int(hhmm[:2]), minute=int(hhmm[2:]), second=0, microsecond=0, tzinfo=pytz.timezone("America/Chicago")) iemob = Observation("OT0007", "OT", ts) iemob.data['tmpf'] = float(tokens[5]) iemob.data['relh'] = float(tokens[6]) tmpf = temperature(iemob.data['tmpf'], 'F') relh = humidity(iemob.data['relh'], '%') iemob.data['dwpf'] = meteorology.dewpoint(tmpf, relh).value('F') iemob.data['sknt'] = float(tokens[7]) * 1.94 iemob.data['drct'] = tokens[8] iemob.data['pres'] = float(tokens[9]) / 960 * 28.36 iemob.save(icursor) icursor.close() IEM.commit() IEM.close()
def hourly_process(nwsli, maxts): """ Process the hourly file """ #print '-------------- HOURLY PROCESS ---------------' fn = "%s%s" % (BASE, STATIONS[nwsli]['hourly'].split("/")[1]) if not os.path.isfile(fn): return lines = open(fn).readlines() if len(lines) < 5: return # Read header.... headers = [] for col in lines[1].strip().replace('"', '').split(","): headers.append(VARCONV.get(col.lower(), col.lower())) # Read data processed = 0 for i in range(len(lines)-1, 3, -1): tokens = lines[i].strip().replace('"','').split(",") if len(tokens) != len(headers): continue valid = datetime.datetime.strptime(tokens[ headers.index('timestamp')], '%Y-%m-%d %H:%M:%S') valid = valid.replace(tzinfo=pytz.FixedOffset(-360)) if valid <= maxts: break #print valid, tokens[ headers.index('timestamp')] # We are ready for dbinserting! dbcols = "station,valid," + ",".join(headers[2:]) dbvals = "'%s','%s-06'," % (nwsli, valid.strftime("%Y-%m-%d %H:%M:%S")) for v in tokens[2:]: dbvals += "%s," % (formatter(v),) sql = "INSERT into sm_hourly (%s) values (%s)" % (dbcols, dbvals[:-1]) icursor.execute(sql) # Update IEMAccess #print nwsli, valid ob = Observation(nwsli, 'ISUSM', valid.astimezone(pytz.timezone("America/Chicago"))) tmpc = temperature(float(tokens[headers.index('tair_c_avg')]), 'C') if tmpc.value('F') > -50 and tmpc.value('F') < 140: ob.data['tmpf'] = tmpc.value('F') relh = humidity(float(tokens[headers.index('rh')]), '%') ob.data['relh'] = relh.value('%') ob.data['dwpf'] = met.dewpoint(tmpc, relh).value('F') ob.data['srad'] = tokens[headers.index('slrkw_avg')] ob.data['phour'] = round( float(tokens[headers.index('rain_mm_tot')]) / 24.5, 2) ob.data['sknt'] = float(tokens[headers.index('ws_mps_s_wvt')]) * 1.94 ob.data['gust'] = float(tokens[headers.index('ws_mph_max')]) / 1.15 ob.data['drct'] = float(tokens[headers.index('winddir_d1_wvt')]) ob.data['c1tmpf'] = temperature( float(tokens[headers.index('tsoil_c_avg')]), 'C').value('F') ob.data['c2tmpf'] = temperature( float(tokens[headers.index('t12_c_avg')]), 'C').value('F') ob.data['c3tmpf'] = temperature( float(tokens[headers.index('t24_c_avg')]), 'C').value('F') ob.data['c4tmpf'] = temperature( float(tokens[headers.index('t50_c_avg')]), 'C').value('F') ob.data['c2smv'] = float(tokens[headers.index('vwc_12_avg')]) * 100.0 ob.data['c3smv'] = float(tokens[headers.index('vwc_24_avg')]) * 100.0 ob.data['c4smv'] = float(tokens[headers.index('vwc_50_avg')]) * 100.0 ob.save(accesstxn) # print 'soilm_ingest.py station: %s ts: %s hrly updated no data?' % ( # nwsli, valid) processed += 1 return processed
def write_grids(fp, valid, fhour): """Do the write to disk""" gribfn = "%s/%sF%03i.grib2" % (TMP, valid.strftime("%Y%m%d%H%M"), fhour) if not os.path.isfile(gribfn): print("Skipping write_grids because of missing fn: %s" % (gribfn, )) return gribs = pygrib.open(gribfn) grids = dict() for grib in gribs: grids[grib.name] = grib d = dict() if '2 metre temperature' in grids: g = grids['2 metre temperature'] if G['LATS'] is None: G['LATS'], G['LONS'] = g.latlons() vals = temperature(g.values, 'K').value('C') nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()), vals.flatten()) d['tmpc'] = nn(XI, YI) if 'Relative humidity' in grids: g = grids['Relative humidity'] vals = g.values nn = NearestNDInterpolator( (G['LONS'].flatten(), G['LATS'].flatten()), vals.flatten()) rh = nn(XI, YI) d['dwpc'] = dewpoint(temperature(d['tmpc'], 'C'), humidity(rh, '%')).value('C') if ('10 metre U wind component' in grids and '10 metre V wind component' in grids): u = grids['10 metre U wind component'].values v = grids['10 metre V wind component'].values vals = ((u**2) + (v**2))**0.5 nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()), vals.flatten()) d['smps'] = nn(XI, YI) vals = drct(speed(u, 'MPS'), speed(v, 'MPS')).value('deg') nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()), vals.flatten()) d['drct'] = nn(XI, YI) if 'Total Precipitation' in grids: vals = grids['Total Precipitation'].values nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()), vals.flatten()) d['pcpn'] = nn(XI, YI) if 'Visibility' in grids: vals = grids['Visibility'].values / 1000. # km nn = NearestNDInterpolator((G['LONS'].flatten(), G['LATS'].flatten()), vals.flatten()) d['vsby'] = nn(XI, YI) fp.write("""{"forecast_hour": "%03i", "gids": [ """ % (fhour, )) fmt = ('{"gid": %s, "tmpc": %s, "dwpc": %s, ' '"smps": %s, "drct": %s, "vsby": %s, "pcpn": %s}') i = 1 ar = [] def f(label, row, col, fmt): if label not in d: return 'null' return fmt % d[label][row, col] for row in range(len(YAXIS)): for col in range(len(XAXIS)): ar.append( fmt % (i, f('tmpc', row, col, '%.2f'), f('dwpc', row, col, '%.2f'), f('smps', row, col, '%.1f'), f('drct', row, col, '%i'), f('vsby', row, col, '%.3f'), f('pcpn', row, col, '%.2f'))) i += 1 fp.write(",\n".join(ar)) fp.write("]}%s\n" % ("," if fhour != 84 else '', ))
def main(): """Go Main Go""" valid = [] pres = [] sped = [] drct = [] tmpf = [] dwpf = [] pday = [] lines = [] now = datetime.datetime(2018, 4, 13, 0, 0) for line in open('jefferson.txt'): if line.find("Min") > 0 or line.find("Max") > 0: continue ts = datetime.datetime.strptime(line[7:21], '%H:%M %m/%d/%y') if len(lines) == 0: now = ts if ts == now: lines.append(line) continue # Spread obs now = ts delta = 60.0 / len(lines) for line in lines: ts += datetime.timedelta(seconds=delta) t = float(line[42:45]) rhval = float(line[47:50]) tmpf.append(t) dwpf.append(dewpoint(temperature(t, 'F'), humidity(rhval, '%')).value("F")) pres.append(float(line[52:57]) * 33.86375) sped.append(float(line[26:28])) pday.append(float(line[59:64])) drct.append(float(txt2drct[line[22:25].strip()]) - 20.0) # Clock was not in CDT valid.append(ts + datetime.timedelta(minutes=60)) lines = [] sts = datetime.datetime(2018, 4, 13, 17, 0) ets = datetime.datetime(2018, 4, 13, 18, 0) interval = datetime.timedelta(minutes=15) xticks = [] xticklabels = [] now = sts while now < ets: fmt = ":%M" if now.minute == 0 or not xticks: fmt = "%-I:%M %p" xticks.append(now) xticklabels.append(now.strftime(fmt)) now += interval fig, ax = plt.subplots(3, 1, sharex=True) """ fancybox = mpatches.FancyBboxPatch( [mx.DateTime.DateTime(2012,5,3,8,6).ticks(), 40], 60*18, 50, boxstyle='round', alpha=0.2, facecolor='#7EE5DE', edgecolor='#EEEEEE', zorder=1) ax[0].add_patch( fancybox) """ ax[0].plot(valid, np.array(tmpf), label='Air') # print("%s %s" % (tmpf, max(tmpf))) ax[0].plot(valid, np.array(dwpf), color='g', label='Dew Point') # ax[0].set_ylim(40, 92) ax[1].scatter(valid, drct, zorder=2) ax2 = ax[1].twinx() ax2.plot(valid, sped, 'r') ax2.set_ylabel("Wind Speed [mph]") ax[2].plot(valid, pres, 'b', label="Pressure [mb]") ax[2].set_ylim(1006, 1014) #ax3 = ax[2].twinx() #ax3.plot(valid, pday, 'r') #ax3.set_ylabel("Daily Precip [inch]") ax[0].set_xticks(xticks) ax[0].set_xticklabels(xticklabels) ax[0].set_xlim(sts, ets) prop = matplotlib.font_manager.FontProperties(size=12) ax[0].legend(loc=2, prop=prop, ncol=2) ax[2].legend(loc=1, prop=prop) ax[0].grid(True) ax[1].grid(True) ax[2].grid(True) ax[0].set_title("Jefferson SchoolNet8 Site (~ $\Delta$6 second data)") ax[2].set_xlabel("Afternoon of 13 Apr 2018") #ax[2].set_ylabel("Pressure [mb]") #[t.set_color('red') for t in ax3.yaxis.get_ticklines()] #[t.set_color('red') for t in ax3.yaxis.get_ticklabels()] [t.set_color('red') for t in ax2.yaxis.get_ticklines()] [t.set_color('red') for t in ax2.yaxis.get_ticklabels()] [t.set_color('b') for t in ax[2].yaxis.get_ticklines()] [t.set_color('b') for t in ax[2].yaxis.get_ticklabels()] #[t.set_color('b') for t in ax[0].yaxis.get_ticklines()] #[t.set_color('b') for t in ax[0].yaxis.get_ticklabels()] [t.set_color('b') for t in ax[1].yaxis.get_ticklines()] [t.set_color('b') for t in ax[1].yaxis.get_ticklabels()] ax[1].set_ylabel("Wind Direction") ax[0].set_ylabel("Temperature $^{\circ}\mathrm{F}$") ax[1].set_yticks(np.arange(0, 361, 90)) ax[1].set_ylim(0, 361) ax[1].set_yticklabels(['N', 'E', 'S', 'W', 'N']) ax[2].set_ylim(min(pres) - 2, max(pres) + 4) fig.savefig('test.png')