import sys import interpreter from metar import Metar import metar_scrape # Program entry: if __name__ == "__main__": try: targetStation = sys.argv[1] except IndexError: print("Input 4-letter station id as command line arg") print("Example usage: metar_hub.py ksgf") exit(0) # get metar report and put it into a Metar object report = Metar(metar_scrape.getReport(targetStation)) components = [ interpreter.stationID(report.getStationID()), interpreter.dateTime(report.getDateTime()), interpreter.reportModifier(report.getModifier()), interpreter.windGroup(report.getWind()), interpreter.visibilityGroup(report.getVisibility()), interpreter.runwayVisibilityRange(report.getRunway()), interpreter.skyCondition(report.getSky()), interpreter.tempDewPoint(report.getTempDewPoint()), interpreter.altimeter(report.getAltimeter()) ] print("\n------------------------------------------------------") for component in components:
class GFS(threading.Thread): ''' NOAA GFS download and parse functions. ''' cycles = [0, 6, 12, 18] baseurl = 'https://nomads.ncep.noaa.gov/cgi-bin/filter_gfs_0p50.pl?' params = [ 'leftlon=0', 'rightlon=360', 'toplat=90', 'bottomlat=-90', ] levels = [ '700_mb', # FL100 '600_mb', # FL140 '500_mb', # FL180 '400_mb', # FL235 '300_mb', # FL300 '200_mb', # FL380 '150_mb', # FL443 '100_mb', # FL518 'high_cloud_bottom_level', 'high_cloud_layer', 'high_cloud_top_level', 'low_cloud_bottom_level', 'low_cloud_layer', 'low_cloud_top_level', 'mean_sea_level', 'middle_cloud_bottom_level', 'middle_cloud_layer', 'middle_cloud_top_level', #'surface', ] variables = ['PRES', 'TCDC', 'UGRD', 'VGRD', 'TMP', 'PRMSL', 'RH', ] nwinds, nclouds = 0, 0 downloading = False downloadWait = 0 # wait n seconds to start download lastgrib = False lat, lon, lastlat, lastlon = False, False, False, False cycle = '' lastcycle = '' newGrib = False parsed_latlon = (0, 0) die = threading.Event() dummy = threading.Event() lock = threading.Lock() def __init__(self, conf): self.conf = conf self.lastgrib = self.conf.lastgrib self.wafs = WAFS(conf) self.metar = Metar(conf) threading.Thread.__init__(self) def run(self): # Worker thread while not self.die.wait(self.conf.parserate): if not self.conf.enabled: continue datecycle, cycle, forecast = self.getCycleDate() if self.downloadWait < 1: self.downloadCycle(datecycle, cycle, forecast) elif self.downloadWait > 0: self.downloadWait -= self.conf.parserate # Run WAFS worker self.wafs.run(self.lat, self.lon, self.conf.parserate) # Run Metar worker self.metar.run(self.lat, self.lon, self.conf.parserate) if self.die.isSet(): # Kill downloaders if avaliable if self.wafs and self.wafs.downloading and self.wafs.download: self.wafs.download.die() if self.downloading and self.download: self.download.die() return # flush stdout sys.stdout.flush() def getCycleDate(self): ''' Returns last cycle date avaliable ''' now = datetime.utcnow() #cycle is published with 4 hours 25min delay cnow = now - timedelta(hours=4, minutes=25) #get last cycle for cycle in self.cycles: if cnow.hour >= cycle: lcycle = cycle # Forecast adjs = 0 if cnow.day != now.day: adjs = +24 forecast = (adjs + now.hour - lcycle)/3*3 return ( '%d%02d%02d%02d' % (cnow.year, cnow.month, cnow.day, lcycle), lcycle, forecast) def downloadCycle(self, datecycle, cycle, forecast): ''' Downloads the requested grib file ''' filename = 'gfs.t%02dz.pgrb2full.0p50.f0%02d' % (cycle, forecast) path = os.sep.join([self.conf.cachepath, 'gfs']) cachefile = os.sep.join(['gfs', '%s_%s.grib2' % (datecycle, filename)]) if cachefile == self.lastgrib: # No need to download return if not os.path.exists(path): os.makedirs(path) if self.downloading == True: if not self.download.q.empty(): #Finished downloading lastgrib = self.download.q.get() # Dowload success if lastgrib: if not self.conf.keepOldFiles and self.conf.lastgrib: util.remove(os.sep.join([self.conf.cachepath, self.conf.lastgrib])) self.lastgrib = lastgrib self.conf.lastgrib = self.lastgrib self.newGrib = True #print "new grib file: " + self.lastgrib else: # Wait a minute self.downloadWait = 60 self.downloading = False elif self.conf.download and self.downloadWait < 1: # Download new grib ## Build download url params = self.params; dir = 'dir=%%2Fgfs.%s' % (datecycle) params.append(dir) params.append('file=' + filename) # add variables for level in self.levels: params.append('lev_' + level + '=1') for var in self.variables: params.append('var_' + var + '=1') url = self.baseurl + '&'.join(params) #print('XPGFS: downloading %s' % (url)) self.downloading = True self.download = AsyncDownload(self.conf, url, cachefile) return False def parseGribData(self, filepath, lat, lon): ''' Executes wgrib2 and parses its output ''' args = ['-s', '-lon', '%f' % (lon), '%f' % (lat), os.sep.join([self.conf.cachepath, filepath]) ] if self.conf.spinfo: p = subprocess.Popen([self.conf.wgrib2bin] + args, stdout=subprocess.PIPE, startupinfo=self.conf.spinfo, shell=True) else: p = subprocess.Popen([self.conf.wgrib2bin] + args, stdout=subprocess.PIPE) it = iter(p.stdout) data = {} clouds = {} pressure = False for line in it: r = line[:-1].split(':') # Level, variable, value level, variable, value = [r[4].split(' '), r[3], r[7].split(',')[2].split('=')[1]] if len(level) > 1: if level[1] == 'cloud': #cloud layer clouds.setdefault(level[0], {}) if len(level) > 3 and variable == 'PRES': clouds[level[0]][level[2]] = value else: #level coverage/temperature clouds[level[0]][variable] = value elif level[1] == 'mb': # wind levels data.setdefault(level[0], {}) data[level[0]][variable] = value elif level[0] == 'mean': if variable == 'PRMSL': pressure = c.pa2inhg(float(value)) windlevels = [] cloudlevels = [] # Let data ready to push on datarefs. # Convert wind levels for level in data: wind = data[level] if 'UGRD' in wind and 'VGRD' in wind: hdg, vel = c.c2p(float(wind['UGRD']), float(wind['VGRD'])) #print wind['UGRD'], wind['VGRD'], float(wind['UGRD']), float(wind['VGRD']), hdg, vel alt = c.mb2alt(float(level)) # Optional varialbes temp, rh, dew = False, False, False # Temperature if 'TMP' in wind: temp = float(wind['TMP']) # Relative Humidity if 'RH' in wind: rh = float(wind['RH']) else: temp = False if temp and rh: dew = c.dewpoint(temp, rh) windlevels.append([alt, hdg, c.ms2knots(vel), {'temp': temp, 'rh': rh, 'dew': dew, 'gust': 0}]) #print 'alt: %i rh: %i vis: %i' % (alt, float(wind['RH']), vis) # Convert cloud level for level in clouds: level = clouds[level] if 'top' in level and 'bottom' in level and 'TCDC' in level: top, bottom, cover = float(level['top']), float(level['bottom']), float(level['TCDC']) #print "XPGFS: top: %.0fmbar %.0fm, bottom: %.0fmbar %.0fm %d%%" % (top * 0.01, c.mb2alt(top * 0.01), bottom * 0.01, c.mb2alt(bottom * 0.01), cover) #if bottom > 1 and alt > 1: cloudlevels.append([c.mb2alt(bottom * 0.01) * 0.3048, c.mb2alt(top * 0.01) * 0.3048, cover]) #XP10 #cloudlevels.append((c.mb2alt(bottom * 0.01) * 0.3048, c.mb2alt(top * 0.01) * 0.3048, cover/10)) windlevels.sort() cloudlevels.sort() data = { 'winds': windlevels, 'clouds': cloudlevels, 'pressure': pressure } return data
def __init__(self, conf): self.conf = conf self.lastgrib = self.conf.lastgrib self.wafs = WAFS(conf) self.metar = Metar(conf) threading.Thread.__init__(self)
def report(vis_group): """(Macro) Return Metar object for a report with the vis group.""" return Metar.Metar(sta_time + "09010KT " + vis_group)
class GFS(threading.Thread): """ NOAA GFS download and parse functions. """ cycles = [0, 6, 12, 18] baseurl = "http://nomads.ncep.noaa.gov/cgi-bin/filter_gfs_0p50.pl?" params = ["leftlon=0", "rightlon=360", "toplat=90", "bottomlat=-90"] levels = [ "700_mb", # FL100 "600_mb", # FL140 "500_mb", # FL180 "400_mb", # FL235 "300_mb", # FL300 "200_mb", # FL380 "150_mb", # FL443 "100_mb", # FL518 "high_cloud_bottom_level", "high_cloud_layer", "high_cloud_top_level", "low_cloud_bottom_level", "low_cloud_layer", "low_cloud_top_level", "mean_sea_level", "middle_cloud_bottom_level", "middle_cloud_layer", "middle_cloud_top_level", #'surface', ] variables = ["PRES", "TCDC", "UGRD", "VGRD", "TMP", "PRMSL", "RH"] nwinds, nclouds = 0, 0 downloading = False downloadWait = 0 # wait n seconds to start download lastgrib = False lat, lon, lastlat, lastlon = False, False, False, False cycle = "" lastcycle = "" newGrib = False parsed_latlon = (0, 0) die = threading.Event() dummy = threading.Event() lock = threading.Lock() def __init__(self, conf): self.conf = conf self.lastgrib = self.conf.lastgrib self.wafs = WAFS(conf) self.metar = Metar(conf) threading.Thread.__init__(self) def run(self): # Worker thread while not self.die.wait(self.conf.parserate): if not self.conf.enabled: continue datecycle, cycle, forecast = self.getCycleDate() if self.downloadWait < 1: self.downloadCycle(datecycle, cycle, forecast) elif self.downloadWait > 0: self.downloadWait -= self.conf.parserate # Run WAFS worker self.wafs.run(self.lat, self.lon, self.conf.parserate) # Run Metar worker self.metar.run(self.lat, self.lon, self.conf.parserate) if self.die.isSet(): # Kill downloaders if avaliable if self.wafs and self.wafs.downloading and self.wafs.download: self.wafs.download.die() if self.downloading and self.download: self.download.die() return # flush stdout sys.stdout.flush() def getCycleDate(self): """ Returns last cycle date avaliable """ now = datetime.utcnow() # cycle is published with 4 hours 25min delay cnow = now - timedelta(hours=4, minutes=0) # get last cycle for cycle in self.cycles: if cnow.hour >= cycle: lcycle = cycle # Forecast adjs = 0 if cnow.day != now.day: adjs = +24 forecast = (adjs + now.hour - lcycle) / 3 * 3 return ("%d%02d%02d%02d" % (cnow.year, cnow.month, cnow.day, lcycle), lcycle, forecast) def downloadCycle(self, datecycle, cycle, forecast): """ Downloads the requested grib file """ filename = "gfs.t%02dz.pgrb2full.0p50.f0%02d" % (cycle, forecast) path = os.sep.join([self.conf.cachepath, "gfs"]) cachefile = os.sep.join(["gfs", "%s_%s.grib2" % (datecycle, filename)]) if cachefile == self.lastgrib: # No need to download return if not os.path.exists(path): os.makedirs(path) if self.downloading == True: if not self.download.q.empty(): # Finished downloading lastgrib = self.download.q.get() # Dowload success if lastgrib: if not self.conf.keepOldFiles and self.conf.lastgrib: util.remove(os.sep.join([self.conf.cachepath, self.conf.lastgrib])) self.lastgrib = lastgrib self.conf.lastgrib = self.lastgrib self.newGrib = True # print "new grib file: " + self.lastgrib else: # Wait a minute self.downloadWait = 60 self.downloading = False elif self.conf.download and self.downloadWait < 1: # Download new grib ## Build download url params = self.params dir = "dir=%%2Fgfs.%s" % (datecycle) params.append(dir) params.append("file=" + filename) # add variables for level in self.levels: params.append("lev_" + level + "=1") for var in self.variables: params.append("var_" + var + "=1") url = self.baseurl + "&".join(params) # print 'XPGFS: downloading %s' % (url) self.downloading = True self.download = AsyncDownload(self.conf, url, cachefile) return False def parseGribData(self, filepath, lat, lon): """ Executes wgrib2 and parses its output """ args = ["-s", "-lon", "%f" % (lon), "%f" % (lat), os.sep.join([self.conf.cachepath, filepath])] if self.conf.spinfo: p = subprocess.Popen( [self.conf.wgrib2bin] + args, stdout=subprocess.PIPE, startupinfo=self.conf.spinfo, shell=True ) else: p = subprocess.Popen([self.conf.wgrib2bin] + args, stdout=subprocess.PIPE) it = iter(p.stdout) data = {} clouds = {} pressure = False for line in it: r = line[:-1].split(":") # Level, variable, value level, variable, value = [r[4].split(" "), r[3], r[7].split(",")[2].split("=")[1]] if len(level) > 1: if level[1] == "cloud": # cloud layer clouds.setdefault(level[0], {}) if len(level) > 3 and variable == "PRES": clouds[level[0]][level[2]] = value else: # level coverage/temperature clouds[level[0]][variable] = value elif level[1] == "mb": # wind levels data.setdefault(level[0], {}) data[level[0]][variable] = value elif level[0] == "mean": if variable == "PRMSL": pressure = c.pa2inhg(float(value)) windlevels = [] cloudlevels = [] # Let data ready to push on datarefs. # Convert wind levels for level in data: wind = data[level] if "UGRD" in wind and "VGRD" in wind: hdg, vel = c.c2p(float(wind["UGRD"]), float(wind["VGRD"])) # print wind['UGRD'], wind['VGRD'], float(wind['UGRD']), float(wind['VGRD']), hdg, vel alt = c.mb2alt(float(level)) # Optional varialbes temp, rh, dew = False, False, False # Temperature if "TMP" in wind: temp = float(wind["TMP"]) # Relative Humidity if "RH" in wind: rh = float(wind["RH"]) else: temp = False if temp and rh: dew = c.dewpoint(temp, rh) windlevels.append([alt, hdg, c.ms2knots(vel), {"temp": temp, "rh": rh, "dew": dew, "gust": 0}]) # print 'alt: %i rh: %i vis: %i' % (alt, float(wind['RH']), vis) # Convert cloud level for level in clouds: level = clouds[level] if "top" in level and "bottom" in level and "TCDC" in level: top, bottom, cover = float(level["top"]), float(level["bottom"]), float(level["TCDC"]) # print "XPGFS: top: %.0fmbar %.0fm, bottom: %.0fmbar %.0fm %d%%" % (top * 0.01, c.mb2alt(top * 0.01), bottom * 0.01, c.mb2alt(bottom * 0.01), cover) # if bottom > 1 and alt > 1: cloudlevels.append([c.mb2alt(bottom * 0.01) * 0.3048, c.mb2alt(top * 0.01) * 0.3048, cover]) # XP10 # cloudlevels.append((c.mb2alt(bottom * 0.01) * 0.3048, c.mb2alt(top * 0.01) * 0.3048, cover/10)) windlevels.sort() cloudlevels.sort() data = {"winds": windlevels, "clouds": cloudlevels, "pressure": pressure} return data
def report(wind_group): """(Macro) Return Metar object from parsing the given wind group.""" return Metar.Metar(sta_time + wind_group)
def test_issue51_strict(): """Check that setting strict=False prevents a ParserError""" with warnings.catch_warnings(record=True) as w: report = Metar.Metar(sta_time + "90010KT", strict=False) assert len(w) == 1 assert report.wind_speed is None
def report(mod_group): """(Macro) Return Metar object from parsing the modifier group.""" return Metar.Metar(sta_time + mod_group)
def test_140_parseWind(): """Check parsing of wind groups.""" report = Metar.Metar(sta_time + "09010KT") assert report.decode_completed assert report.wind_dir.value() == 90 assert report.wind_speed.value() == 10 assert report.wind_gust is None assert report.wind_dir_from is None assert report.wind_dir_from is None assert report.wind() == "E at 10 knots" report = Metar.Metar(sta_time + "09010MPS") assert report.decode_completed assert report.wind_speed.value() == 10 assert report.wind_speed.value("KMH") == 36 assert report.wind() == "E at 19 knots" assert report.wind("MPS") == "E at 10 mps" assert report.wind("KMH") == "E at 36 km/h" report = Metar.Metar(sta_time + "09010KMH") assert report.decode_completed assert report.wind_speed.value() == 10 assert report.wind() == "E at 5 knots" assert report.wind("KMH") == "E at 10 km/h" report = Metar.Metar(sta_time + "090010KT") assert report.decode_completed assert report.wind_dir.value() == 90 assert report.wind_speed.value() == 10 report = Metar.Metar(sta_time + "000000KT") assert report.decode_completed assert report.wind_dir.value() == 0 assert report.wind_speed.value() == 0 assert report.wind() == "calm" report = Metar.Metar(sta_time + "VRB03KT") assert report.decode_completed assert report.wind_dir is None assert report.wind_speed.value() == 3 assert report.wind() == "variable at 3 knots" report = Metar.Metar(sta_time + "VRB00KT") assert report.decode_completed assert report.wind() == "calm" report = Metar.Metar(sta_time + "VRB03G40KT") assert report.decode_completed assert report.wind_dir is None assert report.wind_speed.value() == 3 assert report.wind_gust.value() == 40 assert report.wind_dir_from is None assert report.wind_dir_to is None assert report.wind() == "variable at 3 knots, gusting to 40 knots" report = Metar.Metar(sta_time + "21010G30KT") assert report.decode_completed assert report.wind() == "SSW at 10 knots, gusting to 30 knots" report = Metar.Metar(sta_time + "21010KT 180V240") assert report.wind_dir.value() == 210 assert report.wind_speed.value() == 10 assert report.wind_gust is None assert report.wind_dir_from.value() == 180 assert report.wind_dir_to.value() == 240 assert report.wind() == "S to WSW at 10 knots"
def test_041_parseModifier(): """Check parsing of 'modifier' groups.""" assert Metar.Metar(sta_time + "AUTO").mod == "AUTO" assert Metar.Metar(sta_time + "COR").mod == "COR"
def test_040_parseModifier_default(): """Check default 'modifier' value.""" assert Metar.Metar("KEWR").mod == "AUTO"
def raisesParserError(code): """Helper to test the a given code raises a Metar.ParserError.""" with pytest.raises(Metar.ParserError): Metar.Metar(code)
def test_010_parseType_default(): """Check default value of the report type.""" assert Metar.Metar("KEWR").type == "METAR"
def process(ncfn): """Process this file """ IEM = psycopg2.connect(database='iem', host='iemdb') icursor = IEM.cursor() xref = {} icursor.execute("""SELECT id, network from stations where network ~* 'ASOS' or network = 'AWOS' and country = 'US'""") for row in icursor: xref[row[0]] = row[1] icursor.close() nc = netCDF4.Dataset(ncfn) data = {} for vname in [ 'stationId', 'observationTime', 'temperature', 'dewpoint', 'altimeter', # Pa 'windDir', 'windSpeed', # mps 'windGust', 'visibility', # m 'precipAccum', 'presWx', 'skyCvr', 'skyCovLayerBase', 'autoRemark', 'operatorRemark' ]: data[vname] = nc.variables[vname][:] vname += "QCR" if vname in nc.variables: data[vname] = nc.variables[vname][:] for vname in ['temperature', 'dewpoint']: data[vname + "C"] = temperature(data[vname], 'K').value('C') data[vname] = temperature(data[vname], 'K').value('F') for vname in ['windSpeed', 'windGust']: data[vname] = speed(data[vname], 'MPS').value('KT') data['altimeter'] = pressure(data['altimeter'], 'PA').value("IN") data['skyCovLayerBase'] = distance(data['skyCovLayerBase'], 'M').value("FT") data['visibility'] = distance(data['visibility'], 'M').value("MI") data['precipAccum'] = distance(data['precipAccum'], 'MM').value("IN") for i in range(len(data['stationId'])): sid = tostring(data['stationId'][i]) sid3 = sid[1:] if sid[0] == 'K' else sid ts = datetime.datetime(1970, 1, 1) + datetime.timedelta( seconds=data['observationTime'][i]) ts = ts.replace(tzinfo=pytz.timezone("UTC")) mtr = "%s %sZ AUTO " % (sid, ts.strftime("%d%H%M")) network = xref.get(sid3, 'ASOS') iem = Observation(sid3, network, ts.astimezone(TIMEZONES[LOC2TZ.get(sid3, None)])) # 06019G23KT if (data['windDirQCR'][i] == 0 and data['windDir'][i] is not np.ma.masked): iem.data['drct'] = int(data['windDir'][i]) mtr += "%03i" % (iem.data['drct'], ) else: mtr += "///" if (data['windSpeedQCR'][i] == 0 and data['windSpeed'][i] is not np.ma.masked): iem.data['sknt'] = int(data['windSpeed'][i]) mtr += "%02i" % (iem.data['sknt'], ) else: mtr += "//" if (data['windGustQCR'][i] == 0 and data['windGust'][i] is not np.ma.masked and data['windGust'][i] > 0): iem.data['gust'] = int(data['windGust'][i]) mtr += "G%02i" % (iem.data['gust'], ) mtr += "KT " if (data['visibilityQCR'][i] == 0 and data['visibility'][i] is not np.ma.masked): iem.data['vsby'] = float(data['visibility'][i]) mtr += "%sSM " % (vsbyfmt(iem.data['vsby']), ) presentwx = tostring(data['presWx'][i]) if presentwx != '': iem.data['presentwx'] = presentwx mtr += "%s " % (presentwx, ) for _i, (_c, _l) in enumerate( zip(data['skyCvr'][i], data['skyCovLayerBase'][i])): if tostring(_c) != '': skyc = tostring(_c) iem.data['skyc%s' % (_i + 1, )] = skyc if skyc != 'CLR': iem.data['skyl%s' % (_i + 1, )] = int(_l) mtr += "%s%03i " % (tostring(_c), int(_l) / 100) else: mtr += "CLR " t = "" tgroup = "T" if (data['temperatureQCR'][i] == 0 and data['temperature'][i] is not np.ma.masked): # iem.data['tmpf'] = float(data['temperature'][i]) tmpc = float(data['temperatureC'][i]) t = "%s%02i/" % ("M" if tmpc < 0 else "", tmpc if tmpc > 0 else (0 - tmpc)) tgroup += "%s%03i" % ("1" if tmpc < 0 else "0", (tmpc if tmpc > 0 else (0 - tmpc)) * 10.) if (data['dewpointQCR'][i] == 0 and data['dewpoint'][i] is not np.ma.masked): # iem.data['dwpf'] = float(data['dewpoint'][i]) tmpc = float(data['dewpointC'][i]) if t != "": t = "%s%s%02i " % (t, "M" if tmpc < 0 else "", tmpc if tmpc > 0 else 0 - tmpc) tgroup += "%s%03i" % ("1" if tmpc < 0 else "0", (tmpc if tmpc > 0 else (0 - tmpc)) * 10.) if len(t) > 4: mtr += t if (data['altimeterQCR'][i] == 0 and data['altimeter'][i] is not np.ma.masked): iem.data['alti'] = round(data['altimeter'][i], 2) mtr += "A%4i " % (iem.data['alti'] * 100., ) mtr += "RMK " if (data['precipAccumQCR'][i] == 0 and data['precipAccum'][i] is not np.ma.masked): if data['precipAccum'][i] > 0: iem.data['phour'] = round(data['precipAccum'][i], 2) mtr += "P%04i " % (iem.data['phour'] * 100., ) if tgroup != "T": mtr += "%s " % (tgroup, ) autoremark = tostring(data['autoRemark'][i]) opremark = tostring(data['operatorRemark'][i]) if autoremark != '' or opremark != '': mtr += "%s %s " % (autoremark, opremark) mtr += "MADISHF" # Eat our own dogfood try: Metar(mtr) iem.data['raw'] = mtr except: pass icursor = IEM.cursor() if not iem.save(icursor, force_current_log=True, skip_current=True): print(("extract_hfmetar: unknown station? %s %s %s\n%s") % (sid3, network, ts, mtr)) pass icursor.close() IEM.commit()