def parse_page(page):
lines = filter(clean_line, page.split("\n"))
records = map(get_date_and_standard_metar, lines)
data = {
"observation_time": [],
"temperature": [],
"dew_point": [],
"pressure": [],
"humidity": [],
}
for observation_time, raw_metar in records:
try:
m = Metar(raw_metar)
except ParserError as err:
logger.error(
"Error parsing METAR: %s - %s",
observation_time,
raw_metar,
exc_info=True,
)
continue
temperature = m.temp.value()
dew_point = m.dewpt.value()
data["observation_time"].append(observation_time)
data["temperature"].append(temperature)
data["dew_point"].append(dew_point)
data["pressure"].append(m.press.value())
data["humidity"].append(humidity(temperature, dew_point))
return pd.DataFrame.from_dict(data)
def main():
"""Go Main Go"""
pgconn = get_dbconn("asos")
icursor = pgconn.cursor()
icursor.execute("SET TIME ZONE 'UTC'")
icursor2 = pgconn.cursor()
sts = datetime.datetime(2011, 1, 1)
ets = datetime.datetime(2012, 1, 1)
interval = datetime.timedelta(days=1)
now = sts
while now < ets:
icursor.execute("""
select valid, station, metar from t%s
where metar is not null and valid >= '%s' and valid < '%s'
""" % (now.year, now.strftime("%Y-%m-%d"),
(now+interval).strftime("%Y-%m-%d")))
total = 0
for row in icursor:
try:
mtr = Metar(row[2], row[0].month, row[0].year)
except Exception as _exp:
continue
sql = 'update t%s SET ' % (now.year,)
if mtr.max_temp_6hr:
sql += "max_tmpf_6hr = %s," % (mtr.max_temp_6hr.value("F"),)
if mtr.min_temp_6hr:
sql += "min_tmpf_6hr = %s," % (mtr.min_temp_6hr.value("F"),)
if mtr.max_temp_24hr:
sql += "max_tmpf_24hr = %s," % (mtr.max_temp_24hr.value("F"),)
if mtr.min_temp_24hr:
sql += "min_tmpf_24hr = %s," % (mtr.min_temp_24hr.value("F"),)
if mtr.precip_3hr:
sql += "p03i = %s," % (mtr.precip_3hr.value("IN"),)
if mtr.precip_6hr:
sql += "p06i = %s," % (mtr.precip_6hr.value("IN"),)
if mtr.precip_24hr:
sql += "p24i = %s," % (mtr.precip_24hr.value("IN"),)
if mtr.press_sea_level:
sql += "mslp = %s," % (mtr.press_sea_level.value("MB"),)
if mtr.weather:
pwx = []
for x in mtr.weather:
pwx.append(("").join([a for a in x if a is not None]))
sql += "presentwx = '%s'," % ((",".join(pwx))[:24], )
if sql == "update t%s SET " % (now.year,):
continue
sql = "%s WHERE station = '%s' and valid = '%s'" % (sql[:-1],
row[1],
row[0])
# print sql
icursor2.execute(sql)
total += 1
if total % 1000 == 0:
print('Done total: %s now: %s' % (total, now))
pgconn.commit()
now += interval
icursor2.close()
pgconn.commit()
pgconn.close()
def parse_page(page):
lines = filter(clean_line, page.split('\n'))
records = map(get_date_and_standard_metar, lines)
data = {
'observation_time': [],
'temperature': [],
'dew_point': [],
'pressure': [],
'humidity': [],
}
for observation_time, raw_metar in records:
try:
m = Metar(raw_metar)
except ParserError as err:
logger.error(
'Error parsing METAR: %s - %s',
observation_time,
raw_metar,
exc_info=True,
)
continue
temperature = m.temp.value()
dew_point = m.dewpt.value()
data['observation_time'].append(observation_time)
data['temperature'].append(temperature)
data['dew_point'].append(dew_point)
data['pressure'].append(m.press.value())
data['humidity'].append(humidity(temperature, dew_point))
return pd.DataFrame.from_dict(data)
def process_metar(mstr, now):
""" Do the METAR Processing """
mtr = None
while mtr is None:
try:
mtr = Metar(mstr, now.month, now.year)
except MetarParserError as exp:
try:
msg = str(exp)
except Exception as exp:
return None
if msg.find("day is out of range for month") > 0 and now.day == 1:
now -= datetime.timedelta(days=1)
continue
tokens = ERROR_RE.findall(str(exp))
orig_mstr = mstr
if tokens:
for token in tokens[0].split():
mstr = mstr.replace(" %s" % (token, ), "")
if orig_mstr == mstr:
print("Can't fix badly formatted metar: " + mstr)
return None
else:
print("MetarParserError: "+msg)
print(" --> now: %s month: %s, year: %s" % (now, now.month,
now.year))
sys.exit()
return None
except Exception, exp:
print("Double Fail: %s %s" % (mstr, exp))
return None
def process_metar(mstr, now):
""" Do the METAR Processing """
mtr = None
while mtr is None:
try:
mtr = Metar(mstr, now.month, now.year)
except MetarParserError as exp:
try:
msg = str(exp)
except Exception as exp:
return None
tokens = ERROR_RE.findall(str(exp))
orig_mstr = mstr
if tokens:
for token in tokens[0].split():
mstr = mstr.replace(" %s" % (token, ), "")
if orig_mstr == mstr:
print("Can't fix badly formatted metar: " + mstr)
return None
else:
print("MetarParserError: " + msg)
return None
except Exception, exp:
print("Double Fail: %s %s" % (mstr, exp))
return None
def test_set_weather_from_metar(metar, out, weather_test_file, out_file):
in_metar = Metar(metar)
with Miz(weather_test_file) as miz:
_randomize_weather(miz.mission)
miz.zip(out_file)
emiz.weather.mizfile.set_weather_from_metar(metar, out_file, out_file)
out_metar = Metar(
emiz.weather.mizfile.get_metar_from_mission(out_file, 'UGTB'))
assert in_metar.temp.value() == out_metar.temp.value()
assert int(
in_metar.wind_speed.value('MPS')) == out_metar.wind_speed.value('MPS')
assert in_metar.wind_dir.value() == out_metar.wind_dir.value()
with Miz(out_file) as miz:
assert miz.mission.weather.wind_at_ground_level_dir == emiz.weather.utils.reverse_direction(
in_metar.wind_dir.value())
assert miz.mission.weather.wind_at2000_speed >= miz.mission.weather.wind_at_ground_level_speed
assert miz.mission.weather.wind_at8000_speed >= miz.mission.weather.wind_at_ground_level_speed
for key in out:
if isinstance(out[key], tuple):
assert out[key][0] <= getattr(miz.mission.weather,
key) <= out[key][1]
else:
assert getattr(miz.mission.weather, key) == out[key], key
def weather_today_job():
# request data from NOAA server (METAR of Nancy-Essey Airport)
r = requests.get(
'http://tgftp.nws.noaa.gov/data/observations/metar/stations/LFSN.TXT',
timeout=10.0,
headers={'User-Agent': USER_AGENT})
# check error
if r.status_code == 200:
# extract METAR message
metar_msg = r.content.decode().split('\n')[1]
# METAR parse
obs = Metar(metar_msg)
# init and populate d_today dict
d_today = {}
# message date and time
if obs.time:
d_today['update_iso'] = obs.time.strftime('%Y-%m-%dT%H:%M:%SZ')
d_today['update_fr'] = dt_utc_to_local(
obs.time).strftime('%H:%M %d/%m')
# current temperature
if obs.temp:
d_today['temp'] = round(obs.temp.value('C'))
# current dew point
if obs.dewpt:
d_today['dewpt'] = round(obs.dewpt.value('C'))
# current pressure
if obs.press:
d_today['press'] = round(obs.press.value('hpa'))
# current wind speed
if obs.wind_speed:
d_today['w_speed'] = round(obs.wind_speed.value('KMH'))
# current wind gust
if obs.wind_gust:
d_today['w_gust'] = round(obs.wind_gust.value('KMH'))
# current wind direction
if obs.wind_dir:
# replace 'W'est by 'O'uest
d_today['w_dir'] = obs.wind_dir.compass().replace('W', 'O')
# weather status str
d_today['descr'] = 'n/a'
# store to redis
DB.main.set_as_json('json:weather:today:nancy', d_today, ex=2 * 3600)
def add_ground_wind(airport_code, aloft_wind):
"""
Add the wind information at the ground for the given station code.
"""
raw = requests.get(BASE_ADDR.format(airport_code.upper()))
if raw.status_code == 404:
raw = requests.get(BASE_ADDR.format('K' + airport_code.upper()))
if raw.status_code == 404:
raise ValueError('The given station code is invalid.')
ground_wind = {'altitude': 0, 'direction': 0, 'speed': 0}
for line in raw.text.split('\n'):
if airport_code.upper() in line:
line = line.strip()
observation = Metar(line)
ground_wind['direction'] = observation.wind_dir.value()
ground_wind['speed'] = observation.wind_speed.value(units='KT')
return [ground_wind] + aloft_wind
def process_metar(mstr, now):
""" Do the METAR Processing """
mtr = None
while mtr is None:
try:
mtr = Metar(mstr, now.month, now.year)
except MetarParserError as exp:
try:
msg = str(exp)
except Exception as exp:
return None
if msg.find("day is out of range for month") > 0 and now.day == 1:
now -= datetime.timedelta(days=1)
continue
tokens = ERROR_RE.findall(str(exp))
orig_mstr = mstr
if tokens:
for token in tokens[0].split():
mstr = mstr.replace(" %s" % (token, ), "")
if orig_mstr == mstr:
print("Can't fix badly formatted metar: " + mstr)
return None
else:
print("MetarParserError: " + msg)
print(" --> now: %s month: %s, year: %s" %
(now, now.month, now.year))
sys.exit()
return None
except Exception as exp:
print("Double Fail: %s %s" % (mstr, exp))
return None
if mtr is None or mtr.time is None:
return None
ob = OB()
ob.metar = mstr[:254]
gts = datetime.datetime(
mtr.time.year,
mtr.time.month,
mtr.time.day,
mtr.time.hour,
mtr.time.minute,
)
gts = gts.replace(tzinfo=pytz.UTC)
# When processing data on the last day of the month, we get GMT times
# for the first of this month
if gts.day == 1 and now.day > 10:
tm = now + datetime.timedelta(days=1)
gts = gts.replace(year=tm.year, month=tm.month, day=tm.day)
ob.valid = gts
if mtr.temp:
ob.tmpf = mtr.temp.value("F")
if mtr.dewpt:
ob.dwpf = mtr.dewpt.value("F")
if mtr.wind_speed:
ob.sknt = mtr.wind_speed.value("KT")
if mtr.wind_gust:
ob.gust = mtr.wind_gust.value("KT")
if mtr.wind_dir and mtr.wind_dir.value() != "VRB":
ob.drct = mtr.wind_dir.value()
if mtr.vis:
ob.vsby = mtr.vis.value("SM")
if mtr.press:
ob.alti = mtr.press.value("IN")
if mtr.press_sea_level:
ob.mslp = mtr.press_sea_level.value("MB")
if mtr.precip_1hr:
ob.p01i = mtr.precip_1hr.value("IN")
# Do something with sky coverage
for i in range(len(mtr.sky)):
(c, h, _) = mtr.sky[i]
setattr(ob, "skyc%s" % (i + 1), c)
if h is not None:
setattr(ob, "skyl%s" % (i + 1), h.value("FT"))
if mtr.max_temp_6hr:
ob.max_tmpf_6hr = mtr.max_temp_6hr.value("F")
if mtr.min_temp_6hr:
ob.min_tmpf_6hr = mtr.min_temp_6hr.value("F")
if mtr.max_temp_24hr:
ob.max_tmpf_24hr = mtr.max_temp_24hr.value("F")
if mtr.min_temp_24hr:
ob.min_tmpf_6hr = mtr.min_temp_24hr.value("F")
if mtr.precip_3hr:
ob.p03i = mtr.precip_3hr.value("IN")
if mtr.precip_6hr:
ob.p06i = mtr.precip_6hr.value("IN")
if mtr.precip_24hr:
ob.p24i = mtr.precip_24hr.value("IN")
# Presentwx
if mtr.weather:
pwx = []
for x in mtr.weather:
pwx.append(("").join([a for a in x if a is not None]))
ob.wxcodes = pwx
return ob
def process_data(self):
try:
self.log.info('Processing Metar data...')
with open(os.path.join(os.path.dirname(__file__), 'metar/stations.json')) as in_file:
icao = json.load(in_file)
now = arrow.utcnow()
hour = now.hour
minute = now.minute
if minute < 45:
current_cycle = hour
else:
current_cycle = hour + 1 % 24
stations = {}
for cycle in (current_cycle-1, current_cycle):
file = f'http://tgftp.nws.noaa.gov/data/observations/metar/cycles/{cycle:02d}Z.TXT'
self.log.info(f"Processing '{file}' ...")
request = requests.get(file, stream=True, timeout=(self.connect_timeout, self.read_timeout))
for line in request.iter_lines():
if line:
data = line.decode('iso-8859-1')
try:
# Is this line a date with format "2017/05/12 23:55" ?
arrow.get(data, 'YYYY/MM/DD HH:mm')
continue
# Catch also ValueError because https://github.com/crsmithdev/arrow/issues/535
except (arrow.parser.ParserError, ValueError):
try:
metar = Metar(data, strict=False)
# wind_dir could be NONE if 'dir' is 'VRB'
if metar.wind_speed:
if metar.station_id not in stations:
stations[metar.station_id] = {}
key = arrow.get(metar.time).timestamp
stations[metar.station_id][key] = metar
except Exception as e:
self.log.warn(f'Error while parsing METAR data: {e}')
continue
for metar_id in stations:
metar = next(iter(stations[metar_id].values()))
try:
name, short_name, default_name, lat, lon, altitude, tz = None, None, None, None, None, None, None
checkwx_key = f'metar/checkwx/{metar.station_id}'
if not self.redis.exists(checkwx_key):
try:
self.log.info('Calling api.checkwx.com...')
request = requests.get(
f'https://api.checkwx.com/station/{metar.station_id}',
headers={'Accept': 'application/json', 'X-API-Key': self.checkwx_api_key},
timeout=(self.connect_timeout, self.read_timeout)
)
if request.status_code == 401:
raise UsageLimitException(request.json()['errors'][0]['message'])
elif request.status_code == 429:
raise UsageLimitException('api.checkwx.com rate limit exceeded')
try:
checkwx_data = request.json()['data'][0]
if 'icao' not in checkwx_data:
raise ProviderException('Invalid CheckWX data')
except (ValueError, KeyError):
checkwx_json = request.json()
messages = []
if type(checkwx_json['data']) is list:
messages.extend(checkwx_json['data'])
else:
messages.append(checkwx_json['data'])
raise ProviderException(f'CheckWX API error: {",".join(messages)}')
self.add_redis_key(checkwx_key, {
'data': json.dumps(checkwx_data),
'date': arrow.now().format('YYYY-MM-DD HH:mm:ssZZ'),
}, self.checkwx_cache_duration)
except TimeoutError as e:
raise e
except UsageLimitException as e:
self.add_redis_key(checkwx_key, {
'error': repr(e),
'date': arrow.now().format('YYYY-MM-DD HH:mm:ssZZ'),
}, self.usage_limit_cache_duration)
except Exception as e:
if not isinstance(e, ProviderException):
self.log.exception('Error while getting CheckWX data')
else:
self.log.warn(f'Error while getting CheckWX data: {e}')
self.add_redis_key(checkwx_key, {
'error': repr(e),
'date': arrow.now().format('YYYY-MM-DD HH:mm:ssZZ'),
}, self.checkwx_cache_duration)
if not self.redis.hexists(checkwx_key, 'error'):
checkwx_data = json.loads(self.redis.hget(checkwx_key, 'data'))
station_type = checkwx_data.get('type', None)
if station_type:
name = f'{checkwx_data["name"]} {station_type}'
else:
name = checkwx_data['name']
city = checkwx_data.get('city', None)
if city:
if station_type:
short_name = f'{city} {station_type}'
else:
short_name = city
else:
default_name = checkwx_data['name']
lat = checkwx_data['latitude']['decimal']
lon = checkwx_data['longitude']['decimal']
tz = checkwx_data['timezone']['tzid']
elevation = checkwx_data.get('elevation', None)
altitude = None
if elevation:
if 'meters' in elevation:
altitude = Q_(elevation['meters'], ureg.meters)
elif 'feet' in elevation:
altitude = Q_(elevation['feet'], ureg.feet)
if metar.station_id in icao:
lat = lat or icao[metar.station_id]['lat']
lon = lon or icao[metar.station_id]['lon']
default_name = default_name or icao[metar.station_id]['name']
station = self.save_station(
metar.station_id,
short_name,
name,
lat,
lon,
Status.GREEN,
altitude=altitude,
tz=tz,
url=os.path.join(self.provider_url, f'site?id={metar.station_id}&db=metar'),
default_name=default_name or f'{metar.station_id} Airport',
lookup_name=f'{metar.station_id} Airport ICAO')
station_id = station['_id']
if metar.station_id not in icao:
self.log.warn(f"Missing '{metar.station_id}' ICAO in database. Is it '{station['name']}'?")
measures_collection = self.measures_collection(station_id)
new_measures = []
for key in stations[metar_id]:
metar = stations[metar_id][key]
if not self.has_measure(measures_collection, key):
temp = self.get_quantity(metar.temp, self.temperature_units)
dew_point = self.get_quantity(metar.dewpt, self.temperature_units)
humidity = self.compute_humidity(dew_point, temp)
measure = self.create_measure(
station,
key,
self.get_direction(metar.wind_dir),
self.get_quantity(metar.wind_speed, self.speed_units),
self.get_quantity(metar.wind_gust or metar.wind_speed, self.speed_units),
temperature=temp,
humidity=humidity,
pressure=Pressure(qfe=None,
qnh=self.get_quantity(metar.press, self.pressure_units),
qff=self.get_quantity(metar.press_sea_level, self.pressure_units))
)
new_measures.append(measure)
self.insert_new_measures(measures_collection, station, new_measures)
except ProviderException as e:
self.log.warn(f"Error while processing station '{metar_id}': {e}")
except Exception as e:
self.log.exception(f"Error while processing station '{metar_id}': {e}")
except Exception as e:
self.log.exception(f'Error while processing Metar: {e}')
self.log.info('Done !')
def process_metar(mstr, now):
""" Do the METAR Processing """
mtr = None
while mtr is None:
try:
mtr = Metar(mstr, now.month, now.year)
except MetarParserError as exp:
try:
msg = str(exp)
except Exception as exp:
return None
tokens = ERROR_RE.findall(str(exp))
orig_mstr = mstr
if tokens:
for token in tokens[0].split():
mstr = mstr.replace(" %s" % (token, ), "")
if orig_mstr == mstr:
print("Can't fix badly formatted metar: " + mstr)
return None
else:
print("MetarParserError: "+msg)
return None
except Exception as exp:
print("Double Fail: %s %s" % (mstr, exp))
return None
if mtr is None or mtr.time is None:
return None
ob = OB()
ob.metar = mstr[:254]
ob.valid = now
if mtr.temp:
ob.tmpf = mtr.temp.value("F")
if mtr.dewpt:
ob.dwpf = mtr.dewpt.value("F")
if mtr.wind_speed:
ob.sknt = mtr.wind_speed.value("KT")
if mtr.wind_gust:
ob.gust = mtr.wind_gust.value("KT")
if mtr.wind_dir and mtr.wind_dir.value() != "VRB":
ob.drct = mtr.wind_dir.value()
if mtr.vis:
ob.vsby = mtr.vis.value("SM")
# see pull request #38
if mtr.press and mtr.press != mtr.press_sea_level:
ob.alti = mtr.press.value("IN")
if mtr.press_sea_level:
ob.mslp = mtr.press_sea_level.value("MB")
if mtr.precip_1hr:
ob.p01i = mtr.precip_1hr.value("IN")
# Do something with sky coverage
for i in range(len(mtr.sky)):
(c, h, _) = mtr.sky[i]
setattr(ob, 'skyc%s' % (i+1), c)
if h is not None:
setattr(ob, 'skyl%s' % (i+1), h.value("FT"))
if mtr.max_temp_6hr:
ob.max_tmpf_6hr = mtr.max_temp_6hr.value("F")
if mtr.min_temp_6hr:
ob.min_tmpf_6hr = mtr.min_temp_6hr.value("F")
if mtr.max_temp_24hr:
ob.max_tmpf_24hr = mtr.max_temp_24hr.value("F")
if mtr.min_temp_24hr:
ob.min_tmpf_6hr = mtr.min_temp_24hr.value("F")
if mtr.precip_3hr:
ob.p03i = mtr.precip_3hr.value("IN")
if mtr.precip_6hr:
ob.p06i = mtr.precip_6hr.value("IN")
if mtr.precip_24hr:
ob.p24i = mtr.precip_24hr.value("IN")
# Presentwx
if mtr.weather:
pwx = []
for x in mtr.weather:
pwx.append(("").join([a for a in x if a is not None]))
ob.presentwx = (",".join(pwx))[:24]
return ob
def process_metar(mstr, now):
""" Do the METAR Processing """
mtr = None
while mtr is None:
try:
mtr = Metar(mstr, now.month, now.year)
except MetarParserError as exp:
try:
msg = str(exp)
except Exception as exp:
return None
tokens = ERROR_RE.findall(str(exp))
orig_mstr = mstr
if tokens:
for token in tokens[0].split():
mstr = mstr.replace(" %s" % (token, ), "")
if orig_mstr == mstr:
print("Can't fix badly formatted metar: " + mstr)
return None
else:
print("MetarParserError: " + msg)
return None
except Exception as exp:
print("Double Fail: %s %s" % (mstr, exp))
return None
if mtr is None or mtr.time is None:
return None
ob = OB()
ob.metar = mstr[:254]
ob.valid = now
if mtr.temp:
ob.tmpf = mtr.temp.value("F")
if mtr.dewpt:
ob.dwpf = mtr.dewpt.value("F")
if mtr.wind_speed:
ob.sknt = mtr.wind_speed.value("KT")
if mtr.wind_gust:
ob.gust = mtr.wind_gust.value("KT")
# Calc some stuff
if ob.tmpf is not None and ob.dwpf is not None:
ob.relh = relative_humidity_from_dewpoint(
ob.tmpf * units('degF'),
ob.dwpf * units('degF')).to(units('percent')).magnitude
if ob.sknt is not None:
ob.feel = apparent_temperature(ob.tmpf * units('degF'),
ob.relh * units('percent'),
ob.sknt * units('knots')).to(
units('degF')).magnitude
if mtr.wind_dir and mtr.wind_dir.value() != "VRB":
ob.drct = mtr.wind_dir.value()
if mtr.vis:
ob.vsby = mtr.vis.value("SM")
# see pull request #38
if mtr.press and mtr.press != mtr.press_sea_level:
ob.alti = mtr.press.value("IN")
if mtr.press_sea_level:
ob.mslp = mtr.press_sea_level.value("MB")
if mtr.precip_1hr:
ob.p01i = mtr.precip_1hr.value("IN")
# Do something with sky coverage
for i in range(len(mtr.sky)):
(c, h, _) = mtr.sky[i]
setattr(ob, 'skyc%s' % (i + 1), c)
if h is not None:
setattr(ob, 'skyl%s' % (i + 1), h.value("FT"))
if mtr.max_temp_6hr:
ob.max_tmpf_6hr = mtr.max_temp_6hr.value("F")
if mtr.min_temp_6hr:
ob.min_tmpf_6hr = mtr.min_temp_6hr.value("F")
if mtr.max_temp_24hr:
ob.max_tmpf_24hr = mtr.max_temp_24hr.value("F")
if mtr.min_temp_24hr:
ob.min_tmpf_6hr = mtr.min_temp_24hr.value("F")
if mtr.precip_3hr:
ob.p03i = mtr.precip_3hr.value("IN")
if mtr.precip_6hr:
ob.p06i = mtr.precip_6hr.value("IN")
if mtr.precip_24hr:
ob.p24i = mtr.precip_24hr.value("IN")
# Presentwx
if mtr.weather:
pwx = []
for wx in mtr.weather:
val = "".join([a for a in wx if a is not None])
if val == "" or val == len(val) * "/":
continue
pwx.append(val)
ob.wxcodes = pwx
return ob
def parseMetar(self, metarString):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
self.metar = Metar(metarString, strict=False)
def loopRun(self):
# Get sim data.
self.getPyuipcData()
# Get best suitable Airport.
self.getAirport()
# Handle if no airport found.
if self.airport is None:
self.logger.info(
'No airport found, sleeping for {} seconds...'.format(
self.SLEEP_TIME))
return self.SLEEP_TIME
else:
self.logger.info('Airport: {}.'.format(self.airport))
# Get whazzup file
if not self.debug:
self.getWhazzupText()
else:
self.getWhazzupTextDebug()
# Read whazzup text and get a station.
self.parseWhazzupText()
# Check if station online.
if self.atisRaw is not None:
self.logger.info('Station found, decoding Atis.')
else:
# Actions, if no station online.
self.logger.info('No station online, using metar only.')
with warnings.catch_warnings():
warnings.simplefilter("ignore")
self.metar = Metar(self.getAirportMetar(), strict=False)
self.parseVoiceMetar()
# Parse atis voice with metar only.
self.atisVoice = '{}, {}.'.format(
self.airportInfos[self.airport][3], self.metarVoice)
# Read the metar.
self.readVoice()
return self.SLEEP_TIME
# Parse ATIS.
# Information.
self.getInfoIdentifier()
self.parseVoiceInformation()
# Metar.
if not self.ivac2:
self.parseMetar(self.atisRaw[2].strip())
else:
for ar in self.atisRaw:
if ar.startswith('METAR'):
self.parseMetar(ar.replace('METAR ', '').strip())
break
self.parseVoiceMetar()
# Runways / TRL / TA
self.parseRawRwy()
self.parseVoiceRwy()
# comment.
self.parseVoiceComment()
# Compose complete atis voice string.
self.atisVoice = '{} {} {} {} Information {}, out.'.format(
self.informationVoice, self.rwyVoice, self.commentVoice,
self.metarVoice, self.informationIdentifier)
# Read the string.
self.readVoice()
# After successful reading.
return 0
def process_data(self):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
try:
self.log.info("Processing Metar data...")
with open(
os.path.join(os.path.dirname(__file__),
"../metar/stations.json")) as in_file:
icao = json.load(in_file)
now = arrow.utcnow()
hour = now.hour
minute = now.minute
if minute < 45:
current_cycle = hour
else:
current_cycle = hour + 1 % 24
stations = {}
for cycle in (current_cycle - 1, current_cycle):
file = f"http://tgftp.nws.noaa.gov/data/observations/metar/cycles/{cycle:02d}Z.TXT"
self.log.info(f"Processing '{file}' ...")
request = requests.get(file,
stream=True,
timeout=(self.connect_timeout,
self.read_timeout))
for line in request.iter_lines():
if line:
data = line.decode("iso-8859-1")
try:
# Is this line a date with format "2017/05/12 23:55" ?
arrow.get(data, "YYYY/MM/DD HH:mm")
continue
# Catch also ValueError because https://github.com/crsmithdev/arrow/issues/535
except (arrow.parser.ParserError, ValueError):
try:
metar = Metar(data, strict=False)
# wind_dir could be NONE if 'dir' is 'VRB'
if metar.wind_speed:
if metar.station_id not in stations:
stations[metar.station_id] = {}
key = arrow.get(
metar.time).int_timestamp
stations[metar.station_id][key] = metar
except Exception as e:
self.log.warning(
f"Error while parsing METAR data: {e}")
continue
for metar_id in stations:
metar = next(iter(stations[metar_id].values()))
try:
name, short_name, default_name, lat, lon, altitude, tz = (
None,
None,
None,
None,
None,
None,
None,
)
checkwx_key = f"metar/checkwx/{metar.station_id}"
if not self.redis.exists(checkwx_key):
try:
self.log.info("Calling api.checkwx.com...")
request = requests.get(
f"https://api.checkwx.com/station/{metar.station_id}",
headers={
"Accept": "application/json",
"X-API-Key": self.checkwx_api_key
},
timeout=(self.connect_timeout,
self.read_timeout),
)
if request.status_code == 401:
raise UsageLimitException(
request.json()["errors"][0]["message"])
elif request.status_code == 429:
raise UsageLimitException(
"api.checkwx.com rate limit exceeded")
try:
checkwx_data = request.json()["data"][0]
if "icao" not in checkwx_data:
raise ProviderException(
"Invalid CheckWX data")
except (ValueError, KeyError):
checkwx_json = request.json()
messages = []
if type(checkwx_json["data"]) is list:
messages.extend(checkwx_json["data"])
else:
messages.append(checkwx_json["data"])
raise ProviderException(
f'CheckWX API error: {",".join(messages)}'
)
self.add_redis_key(
checkwx_key,
{
"data":
json.dumps(checkwx_data),
"date":
arrow.now().format(
"YYYY-MM-DD HH:mm:ssZZ"),
},
self.checkwx_cache_duration,
)
except TimeoutError as e:
raise e
except UsageLimitException as e:
self.add_redis_key(
checkwx_key,
{
"error":
repr(e),
"date":
arrow.now().format(
"YYYY-MM-DD HH:mm:ssZZ"),
},
self.usage_limit_cache_duration,
)
except Exception as e:
if not isinstance(e, ProviderException):
self.log.exception(
"Error while getting CheckWX data")
else:
self.log.warning(
f"Error while getting CheckWX data: {e}"
)
self.add_redis_key(
checkwx_key,
{
"error":
repr(e),
"date":
arrow.now().format(
"YYYY-MM-DD HH:mm:ssZZ"),
},
self.checkwx_cache_duration,
)
if not self.redis.hexists(checkwx_key, "error"):
checkwx_data = json.loads(
self.redis.hget(checkwx_key, "data"))
station_type = checkwx_data.get("type", None)
if station_type:
name = f'{checkwx_data["name"]} {station_type}'
else:
name = checkwx_data["name"]
city = checkwx_data.get("city", None)
if city:
if station_type:
short_name = f"{city} {station_type}"
else:
short_name = city
else:
default_name = checkwx_data["name"]
lat = checkwx_data["latitude"]["decimal"]
lon = checkwx_data["longitude"]["decimal"]
try:
tz = checkwx_data["timezone"]["tzid"]
except KeyError:
raise ProviderException(
"Unable to get the timezone")
elevation = checkwx_data.get("elevation", None)
altitude = None
if elevation:
if "meters" in elevation:
altitude = Q_(elevation["meters"],
ureg.meters)
elif "feet" in elevation:
altitude = Q_(elevation["feet"], ureg.feet)
if metar.station_id in icao:
lat = lat or icao[metar.station_id]["lat"]
lon = lon or icao[metar.station_id]["lon"]
default_name = default_name or icao[
metar.station_id]["name"]
station = self.save_station(
metar.station_id,
short_name,
name,
lat,
lon,
StationStatus.GREEN,
altitude=altitude,
tz=tz,
url=os.path.join(
self.provider_url,
f"site?id={metar.station_id}&db=metar"),
default_name=default_name
or f"{metar.station_id} Airport",
lookup_name=f"{metar.station_id} Airport ICAO",
)
station_id = station["_id"]
if metar.station_id not in icao:
self.log.warning(
f"Missing '{metar.station_id}' ICAO in database. Is it '{station['name']}'?"
)
measures_collection = self.measures_collection(
station_id)
new_measures = []
for key in stations[metar_id]:
metar = stations[metar_id][key]
if not self.has_measure(measures_collection, key):
temp = self.get_quantity(
metar.temp, self.temperature_units)
dew_point = self.get_quantity(
metar.dewpt, self.temperature_units)
humidity = self.compute_humidity(
dew_point, temp)
measure = self.create_measure(
station,
key,
self.get_direction(metar.wind_dir),
self.get_quantity(metar.wind_speed,
self.speed_units),
self.get_quantity(
metar.wind_gust or metar.wind_speed,
self.speed_units),
temperature=temp,
humidity=humidity,
pressure=Pressure(
qfe=None,
qnh=self.get_quantity(
metar.press, self.pressure_units),
qff=self.get_quantity(
metar.press_sea_level,
self.pressure_units),
),
)
new_measures.append(measure)
self.insert_new_measures(measures_collection, station,
new_measures)
except ProviderException as e:
self.log.warning(
f"Error while processing station '{metar_id}': {e}"
)
except Exception as e:
self.log.exception(
f"Error while processing station '{metar_id}': {e}"
)
except Exception as e:
self.log.exception(f"Error while processing Metar: {e}")
self.log.info("Done !")
