def process_data(self):
try:
self.log.info('Processing Metar data...')
with open(os.path.join(os.path.dirname(__file__), 'metar/stations.json')) as in_file:
icao = json.load(in_file)
now = arrow.utcnow()
hour = now.hour
minute = now.minute
if minute < 45:
current_cycle = hour
else:
current_cycle = hour + 1 % 24
stations = {}
for cycle in (current_cycle-1, current_cycle):
file = f'http://tgftp.nws.noaa.gov/data/observations/metar/cycles/{cycle:02d}Z.TXT'
self.log.info(f"Processing '{file}' ...")
request = requests.get(file, stream=True, timeout=(self.connect_timeout, self.read_timeout))
for line in request.iter_lines():
if line:
data = line.decode('iso-8859-1')
try:
# Is this line a date with format "2017/05/12 23:55" ?
arrow.get(data, 'YYYY/MM/DD HH:mm')
continue
# Catch also ValueError because https://github.com/crsmithdev/arrow/issues/535
except (arrow.parser.ParserError, ValueError):
try:
metar = Metar(data, strict=False)
# wind_dir could be NONE if 'dir' is 'VRB'
if metar.wind_speed:
if metar.station_id not in stations:
stations[metar.station_id] = {}
key = arrow.get(metar.time).timestamp
stations[metar.station_id][key] = metar
except Exception as e:
self.log.warn(f'Error while parsing METAR data: {e}')
continue
for metar_id in stations:
metar = next(iter(stations[metar_id].values()))
try:
name, short_name, default_name, lat, lon, altitude, tz = None, None, None, None, None, None, None
checkwx_key = f'metar/checkwx/{metar.station_id}'
if not self.redis.exists(checkwx_key):
try:
self.log.info('Calling api.checkwx.com...')
request = requests.get(
f'https://api.checkwx.com/station/{metar.station_id}',
headers={'Accept': 'application/json', 'X-API-Key': self.checkwx_api_key},
timeout=(self.connect_timeout, self.read_timeout)
)
if request.status_code == 401:
raise UsageLimitException(request.json()['errors'][0]['message'])
elif request.status_code == 429:
raise UsageLimitException('api.checkwx.com rate limit exceeded')
try:
checkwx_data = request.json()['data'][0]
if 'icao' not in checkwx_data:
raise ProviderException('Invalid CheckWX data')
except (ValueError, KeyError):
checkwx_json = request.json()
messages = []
if type(checkwx_json['data']) is list:
messages.extend(checkwx_json['data'])
else:
messages.append(checkwx_json['data'])
raise ProviderException(f'CheckWX API error: {",".join(messages)}')
self.add_redis_key(checkwx_key, {
'data': json.dumps(checkwx_data),
'date': arrow.now().format('YYYY-MM-DD HH:mm:ssZZ'),
}, self.checkwx_cache_duration)
except TimeoutError as e:
raise e
except UsageLimitException as e:
self.add_redis_key(checkwx_key, {
'error': repr(e),
'date': arrow.now().format('YYYY-MM-DD HH:mm:ssZZ'),
}, self.usage_limit_cache_duration)
except Exception as e:
if not isinstance(e, ProviderException):
self.log.exception('Error while getting CheckWX data')
else:
self.log.warn(f'Error while getting CheckWX data: {e}')
self.add_redis_key(checkwx_key, {
'error': repr(e),
'date': arrow.now().format('YYYY-MM-DD HH:mm:ssZZ'),
}, self.checkwx_cache_duration)
if not self.redis.hexists(checkwx_key, 'error'):
checkwx_data = json.loads(self.redis.hget(checkwx_key, 'data'))
station_type = checkwx_data.get('type', None)
if station_type:
name = f'{checkwx_data["name"]} {station_type}'
else:
name = checkwx_data['name']
city = checkwx_data.get('city', None)
if city:
if station_type:
short_name = f'{city} {station_type}'
else:
short_name = city
else:
default_name = checkwx_data['name']
lat = checkwx_data['latitude']['decimal']
lon = checkwx_data['longitude']['decimal']
tz = checkwx_data['timezone']['tzid']
elevation = checkwx_data.get('elevation', None)
altitude = None
if elevation:
if 'meters' in elevation:
altitude = Q_(elevation['meters'], ureg.meters)
elif 'feet' in elevation:
altitude = Q_(elevation['feet'], ureg.feet)
if metar.station_id in icao:
lat = lat or icao[metar.station_id]['lat']
lon = lon or icao[metar.station_id]['lon']
default_name = default_name or icao[metar.station_id]['name']
station = self.save_station(
metar.station_id,
short_name,
name,
lat,
lon,
Status.GREEN,
altitude=altitude,
tz=tz,
url=os.path.join(self.provider_url, f'site?id={metar.station_id}&db=metar'),
default_name=default_name or f'{metar.station_id} Airport',
lookup_name=f'{metar.station_id} Airport ICAO')
station_id = station['_id']
if metar.station_id not in icao:
self.log.warn(f"Missing '{metar.station_id}' ICAO in database. Is it '{station['name']}'?")
measures_collection = self.measures_collection(station_id)
new_measures = []
for key in stations[metar_id]:
metar = stations[metar_id][key]
if not self.has_measure(measures_collection, key):
temp = self.get_quantity(metar.temp, self.temperature_units)
dew_point = self.get_quantity(metar.dewpt, self.temperature_units)
humidity = self.compute_humidity(dew_point, temp)
measure = self.create_measure(
station,
key,
self.get_direction(metar.wind_dir),
self.get_quantity(metar.wind_speed, self.speed_units),
self.get_quantity(metar.wind_gust or metar.wind_speed, self.speed_units),
temperature=temp,
humidity=humidity,
pressure=Pressure(qfe=None,
qnh=self.get_quantity(metar.press, self.pressure_units),
qff=self.get_quantity(metar.press_sea_level, self.pressure_units))
)
new_measures.append(measure)
self.insert_new_measures(measures_collection, station, new_measures)
except ProviderException as e:
self.log.warn(f"Error while processing station '{metar_id}': {e}")
except Exception as e:
self.log.exception(f"Error while processing station '{metar_id}': {e}")
except Exception as e:
self.log.exception(f'Error while processing Metar: {e}')
self.log.info('Done !')
def process_data(self):
try:
self.log.info('Processing iWeathar data...')
result_tree = etree.parse(
requests.get(
f'https://iweathar.co.za/live_data.php?unit=kmh&key={self.iweathar_key}',
stream=True,
verify=False,
timeout=(self.connect_timeout, self.read_timeout)).raw)
for item in result_tree.xpath('//ITEM'):
station_id = None
try:
iweathar_id = item.xpath('STATION_ID')[0].text
name = item.xpath('LOCATION')[0].text
status = Status.GREEN if item.xpath(
'STATUS')[0].text == 'ON-LINE' else Status.RED
station = self.save_station(
iweathar_id,
name,
name,
item.xpath('LAT')[0].text,
item.xpath('LONG')[0].text,
status,
url=f'{self.provider_url}/display?s_id={iweathar_id}')
station_id = station['_id']
key_attr = item.xpath('UNIX_DATE_STAMP')
wind_dir_attr = item.xpath('WIND_ANG')
wind_avg_attr = item.xpath('WIND_AVG')
wind_max_attr = item.xpath('WIND_MAX')
if not (key_attr and wind_dir_attr and wind_avg_attr
and wind_max_attr):
self.log.warning(
f"Station '{station_id}' has invalid data")
continue
key = int(key_attr[0].text)
measures_collection = self.measures_collection(station_id)
if not self.has_measure(measures_collection, key):
try:
temperature_attr = item.xpath('TEMPERATURE_C')
if temperature_attr and temperature_attr[0].text:
temperature = Q_(temperature_attr[0].text,
ureg.degC)
else:
temperature = None
humidity_attr = item.xpath('HUMIDITY_PERC')
if humidity_attr and humidity_attr[0].text:
humidity = humidity_attr[0].text
else:
humidity = None
pressure_attr = item.xpath('PRESSURE_MB')
if pressure_attr and pressure_attr[0].text:
pressure = Pressure(qfe=pressure_attr[0].text,
qnh=None,
qff=None)
else:
pressure = None
rain_attr = item.xpath('RAINFALL_MM')
if rain_attr and rain_attr[0].text:
rain = Q_(rain_attr[0].text,
ureg.liter / (ureg.meter**2))
else:
rain = None
measure = self.create_measure(
station,
key,
wind_dir_attr[0].text,
wind_avg_attr[0].text,
wind_max_attr[0].text,
temperature=temperature,
humidity=humidity,
pressure=pressure,
rain=rain)
self.insert_new_measures(measures_collection,
station, [measure])
except ProviderException as e:
self.log.warning(
f"Error while processing measure '{key}' for station '{station_id}': {e}"
)
except Exception as e:
self.log.exception(
f"Error while processing measure '{key}' for station '{station_id}': {e}"
)
except ProviderException as e:
self.log.warning(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(f'Error while processing iWeathar: {e}')
self.log.info('...Done!')
def process_data(self):
try:
self.log.info('Processing JDC data...')
@retry(wait=wait_random_exponential(multiplier=2, min=2), stop=stop_after_delay(60),
after=after_log(self.log, logging.WARNING))
def request_data():
# jdc.ch randomly timeout early in the morning... backup tasks?
return requests.get('http://meteo.jdc.ch/API/?Action=StationView&flags=all',
timeout=(self.connect_timeout, self.read_timeout))
result = request_data()
try:
jdc_stations = result.json()['Stations']
except Exception:
raise Exception('Action=StationView returns invalid json response')
for jdc_station in jdc_stations:
station_id = None
try:
jdc_id = jdc_station['serial']
station = self.save_station(
jdc_id,
jdc_station['short-name'],
jdc_station['name'],
jdc_station['latitude'],
jdc_station['longitude'],
self.get_status(jdc_station['status']),
altitude=jdc_station['altitude'],
url=urllib.parse.urljoin(self.provider_url, '/station/' + str(jdc_station['serial'])))
station_id = station['_id']
try:
# Asking 2 days of data
result = requests.get(
f'http://meteo.jdc.ch/API/?Action=DataView&serial={jdc_id}&duration=172800',
timeout=(self.connect_timeout, self.read_timeout))
try:
json = result.json()
except ValueError:
raise Exception('Action=DataView returns invalid json response')
if json['ERROR'] == 'OK':
measures_collection = self.measures_collection(station_id)
measures = json['data']['measurements']
new_measures = []
for jdc_measure in measures:
key = jdc_measure['unix-time']
if not self.has_measure(measures_collection, key):
try:
measure = self.create_measure(
station,
key,
jdc_measure.get('wind-direction'),
jdc_measure.get('wind-average'),
jdc_measure.get('wind-maximum'),
temperature=jdc_measure.get('temperature'),
humidity=jdc_measure.get('humidity'),
pressure=Pressure(
qfe=jdc_measure.get('pressure', None),
qnh=None,
qff=None),
rain=jdc_measure.get('rain', None),
)
new_measures.append(measure)
except ProviderException as e:
self.log.warning(
f"Error while processing measure '{key}' for station '{station_id}': {e}")
except Exception as e:
self.log.exception(
f"Error while processing measure '{key}' for station '{station_id}': {e}")
self.insert_new_measures(measures_collection, station, new_measures)
else:
raise ProviderException(f"Action=Data returns an error: '{json['ERROR']}'")
except ProviderException as e:
self.log.warning(f"Error while processing measures for station '{station_id}': {e}")
except Exception as e:
self.log.exception(f"Error while processing measures for station '{station_id}': {e}")
except ProviderException as e:
self.log.warning(f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(f'Error while processing JDC: {e}')
self.log.info('Done !')
def process_data(self):
try:
self.log.info('Processing MeteoSwiss data...')
base_url = 'https://www.meteoswiss.admin.ch'
session = requests.Session()
session.headers.update(user_agents.chrome)
main_url = '/home/weather/measurement-values/measurement-values-at-meteorological-stations.html'
html_tree = html.fromstring(
session.get(base_url + main_url,
timeout=(self.connect_timeout,
self.read_timeout)).text)
# Main wind data
element = html_tree.xpath(
'//input[@id="measurement__param-radio--wind"]')[0]
wind_datas = session.get(
base_url + element.get('data-json-url'),
timeout=(self.connect_timeout,
self.read_timeout)).json()['stations']
# Temperature data
element = html_tree.xpath(
'//input[@id="measurement__param-radio--temperature"]')[0]
temp_datas = session.get(
base_url + element.get('data-json-url'),
timeout=(self.connect_timeout,
self.read_timeout)).json()['stations']
# Humidity data
element = html_tree.xpath(
'//input[@id="measurement__param-radio--humidity"]')[0]
humidity_datas = session.get(
base_url + element.get('data-json-url'),
timeout=(self.connect_timeout,
self.read_timeout)).json()['stations']
# Pressure data
element = html_tree.xpath(
'//option[@id="measurement__param-option--airpressure-qfe"]'
)[0]
pressure_datas_qfe = session.get(
base_url + element.get('data-json-url'),
timeout=(self.connect_timeout,
self.read_timeout)).json()['stations']
element = html_tree.xpath(
'//option[@id="measurement__param-option--airpressure-qnh"]'
)[0]
pressure_datas_qnh = session.get(
base_url + element.get('data-json-url'),
timeout=(self.connect_timeout,
self.read_timeout)).json()['stations']
element = html_tree.xpath(
'//option[@id="measurement__param-option--airpressure-qff"]'
)[0]
pressure_datas_qff = session.get(
base_url + element.get('data-json-url'),
timeout=(self.connect_timeout,
self.read_timeout)).json()['stations']
# Rain data
element = html_tree.xpath(
'//input[@id="measurement__param-radio--precipitation"]')[0]
rain_datas = session.get(
base_url + element.get('data-json-url'),
timeout=(self.connect_timeout,
self.read_timeout)).json()['stations']
for wind_data in wind_datas:
station_id = None
try:
urls = {
lang: url.format(id=wind_data['id'].lower())
for lang, url in self.provider_urls.items()
}
station = self.save_station(
wind_data['id'],
wind_data['city_name'],
wind_data['city_name'],
ch_to_wgs_lat(int(wind_data['coord_x']),
int(wind_data['coord_y'])),
ch_to_wgs_lon(int(wind_data['coord_x']),
int(wind_data['coord_y'])),
Status.GREEN,
altitude=wind_data['evelation'],
tz='Europe/Zurich',
url=urls)
station_id = station['_id']
key = arrow.get(int(wind_data['date']) / 1000).timestamp
measures_collection = self.measures_collection(station_id)
new_measures = []
if not self.has_measure(measures_collection, key):
temp_data = None
for data in temp_datas:
if data['id'] == wind_data['id'] and data[
'date'] == wind_data['date']:
temp_data = data
humidity_data = None
for data in humidity_datas:
if data['id'] == wind_data['id'] and data[
'date'] == wind_data['date']:
humidity_data = data
pressure_data_qfe = None
for data in pressure_datas_qfe:
if data['id'] == wind_data['id'] and data[
'date'] == wind_data['date']:
pressure_data_qfe = data
pressure_data_qnh = None
for data in pressure_datas_qnh:
if data['id'] == wind_data['id'] and data[
'date'] == wind_data['date']:
pressure_data_qnh = data
pressure_data_qff = None
for data in pressure_datas_qff:
if data['id'] == wind_data['id'] and data[
'date'] == wind_data['date']:
pressure_data_qff = data
rain_data = None
for data in rain_datas:
if data['id'] == wind_data['id'] and data[
'date'] == wind_data['date']:
rain_data = data
wind_dir = None
if wind_data['current_value'][2] is not None:
wind_dir = Q_(wind_data['current_value'][2],
ureg.degree)
wind_avg = None
if wind_data['current_value'][0] is not None:
wind_avg = Q_(wind_data['current_value'][0],
ureg.kilometer / ureg.hour)
wind_max = None
if wind_data['current_value'][1] is not None:
wind_max = Q_(wind_data['current_value'][1],
ureg.kilometer / ureg.hour)
temp = None
if temp_data and temp_data['current_value'] is not None:
temp = Q_(temp_data['current_value'], ureg.degC)
humidity = None
if humidity_data and humidity_data[
'current_value'] is not None:
humidity = humidity_data['current_value']
qfe = None
if pressure_data_qfe and pressure_data_qfe[
'current_value'] is not None:
qfe = Q_(pressure_data_qfe['current_value'],
ureg.hPa)
qnh = None
if pressure_data_qnh and pressure_data_qnh[
'current_value'] is not None:
qnh = Q_(pressure_data_qnh['current_value'],
ureg.hPa)
qff = None
if pressure_data_qff and pressure_data_qff[
'current_value'] is not None:
qff = Q_(pressure_data_qff['current_value'],
ureg.hPa)
rain = None
if rain_data and rain_data['current_value'] is not None:
rain = Q_(rain_data['current_value'],
ureg.liter / (ureg.meter**2))
measure = self.create_measure(
station,
key,
wind_dir,
wind_avg,
wind_max,
temperature=temp,
humidity=humidity,
pressure=Pressure(qfe=qfe, qnh=qnh, qff=qff),
rain=rain,
)
new_measures.append(measure)
self.insert_new_measures(measures_collection, station,
new_measures)
except ProviderException as e:
self.log.warn(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(f'Error while processing MeteoSwiss: {e}')
self.log.info('...Done!')
def process_data(self):
try:
self.log.info('Processing Romma data...')
romma_tz = tz.gettz('Europe/Paris')
result_tree = etree.fromstring(
requests.get(
f'http://romma.fr/releves_romma_xml.php?id={self.romma_key}',
timeout=(self.connect_timeout, self.read_timeout)).text)
for report in result_tree.xpath('//releves/releve'):
station_id = None
try:
romma_id = report.xpath('id')[0].text
name = report.xpath('station')[0].text
status = Status.GREEN if report.xpath(
'valide')[0].text == '1' else Status.RED
station = self.save_station(
romma_id,
name,
name,
report.xpath('latitude')[0].text,
report.xpath('longitude')[0].text,
status,
altitude=report.xpath('altitude')[0].text,
url=urllib.parse.urljoin(
self.provider_url,
f'/station_24.php?id={romma_id}'))
station_id = station['_id']
wind_dir = self.get_value(
report.xpath('direction')[0].text)
if not wind_dir:
self.log.warning(
f"Station '{station_id}' has no wind direction value"
)
continue
measures_collection = self.measures_collection(station_id)
key = arrow.get(
report.xpath('date')[0].text,
'D.MM.YYYY H:mm').replace(tzinfo=romma_tz).timestamp
if not self.has_measure(measures_collection, key):
try:
measure = self.create_measure(
station,
key,
self.wind_directions[wind_dir],
self.get_value(
report.xpath('vent_moyen_10')[0].text),
self.get_value(
report.xpath('rafale_maxi')[0].text),
temperature=self.get_value(
report.xpath('temperature')[0].text),
humidity=self.get_value(
report.xpath('humidite')[0].text),
pressure=Pressure(qfe=self.get_value(
report.xpath('pression')[0].text),
qnh=None,
qff=None))
self.insert_new_measures(measures_collection,
station, [measure])
except ProviderException as e:
self.log.warning(
f"Error while processing measure '{key}' for station '{station_id}': {e}"
)
except Exception as e:
self.log.exception(
f"Error while processing measure '{key}' for station '{station_id}': {e}"
)
except ProviderException as e:
self.log.warning(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(f'Error while processing Romma: {e}')
self.log.info('...Done!')
def process_data(self):
stations = {}
try:
self.log.info('Processing FFVL data...')
result = requests.get(
'http://data.ffvl.fr/json/balises.json', timeout=(self.connect_timeout, self.read_timeout))
ffvl_stations = result.json()
for ffvl_station in ffvl_stations:
ffvl_id = None
try:
ffvl_id = ffvl_station['idBalise']
station = self.save_station(
ffvl_id,
ffvl_station['nom'],
ffvl_station['nom'],
ffvl_station['latitude'],
ffvl_station['longitude'],
Status.GREEN,
altitude=ffvl_station['altitude'],
url=ffvl_station['url'])
stations[station['_id']] = station
except ProviderException as e:
self.log.warning(f"Error while processing station '{ffvl_id}': {e}")
except Exception as e:
self.log.exception(f"Error while processing station '{ffvl_id}': {e}")
except ProviderException as e:
self.log.warning(f'Error while processing stations: {e}')
except Exception as e:
self.log.exception(f'Error while processing stations: {e}')
try:
@retry(wait=wait_random_exponential(multiplier=2, min=2), stop=stop_after_delay(60),
after=after_log(self.log, logging.WARNING))
def request_data():
# data.ffvl.fr randomly returns an empty file instead the json doc
result = requests.get(
'http://data.ffvl.fr/json/relevesmeteo.json', timeout=(self.connect_timeout, self.read_timeout))
return result.json()
ffvl_measures = request_data()
ffvl_tz = tz.gettz('Europe/Paris')
for ffvl_measure in ffvl_measures:
station_id = None
try:
ffvl_id = ffvl_measure['idbalise']
station_id = self.get_station_id(ffvl_id)
if station_id not in stations:
raise ProviderException(f"Unknown station '{station_id}'")
station = stations[station_id]
measures_collection = self.measures_collection(station_id)
new_measures = []
key = arrow.get(ffvl_measure['date'], 'YYYY-MM-DD HH:mm:ss').replace(tzinfo=ffvl_tz).timestamp
if not self.has_measure(measures_collection, key):
measure = self.create_measure(
station,
key,
ffvl_measure['directVentMoy'],
ffvl_measure['vitesseVentMoy'],
ffvl_measure['vitesseVentMax'],
temperature=ffvl_measure['temperature'],
humidity=ffvl_measure['hydrometrie'],
pressure=Pressure(qfe=ffvl_measure['pression'], qnh=None, qff=None)
)
new_measures.append(measure)
self.insert_new_measures(measures_collection, station, new_measures)
except ProviderException as e:
self.log.warning(f"Error while processing measures for station '{station_id}': {e}")
except Exception as e:
self.log.exception(f"Error while processing measures for station '{station_id}': {e}")
except ProviderException as e:
self.log.warning(f'Error while processing FFVL: {e}')
except Exception as e:
self.log.exception(f'Error while processing FFVL: {e}')
self.log.info('...Done!')
def process_data(self):
try:
self.log.info('Processing Holfuy data...')
holfuy_stations = requests.get(
'https://api.holfuy.com/stations/stations.json',
timeout=(self.connect_timeout, self.read_timeout)).json()
holfuy_data = requests.get(
'https://api.holfuy.com/live/?s=all&m=JSON&tu=C&su=km/h&utc',
timeout=(self.connect_timeout, self.read_timeout)).json()
holfuy_measures = {}
for holfuy_measure in holfuy_data['measurements']:
holfuy_measures[holfuy_measure['stationId']] = holfuy_measure
for holfuy_station in holfuy_stations['holfuyStationsList']:
holfuy_id = None
station_id = None
try:
holfuy_id = holfuy_station['id']
name = holfuy_station['name']
location = holfuy_station['location']
latitude = location.get('latitude')
longitude = location.get('longitude')
if (latitude is None
or longitude is None) or (latitude == 0
and longitude == 0):
raise ProviderException('No geolocation found')
altitude = location.get('altitude')
urls = {
lang: url.format(id=holfuy_id)
for lang, url in self.provider_urls.items()
}
station = self.save_station(holfuy_id,
name,
name,
latitude,
longitude,
Status.GREEN,
altitude=altitude,
url=urls)
station_id = station['_id']
measures_collection = self.measures_collection(station_id)
new_measures = []
if holfuy_id not in holfuy_measures:
raise ProviderException(
f"Station '{name}' not found in 'api.holfuy.com/live/'"
)
holfuy_measure = holfuy_measures[holfuy_id]
last_measure_date = arrow.get(holfuy_measure['dateTime'])
key = last_measure_date.timestamp
if not self.has_measure(measures_collection, key):
measure = self.create_measure(
station,
key,
holfuy_measure['wind']['direction'],
Q_(holfuy_measure['wind']['speed'],
ureg.kilometer / ureg.hour),
Q_(holfuy_measure['wind']['gust'],
ureg.kilometer / ureg.hour),
temperature=Q_(holfuy_measure['temperature'],
ureg.degC)
if 'temperature' in holfuy_measure else None,
pressure=Pressure(
qfe=None,
qnh=Q_(holfuy_measure['pressure'], ureg.hPa)
if 'pressure' in holfuy_measure else None,
qff=None))
new_measures.append(measure)
self.insert_new_measures(measures_collection, station,
new_measures)
except ProviderException as e:
self.log.warning(
f"Error while processing station '{station_id or holfuy_id}': {e}"
)
except Exception as e:
self.log.exception(
f"Error while processing station '{station_id or holfuy_id}': {e}"
)
except Exception as e:
self.log.exception(f'Error while processing Holfuy: {e}')
self.log.info('Done !')
def process_data(self):
try:
self.log.info('Processing Pioupiou data...')
result = requests.get(
'http://api.pioupiou.fr/v1/live-with-meta/all',
timeout=(self.connect_timeout, self.read_timeout))
station_id = None
for piou_station in result.json()['data']:
try:
piou_id = piou_station['id']
location = piou_station['location']
latitude = location.get('latitude')
longitude = location.get('longitude')
if (latitude is None
or longitude is None) or (latitude == 0
and longitude == 0):
continue
location_date = None
if location['date']:
try:
location_date = arrow.get(location['date'])
except ParserError:
pass
station = self.save_station(
piou_id,
None,
None,
latitude,
longitude,
self.get_status(station_id,
piou_station['status']['state'],
location_date, location['success']),
url=urllib.parse.urljoin(self.provider_url,
str(piou_id)),
default_name=piou_station.get('meta',
{}).get('name', None))
station_id = station['_id']
measures_collection = self.measures_collection(station_id)
new_measures = []
piou_measure = piou_station['measurements']
last_measure_date = arrow.get(piou_measure['date'])
key = last_measure_date.timestamp
if not self.has_measure(measures_collection, key):
measure = self.create_measure(
station,
key,
piou_measure['wind_heading'],
piou_measure['wind_speed_avg'],
piou_measure['wind_speed_max'],
pressure=Pressure(qfe=piou_measure['pressure'],
qnh=None,
qff=None))
new_measures.append(measure)
self.insert_new_measures(measures_collection, station,
new_measures)
except ProviderException as e:
self.log.warn(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(f'Error while processing Pioupiou: {e}')
self.log.info('Done !')
def save_measures(self, jdc_id, blocs, stations_metadata):
try:
jdc_station = list(
filter(lambda d: str(d['id']) == jdc_id, stations_metadata))[0]
station = self.save_station(jdc_id,
jdc_station['short_name'],
jdc_station['name'],
jdc_station['latitude'],
jdc_station['longitude'],
self.get_status(jdc_station['status']),
altitude=jdc_station['altitude'],
url=urllib.parse.urljoin(
self.provider_url,
'/station/' + jdc_id))
station_id = station['_id']
try:
jdc_measures = blocs[0]['mesures']
except Exception:
self.log.warning('Unable to find a bloc with measures')
jdc_measures = []
self.log.info(
f"Station '{jdc_id}' ({station['name']}) found {len(jdc_measures)} measures"
)
measures_collection = self.measures_collection(station_id)
for jdc_measure in jdc_measures:
key = jdc_measure['datetime']
if not self.has_measure(measures_collection, key):
try:
new_measure = self.create_measure(
station,
key,
self.find_value(jdc_measure, 'Direction du vent'),
self.find_value(jdc_measure, 'Vent moyen'),
self.find_value(jdc_measure, 'Vent max'),
temperature=self.find_value(
jdc_measure, "Température de l'air"),
humidity=self.find_value(jdc_measure, 'Humidité'),
pressure=Pressure(qfe=self.find_value(
jdc_measure, 'Pression atmosphérique'),
qnh=None,
qff=None),
rain=self.find_value(jdc_measure, 'Pluviométrie'))
self.insert_new_measures(measures_collection, station,
[new_measure])
except ProviderException as e:
self.log.warning(
f"Error while processing measure '{key}' for station '{station_id}': {e}"
)
except Exception as e:
self.log.exception(
f"Error while processing measure '{key}' for station '{station_id}': {e}"
)
except ProviderException as e:
self.log.warning(f"Error while processing station '{jdc_id}': {e}")
except Exception as e:
self.log.exception(
f"Error while processing station '{jdc_id}': {e}")
def process_data(self):
try:
self.log.info('Processing MeteoSwiss data...')
url_pattern = 'https://data.geo.admin.ch/ch.meteoschweiz.messwerte-{parameter}/' \
'ch.meteoschweiz.messwerte-{parameter}_en.json'
main_wind = requests.get(url_pattern.format(parameter='windgeschwindigkeit-kmh-10min'), timeout=(
self.connect_timeout, self.read_timeout)).json()
wind_gust = requests.get(url_pattern.format(parameter='wind-boeenspitze-kmh-10min'), timeout=(
self.connect_timeout, self.read_timeout)).json()
temperature = requests.get(url_pattern.format(parameter='lufttemperatur-10min'), timeout=(
self.connect_timeout, self.read_timeout)).json()
humidity = requests.get(url_pattern.format(parameter='luftfeuchtigkeit-10min'), timeout=(
self.connect_timeout, self.read_timeout)).json()
pressure_qfe = requests.get(url_pattern.format(parameter='luftdruck-qfe-10min'), timeout=(
self.connect_timeout, self.read_timeout)).json()
pressure_qnh = requests.get(url_pattern.format(parameter='luftdruck-qnh-10min'), timeout=(
self.connect_timeout, self.read_timeout)).json()
pressure_qff = requests.get(url_pattern.format(parameter='luftdruck-qnh-10min'), timeout=(
self.connect_timeout, self.read_timeout)).json()
rain = requests.get(url_pattern.format(parameter='niederschlag-10min'), timeout=(
self.connect_timeout, self.read_timeout)).json()
if main_wind['creation_time'] != wind_gust['creation_time'] != temperature['creation_time'] != \
humidity['creation_time'] != pressure_qfe['creation_time'] != pressure_qnh['creation_time'] != \
pressure_qff['creation_time'] != rain['creation_time']:
self.log.error('Creation time of parameters files are not the same')
main_wind_data = main_wind['features']
wind_gust_data = self.to_dict(wind_gust['features'])
temperature_data = self.to_dict(temperature['features'])
humidity_data = self.to_dict(humidity['features'])
pressure_qfe_data = self.to_dict(pressure_qfe['features'])
pressure_qnh_data = self.to_dict(pressure_qnh['features'])
pressure_qff_data = self.to_dict(pressure_qff['features'])
rain_data = self.to_dict(rain['features'])
station_id = None
for meteoswiss_station in main_wind_data:
try:
meteoswiss_id = meteoswiss_station['id']
location = meteoswiss_station['geometry']['coordinates']
urls = {lang: url.format(param='messwerte-windgeschwindigkeit-kmh-10min', id=meteoswiss_id) for
lang, url in self.provider_urls.items()}
station = self.save_station(
meteoswiss_id,
meteoswiss_station['properties']['station_name'],
meteoswiss_station['properties']['station_name'],
ch_to_wgs_lat(location[0], location[1]),
ch_to_wgs_lon(location[0], location[1]),
Status.GREEN,
altitude=meteoswiss_station['properties']['altitude'],
tz='Europe/Zurich',
url=urls)
station_id = station['_id']
timestamp = meteoswiss_station['properties'].get('reference_ts', None)
if not timestamp:
self.log.warning(f"'{station_id}' has no timestamp field")
continue
key = arrow.get(meteoswiss_station['properties']['reference_ts'], 'YY-MM-DDTHH:mm:ZZ').timestamp
measures_collection = self.measures_collection(station_id)
new_measures = []
if meteoswiss_id in temperature_data:
temperature = self.get_value(temperature_data[meteoswiss_id]['properties'], unit=ureg.degC)
else:
temperature = None
if meteoswiss_id in humidity_data:
humidity = humidity_data[meteoswiss_id]['properties']['value']
else:
humidity = None
if meteoswiss_id in pressure_qfe_data:
pressure = Pressure(
qfe=self.get_value(pressure_qfe_data[meteoswiss_id]['properties']),
qnh=self.get_value(pressure_qnh_data[meteoswiss_id]['properties']),
qff=self.get_value(pressure_qff_data[meteoswiss_id]['properties']))
else:
pressure = None
if meteoswiss_id in rain_data:
# 1mm = 1 liter/m^2
rain = self.get_value(rain_data[meteoswiss_id]['properties'],
unit=ureg.liter / (ureg.meter ** 2))
else:
rain = None
if not self.has_measure(measures_collection, key):
measure = self.create_measure(
station,
key,
meteoswiss_station['properties']['wind_direction'],
self.get_value(meteoswiss_station['properties']),
self.get_value(wind_gust_data[meteoswiss_id]['properties']),
temperature=temperature,
humidity=humidity,
pressure=pressure,
rain=rain
)
new_measures.append(measure)
self.insert_new_measures(measures_collection, station, new_measures)
except ProviderException as e:
self.log.warning(f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(f'Error while processing MeteoSwiss: {e}')
self.log.info('...Done!')