def process_data(self):
station_id = "unknown"
try:
self.log.info("Processing Pdcs data...")
pdcs_data = requests.get(
"https://ws.lubu.ch/ws/data.php?minutes=20",
timeout=(self.connect_timeout, self.read_timeout)).json()
for pdcs_station in pdcs_data["stations"]:
try:
station = self.save_station(
pdcs_station["id"],
pdcs_station["shortName"],
pdcs_station["name"],
pdcs_station["coords"]["lat"],
pdcs_station["coords"]["lon"],
StationStatus.GREEN,
altitude=pdcs_station["altitude"],
)
station_id = station["_id"]
if pdcs_station["measurement"]:
measure = pdcs_station["measurement"][0]
key = measure["time"]
measures_collection = self.measures_collection(
station_id)
new_measures = []
if not self.has_measure(measures_collection, key):
measure = self.create_measure(
station,
key,
measure["w-dir"],
measure["w-avg"],
measure["w-max"],
pressure=Pressure(qfe=measure["pres"]["qfe"],
qnh=None,
qff=None),
)
new_measures.append(measure)
self.insert_new_measures(measures_collection,
station, new_measures)
except ProviderException as e:
self.log.warning(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(f"Error while processing Pdcs: {e}")
self.log.info("Done !")
def process_data(self):
station_id = 'unknown'
try:
self.log.info('Processing Pdcs data...')
pdcs_data = requests.get(
'https://ws.lubu.ch/ws/data.php?minutes=20',
timeout=(self.connect_timeout, self.read_timeout)).json()
for pdcs_station in pdcs_data['stations']:
try:
station = self.save_station(
pdcs_station['id'],
pdcs_station['shortName'],
pdcs_station['name'],
pdcs_station['coords']['lat'],
pdcs_station['coords']['lon'],
StationStatus.GREEN,
altitude=pdcs_station['altitude'])
station_id = station['_id']
if pdcs_station['measurement']:
measure = pdcs_station['measurement'][0]
key = measure['time']
measures_collection = self.measures_collection(
station_id)
new_measures = []
if not self.has_measure(measures_collection, key):
measure = self.create_measure(
station,
key,
measure['w-dir'],
measure['w-avg'],
measure['w-max'],
pressure=Pressure(qfe=measure['pres']['qfe'],
qnh=None,
qff=None))
new_measures.append(measure)
self.insert_new_measures(measures_collection,
station, new_measures)
except ProviderException as e:
self.log.warning(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(f'Error while processing Pdcs: {e}')
self.log.info('Done !')
def process_data(self):
try:
self.log.info('Processing Pioupiou data...')
result = requests.get(
'https://api.pioupiou.fr/v1/live-with-meta/all',
timeout=(self.connect_timeout, self.read_timeout))
station_id = None
for piou_station in result.json()['data']:
try:
piou_id = piou_station['id']
location = piou_station['location']
latitude = location.get('latitude')
longitude = location.get('longitude')
if (latitude is None
or longitude is None) or (latitude == 0
and longitude == 0):
continue
location_date = None
if location['date']:
try:
location_date = arrow.get(location['date'])
except ParserError:
pass
station = self.save_station(
piou_id,
None,
None,
latitude,
longitude,
self.get_status(station_id,
piou_station['status']['state'],
location_date, location['success']),
url=f'{self.provider_url}/PP{piou_id}',
default_name=piou_station.get('meta',
{}).get('name', None))
station_id = station['_id']
measures_collection = self.measures_collection(station_id)
new_measures = []
piou_measure = piou_station['measurements']
last_measure_date = arrow.get(piou_measure['date'])
key = last_measure_date.timestamp
if not self.has_measure(measures_collection, key):
measure = self.create_measure(
station,
key,
piou_measure['wind_heading'],
piou_measure['wind_speed_avg'],
piou_measure['wind_speed_max'],
pressure=Pressure(qfe=piou_measure['pressure'],
qnh=None,
qff=None))
new_measures.append(measure)
self.insert_new_measures(measures_collection, station,
new_measures)
except ProviderException as e:
self.log.warning(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(f'Error while processing Pioupiou: {e}')
self.log.info('Done !')
def process_data(self):
try:
self.log.info('Processing Metar data...')
with open(
os.path.join(os.path.dirname(__file__),
'metar/stations.json')) as in_file:
icao = json.load(in_file)
now = arrow.utcnow()
hour = now.hour
minute = now.minute
if minute < 45:
current_cycle = hour
else:
current_cycle = hour + 1 % 24
stations = {}
for cycle in (current_cycle - 1, current_cycle):
file = f'http://tgftp.nws.noaa.gov/data/observations/metar/cycles/{cycle:02d}Z.TXT'
self.log.info(f"Processing '{file}' ...")
request = requests.get(file,
stream=True,
timeout=(self.connect_timeout,
self.read_timeout))
for line in request.iter_lines():
if line:
data = line.decode('iso-8859-1')
try:
# Is this line a date with format "2017/05/12 23:55" ?
arrow.get(data, 'YYYY/MM/DD HH:mm')
continue
# Catch also ValueError because https://github.com/crsmithdev/arrow/issues/535
except (arrow.parser.ParserError, ValueError):
try:
metar = Metar(data, strict=False)
# wind_dir could be NONE if 'dir' is 'VRB'
if metar.wind_speed:
if metar.station_id not in stations:
stations[metar.station_id] = {}
key = arrow.get(metar.time).timestamp
stations[metar.station_id][key] = metar
except Exception as e:
self.log.warning(
f'Error while parsing METAR data: {e}')
continue
for metar_id in stations:
metar = next(iter(stations[metar_id].values()))
try:
name, short_name, default_name, lat, lon, altitude, tz = None, None, None, None, None, None, None
checkwx_key = f'metar/checkwx/{metar.station_id}'
if not self.redis.exists(checkwx_key):
try:
self.log.info('Calling api.checkwx.com...')
request = requests.get(
f'https://api.checkwx.com/station/{metar.station_id}',
headers={
'Accept': 'application/json',
'X-API-Key': self.checkwx_api_key
},
timeout=(self.connect_timeout,
self.read_timeout))
if request.status_code == 401:
raise UsageLimitException(
request.json()['errors'][0]['message'])
elif request.status_code == 429:
raise UsageLimitException(
'api.checkwx.com rate limit exceeded')
try:
checkwx_data = request.json()['data'][0]
if 'icao' not in checkwx_data:
raise ProviderException(
'Invalid CheckWX data')
except (ValueError, KeyError):
checkwx_json = request.json()
messages = []
if type(checkwx_json['data']) is list:
messages.extend(checkwx_json['data'])
else:
messages.append(checkwx_json['data'])
raise ProviderException(
f'CheckWX API error: {",".join(messages)}')
self.add_redis_key(
checkwx_key, {
'data':
json.dumps(checkwx_data),
'date':
arrow.now().format(
'YYYY-MM-DD HH:mm:ssZZ'),
}, self.checkwx_cache_duration)
except TimeoutError as e:
raise e
except UsageLimitException as e:
self.add_redis_key(
checkwx_key, {
'error':
repr(e),
'date':
arrow.now().format(
'YYYY-MM-DD HH:mm:ssZZ'),
}, self.usage_limit_cache_duration)
except Exception as e:
if not isinstance(e, ProviderException):
self.log.exception(
'Error while getting CheckWX data')
else:
self.log.warning(
f'Error while getting CheckWX data: {e}')
self.add_redis_key(
checkwx_key, {
'error':
repr(e),
'date':
arrow.now().format(
'YYYY-MM-DD HH:mm:ssZZ'),
}, self.checkwx_cache_duration)
if not self.redis.hexists(checkwx_key, 'error'):
checkwx_data = json.loads(
self.redis.hget(checkwx_key, 'data'))
station_type = checkwx_data.get('type', None)
if station_type:
name = f'{checkwx_data["name"]} {station_type}'
else:
name = checkwx_data['name']
city = checkwx_data.get('city', None)
if city:
if station_type:
short_name = f'{city} {station_type}'
else:
short_name = city
else:
default_name = checkwx_data['name']
lat = checkwx_data['latitude']['decimal']
lon = checkwx_data['longitude']['decimal']
try:
tz = checkwx_data['timezone']['tzid']
except KeyError:
raise ProviderException(
f'Unable to get the timezone')
elevation = checkwx_data.get('elevation', None)
altitude = None
if elevation:
if 'meters' in elevation:
altitude = Q_(elevation['meters'], ureg.meters)
elif 'feet' in elevation:
altitude = Q_(elevation['feet'], ureg.feet)
if metar.station_id in icao:
lat = lat or icao[metar.station_id]['lat']
lon = lon or icao[metar.station_id]['lon']
default_name = default_name or icao[
metar.station_id]['name']
station = self.save_station(
metar.station_id,
short_name,
name,
lat,
lon,
StationStatus.GREEN,
altitude=altitude,
tz=tz,
url=os.path.join(
self.provider_url,
f'site?id={metar.station_id}&db=metar'),
default_name=default_name
or f'{metar.station_id} Airport',
lookup_name=f'{metar.station_id} Airport ICAO')
station_id = station['_id']
if metar.station_id not in icao:
self.log.warning(
f"Missing '{metar.station_id}' ICAO in database. Is it '{station['name']}'?"
)
measures_collection = self.measures_collection(station_id)
new_measures = []
for key in stations[metar_id]:
metar = stations[metar_id][key]
if not self.has_measure(measures_collection, key):
temp = self.get_quantity(metar.temp,
self.temperature_units)
dew_point = self.get_quantity(
metar.dewpt, self.temperature_units)
humidity = self.compute_humidity(dew_point, temp)
measure = self.create_measure(
station,
key,
self.get_direction(metar.wind_dir),
self.get_quantity(metar.wind_speed,
self.speed_units),
self.get_quantity(
metar.wind_gust or metar.wind_speed,
self.speed_units),
temperature=temp,
humidity=humidity,
pressure=Pressure(qfe=None,
qnh=self.get_quantity(
metar.press,
self.pressure_units),
qff=self.get_quantity(
metar.press_sea_level,
self.pressure_units)))
new_measures.append(measure)
self.insert_new_measures(measures_collection, station,
new_measures)
except ProviderException as e:
self.log.warning(
f"Error while processing station '{metar_id}': {e}")
except Exception as e:
self.log.exception(
f"Error while processing station '{metar_id}': {e}")
except Exception as e:
self.log.exception(f'Error while processing Metar: {e}')
self.log.info('Done !')
def process_data(self):
stations = {}
try:
self.log.info("Processing FFVL data...")
result = requests.get("http://data.ffvl.fr/json/balises.json",
timeout=(self.connect_timeout,
self.read_timeout))
ffvl_stations = result.json()
for ffvl_station in ffvl_stations:
ffvl_id = None
try:
type = ffvl_station.get("station_type", "").lower()
if type not in ["holfuy", "pioupiou", "iweathar"]:
ffvl_id = ffvl_station["idBalise"]
station = self.save_station(
ffvl_id,
ffvl_station["nom"],
ffvl_station["nom"],
ffvl_station["latitude"],
ffvl_station["longitude"],
StationStatus.GREEN,
altitude=ffvl_station["altitude"],
url=ffvl_station["url"],
)
stations[station["_id"]] = station
except ProviderException as e:
self.log.warning(
f"Error while processing station '{ffvl_id}': {e}")
except Exception as e:
self.log.exception(
f"Error while processing station '{ffvl_id}': {e}")
except ProviderException as e:
self.log.warning(f"Error while processing stations: {e}")
except Exception as e:
self.log.exception(f"Error while processing stations: {e}")
try:
@retry(
wait=wait_random_exponential(multiplier=2, min=2),
stop=stop_after_delay(60),
after=after_log(self.log, logging.WARNING),
)
def request_data():
# data.ffvl.fr randomly returns an empty file instead the json doc
result = requests.get(
"http://data.ffvl.fr/json/relevesmeteo.json",
timeout=(self.connect_timeout, self.read_timeout))
return result.json()
ffvl_measures = request_data()
ffvl_tz = tz.gettz("Europe/Paris")
for ffvl_measure in ffvl_measures:
station_id = None
try:
ffvl_id = ffvl_measure["idbalise"]
station_id = self.get_station_id(ffvl_id)
if station_id not in stations:
raise ProviderException(
f"Unknown station '{station_id}'")
station = stations[station_id]
measures_collection = self.measures_collection(station_id)
key = arrow.get(ffvl_measure["date"],
"YYYY-MM-DD HH:mm:ss").replace(
tzinfo=ffvl_tz).int_timestamp
if not self.has_measure(measures_collection, key):
measure = self.create_measure(
station,
key,
ffvl_measure["directVentMoy"],
ffvl_measure["vitesseVentMoy"],
ffvl_measure["vitesseVentMax"],
temperature=ffvl_measure["temperature"],
humidity=ffvl_measure["hydrometrie"],
pressure=Pressure(qfe=ffvl_measure["pression"],
qnh=None,
qff=None),
)
self.insert_new_measures(measures_collection, station,
[measure])
except ProviderException as e:
self.log.warning(
f"Error while processing measures for station '{station_id}': {e}"
)
except Exception as e:
self.log.exception(
f"Error while processing measures for station '{station_id}': {e}"
)
except ProviderException as e:
self.log.warning(f"Error while processing FFVL: {e}")
except Exception as e:
self.log.exception(f"Error while processing FFVL: {e}")
self.log.info("...Done!")
def save_measures(self, jdc_id, blocs, stations_metadata):
try:
jdc_station = list(
filter(lambda d: str(d['id']) == jdc_id, stations_metadata))[0]
station = self.save_station(jdc_id,
jdc_station['short_name'],
jdc_station['name'],
jdc_station['latitude'],
jdc_station['longitude'],
self.get_status(jdc_station['status']),
altitude=jdc_station['altitude'],
url=urllib.parse.urljoin(
self.provider_url,
'/station/' + jdc_id))
station_id = station['_id']
try:
jdc_measures = blocs[0]['mesures']
except Exception:
self.log.warning('Unable to find a bloc with measures')
jdc_measures = []
self.log.info(
f"Station '{jdc_id}' ({station['name']}) found {len(jdc_measures)} measures"
)
measures_collection = self.measures_collection(station_id)
for jdc_measure in jdc_measures:
key = jdc_measure['datetime']
if not self.has_measure(measures_collection, key):
try:
new_measure = self.create_measure(
station,
key,
self.find_value(jdc_measure, 'Direction du vent'),
self.find_value(jdc_measure, 'Vent moyen'),
self.find_value(jdc_measure, 'Vent max'),
temperature=self.find_value(
jdc_measure, "Température de l'air"),
humidity=self.find_value(jdc_measure, 'Humidité'),
pressure=Pressure(qfe=self.find_value(
jdc_measure, 'Pression atmosphérique'),
qnh=None,
qff=None),
rain=self.find_value(jdc_measure, 'Pluviométrie'))
self.insert_new_measures(measures_collection, station,
[new_measure])
except ProviderException as e:
self.log.warning(
f"Error while processing measure '{key}' for station '{station_id}': {e}"
)
except Exception as e:
self.log.exception(
f"Error while processing measure '{key}' for station '{station_id}': {e}"
)
except ProviderException as e:
self.log.warning(f"Error while processing station '{jdc_id}': {e}")
except Exception as e:
self.log.exception(
f"Error while processing station '{jdc_id}': {e}")
def process_data(self):
try:
self.log.info("Processing iWeathar data...")
result_tree = etree.parse(
requests.get(
f"https://iweathar.co.za/live_data.php?unit=kmh&key={self.iweathar_key}",
stream=True,
timeout=(self.connect_timeout, self.read_timeout),
).raw)
for item in result_tree.xpath("//ITEM"):
station_id = None
try:
iweathar_id = item.xpath("STATION_ID")[0].text
name = item.xpath("LOCATION")[0].text
status = StationStatus.GREEN if item.xpath(
"STATUS")[0].text == "ON-LINE" else StationStatus.RED
station = self.save_station(
iweathar_id,
name,
name,
item.xpath("LAT")[0].text,
item.xpath("LONG")[0].text,
status,
url=f"{self.provider_url}/display?s_id={iweathar_id}",
)
station_id = station["_id"]
key_attr = item.xpath("UNIX_DATE_STAMP")
wind_dir_attr = item.xpath("WIND_ANG")
wind_avg_attr = item.xpath("WIND_AVG")
wind_max_attr = item.xpath("WIND_MAX")
if not (key_attr and wind_dir_attr and wind_avg_attr
and wind_max_attr):
self.log.warning(
f"Station '{station_id}' has invalid data")
continue
key = int(key_attr[0].text)
measures_collection = self.measures_collection(station_id)
if not self.has_measure(measures_collection, key):
try:
temperature_attr = item.xpath("TEMPERATURE_C")
if temperature_attr and temperature_attr[0].text:
temperature = Q_(temperature_attr[0].text,
ureg.degC)
else:
temperature = None
humidity_attr = item.xpath("HUMIDITY_PERC")
if humidity_attr and humidity_attr[0].text:
humidity = humidity_attr[0].text
else:
humidity = None
pressure_attr = item.xpath("PRESSURE_MB")
if pressure_attr and pressure_attr[0].text:
pressure = Pressure(qfe=pressure_attr[0].text,
qnh=None,
qff=None)
else:
pressure = None
rain_attr = item.xpath("RAINFALL_MM")
if rain_attr and rain_attr[0].text:
rain = Q_(rain_attr[0].text,
ureg.liter / (ureg.meter**2))
else:
rain = None
measure = self.create_measure(
station,
key,
wind_dir_attr[0].text,
wind_avg_attr[0].text,
wind_max_attr[0].text,
temperature=temperature,
humidity=humidity,
pressure=pressure,
rain=rain,
)
self.insert_new_measures(measures_collection,
station, [measure])
except ProviderException as e:
self.log.warning(
f"Error while processing measure '{key}' for station '{station_id}': {e}"
)
except Exception as e:
self.log.exception(
f"Error while processing measure '{key}' for station '{station_id}': {e}"
)
except ProviderException as e:
self.log.warning(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(f"Error while processing iWeathar: {e}")
self.log.info("...Done!")
def process_data(self):
try:
self.log.info('Processing Romma data...')
romma_tz = tz.gettz('Europe/Paris')
result_tree = etree.fromstring(requests.get(f'http://romma.fr/releves_romma_xml.php?id={self.romma_key}',
timeout=(self.connect_timeout, self.read_timeout)).text)
for report in result_tree.xpath('//releves/releve'):
station_id = None
try:
romma_id = report.xpath('id')[0].text
name = report.xpath('station')[0].text
status = StationStatus.GREEN if report.xpath('valide')[0].text == '1' else StationStatus.RED
station = self.save_station(
romma_id,
name,
name,
report.xpath('latitude')[0].text,
report.xpath('longitude')[0].text,
status,
altitude=report.xpath('altitude')[0].text,
url=urllib.parse.urljoin(self.provider_url, f'/station_24.php?id={romma_id}'))
station_id = station['_id']
wind_dir = self.get_value(report.xpath('direction')[0].text)
if not wind_dir:
self.log.warning(f"Station '{station_id}' has no wind direction value")
continue
measures_collection = self.measures_collection(station_id)
key = arrow.get(report.xpath('date')[0].text, 'D.MM.YYYY H:mm').replace(
tzinfo=romma_tz).timestamp
if not self.has_measure(measures_collection, key):
try:
measure = self.create_measure(
station,
key,
self.wind_directions[wind_dir],
self.get_value(report.xpath('vent_moyen_10')[0].text),
self.get_value(report.xpath('rafale_maxi')[0].text),
temperature=self.get_value(report.xpath('temperature')[0].text),
humidity=self.get_value(report.xpath('humidite')[0].text),
pressure=Pressure(
qfe=self.get_value(report.xpath('pression')[0].text),
qnh=None,
qff=None)
)
self.insert_new_measures(measures_collection, station, [measure])
except ProviderException as e:
self.log.warning(f"Error while processing measure '{key}' for station '{station_id}': {e}")
except Exception as e:
self.log.exception(
f"Error while processing measure '{key}' for station '{station_id}': {e}")
except ProviderException as e:
self.log.warning(f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(f'Error while processing Romma: {e}')
self.log.info('...Done!')
def process_data(self):
try:
self.log.info("Processing Pioupiou data...")
result = requests.get(
"https://api.pioupiou.fr/v1/live-with-meta/all",
timeout=(self.connect_timeout, self.read_timeout))
station_id = None
for piou_station in result.json()["data"]:
try:
piou_id = piou_station["id"]
location = piou_station["location"]
latitude = location.get("latitude")
longitude = location.get("longitude")
if (latitude is None
or longitude is None) or (latitude == 0
and longitude == 0):
continue
location_date = None
if location["date"]:
try:
location_date = arrow.get(location["date"])
except ParserError:
pass
station = self.save_station(
piou_id,
None,
None,
latitude,
longitude,
self.get_status(station_id,
piou_station["status"]["state"],
location_date, location["success"]),
url=f"{self.provider_url}/PP{piou_id}",
default_name=piou_station.get("meta",
{}).get("name", None),
)
station_id = station["_id"]
measures_collection = self.measures_collection(station_id)
new_measures = []
piou_measure = piou_station["measurements"]
last_measure_date = arrow.get(piou_measure["date"])
key = last_measure_date.int_timestamp
if not self.has_measure(measures_collection, key):
measure = self.create_measure(
station,
key,
piou_measure["wind_heading"],
piou_measure["wind_speed_avg"],
piou_measure["wind_speed_max"],
pressure=Pressure(qfe=piou_measure["pressure"],
qnh=None,
qff=None),
)
new_measures.append(measure)
self.insert_new_measures(measures_collection, station,
new_measures)
except ProviderException as e:
self.log.warning(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(f"Error while processing Pioupiou: {e}")
self.log.info("Done !")
def process_data(self):
stations = {}
try:
self.log.info('Processing FFVL data...')
result = requests.get(
'http://data.ffvl.fr/json/balises.json', timeout=(self.connect_timeout, self.read_timeout))
ffvl_stations = result.json()
for ffvl_station in ffvl_stations:
ffvl_id = None
try:
type = ffvl_station.get('station_type', '').lower()
if type not in ['holfuy', 'pioupiou', 'iweathar']:
ffvl_id = ffvl_station['idBalise']
station = self.save_station(
ffvl_id,
ffvl_station['nom'],
ffvl_station['nom'],
ffvl_station['latitude'],
ffvl_station['longitude'],
StationStatus.GREEN,
altitude=ffvl_station['altitude'],
url=ffvl_station['url'])
stations[station['_id']] = station
except ProviderException as e:
self.log.warning(f"Error while processing station '{ffvl_id}': {e}")
except Exception as e:
self.log.exception(f"Error while processing station '{ffvl_id}': {e}")
except ProviderException as e:
self.log.warning(f'Error while processing stations: {e}')
except Exception as e:
self.log.exception(f'Error while processing stations: {e}')
try:
@retry(wait=wait_random_exponential(multiplier=2, min=2), stop=stop_after_delay(60),
after=after_log(self.log, logging.WARNING))
def request_data():
# data.ffvl.fr randomly returns an empty file instead the json doc
result = requests.get(
'http://data.ffvl.fr/json/relevesmeteo.json', timeout=(self.connect_timeout, self.read_timeout))
return result.json()
ffvl_measures = request_data()
ffvl_tz = tz.gettz('Europe/Paris')
for ffvl_measure in ffvl_measures:
station_id = None
try:
ffvl_id = ffvl_measure['idbalise']
station_id = self.get_station_id(ffvl_id)
if station_id not in stations:
raise ProviderException(f"Unknown station '{station_id}'")
station = stations[station_id]
measures_collection = self.measures_collection(station_id)
key = arrow.get(ffvl_measure['date'], 'YYYY-MM-DD HH:mm:ss').replace(tzinfo=ffvl_tz).timestamp
if not self.has_measure(measures_collection, key):
measure = self.create_measure(
station,
key,
ffvl_measure['directVentMoy'],
ffvl_measure['vitesseVentMoy'],
ffvl_measure['vitesseVentMax'],
temperature=ffvl_measure['temperature'],
humidity=ffvl_measure['hydrometrie'],
pressure=Pressure(qfe=ffvl_measure['pression'], qnh=None, qff=None)
)
self.insert_new_measures(measures_collection, station, [measure])
except ProviderException as e:
self.log.warning(f"Error while processing measures for station '{station_id}': {e}")
except Exception as e:
self.log.exception(f"Error while processing measures for station '{station_id}': {e}")
except ProviderException as e:
self.log.warning(f'Error while processing FFVL: {e}')
except Exception as e:
self.log.exception(f'Error while processing FFVL: {e}')
self.log.info('...Done!')
def process_data(self):
try:
self.log.info('Processing Holfuy data...')
holfuy_stations = requests.get(
'https://api.holfuy.com/stations/stations.json',
timeout=(self.connect_timeout, self.read_timeout)).json()
holfuy_data = requests.get(
'https://api.holfuy.com/live/?s=all&m=JSON&tu=C&su=km/h&utc',
timeout=(self.connect_timeout, self.read_timeout)).json()
holfuy_measures = {}
for holfuy_measure in holfuy_data['measurements']:
holfuy_measures[holfuy_measure['stationId']] = holfuy_measure
for holfuy_station in holfuy_stations['holfuyStationsList']:
holfuy_id = None
station_id = None
try:
holfuy_id = holfuy_station['id']
name = holfuy_station['name']
location = holfuy_station['location']
latitude = location.get('latitude')
longitude = location.get('longitude')
if (latitude is None
or longitude is None) or (latitude == 0
and longitude == 0):
raise ProviderException('No geolocation found')
altitude = location.get('altitude')
urls = {
lang: url.format(id=holfuy_id)
for lang, url in self.provider_urls.items()
}
station = self.save_station(holfuy_id,
name,
name,
latitude,
longitude,
StationStatus.GREEN,
altitude=altitude,
url=urls)
station_id = station['_id']
measures_collection = self.measures_collection(station_id)
new_measures = []
if holfuy_id not in holfuy_measures:
raise ProviderException(
f"Station '{name}' not found in 'api.holfuy.com/live/'"
)
holfuy_measure = holfuy_measures[holfuy_id]
last_measure_date = arrow.get(holfuy_measure['dateTime'])
key = last_measure_date.timestamp
if not self.has_measure(measures_collection, key):
measure = self.create_measure(
station,
key,
holfuy_measure['wind']['direction'],
Q_(holfuy_measure['wind']['speed'],
ureg.kilometer / ureg.hour),
Q_(holfuy_measure['wind']['gust'],
ureg.kilometer / ureg.hour),
temperature=Q_(holfuy_measure['temperature'],
ureg.degC)
if 'temperature' in holfuy_measure else None,
pressure=Pressure(
qfe=None,
qnh=Q_(holfuy_measure['pressure'], ureg.hPa)
if 'pressure' in holfuy_measure else None,
qff=None))
new_measures.append(measure)
self.insert_new_measures(measures_collection, station,
new_measures)
except ProviderException as e:
self.log.warning(
f"Error while processing station '{station_id or holfuy_id}': {e}"
)
except Exception as e:
self.log.exception(
f"Error while processing station '{station_id or holfuy_id}': {e}"
)
except Exception as e:
self.log.exception(f'Error while processing Holfuy: {e}')
self.log.info('Done !')
def process_data(self):
try:
self.log.info("Processing Holfuy data...")
holfuy_stations = requests.get(
"https://api.holfuy.com/stations/stations.json",
timeout=(self.connect_timeout, self.read_timeout)).json()
holfuy_data = requests.get(
"https://api.holfuy.com/live/?s=all&m=JSON&tu=C&su=km/h&utc",
timeout=(self.connect_timeout, self.read_timeout),
).json()
holfuy_measures = {}
for holfuy_measure in holfuy_data["measurements"]:
holfuy_measures[holfuy_measure["stationId"]] = holfuy_measure
for holfuy_station in holfuy_stations["holfuyStationsList"]:
holfuy_id = None
station_id = None
try:
holfuy_id = holfuy_station["id"]
name = holfuy_station["name"]
location = holfuy_station["location"]
latitude = location.get("latitude")
longitude = location.get("longitude")
if (latitude is None
or longitude is None) or (latitude == 0
and longitude == 0):
raise ProviderException("No geolocation found")
altitude = location.get("altitude")
urls = {
lang: url.format(id=holfuy_id)
for lang, url in self.provider_urls.items()
}
station = self.save_station(holfuy_id,
name,
name,
latitude,
longitude,
StationStatus.GREEN,
altitude=altitude,
url=urls)
station_id = station["_id"]
measures_collection = self.measures_collection(station_id)
new_measures = []
if holfuy_id not in holfuy_measures:
raise ProviderException(
f"Station '{name}' not found in 'api.holfuy.com/live/'"
)
holfuy_measure = holfuy_measures[holfuy_id]
last_measure_date = arrow.get(holfuy_measure["dateTime"])
key = last_measure_date.int_timestamp
if not self.has_measure(measures_collection, key):
measure = self.create_measure(
station,
key,
holfuy_measure["wind"]["direction"],
Q_(holfuy_measure["wind"]["speed"],
ureg.kilometer / ureg.hour),
Q_(holfuy_measure["wind"]["gust"],
ureg.kilometer / ureg.hour),
temperature=Q_(holfuy_measure["temperature"],
ureg.degC)
if "temperature" in holfuy_measure else None,
pressure=Pressure(
qfe=None,
qnh=Q_(holfuy_measure["pressure"], ureg.hPa)
if "pressure" in holfuy_measure else None,
qff=None,
),
)
new_measures.append(measure)
self.insert_new_measures(measures_collection, station,
new_measures)
except ProviderException as e:
self.log.warning(
f"Error while processing station '{station_id or holfuy_id}': {e}"
)
except Exception as e:
self.log.exception(
f"Error while processing station '{station_id or holfuy_id}': {e}"
)
except Exception as e:
self.log.exception(f"Error while processing Holfuy: {e}")
self.log.info("Done !")
def process_data(self):
try:
self.log.info("Processing MeteoSwiss data...")
url_pattern = (
"https://data.geo.admin.ch/ch.meteoschweiz.messwerte-{parameter}/"
"ch.meteoschweiz.messwerte-{parameter}_en.json"
)
main_wind = requests.get(
url_pattern.format(parameter="windgeschwindigkeit-kmh-10min"),
timeout=(self.connect_timeout, self.read_timeout),
).json()
wind_gust = requests.get(
url_pattern.format(parameter="wind-boeenspitze-kmh-10min"),
timeout=(self.connect_timeout, self.read_timeout),
).json()
temperature = requests.get(
url_pattern.format(parameter="lufttemperatur-10min"), timeout=(self.connect_timeout, self.read_timeout)
).json()
humidity = requests.get(
url_pattern.format(parameter="luftfeuchtigkeit-10min"),
timeout=(self.connect_timeout, self.read_timeout),
).json()
pressure_qfe = requests.get(
url_pattern.format(parameter="luftdruck-qfe-10min"), timeout=(self.connect_timeout, self.read_timeout)
).json()
pressure_qnh = requests.get(
url_pattern.format(parameter="luftdruck-qnh-10min"), timeout=(self.connect_timeout, self.read_timeout)
).json()
pressure_qff = requests.get(
url_pattern.format(parameter="luftdruck-qff-10min"), timeout=(self.connect_timeout, self.read_timeout)
).json()
rain = requests.get(
url_pattern.format(parameter="niederschlag-10min"), timeout=(self.connect_timeout, self.read_timeout)
).json()
if (
main_wind["creation_time"]
!= wind_gust["creation_time"]
!= temperature["creation_time"]
!= humidity["creation_time"]
!= pressure_qfe["creation_time"]
!= pressure_qnh["creation_time"]
!= pressure_qff["creation_time"]
!= rain["creation_time"]
):
self.log.error("Creation time of parameters files are not the same")
main_wind_data = main_wind["features"]
wind_gust_data = self.to_dict(wind_gust["features"])
temperature_data = self.to_dict(temperature["features"])
humidity_data = self.to_dict(humidity["features"])
pressure_qfe_data = self.to_dict(pressure_qfe["features"])
pressure_qnh_data = self.to_dict(pressure_qnh["features"])
pressure_qff_data = self.to_dict(pressure_qff["features"])
rain_data = self.to_dict(rain["features"])
station_id = None
for meteoswiss_station in main_wind_data:
try:
meteoswiss_id = meteoswiss_station["id"]
location = meteoswiss_station["geometry"]["coordinates"]
urls = {
lang: url.format(param="messwerte-windgeschwindigkeit-kmh-10min", id=meteoswiss_id)
for lang, url in self.provider_urls.items()
}
lat, lon = self.lv85_to_wgs84.transform(location[0], location[1])
station = self.save_station(
meteoswiss_id,
meteoswiss_station["properties"]["station_name"],
meteoswiss_station["properties"]["station_name"],
lat,
lon,
StationStatus.GREEN,
altitude=meteoswiss_station["properties"]["altitude"],
tz="Europe/Zurich",
url=urls,
)
station_id = station["_id"]
timestamp = meteoswiss_station["properties"].get("reference_ts", None)
if not timestamp or timestamp == "-":
self.log.warning(f"'{station_id}' has no timestamp field")
continue
key = arrow.get(
meteoswiss_station["properties"]["reference_ts"], "YYYY-MM-DDTHH:mm:ssZ"
).int_timestamp
measures_collection = self.measures_collection(station_id)
new_measures = []
if meteoswiss_id in temperature_data:
temperature = self.get_value(temperature_data[meteoswiss_id]["properties"], unit=ureg.degC)
else:
temperature = None
if meteoswiss_id in humidity_data:
humidity = humidity_data[meteoswiss_id]["properties"]["value"]
else:
humidity = None
if meteoswiss_id in pressure_qfe_data:
pressure = Pressure(
qfe=self.get_pressure_value(pressure_qfe_data, meteoswiss_id),
qnh=self.get_pressure_value(pressure_qnh_data, meteoswiss_id),
qff=self.get_pressure_value(pressure_qff_data, meteoswiss_id),
)
else:
pressure = None
if meteoswiss_id in rain_data:
# 1mm = 1 liter/m^2
rain = self.get_value(
rain_data[meteoswiss_id]["properties"], unit=ureg.liter / (ureg.meter ** 2)
)
else:
rain = None
if not self.has_measure(measures_collection, key):
measure = self.create_measure(
station,
key,
meteoswiss_station["properties"]["wind_direction"],
self.get_value(meteoswiss_station["properties"]),
self.get_value(wind_gust_data[meteoswiss_id]["properties"]),
temperature=temperature,
humidity=humidity,
pressure=pressure,
rain=rain,
)
new_measures.append(measure)
self.insert_new_measures(measures_collection, station, new_measures)
except ProviderException as e:
self.log.warning(f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(f"Error while processing MeteoSwiss: {e}")
self.log.info("...Done!")
def process_data(self):
try:
self.log.info('Processing MeteoSwiss data...')
url_pattern = 'https://data.geo.admin.ch/ch.meteoschweiz.messwerte-{parameter}/' \
'ch.meteoschweiz.messwerte-{parameter}_en.json'
main_wind = requests.get(
url_pattern.format(parameter='windgeschwindigkeit-kmh-10min'),
timeout=(self.connect_timeout, self.read_timeout)).json()
wind_gust = requests.get(
url_pattern.format(parameter='wind-boeenspitze-kmh-10min'),
timeout=(self.connect_timeout, self.read_timeout)).json()
temperature = requests.get(
url_pattern.format(parameter='lufttemperatur-10min'),
timeout=(self.connect_timeout, self.read_timeout)).json()
humidity = requests.get(
url_pattern.format(parameter='luftfeuchtigkeit-10min'),
timeout=(self.connect_timeout, self.read_timeout)).json()
pressure_qfe = requests.get(
url_pattern.format(parameter='luftdruck-qfe-10min'),
timeout=(self.connect_timeout, self.read_timeout)).json()
pressure_qnh = requests.get(
url_pattern.format(parameter='luftdruck-qnh-10min'),
timeout=(self.connect_timeout, self.read_timeout)).json()
pressure_qff = requests.get(
url_pattern.format(parameter='luftdruck-qff-10min'),
timeout=(self.connect_timeout, self.read_timeout)).json()
rain = requests.get(
url_pattern.format(parameter='niederschlag-10min'),
timeout=(self.connect_timeout, self.read_timeout)).json()
if main_wind['creation_time'] != wind_gust['creation_time'] != temperature['creation_time'] != \
humidity['creation_time'] != pressure_qfe['creation_time'] != pressure_qnh['creation_time'] != \
pressure_qff['creation_time'] != rain['creation_time']:
self.log.error(
'Creation time of parameters files are not the same')
main_wind_data = main_wind['features']
wind_gust_data = self.to_dict(wind_gust['features'])
temperature_data = self.to_dict(temperature['features'])
humidity_data = self.to_dict(humidity['features'])
pressure_qfe_data = self.to_dict(pressure_qfe['features'])
pressure_qnh_data = self.to_dict(pressure_qnh['features'])
pressure_qff_data = self.to_dict(pressure_qff['features'])
rain_data = self.to_dict(rain['features'])
station_id = None
for meteoswiss_station in main_wind_data:
try:
meteoswiss_id = meteoswiss_station['id']
location = meteoswiss_station['geometry']['coordinates']
urls = {
lang: url.format(
param='messwerte-windgeschwindigkeit-kmh-10min',
id=meteoswiss_id)
for lang, url in self.provider_urls.items()
}
lat, lon = self.lv85_to_wgs84.transform(
location[0], location[1])
station = self.save_station(
meteoswiss_id,
meteoswiss_station['properties']['station_name'],
meteoswiss_station['properties']['station_name'],
lat,
lon,
StationStatus.GREEN,
altitude=meteoswiss_station['properties']['altitude'],
tz='Europe/Zurich',
url=urls)
station_id = station['_id']
timestamp = meteoswiss_station['properties'].get(
'reference_ts', None)
if not timestamp:
self.log.warning(
f"'{station_id}' has no timestamp field")
continue
key = arrow.get(
meteoswiss_station['properties']['reference_ts'],
'YY-MM-DDTHH:mm:ZZ').timestamp
measures_collection = self.measures_collection(station_id)
new_measures = []
if meteoswiss_id in temperature_data:
temperature = self.get_value(
temperature_data[meteoswiss_id]['properties'],
unit=ureg.degC)
else:
temperature = None
if meteoswiss_id in humidity_data:
humidity = humidity_data[meteoswiss_id]['properties'][
'value']
else:
humidity = None
if meteoswiss_id in pressure_qfe_data:
pressure = Pressure(
qfe=self.get_value(pressure_qfe_data[meteoswiss_id]
['properties']),
qnh=self.get_value(pressure_qnh_data[meteoswiss_id]
['properties']),
qff=self.get_value(pressure_qff_data[meteoswiss_id]
['properties']))
else:
pressure = None
if meteoswiss_id in rain_data:
# 1mm = 1 liter/m^2
rain = self.get_value(
rain_data[meteoswiss_id]['properties'],
unit=ureg.liter / (ureg.meter**2))
else:
rain = None
if not self.has_measure(measures_collection, key):
measure = self.create_measure(
station,
key,
meteoswiss_station['properties']['wind_direction'],
self.get_value(meteoswiss_station['properties']),
self.get_value(
wind_gust_data[meteoswiss_id]['properties']),
temperature=temperature,
humidity=humidity,
pressure=pressure,
rain=rain)
new_measures.append(measure)
self.insert_new_measures(measures_collection, station,
new_measures)
except ProviderException as e:
self.log.warning(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(
f"Error while processing station '{station_id}': {e}")
except Exception as e:
self.log.exception(f'Error while processing MeteoSwiss: {e}')
self.log.info('...Done!')