def get_readings_from_api(self):
# Fetch data from remote API.
log.info('Requesting luftdaten.info live API at {}'.format(self.uri))
payload = self.session.get(self.uri).content.decode('utf-8')
data = json.loads(payload)
#pprint(data)
# Mungle timestamp to be formally in ISO 8601 format (UTC).
timestamp = self.convert_timestamp(data[0]['timestamp'])
log.info('Timestamp of first record: {}'.format(timestamp))
# Apply data filter.
data = self.apply_filter(data)
# Transform live API items to actual readings while optionally
# applying a number of transformation and enrichment steps.
for item in self.wrap_progress(data):
try:
reading = self.make_reading(item)
if reading is None:
continue
if not self.make_observations_from_api(item, reading):
continue
log.debug(f'API reading:\n{json.dumps(reading, indent=2)}')
yield reading
except Exception as ex:
log.warning('Could not make reading from {}.\n{}'.format(
item, exception_traceback()))
def request_live_data(self):
log.info('Requesting Live API at {}'.format(self.uri))
payload = self.session.get(self.uri).content.decode('utf-8')
data = json.loads(payload)
#pprint(data)
timestamp = self.convert_timestamp(data[0]['timestamp'])
log.info('Timestamp of first record: {}'.format(timestamp))
iterator = data
if self.progressbar:
iterator = tqdm(data)
for item in iterator:
try:
reading = self.make_reading(item)
if reading is None:
continue
yield reading
except Exception as ex:
log.warning('Could not make reading from {}.\n{}'.format(
item, exception_traceback()))
def get_current_measurement_readings(self):
"""
Acquire data from the "measurements" API.
https://docs.openaq.org/#api-Measurements
"""
# Fetch data from remote API.
log.info('Requesting measurement data from OpenAQ')
api = openaq.OpenAQ()
params = {}
if self.filter and 'country' in self.filter:
params['country'] = self.filter['country']
# TODO: What to do with readings which do not have any geographic information?
# TODO: Raise limit by introducing result paging.
date_from = self.last_hour()
status, response = api.measurements(date_from=date_from,
has_geo=True,
limit=10000,
include_fields=['attribution'],
**params)
data = response['results']
if not data:
log.warning('No records found beginning {} with filter {}'.format(
date_from, params))
return
# Mungle timestamp to be formally in ISO 8601 format (UTC).
timestamp = data[0]['date']['utc']
log.info('Timestamp of first record: {}'.format(timestamp))
# Apply data filter.
#data = self.apply_filter(data)
# Transform live API items to actual readings while optionally
# applying a number of transformation and enrichment steps.
readings = {}
for item in self.wrap_progress(data):
try:
self.process_measurement(readings, item)
except Exception as ex:
log.warning('Could not use observation from {}.\n{}'.format(
item, exception_traceback()))
for reading in readings.values():
has_data = False
for observation in reading.observations:
if observation['data']:
has_data = True
break
if has_data:
yield reading
def enrich_station(self, station):
# Sanity checks.
if ('latitude' not in station.position or 'longitude' not in station.position) or \
(station.position.latitude is None or station.position.longitude is None):
# TODO: Just emit this message once per X.
log.warning(
'Incomplete station position, skipping geospatial enrichment. Station: {}'
.format(station))
return
# Compute geohash.
station.position.geohash = geohash_encode(station.position.latitude,
station.position.longitude)
# Compute human readable location name.
if self.reverse_geocode:
try:
# Reverse-geocode position.
station.location = resolve_location(
latitude=station.position.latitude,
longitude=station.position.longitude,
geohash=station.position.geohash,
country_code=station.position.country,
)
# Improve location information.
improve_location(station.location)
# Format address into single label.
station.name = format_address(station.location)
try:
if station.name.lower() == station.position.country.lower(
):
del station['name']
except:
pass
except Exception as ex:
log.error(
u'Problem with reverse geocoder for station {}: {}\n{}'.
format(station, ex, exception_traceback()))
if 'name' not in station:
station.name = u'Station #{}'.format(station.station_id)
try:
station.name += ', ' + station.position.country
except:
pass
def get_latest_readings(self):
"""
Acquire data from the "latest" API.
https://docs.openaq.org/#api-Latest
"""
# Fetch data from remote API.
log.info('Requesting latest data from OpenAQ')
api = openaq.OpenAQ()
# Example.
#res = api.latest(city='Delhi', parameter='pm25', df=True)
#print(res.columns)
params = {}
if self.filter and 'country' in self.filter:
params['country'] = self.filter['country']
# TODO: What to do with readings which do not have any geographic information?
# TODO: Raise limit by introducing result paging.
status, response = api.latest(
has_geo=True,
limit=10000,
include_fields=['attribution', 'averagingPeriod', 'sourceName'],
**params)
data = response['results']
# Mungle timestamp to be formally in ISO 8601 format (UTC).
timestamp = data[0]['measurements'][0]['lastUpdated']
log.info('Timestamp of first record: {}'.format(timestamp))
# Apply data filter.
#data = self.apply_filter(data)
# Transform live API items to actual readings while optionally
# applying a number of transformation and enrichment steps.
for item in self.wrap_progress(data):
try:
reading = self.make_reading_from_latest(item)
if reading is None:
continue
log.debug(f'API reading:\n{json.dumps(reading, indent=2)}')
yield reading
except Exception as ex:
log.warning('Could not make reading from {}.\n{}'.format(
item, exception_traceback()))
def csv_reader(self, csvpath, fieldnames):
payload = None
if self.quick_mode:
try:
payload = open(csvpath).read(256)
payload = '\n'.join(payload.split('\n')[:2])
except:
pass
else:
payload = open(csvpath).read()
if payload is None:
log.error('Could not read CSV file %s', csvpath)
return
# Read CSV file into tablib's Dataset and cast to dictionary representation.
imported_data = Dataset().load(payload, format='csv', delimiter=';')
records = imported_data.dict
# Iterate all CSV records.
for record in records:
item = self.make_item(record)
try:
reading = self.make_reading(item)
if reading is None:
continue
if not self.csvdata_to_reading(record, reading, fieldnames):
continue
yield reading
except Exception as ex:
log.warning('Could not make reading from {}.\n{}'.format(
item, exception_traceback()))