def push_smartcitizenkits(url, first_date_to_retrieve_sck,
last_date_to_retrieve_sck, id_dataset, id_sck,
geohashes, center):
"""
Push Smart Citizen Kit data to the necessary datasets
Keyword arguments:
url -- the url to get the data
first_date_to_retrieve_sck -- the start of the hour
last_date_to_retrieve_sck -- the end of the hour
id_dataset -- the id of the dataset to push data every hour
id_sck -- id of the sck dataset
geohashes -- geohashes to cover
center -- center of the area
"""
page = 1
finish = False
while not finish:
result_from_get = get_smartcitizenkits(url, page,
first_date_to_retrieve_sck,
last_date_to_retrieve_sck,
geohashes, center)
if result_from_get is not None:
if result_from_get["records"] is not None:
utils.ckan_upsert(id_sck, result_from_get["records"])
push_to_hourly(id_dataset, result_from_get["records"], id_sck)
if result_from_get["finish"]:
finish = True
else:
page += 1
else:
finish = True
def insert_necessary_fields():
try:
utils.ckan_upsert(ID_BRISTOL_NECESSARY_FIELDS,
RECORDS_BRISTOL_NECESSARY_FIELDS)
print("done")
except Exception as exception_returned:
print(exception_returned)
def push_to_daily(id_dataset, records, id_defra_aurn):
"""
Push the data every day to the aggregated dataset
Keyword arguments:
id_dataset -- the id of the dataset where to push data
id_defra_aurn -- id of the aurn dataset
"""
for record in records:
record["dataset_id"] = id_defra_aurn
record["dataset_name"] = "automatic-urban-and-rural-network-aurn"
utils.ckan_upsert(id_dataset, records)
def push_luftdaten(url, id_dataset, id_luftdaten):
"""
Push Luftdaten data to the necessary datasets
Keyword arguments:
url -- the url to get the data
id_dataset -- the id of the dataset to push data every hour
id_luftdaten -- the id of the luftdaten dataset
"""
to_push = get_luftdaten(url)
if to_push is not None:
utils.ckan_upsert(id_luftdaten, to_push)
push_to_hourly(id_dataset, to_push, id_luftdaten)
def push_to_hourly(id_dataset, records, id_luftdaten):
"""
Push the data every hour to the aggregated dataset
Keyword arguments:
id_dataset -- the id of the dataset where to push data
records -- records that were pushed to the Luftdaten dataset
id_luftdaten -- the id of the luftdaten dataset
"""
for record in records:
record["dataset_id"] = id_luftdaten
record["dataset_name"] = "luftdaten"
utils.ckan_upsert(id_dataset, records)
def push_to_yearly_poly_popest(id_yearlypolygondataset, records, id_population_estimates):
"""
Push the data every year to the aggregated dataset
Keyword arguments:
id_yearlypolygondataset -- the id of the dataset where to push data
records -- records that were pushed to the dataset
id_population_estimates -- the id of the population estimates dataset
"""
for record in records:
record["dataset_id"] = id_population_estimates
record["dataset_name"] = "population-estimates"
utils.ckan_upsert(id_yearlypolygondataset, records)
def push_to_yearly_point(id_yearlypointdataset, records, id_no2):
"""
Push the data every year to the aggregated dataset
Keyword arguments:
id_yearlypointdataset -- the id of the dataset where to push data
records -- records that were pushed to the dataset
id_no2 -- the id of the no2 dataset
"""
for record in records:
record["dataset_id"] = id_no2
record["dataset_name"] = "air-quality-no2-diffusion-tube-data"
utils.ckan_upsert(id_yearlypointdataset, records)
def push_to_hourly(id_hourlypointdataset, records, id_air_quality_data_continuous):
"""
Push the data every hour to the aggregated dataset
Keyword arguments:
id_hourlypointdataset -- the id of the dataset where to push data
records -- records that were pushed to the dataset
id_air_quality_data_continuous -- id of the air quality continuous dataset
"""
for record in records:
record["dataset_id"] = id_air_quality_data_continuous
record["dataset_name"] = "air-quality-data-continuous"
utils.ckan_upsert(id_hourlypointdataset, records)
def push_wards(url, id_wards):
"""
Push data to the necessary datasets
Keyword arguments:
url -- the url to get the data
id_wards -- the id of the wards dataset
"""
to_push = get_wards(url)
if to_push is not None:
utils.ckan_upsert(id_wards, to_push)
return to_push
else:
return None
def push_populationestimates(url, last_year_to_retrieve, id_yearlypolygondataset, id_population_estimates):
"""
Push data to the necessary datasets
Keyword arguments:
url -- the url to get the data
last_year_to_retrieve -- year to retrieve
id_yearlypolygondataset -- the id of the dataset to push data every year
id_population_estimates -- the id of the population estimates dataset
"""
to_push = get_populationestimates(url, last_year_to_retrieve)
if to_push is not None:
utils.ckan_upsert(id_population_estimates, to_push)
push_to_yearly_poly_popest(id_yearlypolygondataset, to_push, id_population_estimates)
def push_no2(url, last_year_to_retrieve, id_yearlypointdataset, id_no2):
"""
Push data to the necessary datasets
Keyword arguments:
url -- the url to get the data
last_year_to_retrieve -- year to retrieve
id_yearlypointdataset -- the id of the dataset to push data every year
id_no2 -- the id of the no2 dataset
"""
to_push = get_no2(url, last_year_to_retrieve)
if to_push is not None:
utils.ckan_upsert(id_no2, to_push)
push_to_yearly_point(id_yearlypointdataset, to_push, id_no2)
def push_caravailability(url, id_car_availability):
"""
Push data to the necessary datasets
Keyword arguments:
url -- the url to get the data
id_car_availability -- the id of the car availaibility dataset
"""
to_push = get_caravailability(url)
if to_push is not None:
utils.ckan_upsert(id_car_availability, to_push)
return to_push
else:
return None
def push_airqualitydatacontinous(url, last_date_to_retrieve, id_hourlypointdataset, id_air_quality_data_continuous):
"""
Push data to the necessary datasets
Keyword arguments:
url -- the url to get the data
last_date_to_retrieve -- date to retrieve
id_dataset -- the id of the dataset to push data every hour
id_air_quality_data_continuous -- id of the air quality continuous dataset
"""
to_push = get_airqualitydatacontinous(url, last_date_to_retrieve)
if to_push is not None:
utils.ckan_upsert(id_air_quality_data_continuous, to_push)
push_to_hourly(id_hourlypointdataset, to_push, id_air_quality_data_continuous)
def push_aurn(id_dataset, files_folder, id_defra_aurn, stations):
"""
Push AURN data to the necessary datasets
Keyword arguments:
id_dataset -- the id of the dataset to push data every day
files_folder -- folder of the files
id_defra_aurn -- id of the aurn dataset
stations -- stations to check (files to check in folder)
"""
for file_aurn in stations:
to_push = get_aurn(file_aurn, files_folder)
if to_push is not None:
utils.ckan_upsert(id_defra_aurn, to_push)
push_to_daily(id_dataset, to_push, id_defra_aurn)
def push_to_hourly(id_dataset, records, id_sck):
"""
Push the data every hour to the aggregated dataset
Keyword arguments:
id_dataset -- the id of the dataset where to push data
records -- records that were pushed to the Smart Citizen dataset
id_sck -- id of the sck dataset
"""
for record in records:
try:
record["recordid"] = record.pop("deviceid")
except KeyError:
pass
record["dataset_id"] = id_sck
record["dataset_name"] = "smart-citizen-kits"
utils.ckan_upsert(id_dataset, records)
def push_to_yrly_caravail_wth_wards(id_yearlypolygondataset, wards, cars, id_car_availability):
"""
Push the data every year to the aggregated dataset
Keyword arguments:
id_yearlypolygondataset -- the id of the dataset where to push data
wards -- wards that were pushed to the dataset
cars -- cars data that were pushed to the dataset
id_car_availability -- the id of the car availaibility dataset
"""
new_cars = []
for car in cars:
found = False
for ward in wards:
if car["wardid"] == ward["wardid"]:
found = True
if "geojson" in ward:
car["geojson"] = ward["geojson"]
if "ward_center" in ward:
car["ward_center"] = ward["ward_center"]
if "objectid" in ward:
car["objectid"] = ward["objectid"]
if found:
new_cars.append(car)
for new_car in new_cars:
try:
if new_car["date_time"] == "2019-04-10T12:46:07.442000+00:00":
new_car["date_time"] = "2019-04-10T12:46:07+00:00"
LOGGER.info("Car Availablity Data record_timestamp is in wrong format--> Hacking here")
new_car["year"] = int(datetime.strptime(new_car.pop("date_time"),
'%Y-%m-%dT%H:%M:%S+00:00').year)
except KeyError:
pass
new_car["dataset_id"] = id_car_availability
new_car["dataset_name"] = "car-availability-by-ward"
utils.ckan_upsert(id_yearlypolygondataset, new_cars)
def insert_units():
try:
utils.ckan_upsert(ID_UNITS, RECORDS_POINT)
utils.ckan_upsert(ID_UNITS, RECORDS_POLYGON)
except Exception as exception_returned:
print(exception_returned)
