def push_smartcitizenkits(url, first_date_to_retrieve_sck, last_date_to_retrieve_sck, id_dataset, id_sck, geohashes, center): """ Push Smart Citizen Kit data to the necessary datasets Keyword arguments: url -- the url to get the data first_date_to_retrieve_sck -- the start of the hour last_date_to_retrieve_sck -- the end of the hour id_dataset -- the id of the dataset to push data every hour id_sck -- id of the sck dataset geohashes -- geohashes to cover center -- center of the area """ page = 1 finish = False while not finish: result_from_get = get_smartcitizenkits(url, page, first_date_to_retrieve_sck, last_date_to_retrieve_sck, geohashes, center) if result_from_get is not None: if result_from_get["records"] is not None: utils.ckan_upsert(id_sck, result_from_get["records"]) push_to_hourly(id_dataset, result_from_get["records"], id_sck) if result_from_get["finish"]: finish = True else: page += 1 else: finish = True
def insert_necessary_fields(): try: utils.ckan_upsert(ID_BRISTOL_NECESSARY_FIELDS, RECORDS_BRISTOL_NECESSARY_FIELDS) print("done") except Exception as exception_returned: print(exception_returned)
def push_to_daily(id_dataset, records, id_defra_aurn): """ Push the data every day to the aggregated dataset Keyword arguments: id_dataset -- the id of the dataset where to push data id_defra_aurn -- id of the aurn dataset """ for record in records: record["dataset_id"] = id_defra_aurn record["dataset_name"] = "automatic-urban-and-rural-network-aurn" utils.ckan_upsert(id_dataset, records)
def push_luftdaten(url, id_dataset, id_luftdaten): """ Push Luftdaten data to the necessary datasets Keyword arguments: url -- the url to get the data id_dataset -- the id of the dataset to push data every hour id_luftdaten -- the id of the luftdaten dataset """ to_push = get_luftdaten(url) if to_push is not None: utils.ckan_upsert(id_luftdaten, to_push) push_to_hourly(id_dataset, to_push, id_luftdaten)
def push_to_hourly(id_dataset, records, id_luftdaten): """ Push the data every hour to the aggregated dataset Keyword arguments: id_dataset -- the id of the dataset where to push data records -- records that were pushed to the Luftdaten dataset id_luftdaten -- the id of the luftdaten dataset """ for record in records: record["dataset_id"] = id_luftdaten record["dataset_name"] = "luftdaten" utils.ckan_upsert(id_dataset, records)
def push_to_yearly_poly_popest(id_yearlypolygondataset, records, id_population_estimates): """ Push the data every year to the aggregated dataset Keyword arguments: id_yearlypolygondataset -- the id of the dataset where to push data records -- records that were pushed to the dataset id_population_estimates -- the id of the population estimates dataset """ for record in records: record["dataset_id"] = id_population_estimates record["dataset_name"] = "population-estimates" utils.ckan_upsert(id_yearlypolygondataset, records)
def push_to_yearly_point(id_yearlypointdataset, records, id_no2): """ Push the data every year to the aggregated dataset Keyword arguments: id_yearlypointdataset -- the id of the dataset where to push data records -- records that were pushed to the dataset id_no2 -- the id of the no2 dataset """ for record in records: record["dataset_id"] = id_no2 record["dataset_name"] = "air-quality-no2-diffusion-tube-data" utils.ckan_upsert(id_yearlypointdataset, records)
def push_to_hourly(id_hourlypointdataset, records, id_air_quality_data_continuous): """ Push the data every hour to the aggregated dataset Keyword arguments: id_hourlypointdataset -- the id of the dataset where to push data records -- records that were pushed to the dataset id_air_quality_data_continuous -- id of the air quality continuous dataset """ for record in records: record["dataset_id"] = id_air_quality_data_continuous record["dataset_name"] = "air-quality-data-continuous" utils.ckan_upsert(id_hourlypointdataset, records)
def push_wards(url, id_wards): """ Push data to the necessary datasets Keyword arguments: url -- the url to get the data id_wards -- the id of the wards dataset """ to_push = get_wards(url) if to_push is not None: utils.ckan_upsert(id_wards, to_push) return to_push else: return None
def push_populationestimates(url, last_year_to_retrieve, id_yearlypolygondataset, id_population_estimates): """ Push data to the necessary datasets Keyword arguments: url -- the url to get the data last_year_to_retrieve -- year to retrieve id_yearlypolygondataset -- the id of the dataset to push data every year id_population_estimates -- the id of the population estimates dataset """ to_push = get_populationestimates(url, last_year_to_retrieve) if to_push is not None: utils.ckan_upsert(id_population_estimates, to_push) push_to_yearly_poly_popest(id_yearlypolygondataset, to_push, id_population_estimates)
def push_no2(url, last_year_to_retrieve, id_yearlypointdataset, id_no2): """ Push data to the necessary datasets Keyword arguments: url -- the url to get the data last_year_to_retrieve -- year to retrieve id_yearlypointdataset -- the id of the dataset to push data every year id_no2 -- the id of the no2 dataset """ to_push = get_no2(url, last_year_to_retrieve) if to_push is not None: utils.ckan_upsert(id_no2, to_push) push_to_yearly_point(id_yearlypointdataset, to_push, id_no2)
def push_caravailability(url, id_car_availability): """ Push data to the necessary datasets Keyword arguments: url -- the url to get the data id_car_availability -- the id of the car availaibility dataset """ to_push = get_caravailability(url) if to_push is not None: utils.ckan_upsert(id_car_availability, to_push) return to_push else: return None
def push_airqualitydatacontinous(url, last_date_to_retrieve, id_hourlypointdataset, id_air_quality_data_continuous): """ Push data to the necessary datasets Keyword arguments: url -- the url to get the data last_date_to_retrieve -- date to retrieve id_dataset -- the id of the dataset to push data every hour id_air_quality_data_continuous -- id of the air quality continuous dataset """ to_push = get_airqualitydatacontinous(url, last_date_to_retrieve) if to_push is not None: utils.ckan_upsert(id_air_quality_data_continuous, to_push) push_to_hourly(id_hourlypointdataset, to_push, id_air_quality_data_continuous)
def push_aurn(id_dataset, files_folder, id_defra_aurn, stations): """ Push AURN data to the necessary datasets Keyword arguments: id_dataset -- the id of the dataset to push data every day files_folder -- folder of the files id_defra_aurn -- id of the aurn dataset stations -- stations to check (files to check in folder) """ for file_aurn in stations: to_push = get_aurn(file_aurn, files_folder) if to_push is not None: utils.ckan_upsert(id_defra_aurn, to_push) push_to_daily(id_dataset, to_push, id_defra_aurn)
def push_to_hourly(id_dataset, records, id_sck): """ Push the data every hour to the aggregated dataset Keyword arguments: id_dataset -- the id of the dataset where to push data records -- records that were pushed to the Smart Citizen dataset id_sck -- id of the sck dataset """ for record in records: try: record["recordid"] = record.pop("deviceid") except KeyError: pass record["dataset_id"] = id_sck record["dataset_name"] = "smart-citizen-kits" utils.ckan_upsert(id_dataset, records)
def push_to_yrly_caravail_wth_wards(id_yearlypolygondataset, wards, cars, id_car_availability): """ Push the data every year to the aggregated dataset Keyword arguments: id_yearlypolygondataset -- the id of the dataset where to push data wards -- wards that were pushed to the dataset cars -- cars data that were pushed to the dataset id_car_availability -- the id of the car availaibility dataset """ new_cars = [] for car in cars: found = False for ward in wards: if car["wardid"] == ward["wardid"]: found = True if "geojson" in ward: car["geojson"] = ward["geojson"] if "ward_center" in ward: car["ward_center"] = ward["ward_center"] if "objectid" in ward: car["objectid"] = ward["objectid"] if found: new_cars.append(car) for new_car in new_cars: try: if new_car["date_time"] == "2019-04-10T12:46:07.442000+00:00": new_car["date_time"] = "2019-04-10T12:46:07+00:00" LOGGER.info("Car Availablity Data record_timestamp is in wrong format--> Hacking here") new_car["year"] = int(datetime.strptime(new_car.pop("date_time"), '%Y-%m-%dT%H:%M:%S+00:00').year) except KeyError: pass new_car["dataset_id"] = id_car_availability new_car["dataset_name"] = "car-availability-by-ward" utils.ckan_upsert(id_yearlypolygondataset, new_cars)
def insert_units(): try: utils.ckan_upsert(ID_UNITS, RECORDS_POINT) utils.ckan_upsert(ID_UNITS, RECORDS_POLYGON) except Exception as exception_returned: print(exception_returned)