def scrape_opendata(): stations_url = "https://opendata.bristol.gov.uk/explore/dataset/polling-stations/download/?format=geojson&timezone=Europe/London" districts_url = "https://opendata.bristol.gov.uk/explore/dataset/polling-districts/download/?format=geojson&timezone=Europe/London" print(stations_url) stations_scraper = GeoJsonScraper( stations_url, council_id, 'utf-8', 'stations_opend', key='objectid') stations_scraper.scrape() print(districts_url) districts_scraper = GeoJsonScraper( districts_url, council_id, 'utf-8', 'districts_opend', key='objectid') districts_scraper.scrape()
from dc_base_scrapers.geojson_scraper import GeoJsonScraper districts_url = "http://inspire.sthelens.gov.uk/geoserver/ows?service=WFS&version=1.3.0&request=GetFeature&typeNames=INSPIRE%3Apolling_districts&outputFormat=json" council_id = 'E08000013' districts_scraper = GeoJsonScraper(districts_url, council_id, 'utf-8', 'districts') districts_scraper.scrape()
# 'utf-8') # # stations_meta_scraper.scrape() # # districts_meta_scraper = DataPressScraper( # base_url, # council_id, # districts_info['dataset'], # districts_info['return_format'], # districts_info['extra_fields'], # 'utf-8') # districts_meta_scraper.scrape() # stations_url = "%s%s" % (base_url, stations_info['dataset']) stations_url = "https://dataworks.calderdale.gov.uk/download/polling-stations/f9b00312-f330-4d18-a944-7cfd36c8d0eb/Polling%20stations.geojson" # districts_url = "%s%s" % (base_url, districts_info['dataset']) districts_url = "https://dataworks.calderdale.gov.uk/download/polling-station-districts/2482d9f3-7eae-4ea6-980c-718e9723e64a/Polling%20districts.geojson" # stations_scraper = HashOnlyScraper(stations_url, council_id, 'stations', 'json') stations_scraper = GeoJsonScraper(stations_url, council_id, 'utf-8', 'stations', key='POLLING_LETTERS') stations_scraper.scrape() districts_scraper = GeoJsonScraper(districts_url, council_id, 'utf-8', 'districts', key='POLLING_LETTERS') districts_scraper.scrape()
council_id = 'YOR' stations_meta_scraper = CkanScraper(base_url, council_id, stations_info['dataset'], stations_info['return_format'], stations_info['extra_fields'], 'utf-8') stations_url = stations_meta_scraper.scrape() districts_meta_scraper = CkanScraper(base_url, council_id, districts_info['dataset'], districts_info['return_format'], districts_info['extra_fields'], 'utf-8') districts_url = districts_meta_scraper.scrape() if stations_url: stations_scraper = GeoJsonScraper(stations_url, council_id, 'utf-8', 'stations', key='OBJECTID_1') stations_scraper.scrape() if districts_url: districts_scraper = GeoJsonScraper(districts_url, council_id, 'utf-8', 'districts', key='OBJECTID_1') districts_scraper.scrape()
from dc_base_scrapers.geojson_scraper import GeoJsonScraper from dc_base_scrapers.hashonly_scraper import HashOnlyScraper search_url = "https://opendata.arcgis.com/api/v2/datasets?filter%5Bcatalogs%5D=maps-south-ayrshire.opendata.arcgis.com&include=organizations%2Cgroups&page%5Bnumber%5D=1&page%5Bsize%5D=10&q=polling&sort=-updatedAt" stations_url = "https://opendata.arcgis.com/datasets/72bef698f90b4da7b28ee403598e4403_29.geojson" districts_url = "https://opendata.arcgis.com/datasets/80572b43c4b24d73ad2c4851aaeb9151_30.geojson" council_id = 'S12000028' search_scraper = HashOnlyScraper(search_url, council_id, 'datasets', 'json') search_scraper.scrape() stations_scraper = GeoJsonScraper(stations_url, council_id, 'utf-8', 'stations', key='objectid') stations_scraper.scrape() districts_scraper = GeoJsonScraper(districts_url, council_id, 'utf-8', 'districts', key='objectid') districts_scraper.scrape()