def handler_dataset_landing_page_fallback(scraper, this_dataset_page, tree): """ At time of writing there's an issue with the latest version of datasets 404'ing on the versions page. this function will create what the latest version should be, using the information on the base dataset landing page. """ logging.warning( "Using fallback logic to scrape latest distribution from dataset landing page (rather " "than previous page). This scrape will only have a single distribution of xls." ) this_distribution = Distribution(scraper) release_date = this_dataset_page["description"]["releaseDate"] this_distribution.issued = parse(release_date.strip()).date() # gonna have to go via html ... download_url = tree.xpath("//a[text()='xls']/@href") this_distribution.downloadURL = download_url media_type = Excel this_distribution.mediaType = media_type this_distribution.title = scraper.dataset.title this_distribution.description = scraper.dataset.description this_distribution.contactPoint = scraper.dataset.contactPoint logging.debug( "Created distribution for download '{}'.".format(download_url)) scraper.distributions.append(this_distribution)
def scrape(scraper, tree): """ Scraper for https://www.lowcarboncontracts.uk/data-portal/dataset/* Example: https://www.lowcarboncontracts.uk/data-portal/dataset/actual-ilr-income """ article = assert_get_one(tree.xpath('//article'), "article element") title_element = assert_get_one(article.xpath('./div/h1'), 'title element') scraper.dataset.title = title_element.text.strip() description_elements = article.xpath('./div/div/p') scraper.dataset.description = "\n\n".join( [x.text.strip() for x in description_elements]) issued_element = assert_get_one( article.xpath('./div/section/table/tbody/tr[1]/td/span'), "issued element") scraper.dataset.issued = parse(issued_element.text.split("(")[0].strip()) scraper.dataset.license = "http://reference.data.gov.uk/id/open-government-licence" for resource in assert_get_one(article.xpath('./div/section[1]/ul[1]'), "resource list").xpath('./li/a'): distro = Distribution(scraper) url = f'https://www.lowcarboncontracts.uk/{resource.get("href")}' resp = scraper.session.get(url) if resp.status_code != 200: raise Exception(f'Failed to get url resource {url}') distro_tree = html.fromstring(resp.text) section = assert_get_one( distro_tree.xpath( '/html[1]/body[1]/div[3]/div[1]/div[3]/section[1]'), "section of distro") distro_title_element = assert_get_one(section.xpath('./div/h1'), "distro title") distro.title = distro_title_element.text distro_description_element = assert_get_one( section.xpath('./div/div/blockquote[1]'), "distro description") distro.description = distro_description_element.text distro_download_url_element = assert_get_one( section.xpath('./div/p/a'), "download url") distro.downloadURL = distro_download_url_element.text # Note: issued is the one thing not in the "section" element, so xpathing the whole distro tree distro_issued_element = assert_get_one( distro_tree.xpath('//table[1]/tbody[1]/tr[2]/td[1]'), "issued") distro.issued = parse(distro_issued_element.text) media_type, _ = mimetypes.guess_type(distro.downloadURL) distro.mediaType = media_type if media_type is not None else CSV # the default/not-specified offering is csv scraper.distributions.append(distro)
def opendata_nhs(scraper, tree): # TODO - this feels more like a catalogue than a list of distributions, investigate # Populate the dataset details = tree.xpath('//tr/td[@class="dataset-details"]') dates = tree.xpath('//span[@class="automatic-local-datetime"]/text()') date_updated = parse(" ".join([ x.replace("\n", "").replace("(BST)", "").strip() for x in dates[0].split(" ") ])) date_created = parse(" ".join([ x.replace("\n", "").replace("(BST)", "").strip() for x in dates[1].split(" ") ])) # Populate distributions distro_resources = tree.xpath('//li[@class="resource-item"]') for dr in distro_resources: download = dr.xpath( 'div/ul/li/a[contains(@class, "resource-url-analytics")]/@href')[0] # Need to go to the preview page for full description and title as they've helpfully truncated both... preview_url = "https://www.opendata.nhs.scot" + dr.xpath( 'div/ul[@class="dropdown-menu"]/li/a/@href')[0] r = scraper.session.get(preview_url) if r.status_code != 200: raise Exception( "Unable to follow url to get full description, url: '{}', status code '{}'." .format(preview_url, r.status_code)) preview_tree = html.fromstring(r.text) description1 = preview_tree.xpath( '//div[contains(@class, "prose notes")]/p/text()')[0] # Some (but not all) descriptions have some additional itallic information try: description2 = preview_tree.xpath( '//div[contains(@class, "prose notes")]/p/em/text()')[0] except IndexError: description2 = "" description = description1 + "\n\n" + description2 description.strip("\n") title = preview_tree.xpath('//title/text()')[0] this_distribution = Distribution(scraper) this_distribution.issued = date_updated this_distribution.downloadURL = download this_distribution.mediaType = CSV this_distribution.title = title.strip() this_distribution.description = description scraper.distributions.append(this_distribution)
def handler_static_adhoc(scraper, landing_page, tree): # A static adhoc is a one-off unscheduled release # These pages should be simpler and should lack the historical distributions for download in landing_page["downloads"]: title = download["title"] file = download["file"] # Create an empty Distribution object to represent this distribution # from here we're just looking to fill in it's fields this_distribution = Distribution(scraper) # if we can't get the release date, continue but throw a warning. try: this_distribution.issued = parse( landing_page["description"]["releaseDate"]).date() except KeyError: logging.warning("Unable to acquire or parse release date") download_url = ONS_DOWNLOAD_PREFIX + landing_page["uri"] + "/" + file this_distribution.downloadURL = download_url # TODO - we're doing this in two place, pull it out # we've had some issues with type-guessing so we're getting the media type # by checking the download url ending if download_url.endswith(".csdb"): media_type = CSDB elif download_url.endswith(".csv"): media_type = CSV elif download_url.endswith(".xlsx"): media_type = Excel elif download_url.endswith(".ods"): media_type = ODS else: media_type, _ = mimetypes.guess_type(download_url) this_distribution.mediaType = media_type this_distribution.title = title # inherit metadata from the dataset where it hasn't explicitly been changed this_distribution.description = scraper.dataset.description logging.debug( "Created distribution for download '{}'.".format(download_url)) scraper.distributions.append(this_distribution)
def handler_dataset_landing_page(scraper, landing_page, tree): # A dataset landing page has uri's to one or more datasets via it's "datasets" field. # We need to look at each in turn, this is an example one as json: # https://www.ons.gov.uk//businessindustryandtrade/internationaltrade/datasets/uktradeingoodsbyclassificationofproductbyactivity/current/data for dataset_page_url in landing_page["datasets"]: # Get the page as json. Throw an information error if we fail for whatever reason dataset_page_json_url = ONS_PREFIX + dataset_page_url["uri"] + "/data" r = scraper.session.get(dataset_page_json_url) if r.status_code != 200: raise ValueError("Scrape of url '{}' failed with status code {}." \ .format(dataset_page_json_url, r.status_code)) # get the response json into a python dict this_dataset_page = r.json() # start a list of dataset versions (to hold current + all previous) as a list # we'll start with just the current/latest version versions_list = [ONS_PREFIX + this_dataset_page["uri"] + "/data"] # if there are older versions of this datasets availible. # iterate and add their uri's to the versions list try: for version_as_dict in this_dataset_page["versions"]: versions_list.append(ONS_PREFIX + version_as_dict["uri"] + "/data") except KeyError: logging.debug( "No older versions found for {}.".format(dataset_page_url)) # NOTE - we've had an issue with the very latest dataset not being updated on the previous versions # page (the page we're getting the distributions from) so we're taking the details for it from # the landing page to use as a fallback in that scenario. # iterate through the lot, we're aiming to create at least one distribution object for each for i, version_url in enumerate(versions_list): logging.debug( "Identified distribution url, building distribution object for: " + version_url) r = scraper.session.get(version_url) if r.status_code != 200: # If we've got a 404 on the latest, fallback on using the details from the # landing page instead if r.status_code == 404 and i == len(versions_list) - 1: handler_dataset_landing_page_fallback( scraper, this_dataset_page, tree) continue else: raise Exception("Scraper unable to acquire the page: {} with http code {}." \ .format(version_url, r.status_code)) # get the response json into a python dict this_page = r.json() # Get the download urls, if there's more than 1 format of this version of the dataset # each forms a separate distribution distribution_formats = this_page["downloads"] for dl in distribution_formats: # Create an empty Distribution object to represent this distribution # from here we're just looking to fill in it's fields this_distribution = Distribution(scraper) # Every distribution SHOULD have a release date, but it seems they're not # always included. If it happens continue but throw a warning. try: release_date = this_page["description"]["releaseDate"] this_distribution.issued = parse_as_local_date( release_date.strip()) except KeyError: logging.warning( "Download {}. Of datasset versions {} of dataset {} does not have " "a release date".format(distribution_formats, version_url, dataset_page_url)) # I don't trust dicts with one constant field (they don't make sense), so just in case... try: download_url = ONS_DOWNLOAD_PREFIX + this_page[ "uri"] + "/" + dl["file"].strip() this_distribution.downloadURL = download_url except: # raise up this time. If we don't have a downloadURL it's not much use raise ValueError("Unable to create complete download url for {} on page {}" \ .format(dl, version_url)) # we've had some issues with type-guessing so we're getting the media type # by checking the download url ending if download_url.endswith(".csdb"): media_type = CSDB elif download_url.endswith(".csv"): media_type = CSV elif download_url.endswith(".xlsx"): media_type = Excel elif download_url.endswith(".ods"): media_type = ODS else: media_type, _ = mimetypes.guess_type(download_url) this_distribution.mediaType = media_type # inherit metadata from the dataset where it hasn't explicitly been changed this_distribution.title = scraper.dataset.title this_distribution.description = scraper.dataset.description logging.debug("Created distribution for download '{}'.".format( download_url)) scraper.distributions.append(this_distribution)
def scrape(scraper, tree): # A quick safety in case people are using this scraper incorrectly if "?search=" not in scraper.uri: raise Exception( """Aborting. This scraper is intended to run off the DCNI seach page. Please modify your url to use the site search. If in doubt, work from this page, change the quoted search text and capture the url https://www.communities-ni.gov.uk/publications/topic/8182?search=%22Northern+Ireland+Housing+Bulletin%22&Search-exposed-form=Go&sort_by=field_published_date """) scraper.dataset.publisher = GOV[ 'department-for-communities-northern-ireland'] scraper.dataset.license = 'http://www.nationalarchives.gov.uk/doc/open-" \ "government-licence/version/3/' # We're taking each search result as a distribution search_result_urls = [] for linkObj in tree.xpath("//h3/a"): # linkObj.items() is eg ("href", "www.foo.com") where we want a url href = [x[1] for x in linkObj.items() if x[0] == "href"][0] # Add to distributions url list, get the root from the original url search_result_urls.append( scraper.uri.split("/publications/topic")[0] + href) # keep track of dates issued so we can find the latest last_issued = None for url in search_result_urls: # Get the distribution page page = scraper.session.get(url) distro_tree = html.fromstring(page.text) # Get any spreadsheets (ods or excel) linked on the page spreadsheet_files = [ x for x in distro_tree.xpath('//a/@href') if x.lower().endswith(".ods") or x.lower().endswith(".xlsx") ] # Now map them together, so we have the supporting info for each relevent download # TODO - make better, kinda nasty title_download_map = {} for spreadsheet_file in spreadsheet_files: # Create our new distribution object this_distribution = Distribution(scraper) # Identify the correct title this_distribution.title = distro_tree.xpath( "//a[@href='" + spreadsheet_file + "']/text()".format(spreadsheet_file))[0] this_distribution.downloadURL = spreadsheet_file if this_distribution.downloadURL.lower().endswith(".xlsx"): media_type = Excel elif this_distribution.downloadURL.lower().endswith(".ods"): media_type = ODS else: raise Exception( "Aborting. Unexpected media type for url: '{}'".format( this_distribution.downloadURL)) this_distribution.mediaType = media_type # Published and modifed time this_distribution.issued = isoparse( distro_tree.xpath( "//*[@property='article:published_time']/@content") [0]).date() this_distribution.modified = isoparse( distro_tree.xpath( "//*[@property='article:modified_time']/@content") [0]).date() this_distribution.description = distro_tree.xpath( "//*[@class='field-summary']/p/text()")[0] if last_issued is None: last_issued = this_distribution.issued elif this_distribution.issued > last_issued: last_issued = this_distribution.issued scraper.distributions.append(this_distribution) # Whatever date the latest distribution was issued, is the last issued date for this "dataset" scraper.dataset.issued = last_issued
def handler_dataset_landing_page(scraper, landing_page, tree): # A dataset landing page has uri's to one or more datasets via it's "datasets" field. # We need to look at each in turn, this is an example one as json: # https://www.ons.gov.uk//businessindustryandtrade/internationaltrade/datasets/uktradeingoodsbyclassificationofproductbyactivity/current/data for dataset_page_url in landing_page["datasets"]: this_dataset_page = get_dict_from_json_url(ONS_PREFIX + dataset_page_url["uri"] + "/data", scraper) # create a list, with each entry a dict of a versions url and update date versions_dict_list = [] # Where the dataset is versioned, use the versions as the distributions try: all_versions = this_dataset_page["versions"] except KeyError: all_versions = [] # Release dates: # -------------- # ONS does this odd thing where each version on the /data api # has a updateDate field which is actually the date THE DATA # WAS SUPERCEDED (so the release fate of the NEXT version of the data). # ......this takes a bit of unpicking. # If no initial release date for the dataset has been provided # We're just going to ignore v1, we don't have a use for it # and with no provided release date ... not a lot to be done initial_release = this_dataset_page["description"].get("releaseDate", None) next_release = None # Where there's multiple versions, iterate all and populate a list if len(all_versions) != 0: try: for version_as_dict in all_versions: if next_release is None: release_date = initial_release else: release_date = next_release if release_date is not None: versions_dict_list.append({ "url": ONS_PREFIX + version_as_dict["uri"] + "/data", "issued": release_date }) next_release = version_as_dict["updateDate"] except KeyError: logging.debug("No older versions found for {}.".format(dataset_page_url)) # Add the current release versions_dict_list.append({ "url": ONS_PREFIX + this_dataset_page["uri"] + "/data", "issued": initial_release if next_release is None else next_release }) # NOTE - we've had an issue with the very latest dataset not being updated on the previous versions # page (the page we're getting the distributions from) so we're taking the details for it from # the landing page to use as a fallback in that scenario. # iterate through the lot, we're aiming to create at least one distribution object for each for i, version_dict in enumerate(versions_dict_list): version_url = version_dict["url"] issued = version_dict["issued"] logging.debug("Identified distribution url, building distribution object for: " + version_url) # get the response json into a python dict this_page = get_dict_from_json_url(version_url, scraper) # Get the download urls, if there's more than 1 format of this version of the dataset # each forms a separate distribution distribution_formats = this_page["downloads"] for dl in distribution_formats: # Create an empty Distribution object to represent this distribution # from here we're just looking to fill in it's fields this_distribution = Distribution(scraper) this_distribution.issued = parse_as_local_date(issued) # I don't trust dicts with one constant field (they don't make sense), so just in case... try: download_url = ONS_DOWNLOAD_PREFIX + this_page["uri"] + "/" + dl["file"].strip() this_distribution.downloadURL = download_url except: # Throw a warning and abandon this distribution, ff we don't have a downloadURL it's not much use logging.warning("Unable to create complete download url for {} on page {}" .format(dl, version_url)) continue # we've had some issues with type-guessing so we're getting the media type # by checking the download url ending if download_url.endswith(".csdb"): media_type = CSDB else: media_type, _ = mimetypes.guess_type(download_url) this_distribution.mediaType = media_type # inherit metadata from the dataset where it hasn't explicitly been changed this_distribution.title = scraper.dataset.title this_distribution.description = scraper.dataset.description logging.debug("Created distribution for download '{}'.".format(download_url)) scraper.distributions.append(this_distribution)
def scrape(scraper, tree): """ This is a scraper intended to use the ONS cmd (customise my data) functionality. :param scraper: the Scraper object :param landing_page: lxml tree :return: """ dataset_document = request_json_data(scraper, scraper.uri) scraper.dataset.title = dataset_document["id"] scraper.dataset.description = dataset_document["description"] # Need to get issued from the assciated publication publication_document = request_json_data( scraper, dataset_document["publications"][0]["href"] + "/data") scraper.dataset.issued = parse( publication_document["description"]["releaseDate"]) # Only take next release it its a date try: next_release = parse(dataset_document["next_release"]) scraper.dataset.updateDueOn = next_release except: pass # it's fine, "unknown" etc # Theoretically you can have more than one contact, but I'm just taking the first scraper.dataset.contactPoint = "mailto:" + dataset_document["contacts"][0][ "email"].strip() scraper.dataset.publisher = 'https://www.gov.uk/government/organisations/office-for-national-statistics' scraper.dataset.license = "http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/" edition_documents = request_json_data(scraper, scraper.uri + "/editions") for edition_document in edition_documents["items"]: edition_name = edition_document["edition"] version_documents = request_json_data( scraper, edition_document["links"]["versions"]["href"]) for version_document in version_documents["items"]: version_name = str(version_document["version"]) this_distribution = Distribution(scraper) this_distribution.issued = version_document["release_date"] this_distribution.downloadURL = version_document["downloads"][ "csv"]["href"] this_distribution.mediaType = CSV this_distribution.title = scraper.dataset.title + ", {}, version {}".format( edition_name, version_name) this_distribution.description = scraper.dataset.description this_distribution.contactPoint = scraper.dataset.contactPoint logging.debug("Created distribution for download '{}'.".format( this_distribution.downloadURL)) scraper.distributions.append(this_distribution)