def collect_bbbike_subregion_catalogue(confirmation_required=True, verbose=False): """ :param confirmation_required: [bool] (default: True) :param verbose: [bool] (default: False) Testing e.g. confirmation_required = True verbose = True collect_bbbike_subregion_catalogue(confirmation_required, verbose) """ if confirmed("To collect BBBike subregion catalogue? ", confirmation_required=confirmation_required): try: home_url = 'http://download.bbbike.org/osm/bbbike/' bbbike_subregion_catalogue = pd.read_html( home_url, header=0, parse_dates=['Last Modified'])[0].drop(0) bbbike_subregion_catalogue.Name = bbbike_subregion_catalogue.Name.map( lambda x: x.strip('/')) save_pickle(bbbike_subregion_catalogue, cd_dat("BBBike-subregion-catalogue.pickle"), verbose=verbose) bbbike_subregion_names = bbbike_subregion_catalogue.Name.tolist() save_pickle(bbbike_subregion_names, cd_dat("BBBike-subregion-name-list.pickle"), verbose=verbose) except Exception as e: print("Failed to get the required information ... {}.".format(e)) else: print( "The information collection process was not activated. The existing local copy will be loaded instead." )
def collect_bbbike_download_catalogue(confirmation_required=True): """ :param confirmation_required: [bool] """ if confirmed("To collect BBBike download dictionary? ", confirmation_required=confirmation_required): try: bbbike_subregion_names = fetch_bbbike_subregion_catalogue("BBBike-subregion-name-list", update=True) download_catalogue = [ fetch_bbbike_subregion_download_catalogue(subregion_name, update=True, confirmation_required=False) for subregion_name in bbbike_subregion_names] subregion_name, subregion_download_catalogue = bbbike_subregion_names[0], download_catalogue[0] # Available file formats file_fmt = [re.sub('{}|CHECKSUM'.format(subregion_name), '', f) for f in subregion_download_catalogue.Filename] save_pickle(file_fmt[:-2], cd_dat("BBBike-osm-file-formats.pickle")) # Available data types data_typ = subregion_download_catalogue.DataType.tolist() save_pickle(data_typ[:-2], cd_dat("BBBike-osm-data-types.pickle")) # available_file_formats = dict(zip(file_fmt, file_ext)) downloads_dictionary = dict(zip(bbbike_subregion_names, download_catalogue)) save_pickle(downloads_dictionary, cd_dat("BBBike-download-catalogue.pickle")) except Exception as e: print("Failed to collect BBBike download dictionary. {}".format(e)) else: print("The information collection process was not activated. The existing local copy will be loaded instead.")
def fetch_bbbike_subregion_catalogue(catalogue_name, update=False, verbose=False): """ :param catalogue_name: [str] "BBBike-subregion-catalogue"; "BBBike-subregion-name-list" :param update: [bool] (default: False) :param verbose: [bool] (default: False) :return: [pd.DataFrame; list] Examples: update = False verbose = True catalogue_name = "BBBike-subregion-catalogue" fetch_bbbike_subregion_catalogue(catalogue_name, update, verbose) catalogue_name = "BBBike-subregion-name-list" fetch_bbbike_subregion_catalogue(catalogue_name, update, verbose) """ available_catalogue = ("BBBike-subregion-catalogue", "BBBike-subregion-name-list") assert catalogue_name in available_catalogue, \ "'catalogue_name' must be one of the following: \n \"{}\".".format("\",\n \"".join(available_catalogue)) path_to_pickle = cd_dat(catalogue_name + ".pickle") if not os.path.isfile(path_to_pickle) or update: collect_bbbike_subregion_catalogue(confirmation_required=False, verbose=verbose) try: bbbike_subregion_catalogue = load_pickle(path_to_pickle, verbose=verbose) return bbbike_subregion_catalogue except Exception as e: print(e)
def fetch_bbbike_download_catalogue(catalogue_name, update=False, verbose=False): """ :param catalogue_name [str] "BBBike-download-catalogue"; "BBBike-osm-file-formats"; "BBBike-osm-data-types" :param update: [bool] (default: False) :param verbose: [bool] (default: False) :return: [dict; list] Examples: verbose = True catalogue_name = "BBBike-download-catalogue" fetch_bbbike_download_catalogue(catalogue_name, update, verbose) # dict catalogue_name = "BBBike-osm-file-formats" fetch_bbbike_download_catalogue(catalogue_name, update, verbose) # list catalogue_name = "BBBike-osm-data-types" fetch_bbbike_download_catalogue(catalogue_name, update, verbose) # list """ available_catalogue = ("BBBike-download-catalogue", "BBBike-osm-file-formats", "BBBike-osm-data-types") assert catalogue_name in available_catalogue, \ "'catalogue_name' must be one of the following: \n \"{}\".".format("\",\n \"".join(available_catalogue)) path_to_file = cd_dat(catalogue_name + ".pickle") if not os.path.isfile(path_to_file) or update: collect_bbbike_download_catalogue(confirmation_required=True, verbose=verbose) try: bbbike_downloads_dictionary = load_pickle(path_to_file, verbose=verbose) return bbbike_downloads_dictionary except Exception as e: print(e)
def fetch_region_subregion_tier(catalogue_name, file_format=".pickle", update=False): """ :param catalogue_name: [str] e.g. "GeoFabrik-region-subregion-tier" :param file_format: [str] ".pickle"(default), or ".json" :param update: [bool] whether to update (i.e. re-collect) all subregion tables for each continent :return: [dict, or list] or null """ available_catalogue = ("GeoFabrik-region-subregion-tier", "GeoFabrik-non-subregion-list") assert catalogue_name in available_catalogue, \ "'catalogue_name' must be one of the following: \n \"{}\".".format("\",\n \"".join(available_catalogue)) available_fmt = (".pickle", ".json") assert file_format in available_fmt, \ "'file_format' must be one of the following: \n \"{}\".".format("\",\n \"".join(available_fmt)) path_to_file = cd_dat(catalogue_name + file_format) if not os.path.isfile(path_to_file) or update: collect_region_subregion_tier(confirmation_required=True) try: index_file = load_pickle( path_to_file) if file_format == ".pickle" else load_json( path_to_file) return index_file except Exception as e: print(e)
def collect_continents_subregion_tables(confirmation_required=True): """ :param confirmation_required: [bool] whether to ask for a confirmation before starting to collect the information """ if confirmed("To collect information about subregions of each continent? ", confirmation_required=confirmation_required): try: home_link = 'https://download.geofabrik.de/' source = requests.get(home_link) soup = bs4.BeautifulSoup(source.text, 'lxml').find_all('td', {'class': 'subregion'}) source.close() continent_names = [td.a.text for td in soup] continent_links = [ urllib.parse.urljoin(home_link, td.a['href']) for td in soup ] subregion_tables = dict( zip(continent_names, [get_subregion_table(url) for url in continent_links])) save_pickle(subregion_tables, cd_dat("GeoFabrik-continents-subregion-tables.pickle")) except Exception as e: print( "Failed to collect the required information ... {}.".format(e)) else: print( "The information collection process was not activated. The existing local copy will be loaded instead." )
def fetch_subregion_info_catalogue(catalogue_name, file_format=".pickle", update=False): """ :param catalogue_name: [str] e.g. "GeoFabrik-subregion-name-list" :param file_format: [str] ".pickle"(default), or ".json" :param update: [bool] whether to update (re-collect) the catalogues of subregion information :return: [list, dict, pandas.DataFrame] or null """ available_catalogue = ("GeoFabrik-subregion-name-list", "GeoFabrik-subregion-name-url-dictionary", "GeoFabrik-subregion-downloads-catalogue") assert catalogue_name in available_catalogue, \ "'catalogue_name' must be one of the following: \n \"{}\".".format("\",\n \"".join(available_catalogue)) available_fmt = (".pickle", ".json") assert file_format in available_fmt, \ "'file_format' must be one of the following: \n \"{}\".".format("\",\n \"".join(available_fmt)) path_to_catalogue = cd_dat(catalogue_name + file_format) if not os.path.isfile( path_to_catalogue) or update: # all(paths_to_files_exist) and collect_subregion_info_catalogue(confirmation_required=True) try: index_file = load_pickle( path_to_catalogue) if file_format == ".pickle" else load_json( path_to_catalogue) return index_file except Exception as e: print(e)
def fetch_continents_subregion_tables(update=False): """ :param update: [bool] whether to update (i.e. re-collect) all subregion tables for each continent :return: [pandas.DataFrame] or null """ path_to_pickle = cd_dat("GeoFabrik-continents-subregion-tables.pickle") if not os.path.isfile(path_to_pickle) or update: collect_continents_subregion_tables(confirmation_required=True) try: subregion_tables = load_pickle(path_to_pickle) return subregion_tables except Exception as e: print(e)
def fetch_bbbike_subregion_catalogue(catalogue_name, update=False): """ :param catalogue_name: [str] :param update: [bool] :return: [pandas.DataFrame] """ available_catalogue = ("BBBike-subregion-catalogue", "BBBike-subregion-name-list") assert catalogue_name in available_catalogue, \ "'catalogue_name' must be one of the following: \n \"{}\".".format("\",\n \"".join(available_catalogue)) path_to_pickle = cd_dat(catalogue_name + ".pickle") if not os.path.isfile(path_to_pickle) or update: collect_bbbike_subregion_catalogue(confirmation_required=False) try: bbbike_subregion_catalogue = load_pickle(path_to_pickle) return bbbike_subregion_catalogue except Exception as e: print(e)
def fetch_continents_subregion_tables(update=False, verbose=False): """ :param update: [bool] (default: False) whether to update (i.e. re-collect) all subregion tables for each continent :param verbose: [bool] (default: False) :return: [pd.DataFrame] Testing e.g. update = False verbose = True fetch_continents_subregion_tables(update, verbose) """ path_to_pickle = cd_dat("GeoFabrik-continents-subregion-tables.pickle") if not os.path.isfile(path_to_pickle) or update: collect_continents_subregion_tables(confirmation_required=True, verbose=verbose) try: subregion_tables = load_pickle(path_to_pickle, verbose=verbose) return subregion_tables except Exception as e: print(e)
def collect_bbbike_download_catalogue(confirmation_required=True, verbose=False): """ :param confirmation_required: [bool] (default: True) :param verbose: [bool] (default: False) Example: confirmation_required = True verbose = True collect_bbbike_download_catalogue(confirmation_required, verbose) """ # def collect_bbbike_subregion_download_catalogue(subregion_name): """ :param subregion_name: [str] Example: subregion_name = 'leeds' confirmation_required = True verbose = True collect_bbbike_subregion_download_catalogue(subregion_name, confirmation_required, verbose) """ def parse_dlc(dlc): dlc_href = dlc.get('href') # URL filename, download_url = dlc_href.strip( './'), urllib.parse.urljoin(url, dlc_href) if not dlc.has_attr('title'): file_format, file_size, last_update = 'Poly', None, None else: if len(dlc.contents) < 3: file_format, file_size = 'Txt', None else: file_format, file_size, _ = dlc.contents # File type and size file_format, file_size = file_format.strip( ), file_size.text last_update = pd.to_datetime(dlc.get('title')) # Date and time parsed_dat = [ filename, download_url, file_format, file_size, last_update ] return parsed_dat subregion_name_ = regulate_bbbike_input_subregion_name(subregion_name) # try: print(" \"{}\" ... ".format(subregion_name_), end="") if verbose else "" url = 'https://download.bbbike.org/osm/bbbike/{}/'.format( subregion_name_) source = urllib.request.urlopen(url) import bs4 source_soup = bs4.BeautifulSoup(source, 'lxml') download_links_class = source_soup.find_all( name='a', attrs={'class': ['download_link', 'small']}) subregion_downloads_catalogue = pd.DataFrame( parse_dlc(x) for x in download_links_class) subregion_downloads_catalogue.columns = [ 'Filename', 'URL', 'DataType', 'Size', 'LastUpdate' ] # path_to_file = cd_dat_bbbike(subregion_name_, subregion_name_ + "-download-catalogue.pickle") # save_pickle(subregion_downloads_catalogue, path_to_file, verbose=verbose) print("Done. ") if verbose else "" except Exception as e_: subregion_downloads_catalogue = None print("Failed. {}".format(subregion_name_, e_)) if verbose else "" return subregion_downloads_catalogue if confirmed("To collect BBBike download dictionary? ", confirmation_required=confirmation_required): try: bbbike_subregion_names = fetch_bbbike_subregion_catalogue( "BBBike-subregion-name-list", verbose=verbose) print("Collecting BBBike download catalogue for: " ) if verbose else "" download_catalogue = [ collect_bbbike_subregion_download_catalogue(subregion_name) for subregion_name in bbbike_subregion_names ] sr_name, sr_download_catalogue = bbbike_subregion_names[ 0], download_catalogue[0] # Available file formats file_fmt = [ re.sub('{}|CHECKSUM'.format(sr_name), '', f) for f in sr_download_catalogue.Filename ] save_pickle(file_fmt[:-2], cd_dat("BBBike-osm-file-formats.pickle"), verbose=verbose) # Available data types data_typ = sr_download_catalogue.DataType.tolist() save_pickle(data_typ[:-2], cd_dat("BBBike-osm-data-types.pickle"), verbose=verbose) # available_file_formats = dict(zip(file_fmt, file_ext)) downloads_dictionary = dict( zip(bbbike_subregion_names, download_catalogue)) save_pickle(downloads_dictionary, cd_dat("BBBike-download-catalogue.pickle"), verbose=verbose) except Exception as e: print("Failed to collect BBBike download dictionary. {}".format( e)) if verbose else ""
def collect_region_subregion_tier(confirmation_required=True, verbose=False): """ :param confirmation_required: [bool] (default: True) whether to confirm before collecting region-subregion tier :param verbose: [bool] (default: False) Testing e.g. confirmation_required = True verbose = True collect_region_subregion_tier(confirmation_required, verbose) """ # Find out the all regions and their subregions def compile_region_subregion_tier(sub_reg_tbls): """ :param sub_reg_tbls: [pd.DataFrame] obtained from fetch_continents_subregion_tables() :return: ([dict], [list]) a dictionary of region-subregion, and a list of (sub)regions without subregions """ having_subregions = copy.deepcopy(sub_reg_tbls) region_subregion_tiers = copy.deepcopy(sub_reg_tbls) non_subregions_list = [] for k, v in sub_reg_tbls.items(): if v is not None and isinstance(v, pd.DataFrame): region_subregion_tiers = update_nested_dict( sub_reg_tbls, {k: set(v.Subregion)}) else: non_subregions_list.append(k) for x in non_subregions_list: having_subregions.pop(x) having_subregions_temp = copy.deepcopy(having_subregions) while having_subregions_temp: for region_name, subregion_table in having_subregions.items(): # subregion_names, subregion_links = subregion_table.Subregion, subregion_table.SubregionURL sub_subregion_tables = dict( zip(subregion_names, [ get_subregion_table(link) for link in subregion_links ])) subregion_index, without_subregion_ = compile_region_subregion_tier( sub_subregion_tables) non_subregions_list += without_subregion_ region_subregion_tiers.update({region_name: subregion_index}) having_subregions_temp.pop(region_name) # Russian Federation in both pages of Asia and Europe, so that there are duplicates in non_subregions_list non_subregions_list = list( more_itertools.unique_everseen(non_subregions_list)) return region_subregion_tiers, non_subregions_list if confirmed( "To compile a region-subregion tier? (Note that it may take a few minutes.) ", confirmation_required=confirmation_required): try: subregion_tables = fetch_continents_subregion_tables(update=True) region_subregion_tier, non_subregions = compile_region_subregion_tier( subregion_tables) save_pickle(region_subregion_tier, cd_dat("GeoFabrik-region-subregion-tier.pickle"), verbose=verbose) save_json(region_subregion_tier, cd_dat("GeoFabrik-region-subregion-tier.json"), verbose=verbose) save_pickle(non_subregions, cd_dat("GeoFabrik-non-subregion-list.pickle"), verbose=verbose) except Exception as e: print("Failed to get the required information ... {}.".format(e)) else: print("The information collection process was not activated.")
def collect_subregion_info_catalogue(confirmation_required=True, verbose=False): """ :param confirmation_required: [bool] (default: False) whether to confirm before starting to collect information :param verbose: [bool] (default: False) Testing e.g. confirmation_required = True verbose = True collect_subregion_info_catalogue(confirmation_required, verbose) """ if confirmed( "To collect all available subregion links? (Note that it may take a few minutes.) ", confirmation_required=confirmation_required): home_url = 'http://download.geofabrik.de/' try: source = requests.get(home_url) soup = bs4.BeautifulSoup(source.text, 'lxml') source.close() avail_subregions = [ td.a.text for td in soup.find_all('td', {'class': 'subregion'}) ] avail_subregion_urls = [ urllib.parse.urljoin(home_url, td.a['href']) for td in soup.find_all('td', {'class': 'subregion'}) ] avail_subregion_url_tables = [ get_subregion_table(sub_url, verbose) for sub_url in avail_subregion_urls ] avail_subregion_url_tables = [ tbl for tbl in avail_subregion_url_tables if tbl is not None ] subregion_url_tables = list(avail_subregion_url_tables) while subregion_url_tables: subregion_url_tables_ = [] for subregion_url_table in subregion_url_tables: subregions = list(subregion_url_table.Subregion) subregion_urls = list(subregion_url_table.SubregionURL) subregion_url_tables_0 = [ get_subregion_table(subregion_url, verbose) for subregion_url in subregion_urls ] subregion_url_tables_ += [ tbl for tbl in subregion_url_tables_0 if tbl is not None ] # (Note that 'Russian Federation' data is available in both 'Asia' and 'Europe') avail_subregions += subregions avail_subregion_urls += subregion_urls avail_subregion_url_tables += subregion_url_tables_ subregion_url_tables = list(subregion_url_tables_) # Save a list of available subregions locally save_pickle(avail_subregions, cd_dat("GeoFabrik-subregion-name-list.pickle"), verbose=verbose) # Subregion index - {Subregion: URL} subregion_url_index = dict( zip(avail_subregions, avail_subregion_urls)) # Save subregion_index to local disk save_pickle( subregion_url_index, cd_dat("GeoFabrik-subregion-name-url-dictionary.pickle"), verbose=verbose) save_json(subregion_url_index, cd_dat("GeoFabrik-subregion-name-url-dictionary.json"), verbose=verbose) # All available URLs for downloading home_subregion_url_table = get_subregion_table(home_url) avail_subregion_url_tables.append(home_subregion_url_table) subregion_downloads_index = pd.DataFrame( pd.concat(avail_subregion_url_tables, ignore_index=True)) subregion_downloads_index.drop_duplicates(inplace=True) subregion_downloads_index_json = subregion_downloads_index.set_index( 'Subregion').to_json() # Save subregion_index_downloads to local disk save_pickle( subregion_downloads_index, cd_dat("GeoFabrik-subregion-downloads-catalogue.pickle"), verbose=verbose) save_json(subregion_downloads_index_json, cd_dat("GeoFabrik-subregion-downloads-catalogue.json"), verbose=verbose) except Exception as e: print("Failed to get the required information ... {}.".format(e)) else: print("The information collection process was not activated.")
def collect_subregion_info_catalogue(confirmation_required=True, verbose=False): """ :param confirmation_required: [bool] (default: False) whether to confirm before starting to collect information :param verbose: [bool] (default: False) Example: confirmation_required = True verbose = True collect_subregion_info_catalogue(confirmation_required, verbose) """ if confirmed( "To collect all available subregion links? (Note that it may take a few minutes.) ", confirmation_required=confirmation_required): home_url = 'http://download.geofabrik.de/' try: source = requests.get(home_url) soup = bs4.BeautifulSoup(source.text, 'lxml') source.close() # avail_subregions = [td.a.text for td in soup.find_all('td', {'class': 'subregion'})] subregion_href = soup.find_all('td', {'class': 'subregion'}) avail_subregion_urls = (urllib.parse.urljoin( home_url, td.a['href']) for td in subregion_href) avail_subregion_url_tables_0 = (get_subregion_table( sub_url, verbose) for sub_url in avail_subregion_urls) avail_subregion_url_tables = [ tbl for tbl in avail_subregion_url_tables_0 if tbl is not None ] subregion_url_tables = list(avail_subregion_url_tables) while subregion_url_tables: subregion_url_tables_ = [] for subregion_url_table in subregion_url_tables: # subregions = list(subregion_url_table.Subregion) subregion_urls = list(subregion_url_table.SubregionURL) subregion_url_tables_0 = [ get_subregion_table(sr_url, verbose) for sr_url in subregion_urls ] subregion_url_tables_ += [ tbl for tbl in subregion_url_tables_0 if tbl is not None ] # (Note that 'Russian Federation' data is available in both 'Asia' and 'Europe') # avail_subregions += subregions # avail_subregion_urls += subregion_urls avail_subregion_url_tables += subregion_url_tables_ subregion_url_tables = list(subregion_url_tables_) # All available URLs for downloading home_subregion_url_table = get_subregion_table(home_url) avail_subregion_url_tables.append(home_subregion_url_table) subregion_downloads_index = pd.DataFrame( pd.concat(avail_subregion_url_tables, ignore_index=True)) subregion_downloads_index.drop_duplicates(inplace=True) duplicated = subregion_downloads_index[ subregion_downloads_index.Subregion.duplicated(keep=False)] if not duplicated.empty: import humanfriendly for i in range(0, 2, len(duplicated)): temp = duplicated.iloc[i:i + 2] size = temp['.osm.pbf_Size'].map( lambda x: humanfriendly.parse_size( x.strip('(').strip(')').replace('\xa0', ' '))) idx = size[size == size.min()].index subregion_downloads_index.drop(idx, inplace=True) subregion_downloads_index.index = range( len(subregion_downloads_index)) subregion_downloads_index_json = subregion_downloads_index.set_index( 'Subregion').to_json() # Save subregion_index_downloads to local disk save_pickle( subregion_downloads_index, cd_dat("GeoFabrik-subregion-downloads-catalogue.pickle"), verbose=verbose) save_json(subregion_downloads_index_json, cd_dat("GeoFabrik-subregion-downloads-catalogue.json"), verbose=verbose) avail_subregions = list(subregion_downloads_index.Subregion) avail_subregion_urls = list(subregion_downloads_index.SubregionURL) # Subregion index - {Subregion: URL} subregion_url_index = dict( zip(avail_subregions, avail_subregion_urls)) # Save a list of available subregions locally save_pickle(avail_subregions, cd_dat("GeoFabrik-subregion-name-list.pickle"), verbose=verbose) # Save subregion_index to local disk save_pickle( subregion_url_index, cd_dat("GeoFabrik-subregion-name-url-dictionary.pickle"), verbose=verbose) save_json(subregion_url_index, cd_dat("GeoFabrik-subregion-name-url-dictionary.json"), verbose=verbose) except Exception as e: print("Failed to get the required information ... {}.".format(e)) else: print("The information collection process was not activated.")