def main(): """Generate dataset and create it in HDX""" configuration = Configuration.read() resources = configuration["resources"] fields = configuration["fields"] source_directory = configuration["source_directory"] download_url = Path(source_directory).resolve().as_uri() with Download() as downloader: countries, headers, countriesdata = get_countriesdata( download_url, resources, downloader) logger.info("Number of countries: %d" % len(countriesdata)) for info, country in progress_storing_tempdir("UNHCR_population", countries, "iso3"): folder = info["folder"] dataset, showcase = generate_dataset_and_showcase( folder, country, countriesdata[country["iso3"]], headers, resources, fields) if dataset: dataset.update_from_yaml() dataset["notes"] = dataset["notes"].replace( "\n", " \n") # ensure markdown has line breaks dataset.generate_resource_view(1) dataset.create_in_hdx( remove_additional_resources=True, hxl_update=False, updated_by_script="HDX Scraper: UNHCR population", batch=info["batch"], ) showcase.create_in_hdx() showcase.add_dataset(dataset)
def main(): """Generate dataset and create it in HDX""" with Download() as downloader: configuration = Configuration.read() countries_path = join('config', configuration['countries_filename']) indicators_url = configuration['indicators_url'] mvam_url = configuration['mvam_url'] showcase_url = configuration['showcase_url'] countries = get_countries(countries_path, downloader) variables = get_mvamvariables(indicators_url, downloader) logger.info('Number of datasets to upload: %d' % len(countries)) for info, country in progress_storing_tempdir('WFPFoodSecurity', countries, 'iso3'): dataset, showcase, bites_disabled = \ generate_dataset_and_showcase(mvam_url, showcase_url, downloader, info['folder'], country, variables) if dataset: dataset.update_from_yaml() dataset.generate_resource_view(bites_disabled=bites_disabled) dataset.create_in_hdx( remove_additional_resources=True, hxl_update=False, updated_by_script='HDX Scraper: WFP Food Security', batch=info['batch']) showcase.create_in_hdx() showcase.add_dataset(dataset)
def main(): """Generate dataset and create it in HDX""" configuration = Configuration.read() indicators = configuration["indicators"] json_url = configuration["json_url"] with Download() as downloader: indicators_metadata = get_indicators_metadata(json_url, downloader, indicators) countriesdata, countries = get_countriesdata(json_url, downloader, indicators) logger.info(f"Number of countries to upload: {len(countries)}") for info, country in progress_storing_tempdir("WorldPop", countries, "iso3"): countryiso = country["iso3"] datasets, showcases = generate_datasets_and_showcases( downloader, countryiso, indicators_metadata, countriesdata[countryiso]) for dataset in datasets: dataset.update_from_yaml() dataset.create_in_hdx( remove_additional_resources=True, hxl_update=False, updated_by_script="HDX Scraper: WorldPop", batch=info["batch"], ) for showcase in showcases.get(dataset["name"], list()): showcase.create_in_hdx() showcase.add_dataset(dataset)
def main(): """Generate dataset and create it in HDX""" configuration = Configuration.read() base_url = configuration['base_url'] with Download(extra_params_yaml=join(expanduser('~'), '.extraparams.yml'), extra_params_lookup=lookup) as downloader: downloader.session.mount( 'http://', HTTPAdapter(max_retries=1, pool_connections=100, pool_maxsize=100)) downloader.session.mount( 'https://', HTTPAdapter(max_retries=1, pool_connections=100, pool_maxsize=100)) countries = get_countries(base_url, downloader) logger.info('Number of countries: %d' % len(countries)) for folder, country in progress_storing_tempdir( 'DHS', countries, 'iso3'): tags = get_tags(base_url, downloader, country['dhscode']) dataset, subdataset, showcase, bites_disabled = \ generate_datasets_and_showcase(configuration, base_url, downloader, folder, country, tags) if dataset: createdataset(dataset) resource_view = generate_resource_view( dataset, bites_disabled=bites_disabled['national']) resource_view.create_in_hdx() showcase.create_in_hdx() showcase.add_dataset(dataset) if subdataset: createdataset(subdataset) showcase.add_dataset(subdataset) subdataset.generate_resource_view( bites_disabled=bites_disabled['subnational'])
def main(save, use_saved, **ignore): """Generate dataset and create it in HDX""" with Download(extra_params_yaml=join(expanduser('~'), '.extraparams.yml'), extra_params_lookup=lookup) as token_downloader: configuration = Configuration.read() with Download() as downloader: folder = temp_dir(lookup) retriever = Retrieve(downloader, folder, 'saved_data', folder, save, use_saved) wfp = WFPFood(configuration, token_downloader, retriever) countries = wfp.get_countries() logger.info('Number of country datasets to upload: %d' % len(countries)) wfp.build_mappings() for info, country in progress_storing_tempdir( lookup, countries, 'iso3'): dataset, showcase, qc_indicators = wfp.generate_dataset_and_showcase( country['iso3'], info['folder']) if dataset: dataset.update_from_yaml() dataset['notes'] = dataset[ 'notes'] % 'Food Prices data for %s. Food prices data comes from the World Food Programme and covers' % country[ 'name'] dataset.generate_resource_view(-1, indicators=qc_indicators) dataset.create_in_hdx( remove_additional_resources=True, hxl_update=False, updated_by_script='HDX Scraper: WFP Food Prices', batch=info['batch']) showcase.create_in_hdx() showcase.add_dataset(dataset)
def main(): """Generate dataset and create it in HDX""" with Download() as downloader: config = Configuration.read() project_config = { key: value for key, value in config.items() if key.startswith("CV") } qc_indicators = config.get("qc_indicators", {}) countries, countriesdata, headers = get_all_countriesdata( project_config, downloader) logger.info("Number of datasets to upload: %d" % len(countries)) for info, country in progress_storing_tempdir("UNICEFSAM", countries, "iso3"): dataset, showcase, bites_disabled = generate_dataset_and_showcase( info["folder"], country, countriesdata[country["iso3"]], headers, project_config, qc_indicators) if dataset: dataset.update_from_yaml() dataset.generate_resource_view(1, bites_disabled=bites_disabled, indicators=qc_indicators) dataset.create_in_hdx( remove_additional_resources=True, hxl_update=False, updated_by_script="HDX Scraper: UNICEF Sam", batch=info["batch"], ) showcase.create_in_hdx() showcase.add_dataset(dataset)
def main(): """Generate dataset and create it in HDX""" configuration = Configuration.read() download_url = configuration["download_url"] with Download() as downloader: countries, headers, countriesdata = get_countriesdata(download_url, downloader) logger.info(f"Number of countries: {len(countriesdata)}") for info, country in progress_storing_tempdir("UCDP", countries, "iso3"): folder = info["folder"] dataset, showcase = generate_dataset_and_showcase( folder, country, countriesdata[country["iso3"]], headers ) if dataset: dataset.update_from_yaml() dataset["notes"] = dataset["notes"].replace( "\n", " \n" ) # ensure markdown has line breaks dataset.generate_resource_view(1) dataset.create_in_hdx( remove_additional_resources=True, hxl_update=False, updated_by_script="HDX Scraper: UCDP", batch=info["batch"], ) showcase.create_in_hdx() showcase.add_dataset(dataset)
def main(): """Generate dataset and create it in HDX""" configuration = Configuration.read() hdro_url = configuration['hdro_url'] qc_indicators = configuration['qc_indicators'] with Download() as downloader: countriesdata = get_countriesdata(hdro_url, downloader) countries = [{ 'iso3': countryiso } for countryiso in sorted(countriesdata.keys())] logger.info('Number of countries to upload: %d' % len(countries)) for info, country in progress_storing_tempdir('HDRO', countries, 'iso3'): countryiso = country['iso3'] countrydata = countriesdata[countryiso] dataset, showcase, bites_disabled = generate_dataset_and_showcase( info['folder'], countryiso, countrydata, qc_indicators) if dataset: dataset.update_from_yaml() dataset.generate_resource_view(-1, bites_disabled=bites_disabled, indicators=qc_indicators) dataset.create_in_hdx(remove_additional_resources=True, hxl_update=False, updated_by_script='HDX Scraper: HDRO', batch=info['batch']) showcase.create_in_hdx() showcase.add_dataset(dataset)
def main(): """Generate dataset and create it in HDX""" configuration = Configuration.read() download_url = configuration['download_url'] with Download() as downloader: countries, headers, countriesdata = get_countriesdata(download_url, downloader) logger.info('Number of countries: %d' % len(countriesdata)) for folder, country in progress_storing_tempdir('UCDP', countries, 'iso3'): dataset, showcase = generate_dataset_and_showcase(folder, country, countriesdata[country['iso3']], headers) if dataset: dataset.update_from_yaml() dataset['notes'] = dataset['notes'].replace('\n', ' \n') # ensure markdown has line breaks dataset.generate_resource_view(1) dataset.create_in_hdx(remove_additional_resources=True, hxl_update=False, updated_by_script='HDX Scraper: UCDP') showcase.create_in_hdx() showcase.add_dataset(dataset)
def main(): """Generate dataset and create it in HDX""" configuration = Configuration.read() base_url = configuration['base_url'] countries_url = configuration['countries_url'] with Download() as downloader: countries = get_countries(countries_url, downloader) logger.info('Number of datasets to upload: %d' % len(countries)) for info, country in progress_storing_tempdir('ACLED', sorted(countries, key=lambda x: x['iso3']), 'iso3'): folder = info['folder'] dataset, showcase = generate_dataset_and_showcase(base_url, downloader, folder, country) if dataset: dataset.update_from_yaml() dataset['license_other'] = dataset['license_other'].replace('\n', ' \n') # ensure markdown has line breaks dataset.generate_resource_view(1) dataset.create_in_hdx(remove_additional_resources=True, hxl_update=False, updated_by_script='HDX Scraper: ACLED', batch=info['batch']) showcase.create_in_hdx() showcase.add_dataset(dataset)
def main(output_failures=False, **ignore): configuration = Configuration.read() with Download() as downloader: dataset_ids = get_dataset_ids(configuration, downloader) logger.info(f"Number of datasets to upload: {len(dataset_ids)}") for info, dataset_id in progress_storing_tempdir( "UNHCR-MICRODATA", dataset_ids, "id" ): dataset = generate_dataset( dataset_id["id"], configuration, downloader, output_failures ) if dataset: dataset.update_from_yaml() dataset.create_in_hdx( remove_additional_resources=True, hxl_update=False, updated_by_script="HDX Scraper: UNHCR microdata", batch=info["batch"], ) for failure in failures: logger.error(failure)
def test_progress_storing_tempdir(self, monkeypatch): tempfolder = "papa" expected_dir = join(gettempdir(), tempfolder) rmtree(expected_dir, ignore_errors=True) iterator = [ { "iso3": "AFG", "name": "Afghanistan" }, { "iso3": "SDN", "name": "Sudan" }, { "iso3": "YEM", "name": "Yemen" }, { "iso3": "ZAM", "name": "Zambia" }, ] expected_batch_file = join(expected_dir, "batch.txt") result = list() for info, nextdict in progress_storing_tempdir(tempfolder, iterator, "iso3"): assert info["folder"] == expected_dir expected_batch = load_file_to_str(expected_batch_file, strip=True) result.append(nextdict) assert result == iterator assert expected_batch == info["batch"] assert exists(expected_dir) is False monkeypatch.setenv("WHERETOSTART", "iso3=SDN") result = list() for info, nextdict in progress_storing_tempdir(tempfolder, iterator, "iso3"): assert exists(info["folder"]) is True assert info["folder"] == expected_dir expected_batch = load_file_to_str(expected_batch_file, strip=True) result.append(nextdict) assert result == iterator[1:] assert expected_batch == info["batch"] assert exists(expected_dir) is False monkeypatch.delenv("WHERETOSTART") try: for info, nextdict in progress_storing_tempdir( tempfolder, iterator, "iso3"): if nextdict["iso3"] == "YEM": start_batch = info["batch"] raise ValueError("Problem!") except ValueError: pass assert exists(expected_dir) is True result = list() for info, nextdict in progress_storing_tempdir(tempfolder, iterator, "iso3"): assert exists(info["folder"]) is True assert info["folder"] == expected_dir assert info["batch"] == start_batch result.append(nextdict) assert result == iterator[2:] assert exists(expected_dir) is False try: for info, nextdict in progress_storing_tempdir( tempfolder, iterator, "iso3"): if nextdict["iso3"] == "YEM": start_batch = info["batch"] raise ValueError("Problem!") except ValueError: pass assert exists(expected_dir) is True monkeypatch.setenv("WHERETOSTART", "RESET") result = list() for info, nextdict in progress_storing_tempdir(tempfolder, iterator, "iso3"): assert exists(info["folder"]) is True assert info["folder"] == expected_dir assert info["batch"] != start_batch result.append(nextdict) assert result == iterator assert exists(expected_dir) is False monkeypatch.delenv("WHERETOSTART") try: for info, nextdict in progress_storing_tempdir( tempfolder, iterator, "iso3"): if nextdict["iso3"] == "YEM": start_batch = info["batch"] raise ValueError("Problem!") except ValueError: pass assert exists(expected_dir) is True monkeypatch.setenv("WHERETOSTART", "iso3=SDN") result = list() for info, nextdict in progress_storing_tempdir(tempfolder, iterator, "iso3"): assert exists(info["folder"]) is True assert info["folder"] == expected_dir assert info["batch"] == start_batch result.append(nextdict) assert result == iterator[1:] assert exists(expected_dir) is False monkeypatch.delenv("WHERETOSTART") try: for info, nextdict in progress_storing_tempdir( tempfolder, iterator, "iso3"): if nextdict["iso3"] == "YEM": start_batch = info["batch"] raise ValueError("Problem!") except ValueError: pass monkeypatch.setenv("WHERETOSTART", "iso3=NOTFOUND") found = False for _ in progress_storing_tempdir(tempfolder, iterator, "iso3"): found = True assert found is False assert exists(expected_dir) is True batch = load_file_to_str(expected_batch_file, strip=True) assert batch == start_batch monkeypatch.delenv("WHERETOSTART") monkeypatch.setenv("WHERETOSTART", "NOTFOUND=SDN") found = False for _ in progress_storing_tempdir(tempfolder, iterator, "iso3"): found = True assert found is False assert exists(expected_dir) is True batch = load_file_to_str(expected_batch_file, strip=True) assert batch == start_batch monkeypatch.delenv("WHERETOSTART") rmtree(expected_dir, ignore_errors=True)