def main(): """Generate dataset and create it in HDX""" filelist_url = Configuration.read()['filelist_url'] country_group_url = Configuration.read()['country_group_url'] dataset_codes = Configuration.read()['dataset_codes'] showcase_base_url = Configuration.read()['showcase_base_url'] with temp_dir('faostat') as folder: with Download() as downloader: indicatortypes = get_indicatortypesdata(filelist_url, downloader) countriesdata = get_countriesdata(country_group_url, downloader) logger.info('Number of indicator types to upload: %d' % len(dataset_codes)) for dataset_code in dataset_codes: datasets, showcases = generate_datasets_and_showcases( downloader, folder, dataset_codes[dataset_code], indicatortypes[dataset_code], countriesdata, showcase_base_url) logger.info('Number of datasets to upload: %d' % len(datasets)) for i, dataset in enumerate(datasets): logger.info('Creating dataset: %s' % dataset['title']) dataset.preview_off() dataset.create_in_hdx() showcase = showcases[i] showcase.create_in_hdx() showcase.add_dataset(dataset)
def test_user_agent(self, user_agent_config_yaml, user_agent_config2_yaml, user_agent_config3_yaml, empty_yaml, user_agent_config_wrong_yaml, project_config_yaml): Configuration._create(user_agent_config_yaml=user_agent_config_yaml, hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml) version = get_api_version() assert Configuration.read().remoteckan().user_agent == 'lala:HDXPythonLibrary/%s-myua' % version Configuration._create(user_agent_config_yaml=user_agent_config2_yaml, hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml) assert Configuration.read().remoteckan().user_agent == 'HDXPythonLibrary/%s-myuseragent' % version Configuration._create(user_agent_config_yaml=user_agent_config3_yaml, user_agent_lookup='lookup', hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml) assert Configuration.read().remoteckan().user_agent == 'HDXPythonLibrary/%s-mylookupagent' % version Configuration._create(user_agent_config_yaml=user_agent_config3_yaml, user_agent_lookup='lookup2', hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml) assert Configuration.read().remoteckan().user_agent == 'HDXPythonLibrary/%s-mylookupagent2' % version Configuration._create(user_agent='my_ua', preprefix='papa', hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml) assert Configuration.read().remoteckan().user_agent == 'papa:HDXPythonLibrary/%s-my_ua' % version with pytest.raises(UserAgentError): Configuration._create(user_agent_config_yaml=user_agent_config3_yaml, user_agent_lookup='fail', hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml) with pytest.raises(LoadError): Configuration._create(user_agent_config_yaml=empty_yaml, hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml) with pytest.raises(UserAgentError): Configuration._create(user_agent_config_yaml=user_agent_config_wrong_yaml, hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml) with pytest.raises(UserAgentError): Configuration._create(hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
def test_set_hdx_key_value(self, empty_hdx_key_file, project_config_yaml): with pytest.raises(LoadError): Configuration.load_api_key(empty_hdx_key_file) Configuration._create(hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_config_dict={}, project_config_yaml=project_config_yaml) configuration = Configuration.read() assert configuration.get_api_key() == 'TEST_HDX_KEY' configuration.set_api_key('NEW API KEY') assert configuration.get_api_key() == 'NEW API KEY' Configuration._create(hdx_site='prod', hdx_read_only=True, hdx_config_dict={}, project_config_yaml=project_config_yaml) assert Configuration.read().get_api_key() is None configuration = Configuration.read() configuration.set_api_key('TEST API KEY') assert configuration.get_api_key() is None configuration.set_read_only(False) assert configuration.get_api_key() == 'TEST API KEY' configuration.set_read_only(True) assert configuration.get_api_key() is None configuration.set_api_key('NEW API KEY') configuration.set_read_only(False) assert configuration.get_api_key() == 'NEW API KEY'
def test_remoteckan_validlocations(self, project_config_yaml): Configuration._create(hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_config_dict={}, project_config_yaml=project_config_yaml) remoteckan = ckanapi.RemoteCKAN('http://lalala', apikey='12345', user_agent='HDXPythonLibrary/1.0') Configuration.read().setup_remoteckan(remoteckan) assert Configuration.read().remoteckan() == remoteckan remoteckan = ckanapi.RemoteCKAN('http://hahaha', apikey='54321', user_agent='HDXPythonLibrary/0.5') Configuration._create(remoteckan=remoteckan, hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_config_dict={}, project_config_yaml=project_config_yaml) assert Configuration.read().remoteckan() == remoteckan Configuration.read()._remoteckan = None with pytest.raises(ConfigurationError): Configuration.read().remoteckan() Configuration.delete() with pytest.raises(ConfigurationError): Configuration.read().remoteckan()
def main(): """Generate dataset and create it in HDX""" with Download() as downloader: indicators = Configuration.read()['indicators'] tags = Configuration.read()['tags'] folder = get_temp_dir('IDMC') datasets, showcase, headersdata, countriesdata = generate_indicator_datasets_and_showcase( downloader, folder, indicators, tags) showcase_not_added = True countries = [{'iso3': x} for x in sorted(countriesdata)] logger.info('Number of indicator datasets to upload: %d' % len(indicators)) logger.info('Number of country datasets to upload: %d' % len(countries)) for i, info, nextdict in multiple_progress_storing_tempdir( 'IDMC', [indicators, countries], ['name', 'iso3']): folder = info['folder'] batch = info['batch'] if i == 0: if showcase_not_added: showcase.create_in_hdx() showcase_not_added = False dataset = datasets[nextdict['name']] dataset.update_from_yaml() dataset.generate_resource_view( join('config', nextdict['resourceview'])) dataset.create_in_hdx(remove_additional_resources=True, hxl_update=False, updated_by_script='HDX Scraper: IDMC', batch=batch) showcase.add_dataset(dataset) else: countryiso = nextdict['iso3'] countrydata = countriesdata[countryiso] dataset, showcase, bites_disabled = \ generate_country_dataset_and_showcase(downloader, folder, headersdata, countryiso, countrydata, datasets, tags) if dataset: dataset.update_from_yaml() dataset.generate_resource_view( bites_disabled=bites_disabled) dataset.create_in_hdx( remove_additional_resources=True, hxl_update=False, updated_by_script='HDX Scraper: IDMC', batch=batch) resources = dataset.get_resources() resource_ids = [ x['id'] for x in sorted(resources, key=lambda x: len(x['name']), reverse=True) ] dataset.reorder_resources(resource_ids, hxl_update=False)
def my_testfnkw(**kwargs): fn = kwargs.get('fn') if fn == 'site': testresult.actual_result = Configuration.read().get_hdx_site_url() elif fn == 'api': testresult.actual_result = Configuration.read().get_api_key() elif fn == 'agent': testresult.actual_result = Configuration.read().user_agent elif fn == 'exc': testresult.actual_result = Configuration.read().get_hdx_site_url() raise ValueError('Some failure!')
def test_get_all_users(self, configuration, post_list, mocksmtp): users = User.get_all_users() assert len(users) == 2 config = Configuration.read() config.setup_emailer(email_config_dict=TestUser.email_config_dict) User.email_users(users, TestUser.subject, TestUser.body, sender=TestUser.sender, mail_options=TestUser.mail_options, rcpt_options=TestUser.rcpt_options) email = config.emailer() assert email.server.type == 'smtpssl' assert email.server.initargs == TestUser.smtp_initargs assert email.server.username == TestUser.username assert email.server.password == TestUser.password assert email.server.sender == TestUser.sender assert email.server.recipients == ['*****@*****.**', '*****@*****.**'] assert email.server.msg == '''Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: hello From: [email protected] To: [email protected], [email protected] hello there''' assert email.server.send_args == {'mail_options': ['a', 'b'], 'rcpt_options': [1, 2]} with pytest.raises(ValueError): User.email_users(list(), TestUser.subject, TestUser.body, sender=TestUser.sender, mail_options=TestUser.mail_options, rcpt_options=TestUser.rcpt_options)
def test_hdx_configuration_yaml(self, hdx_config_yaml, hdx_base_config_yaml, project_config_yaml): Configuration._create(user_agent='test', hdx_config_yaml=hdx_config_yaml, hdx_base_config_yaml=hdx_base_config_yaml, project_config_yaml=project_config_yaml) expected_configuration = { 'hdx_site': 'prod', 'hdx_read_only': False, 'hdx_key': '12345', 'hdx_prod_site': { 'url': 'https://data.humdata.org', }, 'hdx_test_site': { 'url': 'https://test-data.humdata.org', 'username': '******', 'password': '******' }, 'dataset': {'required_fields': [ 'name', 'title', 'dataset_date', ]}, 'resource': {'required_fields': ['package_id', 'name', 'description']}, 'showcase': {'required_fields': ['name', 'title']}, 'approved_tags_vocabulary': 'Topics', 'tags_list_url': 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/Accepted_Tags.csv', 'tags_mapping_url': 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/Tag_Mapping.csv', } assert Configuration.read() == expected_configuration
def test_read_from_hdx(self, configuration, read, mocksmtp): user = User.read_from_hdx('TEST1') assert user['id'] == '9f3e9973-7dbe-4c65-8820-f48578e3ffea' assert user['name'] == 'MyUser1' user = User.read_from_hdx('TEST2') assert user is None user = User.read_from_hdx('TEST3') assert user is None config = Configuration.read() config.setup_emailer(email_config_dict=TestUser.email_config_dict) user = User.read_from_hdx('TEST1') user.email(TestUser.subject, TestUser.body, sender=TestUser.sender, mail_options=TestUser.mail_options, rcpt_options=TestUser.rcpt_options) email = config.emailer() assert email.server.type == 'smtpssl' assert email.server.initargs == TestUser.smtp_initargs assert email.server.username == TestUser.username assert email.server.password == TestUser.password assert email.server.sender == TestUser.sender assert email.server.recipients == ['*****@*****.**'] assert email.server.msg == '''Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: hello From: [email protected] To: [email protected] hello there''' assert email.server.send_args == {'mail_options': ['a', 'b'], 'rcpt_options': [1, 2]}
def post_delete(self): class MockSession(object): @staticmethod def post(url, data, headers, files, allow_redirects, auth=None): decodedata = data.decode('utf-8') datadict = json.loads(decodedata) if url.endswith('show') or 'list' in url: return mockshow(url, datadict) if 'delete' not in url: return MockResponse( 404, '{"success": false, "error": {"message": "TEST ERROR: Not delete", "__type": "TEST ERROR: Not Delete Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_delete"}' ) if datadict['id'] == '05e392bf-04e0-4ca6-848c-4e87bba10746': return MockResponse( 200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_delete"}' % decodedata) return MockResponse( 404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_delete"}' ) Configuration.read().remoteckan().session = MockSession()
def main(): """Generate dataset and create it in HDX""" configuration = Configuration.read() indicators = configuration["indicators"] json_url = configuration["json_url"] with Download() as downloader: indicators_metadata = get_indicators_metadata(json_url, downloader, indicators) countriesdata, countries = get_countriesdata(json_url, downloader, indicators) logger.info(f"Number of countries to upload: {len(countries)}") for info, country in progress_storing_tempdir("WorldPop", countries, "iso3"): countryiso = country["iso3"] datasets, showcases = generate_datasets_and_showcases( downloader, countryiso, indicators_metadata, countriesdata[countryiso]) for dataset in datasets: dataset.update_from_yaml() dataset.create_in_hdx( remove_additional_resources=True, hxl_update=False, updated_by_script="HDX Scraper: WorldPop", batch=info["batch"], ) for showcase in showcases.get(dataset["name"], list()): showcase.create_in_hdx() showcase.add_dataset(dataset)
def main(excel_path, gsheet_auth, updatesheets, updatetabs, scrapers, basic_auths, nojson, **ignore): logger.info('##### hdx-scraper-covid-viz version %.1f ####' % VERSION) configuration = Configuration.read() with Download(rate_limit={'calls': 1, 'period': 0.1}) as downloader: if scrapers: logger.info('Updating only scrapers: %s' % scrapers) tabs = configuration['tabs'] if updatetabs is None: updatetabs = list(tabs.keys()) logger.info('Updating all tabs') else: logger.info('Updating only these tabs: %s' % updatetabs) noout = nooutput(updatetabs) if excel_path: excelout = exceloutput(excel_path, tabs, updatetabs) else: excelout = noout if gsheet_auth: gsheets = googlesheets(configuration, gsheet_auth, updatesheets, tabs, updatetabs) else: gsheets = noout if nojson: jsonout = noout else: jsonout = jsonoutput(configuration, updatetabs) outputs = {'gsheets': gsheets, 'excel': excelout, 'json': jsonout} admininfo = AdminInfo.setup(downloader) get_indicators(configuration, downloader, admininfo, outputs, updatetabs, scrapers, basic_auths) excelout.save() jsonout.add_additional_json(downloader) jsonout.save(hrp_iso3s=admininfo.hrp_iso3s)
def main(): """Generate dataset and create it in HDX""" configuration = Configuration.read() resources = configuration["resources"] fields = configuration["fields"] source_directory = configuration["source_directory"] download_url = Path(source_directory).resolve().as_uri() with Download() as downloader: countries, headers, countriesdata = get_countriesdata( download_url, resources, downloader) logger.info("Number of countries: %d" % len(countriesdata)) for info, country in progress_storing_tempdir("UNHCR_population", countries, "iso3"): folder = info["folder"] dataset, showcase = generate_dataset_and_showcase( folder, country, countriesdata[country["iso3"]], headers, resources, fields) if dataset: dataset.update_from_yaml() dataset["notes"] = dataset["notes"].replace( "\n", " \n") # ensure markdown has line breaks dataset.generate_resource_view(1) dataset.create_in_hdx( remove_additional_resources=True, hxl_update=False, updated_by_script="HDX Scraper: UNHCR population", batch=info["batch"], ) showcase.create_in_hdx() showcase.add_dataset(dataset)
def post_delete(self): class MockSession(object): @staticmethod def post(url, data, headers, files, allow_redirects, auth=None): decodedata = data.decode('utf-8') datadict = json.loads(decodedata) if 'show' in url: return resource_view_mockshow(url, datadict) if 'delete' not in url: return MockResponse( 404, '{"success": false, "error": {"message": "TEST ERROR: Not delete", "__type": "TEST ERROR: Not Delete Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_view_delete"}' ) if datadict['id'] == 'c06b5a0d-1d41-4a74-a196-41c251c76023': return MockResponse( 200, '{"success": true, "result": null, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_view_delete"}' ) return MockResponse( 404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_view_delete"}' ) Configuration.read().remoteckan().session = MockSession()
def main(): """Generate dataset and create it in HDX""" with Download() as downloader: config = Configuration.read() project_config = { key: value for key, value in config.items() if key.startswith("CV") } qc_indicators = config.get("qc_indicators", {}) countries, countriesdata, headers = get_all_countriesdata( project_config, downloader) logger.info("Number of datasets to upload: %d" % len(countries)) for info, country in progress_storing_tempdir("UNICEFSAM", countries, "iso3"): dataset, showcase, bites_disabled = generate_dataset_and_showcase( info["folder"], country, countriesdata[country["iso3"]], headers, project_config, qc_indicators) if dataset: dataset.update_from_yaml() dataset.generate_resource_view(1, bites_disabled=bites_disabled, indicators=qc_indicators) dataset.create_in_hdx( remove_additional_resources=True, hxl_update=False, updated_by_script="HDX Scraper: UNICEF Sam", batch=info["batch"], ) showcase.create_in_hdx() showcase.add_dataset(dataset)
def main(): """Generate dataset and create it in HDX""" configuration = Configuration.read() base_url = configuration['base_url'] with Download(extra_params_yaml=join(expanduser('~'), '.extraparams.yml'), extra_params_lookup=lookup) as downloader: downloader.session.mount( 'http://', HTTPAdapter(max_retries=1, pool_connections=100, pool_maxsize=100)) downloader.session.mount( 'https://', HTTPAdapter(max_retries=1, pool_connections=100, pool_maxsize=100)) countries = get_countries(base_url, downloader) logger.info('Number of countries: %d' % len(countries)) for folder, country in progress_storing_tempdir( 'DHS', countries, 'iso3'): tags = get_tags(base_url, downloader, country['dhscode']) dataset, subdataset, showcase, bites_disabled = \ generate_datasets_and_showcase(configuration, base_url, downloader, folder, country, tags) if dataset: createdataset(dataset) resource_view = generate_resource_view( dataset, bites_disabled=bites_disabled['national']) resource_view.create_in_hdx() showcase.create_in_hdx() showcase.add_dataset(dataset) if subdataset: createdataset(subdataset) showcase.add_dataset(subdataset) subdataset.generate_resource_view( bites_disabled=bites_disabled['subnational'])
def configuration(self): Configuration._create(hdx_read_only=True, user_agent='test', project_config_yaml=join( 'tests', 'config', 'project_configuration.yml')) Locations.set_validlocations([{'name': 'afg', 'title': 'Afghanistan'}]) Country.countriesdata(use_live=False) Vocabulary._tags_dict = True Vocabulary._approved_vocabulary = { 'tags': [{ 'name': 'hxl' }, { 'name': 'indicators' }, { 'name': 'health' }, { 'name': 'education' }, { 'name': 'socioeconomic' }, { 'name': 'demographics' }, { 'name': 'development' }], 'id': '4e61d464-4943-4e97-973a-84673c1aaa87', 'name': 'approved' } return Configuration.read()
def post_delete(self): class MockSession(object): @staticmethod def post(url, data, headers, files, allow_redirects, auth=None): decodedata = data.decode('utf-8') datadict = json.loads(decodedata) if 'show' in url: return organization_mockshow(url, datadict) if 'delete' not in url: return MockResponse( 404, '{"success": false, "error": {"message": "TEST ERROR: Not delete", "__type": "TEST ERROR: Not Delete Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_delete"}' ) if datadict['id'] == 'b67e6c74-c185-4f43-b561-0e114a736f19': return MockResponse( 200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_delete"}' % decodedata) return MockResponse( 404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_delete"}' ) Configuration.read().remoteckan().session = MockSession()
def main(save, use_saved, **ignore): """Generate dataset and create it in HDX""" with Download(extra_params_yaml=join(expanduser('~'), '.extraparams.yml'), extra_params_lookup=lookup) as token_downloader: configuration = Configuration.read() with Download() as downloader: folder = temp_dir(lookup) retriever = Retrieve(downloader, folder, 'saved_data', folder, save, use_saved) wfp = WFPFood(configuration, token_downloader, retriever) countries = wfp.get_countries() logger.info('Number of country datasets to upload: %d' % len(countries)) wfp.build_mappings() for info, country in progress_storing_tempdir( lookup, countries, 'iso3'): dataset, showcase, qc_indicators = wfp.generate_dataset_and_showcase( country['iso3'], info['folder']) if dataset: dataset.update_from_yaml() dataset['notes'] = dataset[ 'notes'] % 'Food Prices data for %s. Food prices data comes from the World Food Programme and covers' % country[ 'name'] dataset.generate_resource_view(-1, indicators=qc_indicators) dataset.create_in_hdx( remove_additional_resources=True, hxl_update=False, updated_by_script='HDX Scraper: WFP Food Prices', batch=info['batch']) showcase.create_in_hdx() showcase.add_dataset(dataset)
def post_create(self): class MockSession(object): @staticmethod def post(url, data, headers, files, allow_redirects, auth=None): datadict = json.loads(data.decode('utf-8')) if 'show' in url: return organization_mockshow(url, datadict) if 'create' not in url: return MockResponse(404, '{"success": false, "error": {"message": "TEST ERROR: Not create", "__type": "TEST ERROR: Not Create Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_create"}') result = json.dumps(resultdict) if datadict['name'] == 'MyOrganization1': return MockResponse(200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_create"}' % result) if datadict['name'] == 'MyOrganization2': return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_create"}') if datadict['name'] == 'MyOrganization3': return MockResponse(200, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_create"}') return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_create"}') Configuration.read().remoteckan().session = MockSession()
def post_update(self): class MockSession(object): @staticmethod def post(url, data, headers, files, allow_redirects, auth): datadict = json.loads(data.decode('utf-8')) if 'show' in url: return user_mockshow(url, datadict) if 'update' not in url: return MockResponse(404, '{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=user_update"}') resultdictcopy = copy.deepcopy(resultdict) merge_two_dictionaries(resultdictcopy, datadict) result = json.dumps(resultdictcopy) if datadict['name'] == 'MyUser1': return MockResponse(200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=user_update"}' % result) if datadict['name'] == 'MyUser2': return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=user_update"}') if datadict['name'] == 'MyUser3': return MockResponse(200, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=user_update"}') return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=user_update"}') Configuration.read().remoteckan().session = MockSession()
def main(): """Generate dataset and create it in HDX""" configuration = Configuration.read() download_url = configuration["download_url"] with Download() as downloader: countries, headers, countriesdata = get_countriesdata(download_url, downloader) logger.info(f"Number of countries: {len(countriesdata)}") for info, country in progress_storing_tempdir("UCDP", countries, "iso3"): folder = info["folder"] dataset, showcase = generate_dataset_and_showcase( folder, country, countriesdata[country["iso3"]], headers ) if dataset: dataset.update_from_yaml() dataset["notes"] = dataset["notes"].replace( "\n", " \n" ) # ensure markdown has line breaks dataset.generate_resource_view(1) dataset.create_in_hdx( remove_additional_resources=True, hxl_update=False, updated_by_script="HDX Scraper: UCDP", batch=info["batch"], ) showcase.create_in_hdx() showcase.add_dataset(dataset)
def __init__(self, downloader): configuration = Configuration.read() admin_info = configuration['admin_info'] self.adm1_name_replacements = configuration['adm1_name_replacements'] self.adm1_fuzzy_ignore = configuration['adm1_fuzzy_ignore'] self.adm_mappings = configuration.get('adm_mappings', [dict(), dict()]) iso3s_no_pcodes = set() countryiso3s = set() for row in admin_info: countryiso3 = row['alpha_3'] countryiso3s.add(countryiso3) pcode = row.get('ADM1_PCODE') if not pcode: iso3s_no_pcodes.add(countryiso3) continue self.pcodes.append(pcode) self.pcode_lengths[countryiso3] = len(pcode) adm1_name = row['ADM1_REF'] self.pcode_to_name[pcode] = adm1_name name_to_pcode = self.name_to_pcode.get(countryiso3, dict()) name_to_pcode[unidecode(adm1_name).lower()] = pcode self.name_to_pcode[countryiso3] = name_to_pcode self.pcode_to_iso3[pcode] = countryiso3 self.countryiso3s = sorted(list(countryiso3s)) self.adms = [self.countryiso3s, self.pcodes] self.iso3s_no_pcodes = sorted(list(iso3s_no_pcodes)) self.hrp_iso3s = sorted(list(self.name_to_pcode.keys())) self.regions, self.iso3_to_region, self.iso3_to_region_and_hrp = self.read_regional(configuration, self.countryiso3s, self.hrp_iso3s, downloader) self.init_matches_errors()
def getData(countryISO2): url = Configuration.read()['base_url'] + "&code=" + countryISO2 response = requests.get(url) headers = [ 'category', 'indicator', 'indicator_friendly', 'type_data', 'latitude', 'longitude', 'region_id', 'country_id', 'name', 'year', 'value' ] hxl = [ '#meta+category', '#indicator+name', '#indicator+description', '#indicator+type', '#geo+lat', '#geo+lon', '#region+code', '#country+code+v_iso2', '#country+name', '#date+year', '#indicator+value' ] if response.status_code != 200: print(":(") else: try: with open('data/%s' % countryISO2 + '.csv', 'w') as f: f.write(response.text) except Exception as e: raise e with open('data/indicator_data_%s' % countryISO2 + '.csv', 'w') as hxl_tags: writer = csv.writer(hxl_tags, delimiter=',') writer.writerow(headers) writer.writerow(hxl) data = open('data/%s' % countryISO2 + '.csv', 'r') with data: reader = csv.reader(data) next(reader) for r in reader: writer.writerow(r)
def main(): """Generate dataset and create it in HDX""" configuration = Configuration.read() hdro_url = configuration['hdro_url'] qc_indicators = configuration['qc_indicators'] with Download() as downloader: countriesdata = get_countriesdata(hdro_url, downloader) countries = [{ 'iso3': countryiso } for countryiso in sorted(countriesdata.keys())] logger.info('Number of countries to upload: %d' % len(countries)) for info, country in progress_storing_tempdir('HDRO', countries, 'iso3'): countryiso = country['iso3'] countrydata = countriesdata[countryiso] dataset, showcase, bites_disabled = generate_dataset_and_showcase( info['folder'], countryiso, countrydata, qc_indicators) if dataset: dataset.update_from_yaml() dataset.generate_resource_view(-1, bites_disabled=bites_disabled, indicators=qc_indicators) dataset.create_in_hdx(remove_additional_resources=True, hxl_update=False, updated_by_script='HDX Scraper: HDRO', batch=info['batch']) showcase.create_in_hdx() showcase.add_dataset(dataset)
def test_hdx_configuration_yaml(self, hdx_key_file, hdx_config_yaml, project_config_yaml): Configuration._create(hdx_key_file=hdx_key_file, hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml) expected_configuration = { 'api_key': '12345', 'param_1': 'ABC', 'hdx_prod_site': { 'url': 'https://data.humdata.org/', 'username': None, 'password': None }, 'hdx_test_site': { 'url': 'https://test-data.humdata.org/', 'username': '******', 'password': '******' }, 'dataset': { 'required_fields': [ 'name', 'title', 'dataset_date', ] }, 'resource': { 'required_fields': ['package_id', 'name', 'description'] }, 'showcase': { 'required_fields': ['name', 'title'] }, } assert Configuration.read() == expected_configuration
def post_update(self): class MockSession(object): @staticmethod def post(url, data, headers, files, allow_redirects, auth=None): datadict = json.loads(data.decode('utf-8')) if url.endswith('show') or 'list' in url: return mockshow(url, datadict) if 'update' not in url: return MockResponse(404, '{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}') resultdictcopy = copy.deepcopy(showcase_resultdict) merge_two_dictionaries(resultdictcopy, datadict) result = json.dumps(resultdictcopy) if datadict['title'] == 'MyShowcase1': return MockResponse(200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}' % result) if datadict['title'] == 'MyShowcase2': return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}') if datadict['title'] == 'MyShowcase3': return MockResponse(200, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}') return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}') Configuration.read().remoteckan().session = MockSession()
def test_create_set_configuration(self, project_config_yaml): Configuration._create(user_agent='test', hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml) with pytest.raises(ConfigurationError): Configuration.create(user_agent='test', hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml) configuration = Configuration(user_agent='test', hdx_site='test', hdx_key='OTHER_TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml) Configuration.setup(configuration) assert Configuration.read() == configuration Configuration.delete() with pytest.raises(ConfigurationError): Configuration.read() Configuration.create(user_agent='test', hdx_site='prod', hdx_key='TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml) assert Configuration.read().get_api_key() == 'TEST_HDX_KEY'
def main(excel_path, gsheet_auth, updatesheets, updatetabs, scrapers, basic_auths, other_auths, nojson, countries_override, save, use_saved, **ignore): logger.info('##### hdx-scraper-covid-viz version %.1f ####' % VERSION) configuration = Configuration.read() with temp_dir() as temp_folder: with Download(rate_limit={'calls': 1, 'period': 0.1}) as downloader: retriever = Retrieve(downloader, temp_folder, 'saved_data', temp_folder, save, use_saved) if scrapers: logger.info('Updating only scrapers: %s' % scrapers) tabs = configuration['tabs'] if updatetabs is None: updatetabs = list(tabs.keys()) logger.info('Updating all tabs') else: logger.info('Updating only these tabs: %s' % updatetabs) noout = NoOutput(updatetabs) if excel_path: excelout = ExcelOutput(excel_path, tabs, updatetabs) else: excelout = noout if gsheet_auth: gsheets = GoogleSheets(configuration, gsheet_auth, updatesheets, tabs, updatetabs) else: gsheets = noout if nojson: jsonout = noout else: jsonout = JsonOutput(configuration, updatetabs) outputs = {'gsheets': gsheets, 'excel': excelout, 'json': jsonout} today = datetime.now() countries_to_save = get_indicators(configuration, today, retriever, outputs, updatetabs, scrapers, basic_auths, other_auths, countries_override) jsonout.add_additional_json(downloader, today=today) jsonout.save(countries_to_save=countries_to_save) excelout.save()
def main(): """Generate dataset and create it in HDX""" with Download() as downloader: configuration = Configuration.read() countries_path = join('config', configuration['countries_filename']) indicators_url = configuration['indicators_url'] mvam_url = configuration['mvam_url'] showcase_url = configuration['showcase_url'] countries = get_countries(countries_path, downloader) variables = get_mvamvariables(indicators_url, downloader) logger.info('Number of datasets to upload: %d' % len(countries)) for info, country in progress_storing_tempdir('WFPFoodSecurity', countries, 'iso3'): dataset, showcase, bites_disabled = \ generate_dataset_and_showcase(mvam_url, showcase_url, downloader, info['folder'], country, variables) if dataset: dataset.update_from_yaml() dataset.generate_resource_view(bites_disabled=bites_disabled) dataset.create_in_hdx( remove_additional_resources=True, hxl_update=False, updated_by_script='HDX Scraper: WFP Food Security', batch=info['batch']) showcase.create_in_hdx() showcase.add_dataset(dataset)
def post_delete(self): class MockSession(object): @staticmethod def post(url, data, headers, files, allow_redirects, auth): decodedata = data.decode('utf-8') datadict = json.loads(decodedata) if 'show' in url: return mockshow(url, datadict) if 'delete' not in url: return MockResponse( 404, '{"success": false, "error": {"message": "TEST ERROR: Not delete", "__type": "TEST ERROR: Not Delete Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_delete"}' ) if datadict['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5': return MockResponse( 200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_delete"}' % decodedata) return MockResponse( 404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_delete"}' ) Configuration.read().remoteckan().session = MockSession()
def post_resourceview(self): class MockSession(object): @staticmethod def post(url, data, headers, files, allow_redirects, auth=None): decodedata = data.decode('utf-8') return mockresourceview(url, decodedata) Configuration.read().remoteckan().session = MockSession()
def read(self): class MockSession(object): @staticmethod def post(url, data, headers, files, allow_redirects, auth=None): datadict = json.loads(data.decode('utf-8')) return organization_mockshow(url, datadict) Configuration.read().remoteckan().session = MockSession()
def read(self): class MockSession(object): @staticmethod def post(url, data, headers, files, allow_redirects, auth=None): datadict = json.loads(data.decode('utf-8')) return organization_mockshow(url, datadict) Configuration.read().remoteckan().session = MockSession()
def main(): """Generate dataset and create it in HDX""" acled_url = Configuration.read()['acled_url'] countries_url = Configuration.read()['countries_url'] hxlproxy_url = Configuration.read()['hxlproxy_url'] with Download() as downloader: countriesdata = get_countriesdata(countries_url, downloader) logger.info('Number of datasets to upload: %d' % len(countriesdata)) for countrydata in sorted(countriesdata, key=lambda x: x['iso3']): dataset, showcase = generate_dataset_and_showcase(acled_url, hxlproxy_url, downloader, countrydata) if dataset: dataset.update_from_yaml() dataset.create_in_hdx() resource_view = generate_resource_view(dataset) resource_view.create_in_hdx() showcase.create_in_hdx() showcase.add_dataset(dataset) sleep(1)
def test_generate_dataset_and_showcase(self, configuration, downloader): hxlproxy_url = Configuration.read()['hxlproxy_url'] dataset, showcase = generate_dataset_and_showcase('http://lala?', hxlproxy_url, downloader, TestAcledAfrica.countrydata) assert dataset == TestAcledAfrica.dataset resources = dataset.get_resources() assert resources == [TestAcledAfrica.resource] assert showcase == {'name': 'acled-data-for-cameroon-showcase', 'notes': 'Conflict Data Dashboard for Cameroon', 'url': 'https://www.acleddata.com/dashboard/#120', 'tags': [{'name': 'HXL'}, {'name': 'conflicts'}, {'name': 'political violence'}, {'name': 'protests'}], 'title': 'Dashboard for Cameroon', 'image_url': 'https://www.acleddata.com/wp-content/uploads/2018/01/dash.png'} dataset, showcase = generate_dataset_and_showcase('http://lala?', hxlproxy_url, downloader, {'m49': 4, 'iso3': 'AFG', 'countryname': 'Afghanistan'}) assert dataset is None
def tagscleanupdicts(configuration=None, url=None, keycolumn=5, failchained=True): # type: (Optional[Configuration], Optional[str], int, bool) -> Tuple[Dict,List] """ Get tags cleanup dictionaries Args: configuration (Optional[Configuration]): HDX configuration. Defaults to global configuration. url (Optional[str]): Url of tags cleanup spreadsheet. Defaults to None (internal configuration parameter). keycolumn (int): Column number of tag column in spreadsheet. Defaults to 5. failchained (bool): Fail if chained rules found. Defaults to True. Returns: Tuple[Dict,List]: Returns (Tags dictionary, Wildcard tags list) """ if not Tags._tags_dict: if configuration is None: configuration = Configuration.read() with Download(full_agent=configuration.get_user_agent()) as downloader: if url is None: url = configuration['tags_cleanup_url'] Tags._tags_dict = downloader.download_tabular_rows_as_dicts(url, keycolumn=keycolumn) keys = Tags._tags_dict.keys() chainerror = False for i, tag in enumerate(keys): whattodo = Tags._tags_dict[tag] action = whattodo[u'action'] final_tags = whattodo[u'final tags (semicolon separated)'] for final_tag in final_tags.split(';'): if final_tag in keys: index = list(keys).index(final_tag) if index != i: whattodo2 = Tags._tags_dict[final_tag] action2 = whattodo2[u'action'] if action2 != 'OK' and action2 != 'Other': final_tags2 = whattodo2[u'final tags (semicolon separated)'] if final_tag not in final_tags2.split(';'): chainerror = True if failchained: logger.error('Chained rules: %s (%s -> %s) | %s (%s -> %s)' % (action, tag, final_tags, action2, final_tag, final_tags2)) if failchained and chainerror: raise ChainRuleError('Chained rules for tags detected!') Tags._wildcard_tags = list() for tag in Tags._tags_dict: if '*' in tag: Tags._wildcard_tags.append(tag) return Tags._tags_dict, Tags._wildcard_tags
def read(self): class MockSession(object): @staticmethod def post(url, data, headers, files, allow_redirects, auth=None): datadict = json.loads(data.decode('utf-8')) if 'association_delete' in url: TestShowcase.association = 'delete' return MockResponse(200, '{"success": true, "result": null, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_package_association_delete"}') elif 'association_create' in url: TestShowcase.association = 'create' result = json.dumps(datadict) return MockResponse(200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_package_association_create"}' % result) return mockshow(url, datadict) Configuration.read().remoteckan().session = MockSession()
def test_read_from_hdx(self, configuration, read, mocksmtp): user = User.read_from_hdx('9f3e9973-7dbe-4c65-8820-f48578e3ffea') assert user['id'] == '9f3e9973-7dbe-4c65-8820-f48578e3ffea' assert user['name'] == 'MyUser1' user = User.read_from_hdx('TEST2') assert user is None user = User.read_from_hdx('TEST3') assert user is None config = Configuration.read() config.setup_emailer(email_config_dict=TestUser.email_config_dict) user = User.read_from_hdx('9f3e9973-7dbe-4c65-8820-f48578e3ffea') user.email(TestUser.subject, TestUser.text_body, html_body=TestUser.html_body, sender=TestUser.sender, mail_options=TestUser.mail_options, rcpt_options=TestUser.rcpt_options) email = config.emailer() assert email.server.type == 'smtpssl' assert email.server.initargs == TestUser.smtp_initargs assert email.server.username == TestUser.username assert email.server.password == TestUser.password assert email.server.sender == TestUser.sender assert email.server.recipients == ['*****@*****.**'] assert 'Content-Type: multipart/alternative;' in email.server.msg assert '''\ MIME-Version: 1.0 Subject: hello From: [email protected] To: [email protected]''' in email.server.msg assert '''\ Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit hello there''' in email.server.msg assert '''\ Content-Type: text/html; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit <html> <head></head> <body> <p>Hi!<br> </p> </body> </html>''' in email.server.msg assert email.server.send_args == {'mail_options': ['a', 'b'], 'rcpt_options': [1, 2]}
def post_delete(self): class MockSession(object): @staticmethod def post(url, data, headers, files, allow_redirects, auth=None): decodedata = data.decode('utf-8') datadict = json.loads(decodedata) if 'show' in url: return organization_mockshow(url, datadict) if 'delete' not in url: return MockResponse(404, '{"success": false, "error": {"message": "TEST ERROR: Not delete", "__type": "TEST ERROR: Not Delete Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_delete"}') if datadict['id'] == 'b67e6c74-c185-4f43-b561-0e114a736f19': return MockResponse(200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_delete"}' % decodedata) return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_delete"}') Configuration.read().remoteckan().session = MockSession()
def post_delete(self): class MockSession(object): @staticmethod def post(url, data, headers, files, allow_redirects, auth=None): decodedata = data.decode('utf-8') datadict = json.loads(decodedata) if url.endswith('show') or 'list' in url: return mockshow(url, datadict) if 'delete' not in url: return MockResponse(404, '{"success": false, "error": {"message": "TEST ERROR: Not delete", "__type": "TEST ERROR: Not Delete Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_delete"}') if datadict['id'] == '05e392bf-04e0-4ca6-848c-4e87bba10746': return MockResponse(200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_delete"}' % decodedata) return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_delete"}') Configuration.read().remoteckan().session = MockSession()
def test_get_all_users(self, configuration, post_list, mocksmtp): users = User.get_all_users() assert len(users) == 2 config = Configuration.read() config.setup_emailer(email_config_dict=TestUser.email_config_dict) User.email_users(users, TestUser.subject, TestUser.text_body, html_body=TestUser.html_body, sender=TestUser.sender, mail_options=TestUser.mail_options, rcpt_options=TestUser.rcpt_options) email = config.emailer() assert email.server.type == 'smtpssl' assert email.server.initargs == TestUser.smtp_initargs assert email.server.username == TestUser.username assert email.server.password == TestUser.password assert email.server.sender == TestUser.sender assert email.server.recipients == ['*****@*****.**', '*****@*****.**'] assert 'Content-Type: multipart/alternative;' in email.server.msg assert '''\ MIME-Version: 1.0 Subject: hello From: [email protected] To: [email protected], [email protected]''' in email.server.msg assert '''\ Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit hello there''' in email.server.msg assert '''\ Content-Type: text/html; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit <html> <head></head> <body> <p>Hi!<br> </p> </body> </html>''' in email.server.msg assert email.server.send_args == {'mail_options': ['a', 'b'], 'rcpt_options': [1, 2]} with pytest.raises(ValueError): User.email_users(list(), TestUser.subject, TestUser.text_body, sender=TestUser.sender, mail_options=TestUser.mail_options, rcpt_options=TestUser.rcpt_options)
def post_listorgs(self): class MockSession(object): @staticmethod def post(url, data, headers, files, allow_redirects, auth=None): decodedata = data.decode('utf-8') datadict = json.loads(decodedata) if 'user' in url: if 'show' in url: return user_mockshow(url, datadict) elif 'list' in url: return MockResponse(200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_list"}' % json.dumps(orglist)) elif 'organization' in url: if 'show' in url: result = json.dumps(orgdict) if datadict['id'] == 'b67e6c74-c185-4f43-b561-0e114a736f19' or datadict['id'] == 'TEST1': return MockResponse(200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_show"}' % result) Configuration.read().remoteckan().session = MockSession()
def post_update(self): class MockSession(object): @staticmethod def post(url, data, headers, files, allow_redirects, auth=None): if isinstance(data, dict): datadict = {k.decode('utf8'): v.decode('utf8') for k, v in data.items()} else: datadict = json.loads(data.decode('utf-8')) if 'show' in url: return mockshow(url, datadict) if 'resource_id' in datadict: if datadict['resource_id'] == '74b74ae1-df0c-4716-829f-4f939a046811': return MockResponse(200, '{"success": true, "result": {"fields": [{"type": "text", "id": "code"}, {"type": "text", "id": "title"}, {"type": "float", "id": "value"}, {"type": "timestamp", "id": "latest_date"}, {"type": "text", "id": "source"}, {"type": "text", "id": "source_link"}, {"type": "text", "id": "notes"}, {"type": "text", "id": "explore"}, {"type": "text", "id": "units"}], "method": "insert", "primary_key": "code", "resource_id": "bfa6b55f-10b6-4ba2-8470-33bb9a5194a5"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_create"}') if 'update' not in url: return MockResponse(404, '{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}') if datadict['name'] == 'MyResource1': resultdictcopy = copy.deepcopy(resultdict) merge_two_dictionaries(resultdictcopy, datadict) if files is not None: resultdictcopy['url_type'] = 'upload' resultdictcopy['resource_type'] = 'file.upload' filename = os.path.basename(files[0][1].name) resultdictcopy[ 'url'] = 'http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/%s' % filename result = json.dumps(resultdictcopy) return MockResponse(200, '{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}' % result) if datadict['name'] == 'MyResource2': return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}') if datadict['name'] == 'MyResource3': return MockResponse(200, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}') return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}') Configuration.read().remoteckan().session = MockSession()
def post_datastore(self): class MockSession(object): @staticmethod def post(url, data, headers, files, allow_redirects, auth=None): decodedata = data.decode('utf-8') datadict = json.loads(decodedata) if 'show' in url: return mockshow(url, datadict) if 'create' not in url and 'insert' not in url and 'upsert' not in url and 'delete' not in url and 'search' not in url: return MockResponse(404, '{"success": false, "error": {"message": "TEST ERROR: Not create or delete", "__type": "TEST ERROR: Not Create or Delete Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_action"}') if 'delete' in url and datadict['resource_id'] == 'datastore_unknown_resource': return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_delete"}') if 'delete' in url and datadict['resource_id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5': TestResource.datastore = 'delete' return MockResponse(200, '{"success": true, "result": {"resource_id": "de6549d8-268b-4dfe-adaf-a4ae5c8510d5"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_delete"}') if 'create' in url and datadict['resource_id'] == 'datastore_unknown_resource': return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_create"}') if 'search' in url and datadict['resource_id'] == 'datastore_unknown_resource': return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_create"}') if 'search' in url and datadict['resource_id'] == '_table_metadata': return MockResponse(200, '{"success": true, "result": {"include_total": true, "resource_id": "_table_metadata", "fields": [{"type": "int", "id": "_id"}, {"type": "name", "id": "name"}, {"type": "oid", "id": "oid"}, {"type": "name", "id": "alias_of"}], "records_format": "objects", "records": [{"_id":"f9cd60f3d7f2f6d0","name":"f9228459-d808-4b51-948f-68a5850abfde","oid":"919290","alias_of":null},{"_id":"7ae63490de9b7d7b","name":"af618a0b-09b8-42c8-836f-2be597e1ea34","oid":"135294","alias_of":null},{"_id":"1dc37f4e89988644","name":"748b40dd-7bd3-40a3-941b-e76f0bfbe0eb","oid":"117144","alias_of":null},{"_id":"2a554a61bd366206","name":"91c78d24-eab3-40b5-ba91-6b29bcda7178","oid":"116963","alias_of":null},{"_id":"fd787575143afe90","name":"9320cfce-4620-489a-bcbe-25c73867d4fc","oid":"107430","alias_of":null},{"_id":"a70093abd230f647","name":"b9d2eb36-e65c-417a-bc28-f4dadb149302","oid":"107409","alias_of":null},{"_id":"95fbdd2d06c07aea","name":"ca6a0891-8395-4d58-9168-6c44e17e0193","oid":"107385","alias_of":null}], "limit": 10000, "_links": {"start": "/api/action/datastore_search?limit=10000&resource_id=_table_metadata", "next": "/api/action/datastore_search?offset=10000&limit=10000&resource_id=_table_metadata"}, "total": 7}}') if ('create' in url or 'insert' in url or 'upsert' in url or 'search' in url) and datadict[ 'resource_id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5': TestResource.datastore = 'create' return MockResponse(200, '{"success": true, "result": {"fields": [{"type": "text", "id": "code"}, {"type": "text", "id": "title"}, {"type": "float", "id": "value"}, {"type": "timestamp", "id": "latest_date"}, {"type": "text", "id": "source"}, {"type": "text", "id": "source_link"}, {"type": "text", "id": "notes"}, {"type": "text", "id": "explore"}, {"type": "text", "id": "units"}], "method": "insert", "primary_key": "code", "resource_id": "bfa6b55f-10b6-4ba2-8470-33bb9a5194a5"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_create"}') return MockResponse(404, '{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_delete"}') Configuration.read().remoteckan().session = MockSession()
def facade(projectmainfn, **kwargs): # (Callable[[None], None], Any) -> None """Facade to simplify project setup that calls project main function Args: projectmainfn ((None) -> None): main function of project **kwargs: configuration parameters to pass to HDX Configuration class Returns: None """ # # Setting up configuration # site_url = Configuration._create(**kwargs) logger.info('--------------------------------------------------') logger.info('> Using HDX Python API Library %s' % Configuration.apiversion) logger.info('> HDX Site: %s' % site_url) UserAgent.user_agent = Configuration.read().user_agent projectmainfn()
def my_excfn(): testresult.actual_result = Configuration.read().get_hdx_site_url() raise ValueError('Some failure!')
def my_testuafn(): testresult.actual_result = Configuration.read().user_agent
def my_testkeyfn(): testresult.actual_result = Configuration.read().get_api_key()
def my_testfn(): testresult.actual_result = Configuration.read().get_hdx_site_url()