def main():
"""Generate dataset and create it in HDX"""
filelist_url = Configuration.read()['filelist_url']
country_group_url = Configuration.read()['country_group_url']
dataset_codes = Configuration.read()['dataset_codes']
showcase_base_url = Configuration.read()['showcase_base_url']
with temp_dir('faostat') as folder:
with Download() as downloader:
indicatortypes = get_indicatortypesdata(filelist_url, downloader)
countriesdata = get_countriesdata(country_group_url, downloader)
logger.info('Number of indicator types to upload: %d' %
len(dataset_codes))
for dataset_code in dataset_codes:
datasets, showcases = generate_datasets_and_showcases(
downloader, folder, dataset_codes[dataset_code],
indicatortypes[dataset_code], countriesdata,
showcase_base_url)
logger.info('Number of datasets to upload: %d' % len(datasets))
for i, dataset in enumerate(datasets):
logger.info('Creating dataset: %s' % dataset['title'])
dataset.preview_off()
dataset.create_in_hdx()
showcase = showcases[i]
showcase.create_in_hdx()
showcase.add_dataset(dataset)
def test_user_agent(self, user_agent_config_yaml, user_agent_config2_yaml, user_agent_config3_yaml,
empty_yaml, user_agent_config_wrong_yaml, project_config_yaml):
Configuration._create(user_agent_config_yaml=user_agent_config_yaml, hdx_site='prod', hdx_key='TEST_HDX_KEY',
hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
version = get_api_version()
assert Configuration.read().remoteckan().user_agent == 'lala:HDXPythonLibrary/%s-myua' % version
Configuration._create(user_agent_config_yaml=user_agent_config2_yaml, hdx_site='prod', hdx_key='TEST_HDX_KEY',
hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
assert Configuration.read().remoteckan().user_agent == 'HDXPythonLibrary/%s-myuseragent' % version
Configuration._create(user_agent_config_yaml=user_agent_config3_yaml, user_agent_lookup='lookup', hdx_site='prod',
hdx_key='TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
assert Configuration.read().remoteckan().user_agent == 'HDXPythonLibrary/%s-mylookupagent' % version
Configuration._create(user_agent_config_yaml=user_agent_config3_yaml, user_agent_lookup='lookup2', hdx_site='prod',
hdx_key='TEST_HDX_KEY', hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
assert Configuration.read().remoteckan().user_agent == 'HDXPythonLibrary/%s-mylookupagent2' % version
Configuration._create(user_agent='my_ua', preprefix='papa', hdx_site='prod', hdx_key='TEST_HDX_KEY',
hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
assert Configuration.read().remoteckan().user_agent == 'papa:HDXPythonLibrary/%s-my_ua' % version
with pytest.raises(UserAgentError):
Configuration._create(user_agent_config_yaml=user_agent_config3_yaml, user_agent_lookup='fail',
hdx_site='prod', hdx_key='TEST_HDX_KEY',
hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
with pytest.raises(LoadError):
Configuration._create(user_agent_config_yaml=empty_yaml, hdx_site='prod', hdx_key='TEST_HDX_KEY',
hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
with pytest.raises(UserAgentError):
Configuration._create(user_agent_config_yaml=user_agent_config_wrong_yaml, hdx_site='prod', hdx_key='TEST_HDX_KEY',
hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
with pytest.raises(UserAgentError):
Configuration._create(hdx_site='prod', hdx_key='TEST_HDX_KEY',
hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
def test_set_hdx_key_value(self, empty_hdx_key_file, project_config_yaml):
with pytest.raises(LoadError):
Configuration.load_api_key(empty_hdx_key_file)
Configuration._create(hdx_site='prod',
hdx_key='TEST_HDX_KEY',
hdx_config_dict={},
project_config_yaml=project_config_yaml)
configuration = Configuration.read()
assert configuration.get_api_key() == 'TEST_HDX_KEY'
configuration.set_api_key('NEW API KEY')
assert configuration.get_api_key() == 'NEW API KEY'
Configuration._create(hdx_site='prod',
hdx_read_only=True,
hdx_config_dict={},
project_config_yaml=project_config_yaml)
assert Configuration.read().get_api_key() is None
configuration = Configuration.read()
configuration.set_api_key('TEST API KEY')
assert configuration.get_api_key() is None
configuration.set_read_only(False)
assert configuration.get_api_key() == 'TEST API KEY'
configuration.set_read_only(True)
assert configuration.get_api_key() is None
configuration.set_api_key('NEW API KEY')
configuration.set_read_only(False)
assert configuration.get_api_key() == 'NEW API KEY'
def test_remoteckan_validlocations(self, project_config_yaml):
Configuration._create(hdx_site='prod',
hdx_key='TEST_HDX_KEY',
hdx_config_dict={},
project_config_yaml=project_config_yaml)
remoteckan = ckanapi.RemoteCKAN('http://lalala',
apikey='12345',
user_agent='HDXPythonLibrary/1.0')
Configuration.read().setup_remoteckan(remoteckan)
assert Configuration.read().remoteckan() == remoteckan
remoteckan = ckanapi.RemoteCKAN('http://hahaha',
apikey='54321',
user_agent='HDXPythonLibrary/0.5')
Configuration._create(remoteckan=remoteckan,
hdx_site='prod',
hdx_key='TEST_HDX_KEY',
hdx_config_dict={},
project_config_yaml=project_config_yaml)
assert Configuration.read().remoteckan() == remoteckan
Configuration.read()._remoteckan = None
with pytest.raises(ConfigurationError):
Configuration.read().remoteckan()
Configuration.delete()
with pytest.raises(ConfigurationError):
Configuration.read().remoteckan()
def main():
"""Generate dataset and create it in HDX"""
with Download() as downloader:
indicators = Configuration.read()['indicators']
tags = Configuration.read()['tags']
folder = get_temp_dir('IDMC')
datasets, showcase, headersdata, countriesdata = generate_indicator_datasets_and_showcase(
downloader, folder, indicators, tags)
showcase_not_added = True
countries = [{'iso3': x} for x in sorted(countriesdata)]
logger.info('Number of indicator datasets to upload: %d' %
len(indicators))
logger.info('Number of country datasets to upload: %d' %
len(countries))
for i, info, nextdict in multiple_progress_storing_tempdir(
'IDMC', [indicators, countries], ['name', 'iso3']):
folder = info['folder']
batch = info['batch']
if i == 0:
if showcase_not_added:
showcase.create_in_hdx()
showcase_not_added = False
dataset = datasets[nextdict['name']]
dataset.update_from_yaml()
dataset.generate_resource_view(
join('config', nextdict['resourceview']))
dataset.create_in_hdx(remove_additional_resources=True,
hxl_update=False,
updated_by_script='HDX Scraper: IDMC',
batch=batch)
showcase.add_dataset(dataset)
else:
countryiso = nextdict['iso3']
countrydata = countriesdata[countryiso]
dataset, showcase, bites_disabled = \
generate_country_dataset_and_showcase(downloader, folder, headersdata, countryiso, countrydata, datasets, tags)
if dataset:
dataset.update_from_yaml()
dataset.generate_resource_view(
bites_disabled=bites_disabled)
dataset.create_in_hdx(
remove_additional_resources=True,
hxl_update=False,
updated_by_script='HDX Scraper: IDMC',
batch=batch)
resources = dataset.get_resources()
resource_ids = [
x['id'] for x in sorted(resources,
key=lambda x: len(x['name']),
reverse=True)
]
dataset.reorder_resources(resource_ids, hxl_update=False)
def my_testfnkw(**kwargs):
fn = kwargs.get('fn')
if fn == 'site':
testresult.actual_result = Configuration.read().get_hdx_site_url()
elif fn == 'api':
testresult.actual_result = Configuration.read().get_api_key()
elif fn == 'agent':
testresult.actual_result = Configuration.read().user_agent
elif fn == 'exc':
testresult.actual_result = Configuration.read().get_hdx_site_url()
raise ValueError('Some failure!')
def test_get_all_users(self, configuration, post_list, mocksmtp):
users = User.get_all_users()
assert len(users) == 2
config = Configuration.read()
config.setup_emailer(email_config_dict=TestUser.email_config_dict)
User.email_users(users, TestUser.subject, TestUser.body, sender=TestUser.sender, mail_options=TestUser.mail_options,
rcpt_options=TestUser.rcpt_options)
email = config.emailer()
assert email.server.type == 'smtpssl'
assert email.server.initargs == TestUser.smtp_initargs
assert email.server.username == TestUser.username
assert email.server.password == TestUser.password
assert email.server.sender == TestUser.sender
assert email.server.recipients == ['*****@*****.**', '*****@*****.**']
assert email.server.msg == '''Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: hello
From: [email protected]
To: [email protected], [email protected]
hello there'''
assert email.server.send_args == {'mail_options': ['a', 'b'], 'rcpt_options': [1, 2]}
with pytest.raises(ValueError):
User.email_users(list(), TestUser.subject, TestUser.body, sender=TestUser.sender,
mail_options=TestUser.mail_options, rcpt_options=TestUser.rcpt_options)
def test_hdx_configuration_yaml(self, hdx_config_yaml, hdx_base_config_yaml, project_config_yaml):
Configuration._create(user_agent='test', hdx_config_yaml=hdx_config_yaml,
hdx_base_config_yaml=hdx_base_config_yaml, project_config_yaml=project_config_yaml)
expected_configuration = {
'hdx_site': 'prod',
'hdx_read_only': False,
'hdx_key': '12345',
'hdx_prod_site': {
'url': 'https://data.humdata.org',
},
'hdx_test_site': {
'url': 'https://test-data.humdata.org',
'username': '******',
'password': '******'
},
'dataset': {'required_fields': [
'name',
'title',
'dataset_date',
]},
'resource': {'required_fields': ['package_id', 'name', 'description']},
'showcase': {'required_fields': ['name', 'title']},
'approved_tags_vocabulary': 'Topics',
'tags_list_url': 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/Accepted_Tags.csv',
'tags_mapping_url': 'https://raw.githubusercontent.com/OCHA-DAP/hdx-python-api/master/tests/fixtures/Tag_Mapping.csv',
}
assert Configuration.read() == expected_configuration
def test_read_from_hdx(self, configuration, read, mocksmtp):
user = User.read_from_hdx('TEST1')
assert user['id'] == '9f3e9973-7dbe-4c65-8820-f48578e3ffea'
assert user['name'] == 'MyUser1'
user = User.read_from_hdx('TEST2')
assert user is None
user = User.read_from_hdx('TEST3')
assert user is None
config = Configuration.read()
config.setup_emailer(email_config_dict=TestUser.email_config_dict)
user = User.read_from_hdx('TEST1')
user.email(TestUser.subject, TestUser.body, sender=TestUser.sender, mail_options=TestUser.mail_options,
rcpt_options=TestUser.rcpt_options)
email = config.emailer()
assert email.server.type == 'smtpssl'
assert email.server.initargs == TestUser.smtp_initargs
assert email.server.username == TestUser.username
assert email.server.password == TestUser.password
assert email.server.sender == TestUser.sender
assert email.server.recipients == ['*****@*****.**']
assert email.server.msg == '''Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: hello
From: [email protected]
To: [email protected]
hello there'''
assert email.server.send_args == {'mail_options': ['a', 'b'], 'rcpt_options': [1, 2]}
def post_delete(self):
class MockSession(object):
@staticmethod
def post(url, data, headers, files, allow_redirects, auth=None):
decodedata = data.decode('utf-8')
datadict = json.loads(decodedata)
if url.endswith('show') or 'list' in url:
return mockshow(url, datadict)
if 'delete' not in url:
return MockResponse(
404,
'{"success": false, "error": {"message": "TEST ERROR: Not delete", "__type": "TEST ERROR: Not Delete Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_delete"}'
)
if datadict['id'] == '05e392bf-04e0-4ca6-848c-4e87bba10746':
return MockResponse(
200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_delete"}'
% decodedata)
return MockResponse(
404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_delete"}'
)
Configuration.read().remoteckan().session = MockSession()
def main():
"""Generate dataset and create it in HDX"""
configuration = Configuration.read()
indicators = configuration["indicators"]
json_url = configuration["json_url"]
with Download() as downloader:
indicators_metadata = get_indicators_metadata(json_url, downloader,
indicators)
countriesdata, countries = get_countriesdata(json_url, downloader,
indicators)
logger.info(f"Number of countries to upload: {len(countries)}")
for info, country in progress_storing_tempdir("WorldPop", countries,
"iso3"):
countryiso = country["iso3"]
datasets, showcases = generate_datasets_and_showcases(
downloader, countryiso, indicators_metadata,
countriesdata[countryiso])
for dataset in datasets:
dataset.update_from_yaml()
dataset.create_in_hdx(
remove_additional_resources=True,
hxl_update=False,
updated_by_script="HDX Scraper: WorldPop",
batch=info["batch"],
)
for showcase in showcases.get(dataset["name"], list()):
showcase.create_in_hdx()
showcase.add_dataset(dataset)
def main(excel_path, gsheet_auth, updatesheets, updatetabs, scrapers,
basic_auths, nojson, **ignore):
logger.info('##### hdx-scraper-covid-viz version %.1f ####' % VERSION)
configuration = Configuration.read()
with Download(rate_limit={'calls': 1, 'period': 0.1}) as downloader:
if scrapers:
logger.info('Updating only scrapers: %s' % scrapers)
tabs = configuration['tabs']
if updatetabs is None:
updatetabs = list(tabs.keys())
logger.info('Updating all tabs')
else:
logger.info('Updating only these tabs: %s' % updatetabs)
noout = nooutput(updatetabs)
if excel_path:
excelout = exceloutput(excel_path, tabs, updatetabs)
else:
excelout = noout
if gsheet_auth:
gsheets = googlesheets(configuration, gsheet_auth, updatesheets,
tabs, updatetabs)
else:
gsheets = noout
if nojson:
jsonout = noout
else:
jsonout = jsonoutput(configuration, updatetabs)
outputs = {'gsheets': gsheets, 'excel': excelout, 'json': jsonout}
admininfo = AdminInfo.setup(downloader)
get_indicators(configuration, downloader, admininfo, outputs,
updatetabs, scrapers, basic_auths)
excelout.save()
jsonout.add_additional_json(downloader)
jsonout.save(hrp_iso3s=admininfo.hrp_iso3s)
def main():
"""Generate dataset and create it in HDX"""
configuration = Configuration.read()
resources = configuration["resources"]
fields = configuration["fields"]
source_directory = configuration["source_directory"]
download_url = Path(source_directory).resolve().as_uri()
with Download() as downloader:
countries, headers, countriesdata = get_countriesdata(
download_url, resources, downloader)
logger.info("Number of countries: %d" % len(countriesdata))
for info, country in progress_storing_tempdir("UNHCR_population",
countries, "iso3"):
folder = info["folder"]
dataset, showcase = generate_dataset_and_showcase(
folder, country, countriesdata[country["iso3"]], headers,
resources, fields)
if dataset:
dataset.update_from_yaml()
dataset["notes"] = dataset["notes"].replace(
"\n", " \n") # ensure markdown has line breaks
dataset.generate_resource_view(1)
dataset.create_in_hdx(
remove_additional_resources=True,
hxl_update=False,
updated_by_script="HDX Scraper: UNHCR population",
batch=info["batch"],
)
showcase.create_in_hdx()
showcase.add_dataset(dataset)
def post_delete(self):
class MockSession(object):
@staticmethod
def post(url, data, headers, files, allow_redirects, auth=None):
decodedata = data.decode('utf-8')
datadict = json.loads(decodedata)
if 'show' in url:
return resource_view_mockshow(url, datadict)
if 'delete' not in url:
return MockResponse(
404,
'{"success": false, "error": {"message": "TEST ERROR: Not delete", "__type": "TEST ERROR: Not Delete Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_view_delete"}'
)
if datadict['id'] == 'c06b5a0d-1d41-4a74-a196-41c251c76023':
return MockResponse(
200,
'{"success": true, "result": null, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_view_delete"}'
)
return MockResponse(
404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_view_delete"}'
)
Configuration.read().remoteckan().session = MockSession()
def main():
"""Generate dataset and create it in HDX"""
with Download() as downloader:
config = Configuration.read()
project_config = {
key: value
for key, value in config.items() if key.startswith("CV")
}
qc_indicators = config.get("qc_indicators", {})
countries, countriesdata, headers = get_all_countriesdata(
project_config, downloader)
logger.info("Number of datasets to upload: %d" % len(countries))
for info, country in progress_storing_tempdir("UNICEFSAM", countries,
"iso3"):
dataset, showcase, bites_disabled = generate_dataset_and_showcase(
info["folder"], country, countriesdata[country["iso3"]],
headers, project_config, qc_indicators)
if dataset:
dataset.update_from_yaml()
dataset.generate_resource_view(1,
bites_disabled=bites_disabled,
indicators=qc_indicators)
dataset.create_in_hdx(
remove_additional_resources=True,
hxl_update=False,
updated_by_script="HDX Scraper: UNICEF Sam",
batch=info["batch"],
)
showcase.create_in_hdx()
showcase.add_dataset(dataset)
def main():
"""Generate dataset and create it in HDX"""
configuration = Configuration.read()
base_url = configuration['base_url']
with Download(extra_params_yaml=join(expanduser('~'), '.extraparams.yml'),
extra_params_lookup=lookup) as downloader:
downloader.session.mount(
'http://',
HTTPAdapter(max_retries=1, pool_connections=100, pool_maxsize=100))
downloader.session.mount(
'https://',
HTTPAdapter(max_retries=1, pool_connections=100, pool_maxsize=100))
countries = get_countries(base_url, downloader)
logger.info('Number of countries: %d' % len(countries))
for folder, country in progress_storing_tempdir(
'DHS', countries, 'iso3'):
tags = get_tags(base_url, downloader, country['dhscode'])
dataset, subdataset, showcase, bites_disabled = \
generate_datasets_and_showcase(configuration, base_url, downloader, folder, country, tags)
if dataset:
createdataset(dataset)
resource_view = generate_resource_view(
dataset, bites_disabled=bites_disabled['national'])
resource_view.create_in_hdx()
showcase.create_in_hdx()
showcase.add_dataset(dataset)
if subdataset:
createdataset(subdataset)
showcase.add_dataset(subdataset)
subdataset.generate_resource_view(
bites_disabled=bites_disabled['subnational'])
def configuration(self):
Configuration._create(hdx_read_only=True,
user_agent='test',
project_config_yaml=join(
'tests', 'config',
'project_configuration.yml'))
Locations.set_validlocations([{'name': 'afg', 'title': 'Afghanistan'}])
Country.countriesdata(use_live=False)
Vocabulary._tags_dict = True
Vocabulary._approved_vocabulary = {
'tags': [{
'name': 'hxl'
}, {
'name': 'indicators'
}, {
'name': 'health'
}, {
'name': 'education'
}, {
'name': 'socioeconomic'
}, {
'name': 'demographics'
}, {
'name': 'development'
}],
'id':
'4e61d464-4943-4e97-973a-84673c1aaa87',
'name':
'approved'
}
return Configuration.read()
def post_delete(self):
class MockSession(object):
@staticmethod
def post(url, data, headers, files, allow_redirects, auth=None):
decodedata = data.decode('utf-8')
datadict = json.loads(decodedata)
if 'show' in url:
return organization_mockshow(url, datadict)
if 'delete' not in url:
return MockResponse(
404,
'{"success": false, "error": {"message": "TEST ERROR: Not delete", "__type": "TEST ERROR: Not Delete Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_delete"}'
)
if datadict['id'] == 'b67e6c74-c185-4f43-b561-0e114a736f19':
return MockResponse(
200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_delete"}'
% decodedata)
return MockResponse(
404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_delete"}'
)
Configuration.read().remoteckan().session = MockSession()
def main(save, use_saved, **ignore):
"""Generate dataset and create it in HDX"""
with Download(extra_params_yaml=join(expanduser('~'), '.extraparams.yml'),
extra_params_lookup=lookup) as token_downloader:
configuration = Configuration.read()
with Download() as downloader:
folder = temp_dir(lookup)
retriever = Retrieve(downloader, folder, 'saved_data', folder,
save, use_saved)
wfp = WFPFood(configuration, token_downloader, retriever)
countries = wfp.get_countries()
logger.info('Number of country datasets to upload: %d' %
len(countries))
wfp.build_mappings()
for info, country in progress_storing_tempdir(
lookup, countries, 'iso3'):
dataset, showcase, qc_indicators = wfp.generate_dataset_and_showcase(
country['iso3'], info['folder'])
if dataset:
dataset.update_from_yaml()
dataset['notes'] = dataset[
'notes'] % 'Food Prices data for %s. Food prices data comes from the World Food Programme and covers' % country[
'name']
dataset.generate_resource_view(-1,
indicators=qc_indicators)
dataset.create_in_hdx(
remove_additional_resources=True,
hxl_update=False,
updated_by_script='HDX Scraper: WFP Food Prices',
batch=info['batch'])
showcase.create_in_hdx()
showcase.add_dataset(dataset)
def post_create(self):
class MockSession(object):
@staticmethod
def post(url, data, headers, files, allow_redirects, auth=None):
datadict = json.loads(data.decode('utf-8'))
if 'show' in url:
return organization_mockshow(url, datadict)
if 'create' not in url:
return MockResponse(404,
'{"success": false, "error": {"message": "TEST ERROR: Not create", "__type": "TEST ERROR: Not Create Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_create"}')
result = json.dumps(resultdict)
if datadict['name'] == 'MyOrganization1':
return MockResponse(200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_create"}' % result)
if datadict['name'] == 'MyOrganization2':
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_create"}')
if datadict['name'] == 'MyOrganization3':
return MockResponse(200,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_create"}')
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_create"}')
Configuration.read().remoteckan().session = MockSession()
def post_update(self):
class MockSession(object):
@staticmethod
def post(url, data, headers, files, allow_redirects, auth):
datadict = json.loads(data.decode('utf-8'))
if 'show' in url:
return user_mockshow(url, datadict)
if 'update' not in url:
return MockResponse(404,
'{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=user_update"}')
resultdictcopy = copy.deepcopy(resultdict)
merge_two_dictionaries(resultdictcopy, datadict)
result = json.dumps(resultdictcopy)
if datadict['name'] == 'MyUser1':
return MockResponse(200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=user_update"}' % result)
if datadict['name'] == 'MyUser2':
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=user_update"}')
if datadict['name'] == 'MyUser3':
return MockResponse(200,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=user_update"}')
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=user_update"}')
Configuration.read().remoteckan().session = MockSession()
def main():
"""Generate dataset and create it in HDX"""
configuration = Configuration.read()
download_url = configuration["download_url"]
with Download() as downloader:
countries, headers, countriesdata = get_countriesdata(download_url, downloader)
logger.info(f"Number of countries: {len(countriesdata)}")
for info, country in progress_storing_tempdir("UCDP", countries, "iso3"):
folder = info["folder"]
dataset, showcase = generate_dataset_and_showcase(
folder, country, countriesdata[country["iso3"]], headers
)
if dataset:
dataset.update_from_yaml()
dataset["notes"] = dataset["notes"].replace(
"\n", " \n"
) # ensure markdown has line breaks
dataset.generate_resource_view(1)
dataset.create_in_hdx(
remove_additional_resources=True,
hxl_update=False,
updated_by_script="HDX Scraper: UCDP",
batch=info["batch"],
)
showcase.create_in_hdx()
showcase.add_dataset(dataset)
def __init__(self, downloader):
configuration = Configuration.read()
admin_info = configuration['admin_info']
self.adm1_name_replacements = configuration['adm1_name_replacements']
self.adm1_fuzzy_ignore = configuration['adm1_fuzzy_ignore']
self.adm_mappings = configuration.get('adm_mappings', [dict(), dict()])
iso3s_no_pcodes = set()
countryiso3s = set()
for row in admin_info:
countryiso3 = row['alpha_3']
countryiso3s.add(countryiso3)
pcode = row.get('ADM1_PCODE')
if not pcode:
iso3s_no_pcodes.add(countryiso3)
continue
self.pcodes.append(pcode)
self.pcode_lengths[countryiso3] = len(pcode)
adm1_name = row['ADM1_REF']
self.pcode_to_name[pcode] = adm1_name
name_to_pcode = self.name_to_pcode.get(countryiso3, dict())
name_to_pcode[unidecode(adm1_name).lower()] = pcode
self.name_to_pcode[countryiso3] = name_to_pcode
self.pcode_to_iso3[pcode] = countryiso3
self.countryiso3s = sorted(list(countryiso3s))
self.adms = [self.countryiso3s, self.pcodes]
self.iso3s_no_pcodes = sorted(list(iso3s_no_pcodes))
self.hrp_iso3s = sorted(list(self.name_to_pcode.keys()))
self.regions, self.iso3_to_region, self.iso3_to_region_and_hrp = self.read_regional(configuration, self.countryiso3s, self.hrp_iso3s, downloader)
self.init_matches_errors()
def getData(countryISO2):
url = Configuration.read()['base_url'] + "&code=" + countryISO2
response = requests.get(url)
headers = [
'category', 'indicator', 'indicator_friendly', 'type_data', 'latitude',
'longitude', 'region_id', 'country_id', 'name', 'year', 'value'
]
hxl = [
'#meta+category', '#indicator+name', '#indicator+description',
'#indicator+type', '#geo+lat', '#geo+lon', '#region+code',
'#country+code+v_iso2', '#country+name', '#date+year',
'#indicator+value'
]
if response.status_code != 200:
print(":(")
else:
try:
with open('data/%s' % countryISO2 + '.csv', 'w') as f:
f.write(response.text)
except Exception as e:
raise e
with open('data/indicator_data_%s' % countryISO2 + '.csv',
'w') as hxl_tags:
writer = csv.writer(hxl_tags, delimiter=',')
writer.writerow(headers)
writer.writerow(hxl)
data = open('data/%s' % countryISO2 + '.csv', 'r')
with data:
reader = csv.reader(data)
next(reader)
for r in reader:
writer.writerow(r)
def main():
"""Generate dataset and create it in HDX"""
configuration = Configuration.read()
hdro_url = configuration['hdro_url']
qc_indicators = configuration['qc_indicators']
with Download() as downloader:
countriesdata = get_countriesdata(hdro_url, downloader)
countries = [{
'iso3': countryiso
} for countryiso in sorted(countriesdata.keys())]
logger.info('Number of countries to upload: %d' % len(countries))
for info, country in progress_storing_tempdir('HDRO', countries,
'iso3'):
countryiso = country['iso3']
countrydata = countriesdata[countryiso]
dataset, showcase, bites_disabled = generate_dataset_and_showcase(
info['folder'], countryiso, countrydata, qc_indicators)
if dataset:
dataset.update_from_yaml()
dataset.generate_resource_view(-1,
bites_disabled=bites_disabled,
indicators=qc_indicators)
dataset.create_in_hdx(remove_additional_resources=True,
hxl_update=False,
updated_by_script='HDX Scraper: HDRO',
batch=info['batch'])
showcase.create_in_hdx()
showcase.add_dataset(dataset)
def test_hdx_configuration_yaml(self, hdx_key_file, hdx_config_yaml,
project_config_yaml):
Configuration._create(hdx_key_file=hdx_key_file,
hdx_config_yaml=hdx_config_yaml,
project_config_yaml=project_config_yaml)
expected_configuration = {
'api_key': '12345',
'param_1': 'ABC',
'hdx_prod_site': {
'url': 'https://data.humdata.org/',
'username': None,
'password': None
},
'hdx_test_site': {
'url': 'https://test-data.humdata.org/',
'username': '******',
'password': '******'
},
'dataset': {
'required_fields': [
'name',
'title',
'dataset_date',
]
},
'resource': {
'required_fields': ['package_id', 'name', 'description']
},
'showcase': {
'required_fields': ['name', 'title']
},
}
assert Configuration.read() == expected_configuration
def post_update(self):
class MockSession(object):
@staticmethod
def post(url, data, headers, files, allow_redirects, auth=None):
datadict = json.loads(data.decode('utf-8'))
if url.endswith('show') or 'list' in url:
return mockshow(url, datadict)
if 'update' not in url:
return MockResponse(404,
'{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}')
resultdictcopy = copy.deepcopy(showcase_resultdict)
merge_two_dictionaries(resultdictcopy, datadict)
result = json.dumps(resultdictcopy)
if datadict['title'] == 'MyShowcase1':
return MockResponse(200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}' % result)
if datadict['title'] == 'MyShowcase2':
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}')
if datadict['title'] == 'MyShowcase3':
return MockResponse(200,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}')
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_update"}')
Configuration.read().remoteckan().session = MockSession()
def test_create_set_configuration(self, project_config_yaml):
Configuration._create(user_agent='test', hdx_site='prod', hdx_key='TEST_HDX_KEY',
hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
with pytest.raises(ConfigurationError):
Configuration.create(user_agent='test', hdx_site='prod', hdx_key='TEST_HDX_KEY',
hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
configuration = Configuration(user_agent='test', hdx_site='test', hdx_key='OTHER_TEST_HDX_KEY',
hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
Configuration.setup(configuration)
assert Configuration.read() == configuration
Configuration.delete()
with pytest.raises(ConfigurationError):
Configuration.read()
Configuration.create(user_agent='test', hdx_site='prod', hdx_key='TEST_HDX_KEY',
hdx_base_config_dict={}, project_config_yaml=project_config_yaml)
assert Configuration.read().get_api_key() == 'TEST_HDX_KEY'
def main(excel_path, gsheet_auth, updatesheets, updatetabs, scrapers, basic_auths, other_auths, nojson,
countries_override, save, use_saved, **ignore):
logger.info('##### hdx-scraper-covid-viz version %.1f ####' % VERSION)
configuration = Configuration.read()
with temp_dir() as temp_folder:
with Download(rate_limit={'calls': 1, 'period': 0.1}) as downloader:
retriever = Retrieve(downloader, temp_folder, 'saved_data', temp_folder, save, use_saved)
if scrapers:
logger.info('Updating only scrapers: %s' % scrapers)
tabs = configuration['tabs']
if updatetabs is None:
updatetabs = list(tabs.keys())
logger.info('Updating all tabs')
else:
logger.info('Updating only these tabs: %s' % updatetabs)
noout = NoOutput(updatetabs)
if excel_path:
excelout = ExcelOutput(excel_path, tabs, updatetabs)
else:
excelout = noout
if gsheet_auth:
gsheets = GoogleSheets(configuration, gsheet_auth, updatesheets, tabs, updatetabs)
else:
gsheets = noout
if nojson:
jsonout = noout
else:
jsonout = JsonOutput(configuration, updatetabs)
outputs = {'gsheets': gsheets, 'excel': excelout, 'json': jsonout}
today = datetime.now()
countries_to_save = get_indicators(configuration, today, retriever, outputs, updatetabs, scrapers,
basic_auths, other_auths, countries_override)
jsonout.add_additional_json(downloader, today=today)
jsonout.save(countries_to_save=countries_to_save)
excelout.save()
def main():
"""Generate dataset and create it in HDX"""
with Download() as downloader:
configuration = Configuration.read()
countries_path = join('config', configuration['countries_filename'])
indicators_url = configuration['indicators_url']
mvam_url = configuration['mvam_url']
showcase_url = configuration['showcase_url']
countries = get_countries(countries_path, downloader)
variables = get_mvamvariables(indicators_url, downloader)
logger.info('Number of datasets to upload: %d' % len(countries))
for info, country in progress_storing_tempdir('WFPFoodSecurity',
countries, 'iso3'):
dataset, showcase, bites_disabled = \
generate_dataset_and_showcase(mvam_url, showcase_url, downloader, info['folder'],
country, variables)
if dataset:
dataset.update_from_yaml()
dataset.generate_resource_view(bites_disabled=bites_disabled)
dataset.create_in_hdx(
remove_additional_resources=True,
hxl_update=False,
updated_by_script='HDX Scraper: WFP Food Security',
batch=info['batch'])
showcase.create_in_hdx()
showcase.add_dataset(dataset)
def post_delete(self):
class MockSession(object):
@staticmethod
def post(url, data, headers, files, allow_redirects, auth):
decodedata = data.decode('utf-8')
datadict = json.loads(decodedata)
if 'show' in url:
return mockshow(url, datadict)
if 'delete' not in url:
return MockResponse(
404,
'{"success": false, "error": {"message": "TEST ERROR: Not delete", "__type": "TEST ERROR: Not Delete Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_delete"}'
)
if datadict['id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5':
return MockResponse(
200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_delete"}'
% decodedata)
return MockResponse(
404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_delete"}'
)
Configuration.read().remoteckan().session = MockSession()
def post_resourceview(self):
class MockSession(object):
@staticmethod
def post(url, data, headers, files, allow_redirects, auth=None):
decodedata = data.decode('utf-8')
return mockresourceview(url, decodedata)
Configuration.read().remoteckan().session = MockSession()
def read(self):
class MockSession(object):
@staticmethod
def post(url, data, headers, files, allow_redirects, auth=None):
datadict = json.loads(data.decode('utf-8'))
return organization_mockshow(url, datadict)
Configuration.read().remoteckan().session = MockSession()
def read(self):
class MockSession(object):
@staticmethod
def post(url, data, headers, files, allow_redirects, auth=None):
datadict = json.loads(data.decode('utf-8'))
return organization_mockshow(url, datadict)
Configuration.read().remoteckan().session = MockSession()
def main():
"""Generate dataset and create it in HDX"""
acled_url = Configuration.read()['acled_url']
countries_url = Configuration.read()['countries_url']
hxlproxy_url = Configuration.read()['hxlproxy_url']
with Download() as downloader:
countriesdata = get_countriesdata(countries_url, downloader)
logger.info('Number of datasets to upload: %d' % len(countriesdata))
for countrydata in sorted(countriesdata, key=lambda x: x['iso3']):
dataset, showcase = generate_dataset_and_showcase(acled_url, hxlproxy_url, downloader, countrydata)
if dataset:
dataset.update_from_yaml()
dataset.create_in_hdx()
resource_view = generate_resource_view(dataset)
resource_view.create_in_hdx()
showcase.create_in_hdx()
showcase.add_dataset(dataset)
sleep(1)
def test_generate_dataset_and_showcase(self, configuration, downloader):
hxlproxy_url = Configuration.read()['hxlproxy_url']
dataset, showcase = generate_dataset_and_showcase('http://lala?', hxlproxy_url, downloader, TestAcledAfrica.countrydata)
assert dataset == TestAcledAfrica.dataset
resources = dataset.get_resources()
assert resources == [TestAcledAfrica.resource]
assert showcase == {'name': 'acled-data-for-cameroon-showcase', 'notes': 'Conflict Data Dashboard for Cameroon',
'url': 'https://www.acleddata.com/dashboard/#120',
'tags': [{'name': 'HXL'}, {'name': 'conflicts'}, {'name': 'political violence'}, {'name': 'protests'}],
'title': 'Dashboard for Cameroon', 'image_url': 'https://www.acleddata.com/wp-content/uploads/2018/01/dash.png'}
dataset, showcase = generate_dataset_and_showcase('http://lala?', hxlproxy_url, downloader, {'m49': 4, 'iso3': 'AFG', 'countryname': 'Afghanistan'})
assert dataset is None
def tagscleanupdicts(configuration=None, url=None, keycolumn=5, failchained=True):
# type: (Optional[Configuration], Optional[str], int, bool) -> Tuple[Dict,List]
"""
Get tags cleanup dictionaries
Args:
configuration (Optional[Configuration]): HDX configuration. Defaults to global configuration.
url (Optional[str]): Url of tags cleanup spreadsheet. Defaults to None (internal configuration parameter).
keycolumn (int): Column number of tag column in spreadsheet. Defaults to 5.
failchained (bool): Fail if chained rules found. Defaults to True.
Returns:
Tuple[Dict,List]: Returns (Tags dictionary, Wildcard tags list)
"""
if not Tags._tags_dict:
if configuration is None:
configuration = Configuration.read()
with Download(full_agent=configuration.get_user_agent()) as downloader:
if url is None:
url = configuration['tags_cleanup_url']
Tags._tags_dict = downloader.download_tabular_rows_as_dicts(url, keycolumn=keycolumn)
keys = Tags._tags_dict.keys()
chainerror = False
for i, tag in enumerate(keys):
whattodo = Tags._tags_dict[tag]
action = whattodo[u'action']
final_tags = whattodo[u'final tags (semicolon separated)']
for final_tag in final_tags.split(';'):
if final_tag in keys:
index = list(keys).index(final_tag)
if index != i:
whattodo2 = Tags._tags_dict[final_tag]
action2 = whattodo2[u'action']
if action2 != 'OK' and action2 != 'Other':
final_tags2 = whattodo2[u'final tags (semicolon separated)']
if final_tag not in final_tags2.split(';'):
chainerror = True
if failchained:
logger.error('Chained rules: %s (%s -> %s) | %s (%s -> %s)' %
(action, tag, final_tags, action2, final_tag, final_tags2))
if failchained and chainerror:
raise ChainRuleError('Chained rules for tags detected!')
Tags._wildcard_tags = list()
for tag in Tags._tags_dict:
if '*' in tag:
Tags._wildcard_tags.append(tag)
return Tags._tags_dict, Tags._wildcard_tags
def read(self):
class MockSession(object):
@staticmethod
def post(url, data, headers, files, allow_redirects, auth=None):
datadict = json.loads(data.decode('utf-8'))
if 'association_delete' in url:
TestShowcase.association = 'delete'
return MockResponse(200,
'{"success": true, "result": null, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_package_association_delete"}')
elif 'association_create' in url:
TestShowcase.association = 'create'
result = json.dumps(datadict)
return MockResponse(200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_package_association_create"}' % result)
return mockshow(url, datadict)
Configuration.read().remoteckan().session = MockSession()
def test_read_from_hdx(self, configuration, read, mocksmtp):
user = User.read_from_hdx('9f3e9973-7dbe-4c65-8820-f48578e3ffea')
assert user['id'] == '9f3e9973-7dbe-4c65-8820-f48578e3ffea'
assert user['name'] == 'MyUser1'
user = User.read_from_hdx('TEST2')
assert user is None
user = User.read_from_hdx('TEST3')
assert user is None
config = Configuration.read()
config.setup_emailer(email_config_dict=TestUser.email_config_dict)
user = User.read_from_hdx('9f3e9973-7dbe-4c65-8820-f48578e3ffea')
user.email(TestUser.subject, TestUser.text_body, html_body=TestUser.html_body, sender=TestUser.sender,
mail_options=TestUser.mail_options, rcpt_options=TestUser.rcpt_options)
email = config.emailer()
assert email.server.type == 'smtpssl'
assert email.server.initargs == TestUser.smtp_initargs
assert email.server.username == TestUser.username
assert email.server.password == TestUser.password
assert email.server.sender == TestUser.sender
assert email.server.recipients == ['*****@*****.**']
assert 'Content-Type: multipart/alternative;' in email.server.msg
assert '''\
MIME-Version: 1.0
Subject: hello
From: [email protected]
To: [email protected]''' in email.server.msg
assert '''\
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
hello there''' in email.server.msg
assert '''\
Content-Type: text/html; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
<html>
<head></head>
<body>
<p>Hi!<br>
</p>
</body>
</html>''' in email.server.msg
assert email.server.send_args == {'mail_options': ['a', 'b'], 'rcpt_options': [1, 2]}
def post_delete(self):
class MockSession(object):
@staticmethod
def post(url, data, headers, files, allow_redirects, auth=None):
decodedata = data.decode('utf-8')
datadict = json.loads(decodedata)
if 'show' in url:
return organization_mockshow(url, datadict)
if 'delete' not in url:
return MockResponse(404,
'{"success": false, "error": {"message": "TEST ERROR: Not delete", "__type": "TEST ERROR: Not Delete Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_delete"}')
if datadict['id'] == 'b67e6c74-c185-4f43-b561-0e114a736f19':
return MockResponse(200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_delete"}' % decodedata)
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_delete"}')
Configuration.read().remoteckan().session = MockSession()
def post_delete(self):
class MockSession(object):
@staticmethod
def post(url, data, headers, files, allow_redirects, auth=None):
decodedata = data.decode('utf-8')
datadict = json.loads(decodedata)
if url.endswith('show') or 'list' in url:
return mockshow(url, datadict)
if 'delete' not in url:
return MockResponse(404,
'{"success": false, "error": {"message": "TEST ERROR: Not delete", "__type": "TEST ERROR: Not Delete Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_delete"}')
if datadict['id'] == '05e392bf-04e0-4ca6-848c-4e87bba10746':
return MockResponse(200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_delete"}' % decodedata)
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=ckanext_showcase_delete"}')
Configuration.read().remoteckan().session = MockSession()
def test_get_all_users(self, configuration, post_list, mocksmtp):
users = User.get_all_users()
assert len(users) == 2
config = Configuration.read()
config.setup_emailer(email_config_dict=TestUser.email_config_dict)
User.email_users(users, TestUser.subject, TestUser.text_body, html_body=TestUser.html_body,
sender=TestUser.sender, mail_options=TestUser.mail_options, rcpt_options=TestUser.rcpt_options)
email = config.emailer()
assert email.server.type == 'smtpssl'
assert email.server.initargs == TestUser.smtp_initargs
assert email.server.username == TestUser.username
assert email.server.password == TestUser.password
assert email.server.sender == TestUser.sender
assert email.server.recipients == ['*****@*****.**', '*****@*****.**']
assert 'Content-Type: multipart/alternative;' in email.server.msg
assert '''\
MIME-Version: 1.0
Subject: hello
From: [email protected]
To: [email protected], [email protected]''' in email.server.msg
assert '''\
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
hello there''' in email.server.msg
assert '''\
Content-Type: text/html; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
<html>
<head></head>
<body>
<p>Hi!<br>
</p>
</body>
</html>''' in email.server.msg
assert email.server.send_args == {'mail_options': ['a', 'b'], 'rcpt_options': [1, 2]}
with pytest.raises(ValueError):
User.email_users(list(), TestUser.subject, TestUser.text_body, sender=TestUser.sender,
mail_options=TestUser.mail_options, rcpt_options=TestUser.rcpt_options)
def post_listorgs(self):
class MockSession(object):
@staticmethod
def post(url, data, headers, files, allow_redirects, auth=None):
decodedata = data.decode('utf-8')
datadict = json.loads(decodedata)
if 'user' in url:
if 'show' in url:
return user_mockshow(url, datadict)
elif 'list' in url:
return MockResponse(200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_list"}' % json.dumps(orglist))
elif 'organization' in url:
if 'show' in url:
result = json.dumps(orgdict)
if datadict['id'] == 'b67e6c74-c185-4f43-b561-0e114a736f19' or datadict['id'] == 'TEST1':
return MockResponse(200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=organization_show"}' % result)
Configuration.read().remoteckan().session = MockSession()
def post_update(self):
class MockSession(object):
@staticmethod
def post(url, data, headers, files, allow_redirects, auth=None):
if isinstance(data, dict):
datadict = {k.decode('utf8'): v.decode('utf8') for k, v in data.items()}
else:
datadict = json.loads(data.decode('utf-8'))
if 'show' in url:
return mockshow(url, datadict)
if 'resource_id' in datadict:
if datadict['resource_id'] == '74b74ae1-df0c-4716-829f-4f939a046811':
return MockResponse(200,
'{"success": true, "result": {"fields": [{"type": "text", "id": "code"}, {"type": "text", "id": "title"}, {"type": "float", "id": "value"}, {"type": "timestamp", "id": "latest_date"}, {"type": "text", "id": "source"}, {"type": "text", "id": "source_link"}, {"type": "text", "id": "notes"}, {"type": "text", "id": "explore"}, {"type": "text", "id": "units"}], "method": "insert", "primary_key": "code", "resource_id": "bfa6b55f-10b6-4ba2-8470-33bb9a5194a5"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_create"}')
if 'update' not in url:
return MockResponse(404,
'{"success": false, "error": {"message": "TEST ERROR: Not update", "__type": "TEST ERROR: Not Update Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
if datadict['name'] == 'MyResource1':
resultdictcopy = copy.deepcopy(resultdict)
merge_two_dictionaries(resultdictcopy, datadict)
if files is not None:
resultdictcopy['url_type'] = 'upload'
resultdictcopy['resource_type'] = 'file.upload'
filename = os.path.basename(files[0][1].name)
resultdictcopy[
'url'] = 'http://test-data.humdata.org/dataset/6f36a41c-f126-4b18-aaaf-6c2ddfbc5d4d/resource/de6549d8-268b-4dfe-adaf-a4ae5c8510d5/download/%s' % filename
result = json.dumps(resultdictcopy)
return MockResponse(200,
'{"success": true, "result": %s, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}' % result)
if datadict['name'] == 'MyResource2':
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
if datadict['name'] == 'MyResource3':
return MockResponse(200,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_update"}')
Configuration.read().remoteckan().session = MockSession()
def post_datastore(self):
class MockSession(object):
@staticmethod
def post(url, data, headers, files, allow_redirects, auth=None):
decodedata = data.decode('utf-8')
datadict = json.loads(decodedata)
if 'show' in url:
return mockshow(url, datadict)
if 'create' not in url and 'insert' not in url and 'upsert' not in url and 'delete' not in url and 'search' not in url:
return MockResponse(404,
'{"success": false, "error": {"message": "TEST ERROR: Not create or delete", "__type": "TEST ERROR: Not Create or Delete Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_action"}')
if 'delete' in url and datadict['resource_id'] == 'datastore_unknown_resource':
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_delete"}')
if 'delete' in url and datadict['resource_id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5':
TestResource.datastore = 'delete'
return MockResponse(200,
'{"success": true, "result": {"resource_id": "de6549d8-268b-4dfe-adaf-a4ae5c8510d5"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_delete"}')
if 'create' in url and datadict['resource_id'] == 'datastore_unknown_resource':
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_create"}')
if 'search' in url and datadict['resource_id'] == 'datastore_unknown_resource':
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_create"}')
if 'search' in url and datadict['resource_id'] == '_table_metadata':
return MockResponse(200,
'{"success": true, "result": {"include_total": true, "resource_id": "_table_metadata", "fields": [{"type": "int", "id": "_id"}, {"type": "name", "id": "name"}, {"type": "oid", "id": "oid"}, {"type": "name", "id": "alias_of"}], "records_format": "objects", "records": [{"_id":"f9cd60f3d7f2f6d0","name":"f9228459-d808-4b51-948f-68a5850abfde","oid":"919290","alias_of":null},{"_id":"7ae63490de9b7d7b","name":"af618a0b-09b8-42c8-836f-2be597e1ea34","oid":"135294","alias_of":null},{"_id":"1dc37f4e89988644","name":"748b40dd-7bd3-40a3-941b-e76f0bfbe0eb","oid":"117144","alias_of":null},{"_id":"2a554a61bd366206","name":"91c78d24-eab3-40b5-ba91-6b29bcda7178","oid":"116963","alias_of":null},{"_id":"fd787575143afe90","name":"9320cfce-4620-489a-bcbe-25c73867d4fc","oid":"107430","alias_of":null},{"_id":"a70093abd230f647","name":"b9d2eb36-e65c-417a-bc28-f4dadb149302","oid":"107409","alias_of":null},{"_id":"95fbdd2d06c07aea","name":"ca6a0891-8395-4d58-9168-6c44e17e0193","oid":"107385","alias_of":null}], "limit": 10000, "_links": {"start": "/api/action/datastore_search?limit=10000&resource_id=_table_metadata", "next": "/api/action/datastore_search?offset=10000&limit=10000&resource_id=_table_metadata"}, "total": 7}}')
if ('create' in url or 'insert' in url or 'upsert' in url or 'search' in url) and datadict[
'resource_id'] == 'de6549d8-268b-4dfe-adaf-a4ae5c8510d5':
TestResource.datastore = 'create'
return MockResponse(200,
'{"success": true, "result": {"fields": [{"type": "text", "id": "code"}, {"type": "text", "id": "title"}, {"type": "float", "id": "value"}, {"type": "timestamp", "id": "latest_date"}, {"type": "text", "id": "source"}, {"type": "text", "id": "source_link"}, {"type": "text", "id": "notes"}, {"type": "text", "id": "explore"}, {"type": "text", "id": "units"}], "method": "insert", "primary_key": "code", "resource_id": "bfa6b55f-10b6-4ba2-8470-33bb9a5194a5"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=datastore_create"}')
return MockResponse(404,
'{"success": false, "error": {"message": "Not found", "__type": "Not Found Error"}, "help": "http://test-data.humdata.org/api/3/action/help_show?name=resource_delete"}')
Configuration.read().remoteckan().session = MockSession()
def facade(projectmainfn, **kwargs):
# (Callable[[None], None], Any) -> None
"""Facade to simplify project setup that calls project main function
Args:
projectmainfn ((None) -> None): main function of project
**kwargs: configuration parameters to pass to HDX Configuration class
Returns:
None
"""
#
# Setting up configuration
#
site_url = Configuration._create(**kwargs)
logger.info('--------------------------------------------------')
logger.info('> Using HDX Python API Library %s' % Configuration.apiversion)
logger.info('> HDX Site: %s' % site_url)
UserAgent.user_agent = Configuration.read().user_agent
projectmainfn()
def my_excfn():
testresult.actual_result = Configuration.read().get_hdx_site_url()
raise ValueError('Some failure!')
def my_testuafn():
testresult.actual_result = Configuration.read().user_agent
def my_testkeyfn():
testresult.actual_result = Configuration.read().get_api_key()
def my_testfn():
testresult.actual_result = Configuration.read().get_hdx_site_url()
