def test_exception(self, hdx_config_yaml, project_config_yaml):
UserAgent.clear_global()
testresult.actual_result = None
with pytest.raises(ValueError):
facade(my_testfnkw, user_agent='test', hdx_config_yaml=hdx_config_yaml,
project_config_yaml=project_config_yaml, fn='exc')
UserAgent.clear_global()
with pytest.raises(UserAgentError):
facade(my_testfnkw, hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml, fn='agent')
UserAgent.clear_global()
updatesheets = None
if args.updatetabs:
updatetabs = args.updatetabs.split(',')
else:
updatetabs = None
if args.scrapers:
scrapers = args.scrapers.split(',')
else:
scrapers = None
basic_auths = dict()
ba = args.basic_auths
if ba is None:
ba = getenv('BASIC_AUTHS')
if ba:
for keyvalue in ba.split(','):
key, value = keyvalue.split(':')
basic_auths[key] = value
facade(main,
hdx_read_only=True,
user_agent=user_agent,
preprefix=preprefix,
hdx_site=hdx_site,
project_config_yaml=join('config', 'project_configuration.yml'),
excel_path=args.excel_path,
gsheet_auth=gsheet_auth,
updatesheets=updatesheets,
updatetabs=updatetabs,
scrapers=scrapers,
basic_auths=basic_auths,
nojson=args.nojson)
updatetabs = None
if args.scrapers:
scrapers = args.scrapers.split(',')
else:
scrapers = None
basic_auths = dict()
ba = args.basic_auths
if ba is None:
ba = getenv('BASIC_AUTHS')
if ba:
for keyvalue in ba.split(','):
key, value = keyvalue.split(':')
basic_auths[key] = value
other_auths = dict()
oa = args.other_auths
if oa is None:
oa = getenv('OTHER_AUTHS')
if oa:
for keyvalue in oa.split(','):
key, value = keyvalue.split(':')
other_auths[key] = value
if args.countries_override:
countries_override = args.countries_override.split(',')
else:
countries_override = None
facade(main, hdx_read_only=True, user_agent=user_agent, preprefix=preprefix, hdx_site=hdx_site,
project_config_yaml=join('config', 'project_configuration.yml'), excel_path=args.excel_path,
gsheet_auth=gsheet_auth, updatesheets=updatesheets, updatetabs=updatetabs, scrapers=scrapers,
basic_auths=basic_auths, other_auths=other_auths, nojson=args.nojson,
countries_override=countries_override, save=args.save, use_saved=args.use_saved)
wfp.build_mappings()
for info, country in progress_storing_tempdir(
lookup, countries, 'iso3'):
dataset, showcase, qc_indicators = wfp.generate_dataset_and_showcase(
country['iso3'], info['folder'])
if dataset:
dataset.update_from_yaml()
dataset['notes'] = dataset[
'notes'] % 'Food Prices data for %s. Food prices data comes from the World Food Programme and covers' % country[
'name']
dataset.generate_resource_view(-1,
indicators=qc_indicators)
dataset.create_in_hdx(
remove_additional_resources=True,
hxl_update=False,
updated_by_script='HDX Scraper: WFP Food Prices',
batch=info['batch'])
showcase.create_in_hdx()
showcase.add_dataset(dataset)
if __name__ == '__main__':
args = parse_args()
facade(main,
user_agent_config_yaml=join(expanduser('~'), '.useragents.yml'),
user_agent_lookup=lookup,
project_config_yaml=join('config', 'project_configuration.yml'),
save=args.save,
use_saved=args.use_saved)
parser.add_argument('-hs',
'--hdx_site',
default=None,
help='HDX site to use')
parser.add_argument('-dp',
'--db_params',
default=None,
help='Database connection parameters')
parser.add_argument(
'-sd',
'--start_date',
default=None,
help=
'Add any datasets created or updated after this date. Defaults to one week prior to current date.'
)
args = parser.parse_args()
hdx_site = args.hdx_site
if hdx_site is None:
hdx_site = 'feature'
start_date = args.start_date
if start_date is None:
start_date = datetime.utcnow() - timedelta(days=7)
else:
start_date = parse(start_date)
facade(main,
hdx_key=args.hdx_key,
hdx_site=hdx_site,
user_agent='UNOSAT',
db_params=args.db_params,
start_date=start_date)
if __name__ == "__main__":
args = parse_args()
user_agent = args.user_agent
if user_agent is None:
user_agent = getenv("USER_AGENT")
if user_agent is None:
user_agent = "hdx-scraper-iati-viz"
preprefix = args.preprefix
if preprefix is None:
preprefix = getenv("PREPREFIX")
hdx_site = args.hdx_site
if hdx_site is None:
hdx_site = getenv("HDX_SITE", "prod")
facade(
main,
hdx_read_only=True,
user_agent=user_agent,
preprefix=preprefix,
hdx_site=hdx_site,
project_config_yaml=join("config", "project_configuration.yml"),
output_dir=args.output_dir,
saved_dir=args.saved_dir,
save=args.save,
use_saved=args.use_saved,
dportal_params=args.dportal_params,
whattorun=args.what,
filterdate=args.date_filter,
)
def test_facade(self, monkeypatch, hdx_config_yaml, project_config_yaml):
UserAgent.clear_global()
my_user_agent = 'test'
testresult.actual_result = None
facade(my_testfnkw, user_agent=my_user_agent, hdx_config_yaml=hdx_config_yaml,
project_config_yaml=project_config_yaml, fn='site')
assert testresult.actual_result == 'https://data.humdata.org'
UserAgent.clear_global()
version = get_api_version()
testresult.actual_result = None
facade(my_testfnkw, user_agent=my_user_agent, hdx_config_yaml=hdx_config_yaml,
project_config_yaml=project_config_yaml, fn='agent')
assert testresult.actual_result == 'HDXPythonLibrary/%s-%s' % (version, my_user_agent)
UserAgent.clear_global()
testresult.actual_result = None
my_user_agent = 'lala'
monkeypatch.setenv('USER_AGENT', my_user_agent)
facade(my_testfnkw, hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml, fn='agent')
assert testresult.actual_result == 'HDXPythonLibrary/%s-%s' % (version, my_user_agent)
UserAgent.clear_global()
testresult.actual_result = None
facade(my_testfnkw, user_agent='test', hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml,
fn='agent')
assert testresult.actual_result == 'HDXPythonLibrary/%s-%s' % (version, my_user_agent)
UserAgent.clear_global()
testresult.actual_result = None
my_preprefix = 'haha'
monkeypatch.setenv('PREPREFIX', my_preprefix)
facade(my_testfnkw, user_agent='test', hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml,
fn='agent')
assert testresult.actual_result == '%s:HDXPythonLibrary/%s-%s' % (my_preprefix, version, my_user_agent)
UserAgent.clear_global()
testresult.actual_result = None
my_test_key = '1234'
facade(my_testfnkw, hdx_key=my_test_key, user_agent=my_user_agent, hdx_config_yaml=hdx_config_yaml,
project_config_yaml=project_config_yaml, fn='api')
assert testresult.actual_result == my_test_key
UserAgent.clear_global()
testresult.actual_result = None
monkeypatch.setenv('HDX_KEY', my_test_key)
facade(my_testfnkw, hdx_key='aaaa', user_agent=my_user_agent, hdx_config_yaml=hdx_config_yaml,
project_config_yaml=project_config_yaml, fn='api')
assert testresult.actual_result == my_test_key
UserAgent.clear_global()
testresult.actual_result = None
my_test_hdxsite = 'test'
facade(my_testfnkw, hdx_site=my_test_hdxsite, user_agent=my_user_agent, hdx_config_yaml=hdx_config_yaml,
project_config_yaml=project_config_yaml, fn='site')
assert testresult.actual_result == 'https://%s-data.humdata.org' % my_test_hdxsite
UserAgent.clear_global()
testresult.actual_result = None
monkeypatch.setenv('HDX_SITE', my_test_hdxsite)
facade(my_testfnkw, hdx_site='feature', user_agent=my_user_agent, hdx_config_yaml=hdx_config_yaml,
project_config_yaml=project_config_yaml, fn='site')
assert testresult.actual_result == 'https://%s-data.humdata.org' % my_test_hdxsite
UserAgent.clear_global()
my_test_hdxurl = 'http://other-data.humdata.org'
monkeypatch.setenv('HDX_URL', my_test_hdxurl)
facade(my_testfnkw, hdx_site='feature', user_agent=my_user_agent, hdx_config_yaml=hdx_config_yaml,
project_config_yaml=project_config_yaml, fn='site')
assert testresult.actual_result == my_test_hdxurl
UserAgent.clear_global()
my_test_hdxurl2 = 'http://other-data.humdata.org/'
monkeypatch.setenv('HDX_URL', my_test_hdxurl2)
facade(my_testfnkw, hdx_site='feature', user_agent=my_user_agent, hdx_config_yaml=hdx_config_yaml,
project_config_yaml=project_config_yaml, fn='site')
assert testresult.actual_result == my_test_hdxurl
UserAgent.clear_global()
if __name__ == '__main__':
args = parse_args()
user_agent = args.user_agent
if user_agent is None:
user_agent = getenv('USER_AGENT')
if user_agent is None:
user_agent = 'test'
preprefix = args.preprefix
if preprefix is None:
preprefix = getenv('PREPREFIX')
hdx_site = args.hdx_site
if hdx_site is None:
hdx_site = getenv('HDX_SITE', 'prod')
folder = args.folder
if folder is None:
folder = get_temp_dir('PA-COVID')
gsheet_auth = args.gsheet_auth
if gsheet_auth is None:
gsheet_auth = getenv('GSHEET_AUTH')
facade(main,
hdx_read_only=True,
user_agent=user_agent,
preprefix=preprefix,
hdx_site=hdx_site,
project_config_yaml=join('config', 'project_configuration.yml'),
folder=folder,
keep=args.keep,
gsheet_auth=gsheet_auth,
excel=args.excel,
debug=args.debug)
if gsheet_auth is None:
gsheet_auth = getenv("GSHEET_AUTH")
email_server = args.email_server
if email_server is None:
email_server = getenv("EMAIL_SERVER")
failure_emails = args.failure_emails
if failure_emails is None:
failure_emails = getenv("FAILURE_EMAILS")
sysadmin_emails = args.sysadmin_emails
if sysadmin_emails is None:
sysadmin_emails = getenv("SYSADMIN_EMAILS")
project_config_yaml = script_dir_plus_file("project_configuration.yml",
main)
facade(
main,
hdx_key=hdx_key,
user_agent=user_agent,
preprefix=preprefix,
hdx_site=hdx_site,
project_config_yaml=project_config_yaml,
db_url=db_url,
db_params=args.db_params,
gsheet_auth=gsheet_auth,
email_server=email_server,
failure_emails=failure_emails,
sysadmin_emails=sysadmin_emails,
email_test=args.email_test,
spreadsheet_test=args.spreadsheet_test,
no_spreadsheet=args.no_spreadsheet,
)
)
if dataset:
dataset.update_from_yaml()
dataset.create_in_hdx(
remove_additional_resources=True,
hxl_update=False,
updated_by_script="HDX Scraper: UNHCR microdata",
batch=info["batch"],
)
for failure in failures:
logger.error(failure)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="UNHCR Microdata")
parser.add_argument(
"-of",
"--output_failures",
default=False,
action="store_true",
help="Output failures",
)
args = parser.parse_args()
facade(
main,
user_agent_config_yaml=join(expanduser("~"), ".useragents.yml"),
user_agent_lookup=lookup,
project_config_yaml=join("config", "project_configuration.yml"),
output_failures=args.output_failures,
)
else:
writeparams = {'driver': 'sqlite', 'database': 'output.db'}
logger.info('> Database (to write) parameters: %s' % writeparams)
with Database(**readparams) as readsession:
with Database(**writeparams) as writesession:
dbactions = DatabaseActions(readsession, writesession, run_numbers)
if action == 'duplicate':
dbactions.duplicate()
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Data Freshness DBUtils')
parser.add_argument('-dbr', '--dbread_url', default=None, help='Database connection string')
parser.add_argument('-dpr', '--dbread_params', default=None, help='Database connection parameters. Overrides --dbread_url.')
parser.add_argument('-dbw', '--dbwrite_url', default=None, help='Database connection string')
parser.add_argument('-dpw', '--dbwrite_params', default=None, help='Database connection parameters. Overrides --dbwrite_url.')
parser.add_argument('-act', '--action', default=None, help='Action to take (currently only "duplicate")')
parser.add_argument('-rn', '--run_numbers', default=None, help='Run numbers to use or "latest"')
args = parser.parse_args()
dbread_url = args.dbread_url
if dbread_url and '://' not in dbread_url:
dbread_url = 'postgresql://%s' % dbread_url
dbwrite_url = args.dbwrite_url
if dbwrite_url and '://' not in dbwrite_url:
dbwrite_url = 'postgresql://%s' % dbwrite_url
run_numbers = args.run_numbers
logger.info('> Run numbers: %s' % run_numbers)
run_numbers = run_numbers.split(',')
facade(main, user_agent='test', dbread_url=dbread_url, dbread_params=args.dbread_params,
dbwrite_url=dbwrite_url, dbwrite_params=args.dbwrite_params, action=args.action, run_numbers=run_numbers)