def test_exception(self, hdx_config_yaml, project_config_yaml): UserAgent.clear_global() testresult.actual_result = None with pytest.raises(ValueError): facade(my_testfnkw, user_agent='test', hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml, fn='exc') UserAgent.clear_global() with pytest.raises(UserAgentError): facade(my_testfnkw, hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml, fn='agent') UserAgent.clear_global()
updatesheets = None if args.updatetabs: updatetabs = args.updatetabs.split(',') else: updatetabs = None if args.scrapers: scrapers = args.scrapers.split(',') else: scrapers = None basic_auths = dict() ba = args.basic_auths if ba is None: ba = getenv('BASIC_AUTHS') if ba: for keyvalue in ba.split(','): key, value = keyvalue.split(':') basic_auths[key] = value facade(main, hdx_read_only=True, user_agent=user_agent, preprefix=preprefix, hdx_site=hdx_site, project_config_yaml=join('config', 'project_configuration.yml'), excel_path=args.excel_path, gsheet_auth=gsheet_auth, updatesheets=updatesheets, updatetabs=updatetabs, scrapers=scrapers, basic_auths=basic_auths, nojson=args.nojson)
updatetabs = None if args.scrapers: scrapers = args.scrapers.split(',') else: scrapers = None basic_auths = dict() ba = args.basic_auths if ba is None: ba = getenv('BASIC_AUTHS') if ba: for keyvalue in ba.split(','): key, value = keyvalue.split(':') basic_auths[key] = value other_auths = dict() oa = args.other_auths if oa is None: oa = getenv('OTHER_AUTHS') if oa: for keyvalue in oa.split(','): key, value = keyvalue.split(':') other_auths[key] = value if args.countries_override: countries_override = args.countries_override.split(',') else: countries_override = None facade(main, hdx_read_only=True, user_agent=user_agent, preprefix=preprefix, hdx_site=hdx_site, project_config_yaml=join('config', 'project_configuration.yml'), excel_path=args.excel_path, gsheet_auth=gsheet_auth, updatesheets=updatesheets, updatetabs=updatetabs, scrapers=scrapers, basic_auths=basic_auths, other_auths=other_auths, nojson=args.nojson, countries_override=countries_override, save=args.save, use_saved=args.use_saved)
wfp.build_mappings() for info, country in progress_storing_tempdir( lookup, countries, 'iso3'): dataset, showcase, qc_indicators = wfp.generate_dataset_and_showcase( country['iso3'], info['folder']) if dataset: dataset.update_from_yaml() dataset['notes'] = dataset[ 'notes'] % 'Food Prices data for %s. Food prices data comes from the World Food Programme and covers' % country[ 'name'] dataset.generate_resource_view(-1, indicators=qc_indicators) dataset.create_in_hdx( remove_additional_resources=True, hxl_update=False, updated_by_script='HDX Scraper: WFP Food Prices', batch=info['batch']) showcase.create_in_hdx() showcase.add_dataset(dataset) if __name__ == '__main__': args = parse_args() facade(main, user_agent_config_yaml=join(expanduser('~'), '.useragents.yml'), user_agent_lookup=lookup, project_config_yaml=join('config', 'project_configuration.yml'), save=args.save, use_saved=args.use_saved)
parser.add_argument('-hs', '--hdx_site', default=None, help='HDX site to use') parser.add_argument('-dp', '--db_params', default=None, help='Database connection parameters') parser.add_argument( '-sd', '--start_date', default=None, help= 'Add any datasets created or updated after this date. Defaults to one week prior to current date.' ) args = parser.parse_args() hdx_site = args.hdx_site if hdx_site is None: hdx_site = 'feature' start_date = args.start_date if start_date is None: start_date = datetime.utcnow() - timedelta(days=7) else: start_date = parse(start_date) facade(main, hdx_key=args.hdx_key, hdx_site=hdx_site, user_agent='UNOSAT', db_params=args.db_params, start_date=start_date)
if __name__ == "__main__": args = parse_args() user_agent = args.user_agent if user_agent is None: user_agent = getenv("USER_AGENT") if user_agent is None: user_agent = "hdx-scraper-iati-viz" preprefix = args.preprefix if preprefix is None: preprefix = getenv("PREPREFIX") hdx_site = args.hdx_site if hdx_site is None: hdx_site = getenv("HDX_SITE", "prod") facade( main, hdx_read_only=True, user_agent=user_agent, preprefix=preprefix, hdx_site=hdx_site, project_config_yaml=join("config", "project_configuration.yml"), output_dir=args.output_dir, saved_dir=args.saved_dir, save=args.save, use_saved=args.use_saved, dportal_params=args.dportal_params, whattorun=args.what, filterdate=args.date_filter, )
def test_facade(self, monkeypatch, hdx_config_yaml, project_config_yaml): UserAgent.clear_global() my_user_agent = 'test' testresult.actual_result = None facade(my_testfnkw, user_agent=my_user_agent, hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml, fn='site') assert testresult.actual_result == 'https://data.humdata.org' UserAgent.clear_global() version = get_api_version() testresult.actual_result = None facade(my_testfnkw, user_agent=my_user_agent, hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml, fn='agent') assert testresult.actual_result == 'HDXPythonLibrary/%s-%s' % (version, my_user_agent) UserAgent.clear_global() testresult.actual_result = None my_user_agent = 'lala' monkeypatch.setenv('USER_AGENT', my_user_agent) facade(my_testfnkw, hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml, fn='agent') assert testresult.actual_result == 'HDXPythonLibrary/%s-%s' % (version, my_user_agent) UserAgent.clear_global() testresult.actual_result = None facade(my_testfnkw, user_agent='test', hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml, fn='agent') assert testresult.actual_result == 'HDXPythonLibrary/%s-%s' % (version, my_user_agent) UserAgent.clear_global() testresult.actual_result = None my_preprefix = 'haha' monkeypatch.setenv('PREPREFIX', my_preprefix) facade(my_testfnkw, user_agent='test', hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml, fn='agent') assert testresult.actual_result == '%s:HDXPythonLibrary/%s-%s' % (my_preprefix, version, my_user_agent) UserAgent.clear_global() testresult.actual_result = None my_test_key = '1234' facade(my_testfnkw, hdx_key=my_test_key, user_agent=my_user_agent, hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml, fn='api') assert testresult.actual_result == my_test_key UserAgent.clear_global() testresult.actual_result = None monkeypatch.setenv('HDX_KEY', my_test_key) facade(my_testfnkw, hdx_key='aaaa', user_agent=my_user_agent, hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml, fn='api') assert testresult.actual_result == my_test_key UserAgent.clear_global() testresult.actual_result = None my_test_hdxsite = 'test' facade(my_testfnkw, hdx_site=my_test_hdxsite, user_agent=my_user_agent, hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml, fn='site') assert testresult.actual_result == 'https://%s-data.humdata.org' % my_test_hdxsite UserAgent.clear_global() testresult.actual_result = None monkeypatch.setenv('HDX_SITE', my_test_hdxsite) facade(my_testfnkw, hdx_site='feature', user_agent=my_user_agent, hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml, fn='site') assert testresult.actual_result == 'https://%s-data.humdata.org' % my_test_hdxsite UserAgent.clear_global() my_test_hdxurl = 'http://other-data.humdata.org' monkeypatch.setenv('HDX_URL', my_test_hdxurl) facade(my_testfnkw, hdx_site='feature', user_agent=my_user_agent, hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml, fn='site') assert testresult.actual_result == my_test_hdxurl UserAgent.clear_global() my_test_hdxurl2 = 'http://other-data.humdata.org/' monkeypatch.setenv('HDX_URL', my_test_hdxurl2) facade(my_testfnkw, hdx_site='feature', user_agent=my_user_agent, hdx_config_yaml=hdx_config_yaml, project_config_yaml=project_config_yaml, fn='site') assert testresult.actual_result == my_test_hdxurl UserAgent.clear_global()
if __name__ == '__main__': args = parse_args() user_agent = args.user_agent if user_agent is None: user_agent = getenv('USER_AGENT') if user_agent is None: user_agent = 'test' preprefix = args.preprefix if preprefix is None: preprefix = getenv('PREPREFIX') hdx_site = args.hdx_site if hdx_site is None: hdx_site = getenv('HDX_SITE', 'prod') folder = args.folder if folder is None: folder = get_temp_dir('PA-COVID') gsheet_auth = args.gsheet_auth if gsheet_auth is None: gsheet_auth = getenv('GSHEET_AUTH') facade(main, hdx_read_only=True, user_agent=user_agent, preprefix=preprefix, hdx_site=hdx_site, project_config_yaml=join('config', 'project_configuration.yml'), folder=folder, keep=args.keep, gsheet_auth=gsheet_auth, excel=args.excel, debug=args.debug)
if gsheet_auth is None: gsheet_auth = getenv("GSHEET_AUTH") email_server = args.email_server if email_server is None: email_server = getenv("EMAIL_SERVER") failure_emails = args.failure_emails if failure_emails is None: failure_emails = getenv("FAILURE_EMAILS") sysadmin_emails = args.sysadmin_emails if sysadmin_emails is None: sysadmin_emails = getenv("SYSADMIN_EMAILS") project_config_yaml = script_dir_plus_file("project_configuration.yml", main) facade( main, hdx_key=hdx_key, user_agent=user_agent, preprefix=preprefix, hdx_site=hdx_site, project_config_yaml=project_config_yaml, db_url=db_url, db_params=args.db_params, gsheet_auth=gsheet_auth, email_server=email_server, failure_emails=failure_emails, sysadmin_emails=sysadmin_emails, email_test=args.email_test, spreadsheet_test=args.spreadsheet_test, no_spreadsheet=args.no_spreadsheet, )
) if dataset: dataset.update_from_yaml() dataset.create_in_hdx( remove_additional_resources=True, hxl_update=False, updated_by_script="HDX Scraper: UNHCR microdata", batch=info["batch"], ) for failure in failures: logger.error(failure) if __name__ == "__main__": parser = argparse.ArgumentParser(description="UNHCR Microdata") parser.add_argument( "-of", "--output_failures", default=False, action="store_true", help="Output failures", ) args = parser.parse_args() facade( main, user_agent_config_yaml=join(expanduser("~"), ".useragents.yml"), user_agent_lookup=lookup, project_config_yaml=join("config", "project_configuration.yml"), output_failures=args.output_failures, )
else: writeparams = {'driver': 'sqlite', 'database': 'output.db'} logger.info('> Database (to write) parameters: %s' % writeparams) with Database(**readparams) as readsession: with Database(**writeparams) as writesession: dbactions = DatabaseActions(readsession, writesession, run_numbers) if action == 'duplicate': dbactions.duplicate() if __name__ == '__main__': parser = argparse.ArgumentParser(description='Data Freshness DBUtils') parser.add_argument('-dbr', '--dbread_url', default=None, help='Database connection string') parser.add_argument('-dpr', '--dbread_params', default=None, help='Database connection parameters. Overrides --dbread_url.') parser.add_argument('-dbw', '--dbwrite_url', default=None, help='Database connection string') parser.add_argument('-dpw', '--dbwrite_params', default=None, help='Database connection parameters. Overrides --dbwrite_url.') parser.add_argument('-act', '--action', default=None, help='Action to take (currently only "duplicate")') parser.add_argument('-rn', '--run_numbers', default=None, help='Run numbers to use or "latest"') args = parser.parse_args() dbread_url = args.dbread_url if dbread_url and '://' not in dbread_url: dbread_url = 'postgresql://%s' % dbread_url dbwrite_url = args.dbwrite_url if dbwrite_url and '://' not in dbwrite_url: dbwrite_url = 'postgresql://%s' % dbwrite_url run_numbers = args.run_numbers logger.info('> Run numbers: %s' % run_numbers) run_numbers = run_numbers.split(',') facade(main, user_agent='test', dbread_url=dbread_url, dbread_params=args.dbread_params, dbwrite_url=dbwrite_url, dbwrite_params=args.dbwrite_params, action=args.action, run_numbers=run_numbers)