def test_setup_logging_dict(self, smtp_config_yaml):
handlers = ["error_file_handler", "error_mail_handler"]
logging_config_dict = {
"version": 1,
"handlers": {
"error_file_handler": {
"class": "logging.FileHandler",
"level": "ERROR",
"filename": "errors.log",
"encoding": "utf8",
"mode": "w",
},
"error_mail_handler": {
"class": "logging.handlers.SMTPHandler",
"level": "CRITICAL",
"mailhost": "localhost",
"fromaddr": "noreply@localhost",
"toaddrs": "*****@*****.**",
"subject": "SCRAPER FAILED",
},
},
"root": {
"level": "INFO",
"handlers": handlers
},
}
setup_logging(logging_config_dict=logging_config_dict)
actual_handler_names = self.get_root_handlers_names()
for handler in handlers:
assert handler in actual_handler_names
assert "*****@*****.**" in self.get_root_mail_handler_email()
def test_setup_logging_dict(self, smtp_config_yaml):
logging_config_dict = {
'version': 1,
'handlers': {
'error_file_handler': {
'class': 'logging.FileHandler',
'level': 'ERROR',
'filename': 'errors.log',
'encoding': 'utf8',
'mode': 'w'
},
'error_mail_handler': {
'class': 'logging.handlers.SMTPHandler',
'level': 'CRITICAL',
'mailhost': 'localhost',
'fromaddr': 'noreply@localhost',
'toaddrs': '*****@*****.**',
'subject': 'SCRAPER FAILED'
}
},
'root': {
'level': 'INFO',
'handlers': ['error_file_handler', 'error_mail_handler']
}
}
setup_logging(logging_config_dict=logging_config_dict)
actual_logging_tree = format.build_description()
for handler in ['File', 'SMTP']:
assert handler in actual_logging_tree
assert '*****@*****.**' in actual_logging_tree
def test_setup_logging_env(self, monkeypatch, logging_config_yaml):
monkeypatch.setenv("LOG_FILE_ONLY", "true")
setup_logging(logging_config_yaml=logging_config_yaml)
actual_handler_names = self.get_root_handlers_names()
assert "console" not in actual_handler_names
monkeypatch.setenv("LOG_FILE_ONLY", "false")
setup_logging(logging_config_yaml=logging_config_yaml)
actual_handler_names = self.get_root_handlers_names()
assert "console" in actual_handler_names
def pop_data_download(region_names, wp_year=2017):
from hdx.utilities.easy_logging import setup_logging
setup_logging()
from hdx.hdx_configuration import Configuration
Configuration.create(hdx_site='prod', user_agent='Read-only user', hdx_read_only=True)
from hdx.data.dataset import Dataset
import wpgpDownload
from wpgpDownload.utils.convenience_functions import download_country_covariates as download_worldpop
from wpgpDownload.utils.convenience_functions import refresh_csv
refresh_csv()
hdx_datasets = Dataset.search_in_hdx('hrsl', rows=500)
hdx_resources = Dataset.get_all_resources(hdx_datasets)
print('')
country_names = set([region[0:3] for region in region_names])
for country in country_names:
print(country)
for res in hdx_resources:
if 'population_'+country.lower() in res['name'] and '.zip' in res['name'] and 'csv' not in res['name']:
print('Downloading HRSL',res['name'], end='\r')
url, path = res.download()
print('HRSL',res['name'],'download completed ')
shutil.move(Path(path),Path('./'+country+'/misc_data/population_'+country.lower()+'.zip'))
zipfile.ZipFile(Path('./'+country+'/misc_data/population_'+country.lower()+'.zip'), 'r').extractall(Path('./'+country+'/misc_data'))
for file in Path('./'+country+'/misc_data').iterdir():
if 'population_'+country.lower() in file.name and file.suffix != '.tif':
os.remove(file)
if type(wp_year) == list:
years = wp_year
elif type(wp_year) == int:
years = [wp_year]
#NTL_files = [file for file in Path("./"+country+"/NTL").iterdir() if "NTL" in file.name]
#
#years = []
#for NTL_file in NTL_files:
# years.append(NTL_file.name[4:8])
#years = [year for year in set(years)]
#years.sort()
for year in years:
print('Downloading WorldPop '+country+' '+str(year)+'\t\t',end='\r')
download_worldpop(ISO=country,out_folder='.\\'+country+'\\worldpop',prod_name='ppp_'+str(year))
print('WorldPop '+country+' '+str(year)+' download completed\t\t')
print("")
print('Done')
def test_setup_logging_smtp_dict(self):
smtp_config_dict = {
"handlers": {
"error_mail_handler": {
"toaddrs": "*****@*****.**",
"subject": "lala",
}
}
}
setup_logging(smtp_config_dict=smtp_config_dict)
actual_handler_names = self.get_root_handlers_names()
for handler in ["console", "error_file_handler", "error_mail_handler"]:
assert handler in actual_handler_names
assert "*****@*****.**" in self.get_root_mail_handler_email()
def test_setup_logging_smtp_dict(self):
smtp_config_dict = {
'handlers': {
'error_mail_handler': {
'toaddrs': '*****@*****.**',
'subject': 'lala'
}
}
}
setup_logging(smtp_config_dict=smtp_config_dict)
actual_logging_tree = format.build_description()
for handler in ['Stream', 'File', 'SMTP']:
assert handler in actual_logging_tree
assert '*****@*****.**' in actual_logging_tree
def hdx_acap_connector():
"""Connects to HDX, and fetches acaps covid 19 government measures dataset
Arguments:
None
Returns:
pandas.DataFrame
"""
setup_logging()
Configuration.create(hdx_site='prod',
user_agent='CoronaWhy',
hdx_read_only=True)
dataset = Dataset.read_from_hdx(
'acaps-covid19-government-measures-dataset')
logger.info("Dataset Fetched from: %s", dataset.get_hdx_url())
logger.info('Expected Update Frequency: %s',
dataset.get_expected_update_frequency())
resources = dataset.get_resources()
logger.info('Description: %s', resources[0]['description'])
logger.info('Last Modified: %s, Revision Last Updated: %s',
resources[0]['last_modified'],
resources[0]['revision_last_updated'])
logger.info('Size: %sMb', resources[0]['size'] / (1024**2))
logger.info('Dataset Url: %s', resources[0]['url'])
logger.info('Tags: %s', dataset.get_tags())
resource = Resource.read_from_hdx(resources[0]['id'])
url, absolute_path = resource.download('./')
logger.info('Downloaded dataset at path: %s', absolute_path)
xl = pd.ExcelFile(absolute_path)
logger.info(xl.sheet_names)
df = xl.parse('Database')
return df
def test_setup_logging(
self,
logging_config_json,
logging_config_yaml,
smtp_config_json,
smtp_config_yaml,
):
with pytest.raises(FILENOTFOUND_EXCTYPE):
setup_logging(smtp_config_json="NOT_EXIST")
with pytest.raises(FILENOTFOUND_EXCTYPE):
setup_logging(
logging_config_json="NOT_EXIST",
smtp_config_yaml=smtp_config_yaml,
)
with pytest.raises(FILENOTFOUND_EXCTYPE):
setup_logging(
logging_config_yaml="NOT_EXIST",
smtp_config_yaml=smtp_config_yaml,
)
with pytest.raises(LoggingError):
setup_logging(
logging_config_yaml=logging_config_yaml,
smtp_config_yaml=smtp_config_yaml,
)
with pytest.raises(LoggingError):
setup_logging(
logging_config_dict={"la": "la"},
logging_config_json=logging_config_json,
smtp_config_yaml=smtp_config_yaml,
)
with pytest.raises(LoggingError):
setup_logging(
logging_config_dict={"la": "la"},
logging_config_yaml=logging_config_yaml,
smtp_config_yaml=smtp_config_yaml,
)
with pytest.raises(LoggingError):
setup_logging(
logging_config_json=logging_config_json,
logging_config_yaml=logging_config_yaml,
smtp_config_yaml=smtp_config_yaml,
)
with pytest.raises(LoggingError):
setup_logging(
smtp_config_dict={"la": "la"},
smtp_config_json=smtp_config_json,
)
with pytest.raises(LoggingError):
setup_logging(
smtp_config_dict={"la": "la"},
smtp_config_yaml=smtp_config_yaml,
)
with pytest.raises(LoggingError):
setup_logging(
smtp_config_json=smtp_config_json,
smtp_config_yaml=smtp_config_yaml,
)
def test_setup_logging_smtp_yaml(self, smtp_config_yaml):
setup_logging(smtp_config_yaml=smtp_config_yaml)
actual_handler_names = self.get_root_handlers_names()
for handler in ["console", "error_file_handler", "error_mail_handler"]:
assert handler in actual_handler_names
assert "*****@*****.**" in self.get_root_mail_handler_email()
def test_setup_logging_smtp_json(self, smtp_config_json):
setup_logging(smtp_config_json=smtp_config_json)
actual_handler_names = self.get_root_handlers_names()
for handler in ["console", "error_file_handler", "error_mail_handler"]:
assert handler in actual_handler_names
assert "*****@*****.**" in self.get_root_mail_handler_email()
def test_setup_logging_json(self, logging_config_json):
setup_logging(logging_config_json=logging_config_json)
actual_logging_tree = format.build_description()
for handler in ['Stream', 'File']:
assert handler in actual_logging_tree
def test_setup_logging_json(self, logging_config_json):
setup_logging(logging_config_json=logging_config_json)
actual_handler_names = self.get_root_handlers_names()
for handler in ["console", "error_file_handler"]:
assert handler in actual_handler_names
def test_setup_logging(self, logging_config_json, logging_config_yaml,
smtp_config_json, smtp_config_yaml):
with pytest.raises(FILENOTFOUND_EXCTYPE):
setup_logging(smtp_config_json='NOT_EXIST')
with pytest.raises(FILENOTFOUND_EXCTYPE):
setup_logging(logging_config_json='NOT_EXIST',
smtp_config_yaml=smtp_config_yaml)
with pytest.raises(FILENOTFOUND_EXCTYPE):
setup_logging(logging_config_yaml='NOT_EXIST',
smtp_config_yaml=smtp_config_yaml)
with pytest.raises(LoggingError):
setup_logging(logging_config_yaml=logging_config_yaml,
smtp_config_yaml=smtp_config_yaml)
with pytest.raises(LoggingError):
setup_logging(logging_config_dict={'la': 'la'},
logging_config_json=logging_config_json,
smtp_config_yaml=smtp_config_yaml)
with pytest.raises(LoggingError):
setup_logging(logging_config_dict={'la': 'la'},
logging_config_yaml=logging_config_yaml,
smtp_config_yaml=smtp_config_yaml)
with pytest.raises(LoggingError):
setup_logging(logging_config_json=logging_config_json,
logging_config_yaml=logging_config_yaml,
smtp_config_yaml=smtp_config_yaml)
with pytest.raises(LoggingError):
setup_logging(smtp_config_dict={'la': 'la'},
smtp_config_json=smtp_config_json)
with pytest.raises(LoggingError):
setup_logging(smtp_config_dict={'la': 'la'},
smtp_config_yaml=smtp_config_yaml)
with pytest.raises(LoggingError):
setup_logging(smtp_config_json=smtp_config_json,
smtp_config_yaml=smtp_config_yaml)
def test_setup_logging_smtp_yaml(self, smtp_config_yaml):
setup_logging(smtp_config_yaml=smtp_config_yaml)
actual_logging_tree = format.build_description()
for handler in ['Stream', 'File', 'SMTP']:
assert handler in actual_logging_tree
assert '*****@*****.**' in actual_logging_tree
def test_setup_logging_smtp_json(self, smtp_config_json):
setup_logging(smtp_config_json=smtp_config_json)
actual_logging_tree = format.build_description()
for handler in ['Stream', 'File', 'SMTP']:
assert handler in actual_logging_tree
assert '*****@*****.**' in actual_logging_tree
# -*- coding: utf-8 -*-
"""Facade to simplify project setup that calls project main function"""
import logging
from hdx.utilities.easy_logging import setup_logging
from hdx.utilities.useragent import UserAgent
from hdx.facades import logging_kwargs
from hdx.hdx_configuration import Configuration
logger = logging.getLogger(__name__)
setup_logging(**logging_kwargs)
def facade(projectmainfn, **kwargs):
# (Callable[[None], None], Any) -> None
"""Facade to simplify project setup that calls project main function
Args:
projectmainfn ((None) -> None): main function of project
**kwargs: configuration parameters to pass to HDX Configuration class
Returns:
None
"""
#
# Setting up configuration
#
site_url = Configuration._create(**kwargs)
import shutil
import os
import logging
from hdx.utilities.easy_logging import setup_logging
from hdx.hdx_configuration import Configuration
from hdx.data.dataset import Dataset
HDX_SITE = "prod"
USER_AGENT = "MapAction"
setup_logging()
logger = logging.getLogger()
Configuration.create(hdx_site=HDX_SITE,
user_agent=USER_AGENT,
hdx_read_only=True)
def query_api(hdx_address, directory, resource_format="XLSX"):
dataset = Dataset.read_from_hdx(hdx_address)
resources = dataset.get_resources()
filenames = {}
for resource in resources:
if resource["format"] == resource_format:
_, path = resource.download()
filename = os.path.basename(path)
shutil.move(path, os.path.join(directory, filename))
filenames[resource["name"]] = filename
logging.info(
f'Saved "{resource["name"]}" to "{directory}/{filename}"')
return filenames
def test_setup_logging_yaml(self, monkeypatch, logging_config_yaml):
setup_logging(logging_config_yaml=logging_config_yaml)
actual_handler_names = self.get_root_handlers_names()
for handler in ["console", "error_mail_handler"]:
assert handler in actual_handler_names
assert "*****@*****.**" in self.get_root_mail_handler_email()
