def test_setup_logging_dict(self, smtp_config_yaml): handlers = ["error_file_handler", "error_mail_handler"] logging_config_dict = { "version": 1, "handlers": { "error_file_handler": { "class": "logging.FileHandler", "level": "ERROR", "filename": "errors.log", "encoding": "utf8", "mode": "w", }, "error_mail_handler": { "class": "logging.handlers.SMTPHandler", "level": "CRITICAL", "mailhost": "localhost", "fromaddr": "noreply@localhost", "toaddrs": "*****@*****.**", "subject": "SCRAPER FAILED", }, }, "root": { "level": "INFO", "handlers": handlers }, } setup_logging(logging_config_dict=logging_config_dict) actual_handler_names = self.get_root_handlers_names() for handler in handlers: assert handler in actual_handler_names assert "*****@*****.**" in self.get_root_mail_handler_email()
def test_setup_logging_dict(self, smtp_config_yaml): logging_config_dict = { 'version': 1, 'handlers': { 'error_file_handler': { 'class': 'logging.FileHandler', 'level': 'ERROR', 'filename': 'errors.log', 'encoding': 'utf8', 'mode': 'w' }, 'error_mail_handler': { 'class': 'logging.handlers.SMTPHandler', 'level': 'CRITICAL', 'mailhost': 'localhost', 'fromaddr': 'noreply@localhost', 'toaddrs': '*****@*****.**', 'subject': 'SCRAPER FAILED' } }, 'root': { 'level': 'INFO', 'handlers': ['error_file_handler', 'error_mail_handler'] } } setup_logging(logging_config_dict=logging_config_dict) actual_logging_tree = format.build_description() for handler in ['File', 'SMTP']: assert handler in actual_logging_tree assert '*****@*****.**' in actual_logging_tree
def test_setup_logging_env(self, monkeypatch, logging_config_yaml): monkeypatch.setenv("LOG_FILE_ONLY", "true") setup_logging(logging_config_yaml=logging_config_yaml) actual_handler_names = self.get_root_handlers_names() assert "console" not in actual_handler_names monkeypatch.setenv("LOG_FILE_ONLY", "false") setup_logging(logging_config_yaml=logging_config_yaml) actual_handler_names = self.get_root_handlers_names() assert "console" in actual_handler_names
def pop_data_download(region_names, wp_year=2017): from hdx.utilities.easy_logging import setup_logging setup_logging() from hdx.hdx_configuration import Configuration Configuration.create(hdx_site='prod', user_agent='Read-only user', hdx_read_only=True) from hdx.data.dataset import Dataset import wpgpDownload from wpgpDownload.utils.convenience_functions import download_country_covariates as download_worldpop from wpgpDownload.utils.convenience_functions import refresh_csv refresh_csv() hdx_datasets = Dataset.search_in_hdx('hrsl', rows=500) hdx_resources = Dataset.get_all_resources(hdx_datasets) print('') country_names = set([region[0:3] for region in region_names]) for country in country_names: print(country) for res in hdx_resources: if 'population_'+country.lower() in res['name'] and '.zip' in res['name'] and 'csv' not in res['name']: print('Downloading HRSL',res['name'], end='\r') url, path = res.download() print('HRSL',res['name'],'download completed ') shutil.move(Path(path),Path('./'+country+'/misc_data/population_'+country.lower()+'.zip')) zipfile.ZipFile(Path('./'+country+'/misc_data/population_'+country.lower()+'.zip'), 'r').extractall(Path('./'+country+'/misc_data')) for file in Path('./'+country+'/misc_data').iterdir(): if 'population_'+country.lower() in file.name and file.suffix != '.tif': os.remove(file) if type(wp_year) == list: years = wp_year elif type(wp_year) == int: years = [wp_year] #NTL_files = [file for file in Path("./"+country+"/NTL").iterdir() if "NTL" in file.name] # #years = [] #for NTL_file in NTL_files: # years.append(NTL_file.name[4:8]) #years = [year for year in set(years)] #years.sort() for year in years: print('Downloading WorldPop '+country+' '+str(year)+'\t\t',end='\r') download_worldpop(ISO=country,out_folder='.\\'+country+'\\worldpop',prod_name='ppp_'+str(year)) print('WorldPop '+country+' '+str(year)+' download completed\t\t') print("") print('Done')
def test_setup_logging_smtp_dict(self): smtp_config_dict = { "handlers": { "error_mail_handler": { "toaddrs": "*****@*****.**", "subject": "lala", } } } setup_logging(smtp_config_dict=smtp_config_dict) actual_handler_names = self.get_root_handlers_names() for handler in ["console", "error_file_handler", "error_mail_handler"]: assert handler in actual_handler_names assert "*****@*****.**" in self.get_root_mail_handler_email()
def test_setup_logging_smtp_dict(self): smtp_config_dict = { 'handlers': { 'error_mail_handler': { 'toaddrs': '*****@*****.**', 'subject': 'lala' } } } setup_logging(smtp_config_dict=smtp_config_dict) actual_logging_tree = format.build_description() for handler in ['Stream', 'File', 'SMTP']: assert handler in actual_logging_tree assert '*****@*****.**' in actual_logging_tree
def hdx_acap_connector(): """Connects to HDX, and fetches acaps covid 19 government measures dataset Arguments: None Returns: pandas.DataFrame """ setup_logging() Configuration.create(hdx_site='prod', user_agent='CoronaWhy', hdx_read_only=True) dataset = Dataset.read_from_hdx( 'acaps-covid19-government-measures-dataset') logger.info("Dataset Fetched from: %s", dataset.get_hdx_url()) logger.info('Expected Update Frequency: %s', dataset.get_expected_update_frequency()) resources = dataset.get_resources() logger.info('Description: %s', resources[0]['description']) logger.info('Last Modified: %s, Revision Last Updated: %s', resources[0]['last_modified'], resources[0]['revision_last_updated']) logger.info('Size: %sMb', resources[0]['size'] / (1024**2)) logger.info('Dataset Url: %s', resources[0]['url']) logger.info('Tags: %s', dataset.get_tags()) resource = Resource.read_from_hdx(resources[0]['id']) url, absolute_path = resource.download('./') logger.info('Downloaded dataset at path: %s', absolute_path) xl = pd.ExcelFile(absolute_path) logger.info(xl.sheet_names) df = xl.parse('Database') return df
def test_setup_logging( self, logging_config_json, logging_config_yaml, smtp_config_json, smtp_config_yaml, ): with pytest.raises(FILENOTFOUND_EXCTYPE): setup_logging(smtp_config_json="NOT_EXIST") with pytest.raises(FILENOTFOUND_EXCTYPE): setup_logging( logging_config_json="NOT_EXIST", smtp_config_yaml=smtp_config_yaml, ) with pytest.raises(FILENOTFOUND_EXCTYPE): setup_logging( logging_config_yaml="NOT_EXIST", smtp_config_yaml=smtp_config_yaml, ) with pytest.raises(LoggingError): setup_logging( logging_config_yaml=logging_config_yaml, smtp_config_yaml=smtp_config_yaml, ) with pytest.raises(LoggingError): setup_logging( logging_config_dict={"la": "la"}, logging_config_json=logging_config_json, smtp_config_yaml=smtp_config_yaml, ) with pytest.raises(LoggingError): setup_logging( logging_config_dict={"la": "la"}, logging_config_yaml=logging_config_yaml, smtp_config_yaml=smtp_config_yaml, ) with pytest.raises(LoggingError): setup_logging( logging_config_json=logging_config_json, logging_config_yaml=logging_config_yaml, smtp_config_yaml=smtp_config_yaml, ) with pytest.raises(LoggingError): setup_logging( smtp_config_dict={"la": "la"}, smtp_config_json=smtp_config_json, ) with pytest.raises(LoggingError): setup_logging( smtp_config_dict={"la": "la"}, smtp_config_yaml=smtp_config_yaml, ) with pytest.raises(LoggingError): setup_logging( smtp_config_json=smtp_config_json, smtp_config_yaml=smtp_config_yaml, )
def test_setup_logging_smtp_yaml(self, smtp_config_yaml): setup_logging(smtp_config_yaml=smtp_config_yaml) actual_handler_names = self.get_root_handlers_names() for handler in ["console", "error_file_handler", "error_mail_handler"]: assert handler in actual_handler_names assert "*****@*****.**" in self.get_root_mail_handler_email()
def test_setup_logging_smtp_json(self, smtp_config_json): setup_logging(smtp_config_json=smtp_config_json) actual_handler_names = self.get_root_handlers_names() for handler in ["console", "error_file_handler", "error_mail_handler"]: assert handler in actual_handler_names assert "*****@*****.**" in self.get_root_mail_handler_email()
def test_setup_logging_json(self, logging_config_json): setup_logging(logging_config_json=logging_config_json) actual_logging_tree = format.build_description() for handler in ['Stream', 'File']: assert handler in actual_logging_tree
def test_setup_logging_json(self, logging_config_json): setup_logging(logging_config_json=logging_config_json) actual_handler_names = self.get_root_handlers_names() for handler in ["console", "error_file_handler"]: assert handler in actual_handler_names
def test_setup_logging(self, logging_config_json, logging_config_yaml, smtp_config_json, smtp_config_yaml): with pytest.raises(FILENOTFOUND_EXCTYPE): setup_logging(smtp_config_json='NOT_EXIST') with pytest.raises(FILENOTFOUND_EXCTYPE): setup_logging(logging_config_json='NOT_EXIST', smtp_config_yaml=smtp_config_yaml) with pytest.raises(FILENOTFOUND_EXCTYPE): setup_logging(logging_config_yaml='NOT_EXIST', smtp_config_yaml=smtp_config_yaml) with pytest.raises(LoggingError): setup_logging(logging_config_yaml=logging_config_yaml, smtp_config_yaml=smtp_config_yaml) with pytest.raises(LoggingError): setup_logging(logging_config_dict={'la': 'la'}, logging_config_json=logging_config_json, smtp_config_yaml=smtp_config_yaml) with pytest.raises(LoggingError): setup_logging(logging_config_dict={'la': 'la'}, logging_config_yaml=logging_config_yaml, smtp_config_yaml=smtp_config_yaml) with pytest.raises(LoggingError): setup_logging(logging_config_json=logging_config_json, logging_config_yaml=logging_config_yaml, smtp_config_yaml=smtp_config_yaml) with pytest.raises(LoggingError): setup_logging(smtp_config_dict={'la': 'la'}, smtp_config_json=smtp_config_json) with pytest.raises(LoggingError): setup_logging(smtp_config_dict={'la': 'la'}, smtp_config_yaml=smtp_config_yaml) with pytest.raises(LoggingError): setup_logging(smtp_config_json=smtp_config_json, smtp_config_yaml=smtp_config_yaml)
def test_setup_logging_smtp_yaml(self, smtp_config_yaml): setup_logging(smtp_config_yaml=smtp_config_yaml) actual_logging_tree = format.build_description() for handler in ['Stream', 'File', 'SMTP']: assert handler in actual_logging_tree assert '*****@*****.**' in actual_logging_tree
def test_setup_logging_smtp_json(self, smtp_config_json): setup_logging(smtp_config_json=smtp_config_json) actual_logging_tree = format.build_description() for handler in ['Stream', 'File', 'SMTP']: assert handler in actual_logging_tree assert '*****@*****.**' in actual_logging_tree
# -*- coding: utf-8 -*- """Facade to simplify project setup that calls project main function""" import logging from hdx.utilities.easy_logging import setup_logging from hdx.utilities.useragent import UserAgent from hdx.facades import logging_kwargs from hdx.hdx_configuration import Configuration logger = logging.getLogger(__name__) setup_logging(**logging_kwargs) def facade(projectmainfn, **kwargs): # (Callable[[None], None], Any) -> None """Facade to simplify project setup that calls project main function Args: projectmainfn ((None) -> None): main function of project **kwargs: configuration parameters to pass to HDX Configuration class Returns: None """ # # Setting up configuration # site_url = Configuration._create(**kwargs)
import shutil import os import logging from hdx.utilities.easy_logging import setup_logging from hdx.hdx_configuration import Configuration from hdx.data.dataset import Dataset HDX_SITE = "prod" USER_AGENT = "MapAction" setup_logging() logger = logging.getLogger() Configuration.create(hdx_site=HDX_SITE, user_agent=USER_AGENT, hdx_read_only=True) def query_api(hdx_address, directory, resource_format="XLSX"): dataset = Dataset.read_from_hdx(hdx_address) resources = dataset.get_resources() filenames = {} for resource in resources: if resource["format"] == resource_format: _, path = resource.download() filename = os.path.basename(path) shutil.move(path, os.path.join(directory, filename)) filenames[resource["name"]] = filename logging.info( f'Saved "{resource["name"]}" to "{directory}/{filename}"') return filenames
def test_setup_logging_yaml(self, monkeypatch, logging_config_yaml): setup_logging(logging_config_yaml=logging_config_yaml) actual_handler_names = self.get_root_handlers_names() for handler in ["console", "error_mail_handler"]: assert handler in actual_handler_names assert "*****@*****.**" in self.get_root_mail_handler_email()