def load_data(self): config = cfg.getConfig(AUTH_CONFIG_LOCATION) homePath = cfg.getHomePath() self.pathToCertificate = homePath + config['pathToCert'] self.pathToCertificatePass = homePath + config['pathToCertKey'] self.password = self._read_password_file(config['passwordFile']) return self
def load_data(self): config_dir = pathlib.Path(__file__).parent / "config" config = cfg.getConfig(config_dir / AUTH_CONFIG_FILE) self.certificate_path = self._assure_file_path( config_dir / config['cert_path'], "Certificate") self.certificate_pass_path = self._assure_file_path( config_dir / config['cert_key_path'], "Certificate password") self.password = self._read_password_file( config_dir / config.get('password_file', "")) return self
import ssl import json import aiohttp from config import configUtils from errors.errors import FetchError from dataFetching.authContainer import AuthContainer from errors.errors import ConfigError import time FETCH_CONFIG_LOCATION="dataFetching/config/fetch.config.ini" PATH_STEPS = configUtils.getConfig(FETCH_CONFIG_LOCATION)["matrix_json_path"].split(".") class AsyncRequestExecutor(): def __init__(self, session: aiohttp.ClientSession, authContainer: AuthContainer): self.session = session self.ssl_context = self._init_ssl_context(authContainer) async def get_matrix_from_protected_url(self, url): dataJsonString = await self._get_json_data_from_protected_url(url) dataJson = self._parse_json(dataJsonString, url) return self._navigate_json_to_matrix(dataJson) async def _get_json_data_from_protected_url(self, url): result = await self.session.get(url, ssl=self.ssl_context) return await result.content.read() def _parse_json(self, dataJson, url): try: return json.loads(dataJson) except ValueError as ve: raise FetchError(f"Invalid json structure from url:{url} \n Error: {ve}")
def _get_default_database_config(): config = getConfig(DB_CONFIG_LOCATION) database = config["database"] collection = config["collection"] return database, collection
from config.configUtils import getJsonConfig, getConfig import json import warnings from typing import Generator structureConfig: dict = getJsonConfig( "dataFetching/config/param.list.config.json") url = getConfig("dataFetching/config/fetch.config.ini")["url"] def get_url_generator(identifier: dict): """Generates urls to fetch data for each identifier based on params provided at param.config file""" param_dicts = _get_param_dicts() if (param_dicts is not None): for params in _get_param_dicts(): yield url.format(**identifier, **params), params else: yield url.format(**identifier), None def _get_param_dicts(): parameters = None for key in structureConfig.keys(): if structureConfig[key]["type"] == "static": parameters = _handle_static_params(key, parameters) elif structureConfig[key]["type"] == "dependant": parameters = _handle_dependant_params(key, parameters) else: warnings.warn( f"unrecognized param type {structureConfig[key]['type']}") return parameters