def load_data(self):
config = cfg.getConfig(AUTH_CONFIG_LOCATION)
homePath = cfg.getHomePath()
self.pathToCertificate = homePath + config['pathToCert']
self.pathToCertificatePass = homePath + config['pathToCertKey']
self.password = self._read_password_file(config['passwordFile'])
return self
def load_data(self):
config_dir = pathlib.Path(__file__).parent / "config"
config = cfg.getConfig(config_dir / AUTH_CONFIG_FILE)
self.certificate_path = self._assure_file_path(
config_dir / config['cert_path'], "Certificate")
self.certificate_pass_path = self._assure_file_path(
config_dir / config['cert_key_path'], "Certificate password")
self.password = self._read_password_file(
config_dir / config.get('password_file', ""))
return self
import ssl
import json
import aiohttp
from config import configUtils
from errors.errors import FetchError
from dataFetching.authContainer import AuthContainer
from errors.errors import ConfigError
import time
FETCH_CONFIG_LOCATION="dataFetching/config/fetch.config.ini"
PATH_STEPS = configUtils.getConfig(FETCH_CONFIG_LOCATION)["matrix_json_path"].split(".")
class AsyncRequestExecutor():
def __init__(self, session: aiohttp.ClientSession, authContainer: AuthContainer):
self.session = session
self.ssl_context = self._init_ssl_context(authContainer)
async def get_matrix_from_protected_url(self, url):
dataJsonString = await self._get_json_data_from_protected_url(url)
dataJson = self._parse_json(dataJsonString, url)
return self._navigate_json_to_matrix(dataJson)
async def _get_json_data_from_protected_url(self, url):
result = await self.session.get(url, ssl=self.ssl_context)
return await result.content.read()
def _parse_json(self, dataJson, url):
try:
return json.loads(dataJson)
except ValueError as ve:
raise FetchError(f"Invalid json structure from url:{url} \n Error: {ve}")
def _get_default_database_config():
config = getConfig(DB_CONFIG_LOCATION)
database = config["database"]
collection = config["collection"]
return database, collection
from config.configUtils import getJsonConfig, getConfig
import json
import warnings
from typing import Generator
structureConfig: dict = getJsonConfig(
"dataFetching/config/param.list.config.json")
url = getConfig("dataFetching/config/fetch.config.ini")["url"]
def get_url_generator(identifier: dict):
"""Generates urls to fetch data for each identifier based on params provided at param.config file"""
param_dicts = _get_param_dicts()
if (param_dicts is not None):
for params in _get_param_dicts():
yield url.format(**identifier, **params), params
else:
yield url.format(**identifier), None
def _get_param_dicts():
parameters = None
for key in structureConfig.keys():
if structureConfig[key]["type"] == "static":
parameters = _handle_static_params(key, parameters)
elif structureConfig[key]["type"] == "dependant":
parameters = _handle_dependant_params(key, parameters)
else:
warnings.warn(
f"unrecognized param type {structureConfig[key]['type']}")
return parameters