class ConfigReader:
"""
read user configurations
:param start_time: strftime => start time of the application for log timestamp
:param file: string => name of the configuration file in config/
"""
def __init__(self,
start_time=strftime("%Y-%m-%d %H.%M.%S"),
file="application_configurations.json"):
self._logger = Logger(name=__name__,
start_time=start_time).get_logger()
file_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
separator = SystemSpec().get_separator()
self.configurations = file_root + separator + "config" + separator + file
def get_configurations(self):
try:
with open(self.configurations) as file:
self._logger.info("reading configurations from {}".format(
self.configurations))
data = json.load(file)
return data
except FileNotFoundError:
self._logger.exception("configuration file: {} not found !".format(
self.configurations))
sys.exit(2)
class ScrapperAbstract(ABC):
"""
:param args: list[String] => list of ticker names
:param store_location: String => root directory on hard drive to save output
:param folder_name: String => folder name to be created in the directory of store_location
:param file_save: Boolean => whether to save the output
:param start_time: strftime => start time of the application for log timestamp
:param logger_name : String => default __name__
"""
def __init__(self,
tickers,
store_location,
folder_name='test_folder',
file_save=False,
start_time=strftime("%Y-%m-%d %H.%M.%S"),
logger_name=__name__):
self._tickers = tickers
self._store_location = store_location
self._folder_name = folder_name
self._file_save = file_save
self._separator = SystemSpec.get_separator()
self._logger = Logger(name=logger_name,
start_time=start_time).get_logger()
self._lock = Lock()
self._application_logic = ConfigReader(
file="application_logic.json").get_configurations()
def requester(self, url):
"""
:param url: web url to be requested
:return: parsed html
"""
self._logger.info("Sending request to url: {}".format(url))
try:
r = requests.get(url, timeout=0.5)
while r.status_code != 200:
self._logger.error(
"Attempt failed with status code: {}. Retrying...".format(
r.status_code))
r = requests.get(url, timeout=0.5)
except requests.exceptions.Timeout:
self._logger.error(
"Attempt timed out for url: {}, retrying now...".format(url))
return self.requester(url)
return Soup(r.text, 'html.parser')
@abstractmethod
def data_parser(self, ticker):
pass
@abstractmethod
def run(self):
pass
def test(self):
logger = Logger(__name__).get_logger()
with self.assertLogs(logger) as cm:
logger.info("test message")
self.assertEqual(cm.output, ["INFO:{}:test message".format(__name__)])