def open(self,
user=pika.connection.Parameters.DEFAULT_USERNAME,
password=pika.connection.Parameters.DEFAULT_PASSWORD,
host=pika.connection.Parameters.DEFAULT_PORT,
port=pika.connection.Parameters.DEFAULT_PORT,
virtual_host=pika.connection.Parameters.DEFAULT_VIRTUAL_HOST):
try:
self.__user = user
self.__password = password
self.__host = host
self.__port = port
self.__virtual_host = virtual_host
credentials = pika.PlainCredentials(username=user,
password=password)
params = pika.ConnectionParameters(host=host,
port=port,
virtual_host=virtual_host,
credentials=credentials)
self.connection = pika.BlockingConnection(params)
self.channel = self.connection.channel()
except pika.exceptions.AMQPError as err:
Logger.error(__file__, err.args)
return None
def exchange_unbind(self, destination, source, routing_key=''):
try:
return self.channel.exchange_unbind(destination=destination,
source=source,
routing_key=routing_key)
except pika.exceptions.AMQPError as err:
Logger.error(__file__, err.args)
def queue_delete(self, queue_name, if_unused=False, if_empty=False):
try:
return self.channel.queue_delete(queue_name,
if_unused=if_unused,
if_empty=if_empty)
except pika.exceptions.AMQPError as err:
Logger.error(__file__, err.args)
def get_db_report_date(cls):
from Service.LoggerService.Implementation.DefaultPythonLoggingService import \
DefaultPythonLoggingService as Logger
from Config.Configurations import Configuration
from Config.Configurations import ValuesNames as Values
from Service.DbService.Implementation.MySqlService import MySqlService
from Entities.StatisticsDataStorage import StatisticsDataStorage
Logger.info(__file__, 'Getting statistic from db started')
mysql_settings = Configuration().settings[Values.MYSQL_SECTION_NAME]
mysql = MySqlService(user=mysql_settings[Values.MYSQL_USER],
password=mysql_settings[Values.MYSQL_PASSWORD],
host=mysql_settings[Values.MYSQL_HOST],
port=mysql_settings[Values.MYSQL_PORT],
database=mysql_settings[Values.MYSQL_DB_NAME])
try:
mysql.open_connection()
for (value, name) in mysql.execute(Values.MYSQL_GET_REPORT_QUERY,
select=True):
if name == '1':
name = 'Red zone orders avg amount'
if name == '2':
name = 'Green zone orders avg amount'
if name == '3':
name = 'Blue zone orders avg amount'
StatisticsDataStorage.statistics[name] = value
Logger.info(__file__, 'Database service configurated')
except AttributeError as er:
Logger.error(__file__, er.args)
Logger.info(__file__, 'Sending records to MySQL aborted')
Logger.info(__file__, 'Getting statistic from db finished')
def queue_unbind(self, queue_name, exchange_name, routing_key=None):
try:
return self.channel.queue_unbind(queue_name,
exchange_name,
routing_key=routing_key)
except pika.exceptions.AMQPError as err:
Logger.error(__file__, err.args)
def get_cursor(self):
try:
return self.__conn.cursor()
except mysql.connector.Error as err:
Logger.error(__file__, err.msg)
if not self.is_available():
self.reconnect()
self.get_cursor()
def commit(self):
try:
self.__conn.commit()
return True
except mysql.connector.Error as err:
Logger.error(__file__, err.msg)
self.rollback()
return False
def consume(self, queue_name, on_consume_callback, reconnect=False):
try:
if not reconnect:
self.consume_info[queue_name] = on_consume_callback
self.channel.basic_consume(queue=queue_name,
on_message_callback=on_consume_callback)
except pika.exceptions.AMQPError as err:
Logger.error(__file__, err.args)
def __init__(self, driver_path="chromedriver", options=None):
Logger.info(__file__, "SeleniumCrawler init")
try:
self.driver = Chrome(executable_path=driver_path, options=options)
Logger.info(__file__, "Driver initiated")
except WebDriverException as err:
Logger.error(__file__, err.args)
self.configs = Configuration()
def __open_conn(self):
try:
self.__conn = None
self.__conn = mysql.connector.connect(user=self.__user,
password=self.__password,
host=self.__host,
port=self.__port,
database=self.__database)
except mysql.connector.Error as err:
Logger.error(__file__, err.msg)
def declare_queue(self,
queue_name,
durable=True,
exclusive=False,
auto_delete=False):
try:
return self.channel.queue_declare(queue=queue_name,
durable=durable,
exclusive=exclusive,
auto_delete=auto_delete)
except pika.exceptions.AMQPError as err:
Logger.error(__file__, err.args)
def open(self, path, mode='r'):
if path is None:
if self.path is not None:
raise TypeError('Path parameter of file descriptor must bo not None')
else:
path = self.path
if mode is None:
raise TypeError('Mode parameter of file descriptor must bo not None')
try:
self.__file = open(file=path, mode=mode, newline='')
except Exception as err:
Logger.error(__file__, err.__str__())
def init_rmq(self):
rmq_settings = self.configs.settings[Values.RMQ_SECTION_NAME]
self.rmq = RmqService()
while not self.rmq.open_connection(
host=rmq_settings[Values.RMQ_HOST],
port=rmq_settings[Values.RMQ_PORT],
virtual_host=rmq_settings[Values.RMQ_VIRTUAL_HOST],
user=rmq_settings[Values.RMQ_USER],
password=rmq_settings[Values.RMQ_PASSWORD]):
self.rmq.reconfig()
self.rmq.exchange_delete(
exchange_name=rmq_settings[Values.RMQ_EXCHANGE_NAME])
try:
self.rmq.declare_exchange(
exchange_name=rmq_settings[Values.RMQ_EXCHANGE_NAME],
exchange_type=ExchangeType(
rmq_settings[Values.RMQ_EXCHANGE_TYPE]))
except ValueError as er:
Logger.error(__file__, er.args)
Logger.info(__file__, 'Sending records to RabbitMQ aborted')
return
self.rmq.declare_queue(queue_name=Zone.RED.value,
durable=bool(
rmq_settings[Values.RMQ_DURABLE_QUEUES]))
self.rmq.declare_queue(queue_name=Zone.BLUE.value,
durable=bool(
rmq_settings[Values.RMQ_DURABLE_QUEUES]))
self.rmq.declare_queue(queue_name=Zone.GREEN.value,
durable=bool(
rmq_settings[Values.RMQ_DURABLE_QUEUES]))
self.rmq.queue_bind(
Zone.RED.value, rmq_settings[Values.RMQ_EXCHANGE_NAME],
rmq_settings[Values.RMQ_EXCHANGE_RED_RECORDS_ROUTING_KEY])
self.rmq.queue_bind(
Zone.BLUE.value, rmq_settings[Values.RMQ_EXCHANGE_NAME],
rmq_settings[Values.RMQ_EXCHANGE_BLUE_RECORDS_ROUTING_KEY])
self.rmq.queue_bind(
Zone.GREEN.value, rmq_settings[Values.RMQ_EXCHANGE_NAME],
rmq_settings[Values.RMQ_EXCHANGE_GREEN_RECORDS_ROUTING_KEY])
self.rmq.queue_purge(queue_name=Zone.RED.value)
self.rmq.queue_purge(queue_name=Zone.BLUE.value)
self.rmq.queue_purge(queue_name=Zone.GREEN.value)
def execute_multiple(self, query, params):
try:
while True:
if not self.__connection.is_available():
self.__connection.open()
cursor = self.__connection.get_cursor()
if cursor is not None:
cursor.executemany(query, params)
if self.__connection.commit():
break
except mysql.connector.Error as err:
Logger.error(__file__, err.msg)
def __load_currency_pairs_from_file(self):
'''
Load currency pairs from file by path in config object
'''
Logger.info(
__file__, 'Start loading currency pairs from file {}'.format(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.CURRENCY_PAIRS_FILE_PATH]))
try:
with open(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.CURRENCY_PAIRS_FILE_PATH], "r") as file:
for line in file.readlines():
if re.fullmatch(
r'[a-zA-Z]{3}\/[a-zA-Z]{3};[0-9]+([\.|\,][0-9]{0,5})',
line.replace('\n', '')):
parts = line.split(";")
self.configs.currency_pairs.append({
Values.CURRENCY_PAIR_NAME:
parts[0],
Values.CURRENCY_PAIR_VALUE:
float(parts[1].replace("\n", '').replace(',', '.'))
})
self.configs.settings[Values.CURRENCY_PAIR_GENERATOR][
LCGParams.MODULUS.value] = len(self.configs.currency_pairs)
Logger.info(
__file__,
'Loading currency pairs from file {} finished'.format(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.CURRENCY_PAIRS_FILE_PATH]))
Logger.debug(
__file__, 'Loaded {} currency pairs: {}'.format(
len(self.configs.currency_pairs),
self.configs.currency_pairs))
except:
Logger.error(
__file__, "Loading currency pairs from file {} failed".format(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.CURRENCY_PAIRS_FILE_PATH]))
if len(self.configs.currency_pairs) == 0:
exit(-1)
def __load_tags_from_file(self):
'''
Load tags from file by path in config object
'''
Logger.info(
__file__,
'Start loading tags from file {}'.format(self.configs.settings[
Values.GENERAL_SECTION_NAME][Values.TAGS_FILE_PATH]))
try:
with open(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.TAGS_FILE_PATH], "r") as file:
for line in file.readlines():
if re.fullmatch('[a-zA-Z]{0,10}', line.replace('\n', '')):
self.configs.tags.append(line.replace("\n", ''))
Logger.info(
__file__, 'Loading tags from file {} finished'.format(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.TAGS_FILE_PATH]))
Logger.debug(
__file__,
'Loaded {} tags: {}'.format(len(self.configs.tags),
', '.join(self.configs.tags)))
self.configs.settings[Values.TAGS_GENERATOR_1][
LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5)
self.configs.settings[Values.TAGS_GENERATOR_2][
LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5)
self.configs.settings[Values.TAGS_GENERATOR_3][
LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5)
self.configs.settings[Values.TAGS_GENERATOR_4][
LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5)
self.configs.settings[Values.TAGS_GENERATOR_5][
LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5)
except:
Logger.error(
__file__, "Loading tags from file {} failed".format(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.TAGS_FILE_PATH]))
if len(self.configs.tags) == 0:
exit(-2)
def declare_exchange(self,
exchange_name,
exchange_type,
passive=False,
durable=True,
auto_delete=False):
try:
self.exchanges_bindings[exchange_name] = list()
self.exchanges_bindings[exchange_name].append(exchange_type)
return self.channel.exchange_declare(
exchange=exchange_name,
exchange_type=exchange_type.value,
passive=passive,
durable=durable,
auto_delete=auto_delete)
except pika.exceptions.AMQPError as err:
Logger.error(__file__, err.args)
def execute_many(self, queries):
try:
if not self.__connection.is_available():
self.__connection.open()
cursor = self.__connection.get_cursor()
Logger.debug(__file__, 'Executing many queries started')
if cursor is not None:
for query in queries:
Logger.debug(__file__, 'Executing query:{}'.format(query))
cursor.execute(query)
self.__connection.commit()
except mysql.connector.Error as err:
Logger.error(__file__, err.msg)
Logger.debug(__file__, 'Executing many queries finished')
def execute(self, query, select=False, multi=False, *args, **kwargs):
try:
while True:
if not self.__connection.is_available():
self.__connection.open()
cursor = self.__connection.get_cursor()
if cursor is not None:
Logger.debug(__file__, 'Executing query:{}'.format(query))
cursor.execute(query, multi)
if not select:
if self.__connection.commit():
break
else:
return cursor
except mysql.connector.Error as err:
Logger.error(__file__, err.msg)
def configurate_db_service(self):
Logger.info(__file__, 'Configuration database service...')
mysql_settings = self.configs.settings[Values.MYSQL_SECTION_NAME]
self.mysql = MySqlService(
user=mysql_settings[Values.MYSQL_USER],
password=mysql_settings[Values.MYSQL_PASSWORD],
host=mysql_settings[Values.MYSQL_HOST],
port=mysql_settings[Values.MYSQL_PORT],
database=mysql_settings[Values.MYSQL_DB_NAME])
try:
self.mysql.open_connection()
self.mysql.execute(
f'TRUNCATE `{self.configs.settings[Values.MYSQL_SECTION_NAME][Values.MYSQL_DB_NAME]}`.`history`'
)
Logger.info(__file__, 'Database service configurated')
except AttributeError as er:
Logger.error(__file__, er.args)
Logger.info(__file__, 'Sending records to MySQL aborted')
def rollback(self):
try:
self.__conn.rollback()
except mysql.connector.Error as err:
Logger.error(__file__, err.msg)
def exchange_delete(self, exchange_name=None, if_unused=False):
try:
return self.channel.exchange_delete(exchange=exchange_name,
if_unused=if_unused)
except pika.exceptions.AMQPError as err:
Logger.error(__file__, err.args)
def stop_consuming(self):
try:
self.channel.stop_consuming()
except pika.exceptions.AMQPError as err:
Logger.error(__file__, err.args)
def close(self, *args, **kwargs):
try:
self.__conn.close()
except mysql.connector.Error as err:
Logger.error(__file__, err.msg)
def close(self, *args, **kwargs):
try:
if self.connection.is_open:
self.close()
except pika.exceptions.AMQPError as err:
Logger.error(__file__, err.args)
def close(self, *args, **kwargs):
try:
self.__file.close()
except Exception as err:
Logger.error(__file__, err.__str__())
def queue_purge(self, queue_name):
try:
return self.channel.queue_purge(queue_name)
except pika.exceptions.AMQPError as err:
Logger.error(__file__, err.args)
def get_elements(self, by, value, wait=0):
try:
return WebDriverWait(self.driver, wait).until(
EC.presence_of_all_elements_located((by, value)))
except TimeoutException as err:
Logger.error(__file__, err.args)