def queue_delete(self, queue_name, if_unused=False, if_empty=False):
try:
return self.channel.queue_delete(queue_name,
if_unused=if_unused,
if_empty=if_empty)
except pika.exceptions.AMQPError as err:
Logger.error(__file__, err.args)
def exchange_unbind(self, destination, source, routing_key=''):
try:
return self.channel.exchange_unbind(destination=destination,
source=source,
routing_key=routing_key)
except pika.exceptions.AMQPError as err:
Logger.error(__file__, err.args)
def queue_unbind(self, queue_name, exchange_name, routing_key=None):
try:
return self.channel.queue_unbind(queue_name,
exchange_name,
routing_key=routing_key)
except pika.exceptions.AMQPError as err:
Logger.error(__file__, err.args)
def __generate_orders(self):
'''
Generate green zone orders records with generators configurations
'''
Logger.info(__file__, 'Started generation order history in green zone')
previous_time = datetime.datetime.now()
t = 0
for period in range(len(self.configs.orders_volumes_for_generation)):
zone_index = 0
for zone in self.configs.orders_volumes_for_generation[period]:
for i in range(
int(self.configs.orders_volumes_for_generation[period]
[zone])):
order = self.__generate_general_order_information()
self.history.orders[order.id] = order
records = self.__get_order_in_zone(
order.id, order.status_sequence, period, zone,
order.statuses_in_blue_zone)
self.history.records.extend(records)
self.inc_statistic('Generated orders', 1)
self.inc_statistic('Generated records', len(records))
if len(self.history.orders) % self.configs.settings[
Values.GENERAL_SECTION_NAME][
Values.BATCH_SIZE] == 0:
self.add_time_statistic(
'Order history generation',
(datetime.datetime.now() -
previous_time).total_seconds() * 1000)
self.send_to_rmq()
previous_time = datetime.datetime.now()
zone_index += 1
def report(data):
print()
Logger.info(__file__, 'Start reporting')
print()
print('========== REPORT ==========')
for key in data:
item = data[key]
print(f'----- {key} -----')
if isinstance(item, list):
if len(item) > 1:
max_value = max(item)
min_value = min(item)
avg_value = sum(item) / len(item)
total = sum(item)
print(f'Max: {max_value} ms')
print(f'Min: {min_value} ms')
print(f'Avg: {avg_value} ms')
print(f'Total: {total} ms')
else:
print(f'{key}: {item[0]} ms')
else:
print(item)
print()
print('========== REPORT END ==========')
print()
Logger.info(__file__, 'Reporting finished')
print()
def declare_exchange(self,
exchange_name,
exchange_type,
passive=False,
durable=True,
auto_delete=False):
'''
Declare new exchange
:param exchange_name: exchange name
:param exchange_type: exhange type
:param passive:
:param durable:
:param auto_delete:
:return:
'''
Logger.debug(
__file__, 'Declaring exchange {} with exchange type'.format(
exchange_name, exchange_type.value))
return self.conn.declare_exchange(exchange_name,
exchange_type,
passive=passive,
durable=durable,
auto_delete=auto_delete)
def __calculate_first_generation_period_start_date(self):
'''
Calculate start date of first generation perid
'''
Logger.info(
__file__,
'Starting calculating start date of first period for generation orders'
)
current_date = datetime.datetime.today().replace(hour=0,
minute=0,
second=0,
microsecond=0)
day_of_week = current_date.weekday() + 1
self.configs.is_current_date_in_trading_period = day_of_week in (1, 2,
5)
days_to_last_period = day_of_week + 7 - 5 if day_of_week < 5 else day_of_week - 5
days_to_start_period = days_to_last_period + (
7 * (len(self.configs.orders_volumes_for_generation)))
self.configs.start_date = current_date + datetime.timedelta(
days=-days_to_start_period)
Logger.info(
__file__,
'Calculating start date of first period for generation orders finished. Start at {}'
.format(self.configs.start_date))
def __add_parameter(self, cfg_parser, section_name, parameter_name, value):
cfg_parser.set(section_name, parameter_name, str(value))
Logger.add_to_journal(
__file__, Level.DEBUG,
"Added parameter '{}' to section {} with value: {}".format(
parameter_name, section_name, value))
def execute_generation(self):
'''
Execute generation orders history
'''
self.history.clear_history()
Logger.info(__file__, 'Generating order history started')
self.__generate_orders()
Logger.info(__file__, 'Generating order history finished')
self.rmq.publish(
self.configs.settings[Values.RMQ_SECTION_NAME][
Values.RMQ_EXCHANGE_NAME],
self.configs.settings[Values.RMQ_SECTION_NAME][
Values.RMQ_EXCHANGE_GREEN_RECORDS_ROUTING_KEY], 'stop')
self.rmq.publish(
self.configs.settings[Values.RMQ_SECTION_NAME][
Values.RMQ_EXCHANGE_NAME],
self.configs.settings[Values.RMQ_SECTION_NAME][
Values.RMQ_EXCHANGE_RED_RECORDS_ROUTING_KEY], 'stop')
self.rmq.publish(
self.configs.settings[Values.RMQ_SECTION_NAME][
Values.RMQ_EXCHANGE_NAME],
self.configs.settings[Values.RMQ_SECTION_NAME][
Values.RMQ_EXCHANGE_BLUE_RECORDS_ROUTING_KEY], 'stop')
self.fininsh_event.set()
def __msg_consumer(self, channel, method, header, body):
if body == b'stop':
self.stop_messages_count += 1
if self.stop_messages_count == 3:
self.send_consumed_data_to_mysql()
self.rmq.stop_consuming()
self.finish_event.set()
else:
self.set_stat('Consumed messages', 1)
order_record = Entities.Protobuf.OrderInformation_pb2.OrderInformation(
)
order_record.ParseFromString(body)
self.to_data_list(order_record)
if len(self.consumed_data) == self.configs.settings[
Values.GENERAL_SECTION_NAME][Values.BATCH_SIZE]:
OrderHistoryMaker.add_time_statistic(
'Consuming data from RabbitMQ',
(datetime.datetime.now() -
self.previous_time).total_seconds() * 1000)
Logger.info(__file__, "Batch size data consumed")
self.send_consumed_data_to_mysql()
self.previous_time = datetime.datetime.now()
channel.basic_ack(delivery_tag=method.delivery_tag)
def get_db_report_date(cls):
from Service.LoggerService.Implementation.DefaultPythonLoggingService import \
DefaultPythonLoggingService as Logger
from Config.Configurations import Configuration
from Config.Configurations import ValuesNames as Values
from Service.DbService.Implementation.MySqlService import MySqlService
from Entities.StatisticsDataStorage import StatisticsDataStorage
Logger.info(__file__, 'Getting statistic from db started')
mysql_settings = Configuration().settings[Values.MYSQL_SECTION_NAME]
mysql = MySqlService(user=mysql_settings[Values.MYSQL_USER],
password=mysql_settings[Values.MYSQL_PASSWORD],
host=mysql_settings[Values.MYSQL_HOST],
port=mysql_settings[Values.MYSQL_PORT],
database=mysql_settings[Values.MYSQL_DB_NAME])
try:
mysql.open_connection()
for (value, name) in mysql.execute(Values.MYSQL_GET_REPORT_QUERY,
select=True):
if name == '1':
name = 'Red zone orders avg amount'
if name == '2':
name = 'Green zone orders avg amount'
if name == '3':
name = 'Blue zone orders avg amount'
StatisticsDataStorage.statistics[name] = value
Logger.info(__file__, 'Database service configurated')
except AttributeError as er:
Logger.error(__file__, er.args)
Logger.info(__file__, 'Sending records to MySQL aborted')
Logger.info(__file__, 'Getting statistic from db finished')
def set_general_order_info(self, general_order_info):
if isinstance(general_order_info, GeneralOrderInformation):
self.__order_general_information = general_order_info
Logger.debug(__file__,
'Set general order info parameter to {}'.format(self.__order_general_information.__str__()))
return self
def open(self,
user=pika.connection.Parameters.DEFAULT_USERNAME,
password=pika.connection.Parameters.DEFAULT_PASSWORD,
host=pika.connection.Parameters.DEFAULT_PORT,
port=pika.connection.Parameters.DEFAULT_PORT,
virtual_host=pika.connection.Parameters.DEFAULT_VIRTUAL_HOST):
try:
self.__user = user
self.__password = password
self.__host = host
self.__port = port
self.__virtual_host = virtual_host
credentials = pika.PlainCredentials(username=user,
password=password)
params = pika.ConnectionParameters(host=host,
port=port,
virtual_host=virtual_host,
credentials=credentials)
self.connection = pika.BlockingConnection(params)
self.channel = self.connection.channel()
except pika.exceptions.AMQPError as err:
Logger.error(__file__, err.args)
return None
def queue_unbind(self, queue_name, exchange_name, routing_key=None):
Logger.debug(
__file__,
'Unbinding queue {} from exchange {} with routing key {}'.format(
queue_name, exchange_name, routing_key))
return self.conn.queue_unbind(queue_name=queue_name,
exchange_name=exchange_name,
routing_key=routing_key)
def report(self):
Logger.info(__file__, 'Start reporting')
print('Start getting reporting data at {}'.format(
datetime.datetime.now()))
Utils.get_db_report_date()
ConsoleReporter.report(StatisticsDataStorage.statistics)
print('Reporter finished data at {}'.format(datetime.datetime.now()))
Logger.info(__file__, 'Reporting finished')
def commit(self):
try:
self.__conn.commit()
return True
except mysql.connector.Error as err:
Logger.error(__file__, err.msg)
self.rollback()
return False
def get_cursor(self):
try:
return self.__conn.cursor()
except mysql.connector.Error as err:
Logger.error(__file__, err.msg)
if not self.is_available():
self.reconnect()
self.get_cursor()
def consume(self, queue_name, on_consume_callback, reconnect=False):
try:
if not reconnect:
self.consume_info[queue_name] = on_consume_callback
self.channel.basic_consume(queue=queue_name,
on_message_callback=on_consume_callback)
except pika.exceptions.AMQPError as err:
Logger.error(__file__, err.args)
def start(self):
Logger.add_to_journal(__file__, Level.INFO, 'Launcher started')
self.__load_configs()
self.__execute()
self.generator_and_publisher_thread = None
self.consumer_thread = None
self.generator_and_publisher_event = None
self.consumer_event = None
self.report_thread = None
def __open_conn(self):
try:
self.__conn = None
self.__conn = mysql.connector.connect(user=self.__user,
password=self.__password,
host=self.__host,
port=self.__port,
database=self.__database)
except mysql.connector.Error as err:
Logger.error(__file__, err.msg)
def __get_friends_list(self):
Logger.info(__file__, "Load friends list web elements")
list = self.get_elements(by=By.XPATH,
value=self.__FRIENDS_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME])
Logger.debug(__file__, f"Size of loaded friends list: {len(list)}")
return list
def queue_purge(self, queue_name):
'''
Clearing queue
:param queue_name: queue name
:return:
'''
Logger.debug(__file__, 'Purging queue {}'.format(queue_name))
return self.conn.queue_purge(queue_name)
def open(self):
self.__open_conn()
if not self.is_available():
self.reconnect()
else:
Logger.debug(__file__, 'Created mysql connection with params {} {} {} {}'.format(self.__user,
self.__password,
self.__host,
self.__database))
def declare_queue(self, queue_name, durable):
'''
Declare new queue
:param queue_name: queue name
:return:
'''
Logger.debug(__file__,
'Declaring queue with name {}'.format(queue_name))
return self.conn.declare_queue(queue_name=queue_name, durable=durable)
def declare_queue(self,
queue_name,
durable=True,
exclusive=False,
auto_delete=False):
try:
return self.channel.queue_declare(queue=queue_name,
durable=durable,
exclusive=exclusive,
auto_delete=auto_delete)
except pika.exceptions.AMQPError as err:
Logger.error(__file__, err.args)
def exchange_delete(self, exchange_name=None, if_unused=False):
'''
Delete exchange
:param exchange_name: exchange name
:param if_unused:
:return:
'''
Logger.debug(__file__, 'Deleting exchange {}'.format(exchange_name))
return self.conn.exchange_delete(exchange_name=exchange_name,
if_unused=if_unused)
def exchange_unbind(self, destination, source, routing_key=''):
'''
Unind exchange
'''
Logger.debug(
__file__,
'Unbind exchange: destination {}, sourse, routing_key'.format(
destination, source, routing_key))
return self.conn.exchange_unbind(destination=destination,
source=source,
routing_key=routing_key)
def __init__(self,
user='******',
password='',
host='127.0.0.1',
port='3306',
database=''):
self.__connection = MySqlConnection(user=user,
password=password,
host=host,
port=port,
database=database)
Logger.debug(__file__, 'Created mysql service')
def open(self, path, mode='r'):
if path is None:
if self.path is not None:
raise TypeError('Path parameter of file descriptor must bo not None')
else:
path = self.path
if mode is None:
raise TypeError('Mode parameter of file descriptor must bo not None')
try:
self.__file = open(file=path, mode=mode, newline='')
except Exception as err:
Logger.error(__file__, err.__str__())
def send_consumed_data_to_mysql(self):
Logger.info(__file__, 'Sending readed batch records to MySQL')
self.previous_time = datetime.datetime.now()
self.mysql.execute_multiple(Values.MYSQL_INSERT_QUERY,
self.consumed_data)
OrderHistoryMaker.add_time_statistic(
'Send data to MySQL',
(datetime.datetime.now() - self.previous_time).total_seconds() *
1000)
self.consumed_data.clear()
