def report(data):
print()
Logger.info(__file__, 'Start reporting')
print()
print('========== REPORT ==========')
for key in data:
item = data[key]
print(f'----- {key} -----')
if isinstance(item, list):
if len(item) > 1:
max_value = max(item)
min_value = min(item)
avg_value = sum(item) / len(item)
total = sum(item)
print(f'Max: {max_value} ms')
print(f'Min: {min_value} ms')
print(f'Avg: {avg_value} ms')
print(f'Total: {total} ms')
else:
print(f'{key}: {item[0]} ms')
else:
print(item)
print()
print('========== REPORT END ==========')
print()
Logger.info(__file__, 'Reporting finished')
print()
def __calculate_first_generation_period_start_date(self):
'''
Calculate start date of first generation perid
'''
Logger.info(
__file__,
'Starting calculating start date of first period for generation orders'
)
current_date = datetime.datetime.today().replace(hour=0,
minute=0,
second=0,
microsecond=0)
day_of_week = current_date.weekday() + 1
self.configs.is_current_date_in_trading_period = day_of_week in (1, 2,
5)
days_to_last_period = day_of_week + 7 - 5 if day_of_week < 5 else day_of_week - 5
days_to_start_period = days_to_last_period + (
7 * (len(self.configs.orders_volumes_for_generation)))
self.configs.start_date = current_date + datetime.timedelta(
days=-days_to_start_period)
Logger.info(
__file__,
'Calculating start date of first period for generation orders finished. Start at {}'
.format(self.configs.start_date))
def execute_generation(self):
'''
Execute generation orders history
'''
self.history.clear_history()
Logger.info(__file__, 'Generating order history started')
self.__generate_orders()
Logger.info(__file__, 'Generating order history finished')
self.rmq.publish(
self.configs.settings[Values.RMQ_SECTION_NAME][
Values.RMQ_EXCHANGE_NAME],
self.configs.settings[Values.RMQ_SECTION_NAME][
Values.RMQ_EXCHANGE_GREEN_RECORDS_ROUTING_KEY], 'stop')
self.rmq.publish(
self.configs.settings[Values.RMQ_SECTION_NAME][
Values.RMQ_EXCHANGE_NAME],
self.configs.settings[Values.RMQ_SECTION_NAME][
Values.RMQ_EXCHANGE_RED_RECORDS_ROUTING_KEY], 'stop')
self.rmq.publish(
self.configs.settings[Values.RMQ_SECTION_NAME][
Values.RMQ_EXCHANGE_NAME],
self.configs.settings[Values.RMQ_SECTION_NAME][
Values.RMQ_EXCHANGE_BLUE_RECORDS_ROUTING_KEY], 'stop')
self.fininsh_event.set()
def __generate_orders(self):
'''
Generate green zone orders records with generators configurations
'''
Logger.info(__file__, 'Started generation order history in green zone')
previous_time = datetime.datetime.now()
t = 0
for period in range(len(self.configs.orders_volumes_for_generation)):
zone_index = 0
for zone in self.configs.orders_volumes_for_generation[period]:
for i in range(
int(self.configs.orders_volumes_for_generation[period]
[zone])):
order = self.__generate_general_order_information()
self.history.orders[order.id] = order
records = self.__get_order_in_zone(
order.id, order.status_sequence, period, zone,
order.statuses_in_blue_zone)
self.history.records.extend(records)
self.inc_statistic('Generated orders', 1)
self.inc_statistic('Generated records', len(records))
if len(self.history.orders) % self.configs.settings[
Values.GENERAL_SECTION_NAME][
Values.BATCH_SIZE] == 0:
self.add_time_statistic(
'Order history generation',
(datetime.datetime.now() -
previous_time).total_seconds() * 1000)
self.send_to_rmq()
previous_time = datetime.datetime.now()
zone_index += 1
def report(self):
Logger.info(__file__, 'Start reporting')
print('Start getting reporting data at {}'.format(
datetime.datetime.now()))
Utils.get_db_report_date()
ConsoleReporter.report(StatisticsDataStorage.statistics)
print('Reporter finished data at {}'.format(datetime.datetime.now()))
Logger.info(__file__, 'Reporting finished')
def __init__(self, driver_path="chromedriver", options=None):
Logger.info(__file__, "SeleniumCrawler init")
try:
self.driver = Chrome(executable_path=driver_path, options=options)
Logger.info(__file__, "Driver initiated")
except WebDriverException as err:
Logger.error(__file__, err.args)
self.configs = Configuration()
def __get_friends_list(self):
Logger.info(__file__, "Load friends list web elements")
list = self.get_elements(by=By.XPATH,
value=self.__FRIENDS_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME])
Logger.debug(__file__, f"Size of loaded friends list: {len(list)}")
return list
def __calculate_orders_volumes_for_generations(self):
'''
Calculate orders volumes list for generation
'''
Logger.info(
__file__,
'Starting calculating orders volumes for generating {} orders'.
format(self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.ORDERS_AMOUNT]))
min_percentage_amount = int(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.ORDERS_AMOUNT] /
sum(self.configs.min_orders_volumes.values()))
multiplier = LCG.get_next(Values.PERIODS_SIZE_GENERATOR)
while min_percentage_amount > multiplier:
self.configs.orders_volumes_for_generation.append({
Zone.RED:
self.configs.min_orders_volumes[Values.RED_ZONE_VOLUME] *
multiplier,
Zone.GREEN:
self.configs.min_orders_volumes[Values.GREEN_ZONE_VOLUME] *
multiplier,
Zone.BLUE:
self.configs.min_orders_volumes[Values.BLUE_ZONE_VOLUME] *
multiplier
})
min_percentage_amount -= multiplier
multiplier = LCG.get_next(Values.PERIODS_SIZE_GENERATOR)
if min_percentage_amount > 0:
self.configs.orders_volumes_for_generation.append({
Zone.RED:
self.configs.min_orders_volumes[Values.RED_ZONE_VOLUME] *
min_percentage_amount,
Zone.GREEN:
self.configs.min_orders_volumes[Values.GREEN_ZONE_VOLUME] *
min_percentage_amount,
Zone.BLUE:
self.configs.min_orders_volumes[Values.BLUE_ZONE_VOLUME] *
min_percentage_amount
})
Logger.info(__file__,
"Calculating orders volumes for each period finished")
Logger.debug(
__file__,
'Calculated {} orders volumes for generations: {}'.format(
len(self.configs.orders_volumes_for_generation),
self.configs.orders_volumes_for_generation))
def send_consumed_data_to_mysql(self):
Logger.info(__file__, 'Sending readed batch records to MySQL')
self.previous_time = datetime.datetime.now()
self.mysql.execute_multiple(Values.MYSQL_INSERT_QUERY,
self.consumed_data)
OrderHistoryMaker.add_time_statistic(
'Send data to MySQL',
(datetime.datetime.now() - self.previous_time).total_seconds() *
1000)
self.consumed_data.clear()
def get_db_report_date(cls):
from Service.LoggerService.Implementation.DefaultPythonLoggingService import \
DefaultPythonLoggingService as Logger
from Config.Configurations import Configuration
from Config.Configurations import ValuesNames as Values
from Service.DbService.Implementation.MySqlService import MySqlService
from Entities.StatisticsDataStorage import StatisticsDataStorage
Logger.info(__file__, 'Getting statistic from db started')
mysql_settings = Configuration().settings[Values.MYSQL_SECTION_NAME]
mysql = MySqlService(user=mysql_settings[Values.MYSQL_USER],
password=mysql_settings[Values.MYSQL_PASSWORD],
host=mysql_settings[Values.MYSQL_HOST],
port=mysql_settings[Values.MYSQL_PORT],
database=mysql_settings[Values.MYSQL_DB_NAME])
try:
mysql.open_connection()
for (value, name) in mysql.execute(Values.MYSQL_GET_REPORT_QUERY,
select=True):
if name == '1':
name = 'Red zone orders avg amount'
if name == '2':
name = 'Green zone orders avg amount'
if name == '3':
name = 'Blue zone orders avg amount'
StatisticsDataStorage.statistics[name] = value
Logger.info(__file__, 'Database service configurated')
except AttributeError as er:
Logger.error(__file__, er.args)
Logger.info(__file__, 'Sending records to MySQL aborted')
Logger.info(__file__, 'Getting statistic from db finished')
def report(data):
print()
Logger.info(__file__, 'Start reporting')
print()
print('========== REPORT ==========')
for key in data:
print(key, "-", data[key])
print('========== REPORT END ==========')
print()
Logger.info(__file__, 'Reporting finished')
print()
def init_rmq(self):
rmq_settings = self.configs.settings[Values.RMQ_SECTION_NAME]
self.rmq = RmqService()
while not self.rmq.open_connection(
host=rmq_settings[Values.RMQ_HOST],
port=rmq_settings[Values.RMQ_PORT],
virtual_host=rmq_settings[Values.RMQ_VIRTUAL_HOST],
user=rmq_settings[Values.RMQ_USER],
password=rmq_settings[Values.RMQ_PASSWORD]):
self.rmq.reconfig()
self.rmq.exchange_delete(
exchange_name=rmq_settings[Values.RMQ_EXCHANGE_NAME])
try:
self.rmq.declare_exchange(
exchange_name=rmq_settings[Values.RMQ_EXCHANGE_NAME],
exchange_type=ExchangeType(
rmq_settings[Values.RMQ_EXCHANGE_TYPE]))
except ValueError as er:
Logger.error(__file__, er.args)
Logger.info(__file__, 'Sending records to RabbitMQ aborted')
return
self.rmq.declare_queue(queue_name=Zone.RED.value,
durable=bool(
rmq_settings[Values.RMQ_DURABLE_QUEUES]))
self.rmq.declare_queue(queue_name=Zone.BLUE.value,
durable=bool(
rmq_settings[Values.RMQ_DURABLE_QUEUES]))
self.rmq.declare_queue(queue_name=Zone.GREEN.value,
durable=bool(
rmq_settings[Values.RMQ_DURABLE_QUEUES]))
self.rmq.queue_bind(
Zone.RED.value, rmq_settings[Values.RMQ_EXCHANGE_NAME],
rmq_settings[Values.RMQ_EXCHANGE_RED_RECORDS_ROUTING_KEY])
self.rmq.queue_bind(
Zone.BLUE.value, rmq_settings[Values.RMQ_EXCHANGE_NAME],
rmq_settings[Values.RMQ_EXCHANGE_BLUE_RECORDS_ROUTING_KEY])
self.rmq.queue_bind(
Zone.GREEN.value, rmq_settings[Values.RMQ_EXCHANGE_NAME],
rmq_settings[Values.RMQ_EXCHANGE_GREEN_RECORDS_ROUTING_KEY])
self.rmq.queue_purge(queue_name=Zone.RED.value)
self.rmq.queue_purge(queue_name=Zone.BLUE.value)
self.rmq.queue_purge(queue_name=Zone.GREEN.value)
def __load_currency_pairs_from_file(self):
'''
Load currency pairs from file by path in config object
'''
Logger.info(
__file__, 'Start loading currency pairs from file {}'.format(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.CURRENCY_PAIRS_FILE_PATH]))
try:
with open(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.CURRENCY_PAIRS_FILE_PATH], "r") as file:
for line in file.readlines():
if re.fullmatch(
r'[a-zA-Z]{3}\/[a-zA-Z]{3};[0-9]+([\.|\,][0-9]{0,5})',
line.replace('\n', '')):
parts = line.split(";")
self.configs.currency_pairs.append({
Values.CURRENCY_PAIR_NAME:
parts[0],
Values.CURRENCY_PAIR_VALUE:
float(parts[1].replace("\n", '').replace(',', '.'))
})
self.configs.settings[Values.CURRENCY_PAIR_GENERATOR][
LCGParams.MODULUS.value] = len(self.configs.currency_pairs)
Logger.info(
__file__,
'Loading currency pairs from file {} finished'.format(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.CURRENCY_PAIRS_FILE_PATH]))
Logger.debug(
__file__, 'Loaded {} currency pairs: {}'.format(
len(self.configs.currency_pairs),
self.configs.currency_pairs))
except:
Logger.error(
__file__, "Loading currency pairs from file {} failed".format(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.CURRENCY_PAIRS_FILE_PATH]))
if len(self.configs.currency_pairs) == 0:
exit(-1)
def send_to_rmq(self):
Logger.info(__file__, 'Sending ordres batch information to RabbitMQ')
previous_time = datetime.datetime.now()
for record in self.history.records:
self.rmq.publish(
self.configs.settings[Values.RMQ_SECTION_NAME][
Values.RMQ_EXCHANGE_NAME],
self.get_routing_key_by_zone(record.zone),
self.__order_record_to_proto(record).SerializeToString())
self.add_time_statistic(
'Sending records batch to RabbitMQ',
(datetime.datetime.now() - previous_time).total_seconds() * 1000)
self.history.clear_history()
def __register_lcg_generators(self):
'''
Register all generators setting to LCG generator
'''
Logger.info(
__file__,
"Started registration linear congruential generators configs ")
for section in self.configs.settings:
if 'GENERATOR' in section:
LCG.set_linear_congruential_generator(
section, self.configs.settings[section])
Logger.info(
__file__,
"Registration linear congruential generators configs finished")
def __load_tags_from_file(self):
'''
Load tags from file by path in config object
'''
Logger.info(
__file__,
'Start loading tags from file {}'.format(self.configs.settings[
Values.GENERAL_SECTION_NAME][Values.TAGS_FILE_PATH]))
try:
with open(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.TAGS_FILE_PATH], "r") as file:
for line in file.readlines():
if re.fullmatch('[a-zA-Z]{0,10}', line.replace('\n', '')):
self.configs.tags.append(line.replace("\n", ''))
Logger.info(
__file__, 'Loading tags from file {} finished'.format(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.TAGS_FILE_PATH]))
Logger.debug(
__file__,
'Loaded {} tags: {}'.format(len(self.configs.tags),
', '.join(self.configs.tags)))
self.configs.settings[Values.TAGS_GENERATOR_1][
LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5)
self.configs.settings[Values.TAGS_GENERATOR_2][
LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5)
self.configs.settings[Values.TAGS_GENERATOR_3][
LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5)
self.configs.settings[Values.TAGS_GENERATOR_4][
LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5)
self.configs.settings[Values.TAGS_GENERATOR_5][
LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5)
except:
Logger.error(
__file__, "Loading tags from file {} failed".format(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.TAGS_FILE_PATH]))
if len(self.configs.tags) == 0:
exit(-2)
def prepare_configurations_for_generation(self):
'''
Call all function for prepare to generation ordera
'''
Logger.info(__file__, "Execute preparing to execution")
self.__load_currency_pairs_from_file()
self.__load_tags_from_file()
self.__register_lcg_generators()
self.__calculate_orders_period_volumes()
self.__calculate_orders_volumes_for_generations()
self.__calculate_first_generation_period_start_date()
self.__calculate_avg_values_of_id()
self.init_rmq()
Logger.info(__file__, "Execute preparing finished")
def consume(self):
Logger.info(__file__, 'Configuration consumer...')
rmq_settings = self.configs.settings[Values.RMQ_SECTION_NAME]
self.rmq = RmqService()
while not self.rmq.open_connection(
host=rmq_settings[Values.RMQ_HOST],
port=rmq_settings[Values.RMQ_PORT],
virtual_host=rmq_settings[Values.RMQ_VIRTUAL_HOST],
user=rmq_settings[Values.RMQ_USER],
password=rmq_settings[Values.RMQ_PASSWORD]):
self.rmq.reconfig()
self.rmq.consume(queue_name=Zone.RED.value,
on_consume_callback=self.__msg_consumer)
self.rmq.consume(queue_name=Zone.BLUE.value,
on_consume_callback=self.__msg_consumer)
self.rmq.consume(queue_name=Zone.GREEN.value,
on_consume_callback=self.__msg_consumer)
Logger.info(__file__, 'Consumer configurated')
self.configurate_db_service()
self.previous_time = datetime.datetime.now()
Logger.info(__file__, 'Start consuming')
self.stop_messages_count = 0
self.rmq.start_consuming()
def __execute(self):
self.generator_and_publisher_event = threading.Event()
self.history_maker = OrderHistoryMaker(
self.generator_and_publisher_event)
self.history_maker.prepare_configurations_for_generation()
self.consumer_thread = threading.Thread(target=self.start_consumer)
self.generator_and_publisher_thread = threading.Thread(
target=self.start_orders_history_generation)
self.generator_and_publisher_thread.start()
self.consumer_thread.start()
sec = self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.REPORTER_DELAY]
while not self.generator_and_publisher_event.is_set(
) or not self.consumer_event.is_set():
time.sleep(sec)
self.report()
self.report()
Logger.info(__file__, 'Program finished')
def __calculate_orders_period_volumes(self):
'''
Calculate orders volumes list than equal to percentage of zones:
red = 15%, green = 60% and blue = 25%
'''
red_zone_percent = self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.RED_ZONE_ORDERS_PERCENT]
green_zone_percent = self.configs.settings[
Values.GENERAL_SECTION_NAME][Values.GREEN_ZONE_ORDERS_PERCENT]
blue_zone_percent = self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.BLUE_ZONE_ORDERS_PERCENT]
Logger.info(
__file__,
'Started calculating of orders volumes to period with zones percentes: red = {}, green ={}, blue ={}'
.format(red_zone_percent, green_zone_percent, blue_zone_percent))
for i in range(
1, self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.ORDERS_AMOUNT] + 1):
first_percent = Utils.calculate_percent_from_value(
i, red_zone_percent)
second_percent = Utils.calculate_percent_from_value(
i, green_zone_percent)
third_percent = Utils.calculate_percent_from_value(
i, blue_zone_percent)
if Utils.is_int(first_percent) and Utils.is_int(
second_percent) and Utils.is_int(third_percent):
self.configs.min_orders_volumes = {
Values.RED_ZONE_VOLUME: first_percent,
Values.GREEN_ZONE_VOLUME: second_percent,
Values.BLUE_ZONE_VOLUME: third_percent
}
break
def configurate_db_service(self):
Logger.info(__file__, 'Configuration database service...')
mysql_settings = self.configs.settings[Values.MYSQL_SECTION_NAME]
self.mysql = MySqlService(
user=mysql_settings[Values.MYSQL_USER],
password=mysql_settings[Values.MYSQL_PASSWORD],
host=mysql_settings[Values.MYSQL_HOST],
port=mysql_settings[Values.MYSQL_PORT],
database=mysql_settings[Values.MYSQL_DB_NAME])
try:
self.mysql.open_connection()
self.mysql.execute(
f'TRUNCATE `{self.configs.settings[Values.MYSQL_SECTION_NAME][Values.MYSQL_DB_NAME]}`.`history`'
)
Logger.info(__file__, 'Database service configurated')
except AttributeError as er:
Logger.error(__file__, er.args)
Logger.info(__file__, 'Sending records to MySQL aborted')
def start_orders_history_generation(self):
self.history_maker.execute_generation()
Logger.info(__file__, 'Order history generation finished')
def execute(self):
data = dict()
Logger.info(__file__, f"Opening {self.__PAGE_URL} page")
self.driver.get(self.__PAGE_URL)
Logger.info(__file__, "Finding login input web element")
login_web_element = self.get_element(
by=By.XPATH,
value=self.__EMAIL_INPUT_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME])
Logger.info(__file__, "Send email to login input web element}")
login_web_element.send_keys(
self.configs.settings[Values.SETTINGS][Values.FACEBOOK_LOGIN])
Logger.info(__file__, "Finding password input web element")
password_web_element = self.get_element(
by=By.XPATH,
value=self.__PASSWORD_INPUT_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME])
Logger.info(__file__, "Send password to password input web element}")
password_web_element.send_keys(
self.configs.settings[Values.SETTINGS][Values.FACEBOOK_PASSWORD])
Logger.info(__file__,
"Send Enter key pressing to password input web element}")
password_web_element.send_keys(Keys.ENTER)
Logger.info(__file__, "Finding account link")
account_button_web_element = self.get_element(
by=By.XPATH,
value=self.__ACCOUNT_LINK_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME])
Logger.info(__file__, "Click on account link")
account_button_web_element.click()
Logger.info(__file__, "Extracting total friends amount value")
total_friends_amount = int(
self.get_element(by=By.XPATH,
value=self.__FRIENDS_AMOUNT_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME]).text)
Logger.debug(__file__, f"Extracted {total_friends_amount} value")
Logger.info(__file__, "Finding friends page link")
friends = self.get_element(by=By.XPATH,
value=self.__FRIENDS_LINK_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME])
Logger.info(__file__, "Click on friends page link")
friends.click()
Logger.info(__file__, "Extracting account name")
name = self.get_element(by=By.XPATH,
value=self.__ACCOUNT_NAME,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME]).text
friends = self.__get_friends_list()
previous_friend_amount = 0
current_friends_amount = len(friends)
while previous_friend_amount != current_friends_amount:
Logger.info(__file__, "Friends list can be scroll")
previous_friend_amount = current_friends_amount
Logger.info(__file__, "Scroll to last friends list element")
ActionChains(self.driver).move_to_element(friends[-1]).perform()
time.sleep(1)
friends = self.__get_friends_list()
current_friends_amount = len(friends)
Logger.info(__file__, "Full friends list already loaded")
Logger.info(__file__, "Extract data from friends web elements list")
data["Account name: "] = name
for friend in friends:
link = friend.get_attribute("href")
data[link[:friend.get_attribute('href').find('fref') -
1]] = friend.text
Logger.info(__file__, "Closing web driver")
Logger.info(__file__, "Closing selenium web driver")
self.driver.quit()
data["Total friends amount"] = total_friends_amount
data["Scanned friends amount"] = current_friends_amount
return data