def set_general_order_info(self, general_order_info):
if isinstance(general_order_info, GeneralOrderInformation):
self.__order_general_information = general_order_info
Logger.debug(__file__,
'Set general order info parameter to {}'.format(self.__order_general_information.__str__()))
return self
def declare_exchange(self,
exchange_name,
exchange_type,
passive=False,
durable=True,
auto_delete=False):
'''
Declare new exchange
:param exchange_name: exchange name
:param exchange_type: exhange type
:param passive:
:param durable:
:param auto_delete:
:return:
'''
Logger.debug(
__file__, 'Declaring exchange {} with exchange type'.format(
exchange_name, exchange_type.value))
return self.conn.declare_exchange(exchange_name,
exchange_type,
passive=passive,
durable=durable,
auto_delete=auto_delete)
def queue_unbind(self, queue_name, exchange_name, routing_key=None):
Logger.debug(
__file__,
'Unbinding queue {} from exchange {} with routing key {}'.format(
queue_name, exchange_name, routing_key))
return self.conn.queue_unbind(queue_name=queue_name,
exchange_name=exchange_name,
routing_key=routing_key)
def __get_friends_list(self):
Logger.info(__file__, "Load friends list web elements")
list = self.get_elements(by=By.XPATH,
value=self.__FRIENDS_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME])
Logger.debug(__file__, f"Size of loaded friends list: {len(list)}")
return list
def queue_purge(self, queue_name):
'''
Clearing queue
:param queue_name: queue name
:return:
'''
Logger.debug(__file__, 'Purging queue {}'.format(queue_name))
return self.conn.queue_purge(queue_name)
def open(self):
self.__open_conn()
if not self.is_available():
self.reconnect()
else:
Logger.debug(__file__, 'Created mysql connection with params {} {} {} {}'.format(self.__user,
self.__password,
self.__host,
self.__database))
def declare_queue(self, queue_name, durable):
'''
Declare new queue
:param queue_name: queue name
:return:
'''
Logger.debug(__file__,
'Declaring queue with name {}'.format(queue_name))
return self.conn.declare_queue(queue_name=queue_name, durable=durable)
def exchange_unbind(self, destination, source, routing_key=''):
'''
Unind exchange
'''
Logger.debug(
__file__,
'Unbind exchange: destination {}, sourse, routing_key'.format(
destination, source, routing_key))
return self.conn.exchange_unbind(destination=destination,
source=source,
routing_key=routing_key)
def exchange_delete(self, exchange_name=None, if_unused=False):
'''
Delete exchange
:param exchange_name: exchange name
:param if_unused:
:return:
'''
Logger.debug(__file__, 'Deleting exchange {}'.format(exchange_name))
return self.conn.exchange_delete(exchange_name=exchange_name,
if_unused=if_unused)
def __calculate_orders_volumes_for_generations(self):
'''
Calculate orders volumes list for generation
'''
Logger.info(
__file__,
'Starting calculating orders volumes for generating {} orders'.
format(self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.ORDERS_AMOUNT]))
min_percentage_amount = int(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.ORDERS_AMOUNT] /
sum(self.configs.min_orders_volumes.values()))
multiplier = LCG.get_next(Values.PERIODS_SIZE_GENERATOR)
while min_percentage_amount > multiplier:
self.configs.orders_volumes_for_generation.append({
Zone.RED:
self.configs.min_orders_volumes[Values.RED_ZONE_VOLUME] *
multiplier,
Zone.GREEN:
self.configs.min_orders_volumes[Values.GREEN_ZONE_VOLUME] *
multiplier,
Zone.BLUE:
self.configs.min_orders_volumes[Values.BLUE_ZONE_VOLUME] *
multiplier
})
min_percentage_amount -= multiplier
multiplier = LCG.get_next(Values.PERIODS_SIZE_GENERATOR)
if min_percentage_amount > 0:
self.configs.orders_volumes_for_generation.append({
Zone.RED:
self.configs.min_orders_volumes[Values.RED_ZONE_VOLUME] *
min_percentage_amount,
Zone.GREEN:
self.configs.min_orders_volumes[Values.GREEN_ZONE_VOLUME] *
min_percentage_amount,
Zone.BLUE:
self.configs.min_orders_volumes[Values.BLUE_ZONE_VOLUME] *
min_percentage_amount
})
Logger.info(__file__,
"Calculating orders volumes for each period finished")
Logger.debug(
__file__,
'Calculated {} orders volumes for generations: {}'.format(
len(self.configs.orders_volumes_for_generation),
self.configs.orders_volumes_for_generation))
def __init__(self,
user='******',
password='',
host='127.0.0.1',
port='3306',
database=''):
self.__connection = MySqlConnection(user=user,
password=password,
host=host,
port=port,
database=database)
Logger.debug(__file__, 'Created mysql service')
def __load_currency_pairs_from_file(self):
'''
Load currency pairs from file by path in config object
'''
Logger.info(
__file__, 'Start loading currency pairs from file {}'.format(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.CURRENCY_PAIRS_FILE_PATH]))
try:
with open(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.CURRENCY_PAIRS_FILE_PATH], "r") as file:
for line in file.readlines():
if re.fullmatch(
r'[a-zA-Z]{3}\/[a-zA-Z]{3};[0-9]+([\.|\,][0-9]{0,5})',
line.replace('\n', '')):
parts = line.split(";")
self.configs.currency_pairs.append({
Values.CURRENCY_PAIR_NAME:
parts[0],
Values.CURRENCY_PAIR_VALUE:
float(parts[1].replace("\n", '').replace(',', '.'))
})
self.configs.settings[Values.CURRENCY_PAIR_GENERATOR][
LCGParams.MODULUS.value] = len(self.configs.currency_pairs)
Logger.info(
__file__,
'Loading currency pairs from file {} finished'.format(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.CURRENCY_PAIRS_FILE_PATH]))
Logger.debug(
__file__, 'Loaded {} currency pairs: {}'.format(
len(self.configs.currency_pairs),
self.configs.currency_pairs))
except:
Logger.error(
__file__, "Loading currency pairs from file {} failed".format(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.CURRENCY_PAIRS_FILE_PATH]))
if len(self.configs.currency_pairs) == 0:
exit(-1)
def __load_tags_from_file(self):
'''
Load tags from file by path in config object
'''
Logger.info(
__file__,
'Start loading tags from file {}'.format(self.configs.settings[
Values.GENERAL_SECTION_NAME][Values.TAGS_FILE_PATH]))
try:
with open(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.TAGS_FILE_PATH], "r") as file:
for line in file.readlines():
if re.fullmatch('[a-zA-Z]{0,10}', line.replace('\n', '')):
self.configs.tags.append(line.replace("\n", ''))
Logger.info(
__file__, 'Loading tags from file {} finished'.format(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.TAGS_FILE_PATH]))
Logger.debug(
__file__,
'Loaded {} tags: {}'.format(len(self.configs.tags),
', '.join(self.configs.tags)))
self.configs.settings[Values.TAGS_GENERATOR_1][
LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5)
self.configs.settings[Values.TAGS_GENERATOR_2][
LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5)
self.configs.settings[Values.TAGS_GENERATOR_3][
LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5)
self.configs.settings[Values.TAGS_GENERATOR_4][
LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5)
self.configs.settings[Values.TAGS_GENERATOR_5][
LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5)
except:
Logger.error(
__file__, "Loading tags from file {} failed".format(
self.configs.settings[Values.GENERAL_SECTION_NAME][
Values.TAGS_FILE_PATH]))
if len(self.configs.tags) == 0:
exit(-2)
def queue_bind(self, queue_name, exchange_name, routing_key=None):
'''
Binding queue to exchange
:param queue_name: queue name
:param exchange_name: exchange name
:param routing_key: routing key
:return:
'''
Logger.debug(
__file__,
'Binding queue {} to exchange {} with routing key {}'.format(
queue_name, exchange_name, routing_key))
return self.conn.queue_bind(queue_name=queue_name,
exchange_name=exchange_name,
routing_key=routing_key)
def execute(self, query, select=False, multi=False, *args, **kwargs):
try:
while True:
if not self.__connection.is_available():
self.__connection.open()
cursor = self.__connection.get_cursor()
if cursor is not None:
Logger.debug(__file__, 'Executing query:{}'.format(query))
cursor.execute(query, multi)
if not select:
if self.__connection.commit():
break
else:
return cursor
except mysql.connector.Error as err:
Logger.error(__file__, err.msg)
def open_connection(self,
user=pika.connection.Parameters.DEFAULT_USERNAME,
password=pika.connection.Parameters.DEFAULT_PASSWORD,
host=pika.connection.Parameters.DEFAULT_HOST,
port=pika.connection.Parameters.DEFAULT_PORT,
*args,
**kwargs):
'''
Opening RMQ connection
:param user: username
:param password: password
:param host: host
:param port: port
:param args:
:param kwargs: must containts virtual_host
:return:
'''
Logger.debug(__file__, 'Opening RMQ connection')
self.conn = RmqConnection()
if 'virtual_host' in kwargs:
virtual_host = kwargs['virtual_host']
else:
virtual_host = pika.connection.Parameters.DEFAULT_VIRTUAL_HOST
try:
self.conn.open(host=host,
port=port,
user=user,
password=password,
virtual_host=virtual_host)
return self.conn.is_available()
except:
return False
def build_order_zone(self, id, status_sequence, period_index, zone, statuses_in_unfinished_zone):
Logger.debug(__file__, 'Called building order in green zone')
period_start = self.__configs.start_date + datetime.timedelta(days=period_index * 7)
first_status_day_offset, second_status_day_offset, third_status_day_offset = self.__generate_days_statuses_offsets()
first_status_date = self.__calculate_date(first_status_day_offset, period_start)
second_status_date = self.__calculate_date(second_status_day_offset, first_status_date)
third_status_date = self.__calculate_date(third_status_day_offset, second_status_date)
Logger.debug(__file__,
f'Generated dates for records: {first_status_date}, {second_status_date}, {third_status_date}')
first_status_time, second_status_time, third_status_time = self.__calculate_order_statuses_times(
first_status_date,
second_status_date,
third_status_date)
Logger.debug(__file__,
f'Generated times for records: {first_status_time}, {second_status_time}, {third_status_time}')
Logger.debug(__file__, 'Building order records in green zone finished')
records = [
OrderStatusChangingInfo(id, self.__generate_ms_datetime(first_status_date, first_status_time), Status.NEW,
zone, period_index),
OrderStatusChangingInfo(id, self.__generate_ms_datetime(second_status_date, second_status_time),
Status.TO_PROVIDER, zone, period_index),
OrderStatusChangingInfo(id, self.__generate_ms_datetime(third_status_date, third_status_time),
self.__get_last_status(status_sequence),
zone, period_index)
]
if zone == Zone.GREEN:
return records
elif zone == Zone.BLUE:
if statuses_in_unfinished_zone == 1:
return [records[0]]
else:
return [records[0], records[1]]
else:
if statuses_in_unfinished_zone == 1:
return [records[2]]
else:
return [records[1], records[2]]
def execute_many(self, queries):
try:
if not self.__connection.is_available():
self.__connection.open()
cursor = self.__connection.get_cursor()
Logger.debug(__file__, 'Executing many queries started')
if cursor is not None:
for query in queries:
Logger.debug(__file__, 'Executing query:{}'.format(query))
cursor.execute(query)
self.__connection.commit()
except mysql.connector.Error as err:
Logger.error(__file__, err.msg)
Logger.debug(__file__, 'Executing many queries finished')
def open_connection(self):
Logger.debug(__file__, 'Opening MySQL connection')
self.__connection.open()
def close_connection(self):
Logger.debug(__file__, 'Closing MySQL connection')
self.__connection.close()
def queue_delete(self, queue_name, if_unused=False, if_empty=False):
Logger.debug(__file__, 'Deleting queue {}'.format(queue_name))
return self.conn.queue_delete(queue_name,
if_unused=if_unused,
if_empty=if_empty)
def close(self, *args, **kwargs):
''''
Closing RMQ service
'''
Logger.debug(__file__, 'Closing RMQ connection')
self.conn.close()
def close(self):
Logger.debug(__file__, 'Closing csv file service')
self.__file_descriptor.close()
def execute(self):
data = dict()
Logger.info(__file__, f"Opening {self.__PAGE_URL} page")
self.driver.get(self.__PAGE_URL)
Logger.info(__file__, "Finding login input web element")
login_web_element = self.get_element(
by=By.XPATH,
value=self.__EMAIL_INPUT_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME])
Logger.info(__file__, "Send email to login input web element}")
login_web_element.send_keys(
self.configs.settings[Values.SETTINGS][Values.FACEBOOK_LOGIN])
Logger.info(__file__, "Finding password input web element")
password_web_element = self.get_element(
by=By.XPATH,
value=self.__PASSWORD_INPUT_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME])
Logger.info(__file__, "Send password to password input web element}")
password_web_element.send_keys(
self.configs.settings[Values.SETTINGS][Values.FACEBOOK_PASSWORD])
Logger.info(__file__,
"Send Enter key pressing to password input web element}")
password_web_element.send_keys(Keys.ENTER)
Logger.info(__file__, "Finding account link")
account_button_web_element = self.get_element(
by=By.XPATH,
value=self.__ACCOUNT_LINK_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME])
Logger.info(__file__, "Click on account link")
account_button_web_element.click()
Logger.info(__file__, "Extracting total friends amount value")
total_friends_amount = int(
self.get_element(by=By.XPATH,
value=self.__FRIENDS_AMOUNT_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME]).text)
Logger.debug(__file__, f"Extracted {total_friends_amount} value")
Logger.info(__file__, "Finding friends page link")
friends = self.get_element(by=By.XPATH,
value=self.__FRIENDS_LINK_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME])
Logger.info(__file__, "Click on friends page link")
friends.click()
Logger.info(__file__, "Extracting account name")
name = self.get_element(by=By.XPATH,
value=self.__ACCOUNT_NAME,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME]).text
friends = self.__get_friends_list()
previous_friend_amount = 0
current_friends_amount = len(friends)
while previous_friend_amount != current_friends_amount:
Logger.info(__file__, "Friends list can be scroll")
previous_friend_amount = current_friends_amount
Logger.info(__file__, "Scroll to last friends list element")
ActionChains(self.driver).move_to_element(friends[-1]).perform()
time.sleep(1)
friends = self.__get_friends_list()
current_friends_amount = len(friends)
Logger.info(__file__, "Full friends list already loaded")
Logger.info(__file__, "Extract data from friends web elements list")
data["Account name: "] = name
for friend in friends:
link = friend.get_attribute("href")
data[link[:friend.get_attribute('href').find('fref') -
1]] = friend.text
Logger.info(__file__, "Closing web driver")
Logger.info(__file__, "Closing selenium web driver")
self.driver.quit()
data["Total friends amount"] = total_friends_amount
data["Scanned friends amount"] = current_friends_amount
return data
def __init__(self):
self.__order_general_information = None
self.__configs = Configuration()
Logger.debug(__file__, 'Init OrderRecordsBuilder instance')
