def report(data): print() Logger.info(__file__, 'Start reporting') print() print('========== REPORT ==========') for key in data: item = data[key] print(f'----- {key} -----') if isinstance(item, list): if len(item) > 1: max_value = max(item) min_value = min(item) avg_value = sum(item) / len(item) total = sum(item) print(f'Max: {max_value} ms') print(f'Min: {min_value} ms') print(f'Avg: {avg_value} ms') print(f'Total: {total} ms') else: print(f'{key}: {item[0]} ms') else: print(item) print() print('========== REPORT END ==========') print() Logger.info(__file__, 'Reporting finished') print()
def __calculate_first_generation_period_start_date(self): ''' Calculate start date of first generation perid ''' Logger.info( __file__, 'Starting calculating start date of first period for generation orders' ) current_date = datetime.datetime.today().replace(hour=0, minute=0, second=0, microsecond=0) day_of_week = current_date.weekday() + 1 self.configs.is_current_date_in_trading_period = day_of_week in (1, 2, 5) days_to_last_period = day_of_week + 7 - 5 if day_of_week < 5 else day_of_week - 5 days_to_start_period = days_to_last_period + ( 7 * (len(self.configs.orders_volumes_for_generation))) self.configs.start_date = current_date + datetime.timedelta( days=-days_to_start_period) Logger.info( __file__, 'Calculating start date of first period for generation orders finished. Start at {}' .format(self.configs.start_date))
def execute_generation(self): ''' Execute generation orders history ''' self.history.clear_history() Logger.info(__file__, 'Generating order history started') self.__generate_orders() Logger.info(__file__, 'Generating order history finished') self.rmq.publish( self.configs.settings[Values.RMQ_SECTION_NAME][ Values.RMQ_EXCHANGE_NAME], self.configs.settings[Values.RMQ_SECTION_NAME][ Values.RMQ_EXCHANGE_GREEN_RECORDS_ROUTING_KEY], 'stop') self.rmq.publish( self.configs.settings[Values.RMQ_SECTION_NAME][ Values.RMQ_EXCHANGE_NAME], self.configs.settings[Values.RMQ_SECTION_NAME][ Values.RMQ_EXCHANGE_RED_RECORDS_ROUTING_KEY], 'stop') self.rmq.publish( self.configs.settings[Values.RMQ_SECTION_NAME][ Values.RMQ_EXCHANGE_NAME], self.configs.settings[Values.RMQ_SECTION_NAME][ Values.RMQ_EXCHANGE_BLUE_RECORDS_ROUTING_KEY], 'stop') self.fininsh_event.set()
def __generate_orders(self): ''' Generate green zone orders records with generators configurations ''' Logger.info(__file__, 'Started generation order history in green zone') previous_time = datetime.datetime.now() t = 0 for period in range(len(self.configs.orders_volumes_for_generation)): zone_index = 0 for zone in self.configs.orders_volumes_for_generation[period]: for i in range( int(self.configs.orders_volumes_for_generation[period] [zone])): order = self.__generate_general_order_information() self.history.orders[order.id] = order records = self.__get_order_in_zone( order.id, order.status_sequence, period, zone, order.statuses_in_blue_zone) self.history.records.extend(records) self.inc_statistic('Generated orders', 1) self.inc_statistic('Generated records', len(records)) if len(self.history.orders) % self.configs.settings[ Values.GENERAL_SECTION_NAME][ Values.BATCH_SIZE] == 0: self.add_time_statistic( 'Order history generation', (datetime.datetime.now() - previous_time).total_seconds() * 1000) self.send_to_rmq() previous_time = datetime.datetime.now() zone_index += 1
def report(self): Logger.info(__file__, 'Start reporting') print('Start getting reporting data at {}'.format( datetime.datetime.now())) Utils.get_db_report_date() ConsoleReporter.report(StatisticsDataStorage.statistics) print('Reporter finished data at {}'.format(datetime.datetime.now())) Logger.info(__file__, 'Reporting finished')
def __init__(self, driver_path="chromedriver", options=None): Logger.info(__file__, "SeleniumCrawler init") try: self.driver = Chrome(executable_path=driver_path, options=options) Logger.info(__file__, "Driver initiated") except WebDriverException as err: Logger.error(__file__, err.args) self.configs = Configuration()
def __get_friends_list(self): Logger.info(__file__, "Load friends list web elements") list = self.get_elements(by=By.XPATH, value=self.__FRIENDS_XPATH, wait=self.configs.settings[Values.SETTINGS][ Values.ELEMENT_WAIT_TIME]) Logger.debug(__file__, f"Size of loaded friends list: {len(list)}") return list
def __calculate_orders_volumes_for_generations(self): ''' Calculate orders volumes list for generation ''' Logger.info( __file__, 'Starting calculating orders volumes for generating {} orders'. format(self.configs.settings[Values.GENERAL_SECTION_NAME][ Values.ORDERS_AMOUNT])) min_percentage_amount = int( self.configs.settings[Values.GENERAL_SECTION_NAME][ Values.ORDERS_AMOUNT] / sum(self.configs.min_orders_volumes.values())) multiplier = LCG.get_next(Values.PERIODS_SIZE_GENERATOR) while min_percentage_amount > multiplier: self.configs.orders_volumes_for_generation.append({ Zone.RED: self.configs.min_orders_volumes[Values.RED_ZONE_VOLUME] * multiplier, Zone.GREEN: self.configs.min_orders_volumes[Values.GREEN_ZONE_VOLUME] * multiplier, Zone.BLUE: self.configs.min_orders_volumes[Values.BLUE_ZONE_VOLUME] * multiplier }) min_percentage_amount -= multiplier multiplier = LCG.get_next(Values.PERIODS_SIZE_GENERATOR) if min_percentage_amount > 0: self.configs.orders_volumes_for_generation.append({ Zone.RED: self.configs.min_orders_volumes[Values.RED_ZONE_VOLUME] * min_percentage_amount, Zone.GREEN: self.configs.min_orders_volumes[Values.GREEN_ZONE_VOLUME] * min_percentage_amount, Zone.BLUE: self.configs.min_orders_volumes[Values.BLUE_ZONE_VOLUME] * min_percentage_amount }) Logger.info(__file__, "Calculating orders volumes for each period finished") Logger.debug( __file__, 'Calculated {} orders volumes for generations: {}'.format( len(self.configs.orders_volumes_for_generation), self.configs.orders_volumes_for_generation))
def send_consumed_data_to_mysql(self): Logger.info(__file__, 'Sending readed batch records to MySQL') self.previous_time = datetime.datetime.now() self.mysql.execute_multiple(Values.MYSQL_INSERT_QUERY, self.consumed_data) OrderHistoryMaker.add_time_statistic( 'Send data to MySQL', (datetime.datetime.now() - self.previous_time).total_seconds() * 1000) self.consumed_data.clear()
def get_db_report_date(cls): from Service.LoggerService.Implementation.DefaultPythonLoggingService import \ DefaultPythonLoggingService as Logger from Config.Configurations import Configuration from Config.Configurations import ValuesNames as Values from Service.DbService.Implementation.MySqlService import MySqlService from Entities.StatisticsDataStorage import StatisticsDataStorage Logger.info(__file__, 'Getting statistic from db started') mysql_settings = Configuration().settings[Values.MYSQL_SECTION_NAME] mysql = MySqlService(user=mysql_settings[Values.MYSQL_USER], password=mysql_settings[Values.MYSQL_PASSWORD], host=mysql_settings[Values.MYSQL_HOST], port=mysql_settings[Values.MYSQL_PORT], database=mysql_settings[Values.MYSQL_DB_NAME]) try: mysql.open_connection() for (value, name) in mysql.execute(Values.MYSQL_GET_REPORT_QUERY, select=True): if name == '1': name = 'Red zone orders avg amount' if name == '2': name = 'Green zone orders avg amount' if name == '3': name = 'Blue zone orders avg amount' StatisticsDataStorage.statistics[name] = value Logger.info(__file__, 'Database service configurated') except AttributeError as er: Logger.error(__file__, er.args) Logger.info(__file__, 'Sending records to MySQL aborted') Logger.info(__file__, 'Getting statistic from db finished')
def report(data): print() Logger.info(__file__, 'Start reporting') print() print('========== REPORT ==========') for key in data: print(key, "-", data[key]) print('========== REPORT END ==========') print() Logger.info(__file__, 'Reporting finished') print()
def init_rmq(self): rmq_settings = self.configs.settings[Values.RMQ_SECTION_NAME] self.rmq = RmqService() while not self.rmq.open_connection( host=rmq_settings[Values.RMQ_HOST], port=rmq_settings[Values.RMQ_PORT], virtual_host=rmq_settings[Values.RMQ_VIRTUAL_HOST], user=rmq_settings[Values.RMQ_USER], password=rmq_settings[Values.RMQ_PASSWORD]): self.rmq.reconfig() self.rmq.exchange_delete( exchange_name=rmq_settings[Values.RMQ_EXCHANGE_NAME]) try: self.rmq.declare_exchange( exchange_name=rmq_settings[Values.RMQ_EXCHANGE_NAME], exchange_type=ExchangeType( rmq_settings[Values.RMQ_EXCHANGE_TYPE])) except ValueError as er: Logger.error(__file__, er.args) Logger.info(__file__, 'Sending records to RabbitMQ aborted') return self.rmq.declare_queue(queue_name=Zone.RED.value, durable=bool( rmq_settings[Values.RMQ_DURABLE_QUEUES])) self.rmq.declare_queue(queue_name=Zone.BLUE.value, durable=bool( rmq_settings[Values.RMQ_DURABLE_QUEUES])) self.rmq.declare_queue(queue_name=Zone.GREEN.value, durable=bool( rmq_settings[Values.RMQ_DURABLE_QUEUES])) self.rmq.queue_bind( Zone.RED.value, rmq_settings[Values.RMQ_EXCHANGE_NAME], rmq_settings[Values.RMQ_EXCHANGE_RED_RECORDS_ROUTING_KEY]) self.rmq.queue_bind( Zone.BLUE.value, rmq_settings[Values.RMQ_EXCHANGE_NAME], rmq_settings[Values.RMQ_EXCHANGE_BLUE_RECORDS_ROUTING_KEY]) self.rmq.queue_bind( Zone.GREEN.value, rmq_settings[Values.RMQ_EXCHANGE_NAME], rmq_settings[Values.RMQ_EXCHANGE_GREEN_RECORDS_ROUTING_KEY]) self.rmq.queue_purge(queue_name=Zone.RED.value) self.rmq.queue_purge(queue_name=Zone.BLUE.value) self.rmq.queue_purge(queue_name=Zone.GREEN.value)
def __load_currency_pairs_from_file(self): ''' Load currency pairs from file by path in config object ''' Logger.info( __file__, 'Start loading currency pairs from file {}'.format( self.configs.settings[Values.GENERAL_SECTION_NAME][ Values.CURRENCY_PAIRS_FILE_PATH])) try: with open( self.configs.settings[Values.GENERAL_SECTION_NAME][ Values.CURRENCY_PAIRS_FILE_PATH], "r") as file: for line in file.readlines(): if re.fullmatch( r'[a-zA-Z]{3}\/[a-zA-Z]{3};[0-9]+([\.|\,][0-9]{0,5})', line.replace('\n', '')): parts = line.split(";") self.configs.currency_pairs.append({ Values.CURRENCY_PAIR_NAME: parts[0], Values.CURRENCY_PAIR_VALUE: float(parts[1].replace("\n", '').replace(',', '.')) }) self.configs.settings[Values.CURRENCY_PAIR_GENERATOR][ LCGParams.MODULUS.value] = len(self.configs.currency_pairs) Logger.info( __file__, 'Loading currency pairs from file {} finished'.format( self.configs.settings[Values.GENERAL_SECTION_NAME][ Values.CURRENCY_PAIRS_FILE_PATH])) Logger.debug( __file__, 'Loaded {} currency pairs: {}'.format( len(self.configs.currency_pairs), self.configs.currency_pairs)) except: Logger.error( __file__, "Loading currency pairs from file {} failed".format( self.configs.settings[Values.GENERAL_SECTION_NAME][ Values.CURRENCY_PAIRS_FILE_PATH])) if len(self.configs.currency_pairs) == 0: exit(-1)
def send_to_rmq(self): Logger.info(__file__, 'Sending ordres batch information to RabbitMQ') previous_time = datetime.datetime.now() for record in self.history.records: self.rmq.publish( self.configs.settings[Values.RMQ_SECTION_NAME][ Values.RMQ_EXCHANGE_NAME], self.get_routing_key_by_zone(record.zone), self.__order_record_to_proto(record).SerializeToString()) self.add_time_statistic( 'Sending records batch to RabbitMQ', (datetime.datetime.now() - previous_time).total_seconds() * 1000) self.history.clear_history()
def __register_lcg_generators(self): ''' Register all generators setting to LCG generator ''' Logger.info( __file__, "Started registration linear congruential generators configs ") for section in self.configs.settings: if 'GENERATOR' in section: LCG.set_linear_congruential_generator( section, self.configs.settings[section]) Logger.info( __file__, "Registration linear congruential generators configs finished")
def __load_tags_from_file(self): ''' Load tags from file by path in config object ''' Logger.info( __file__, 'Start loading tags from file {}'.format(self.configs.settings[ Values.GENERAL_SECTION_NAME][Values.TAGS_FILE_PATH])) try: with open( self.configs.settings[Values.GENERAL_SECTION_NAME][ Values.TAGS_FILE_PATH], "r") as file: for line in file.readlines(): if re.fullmatch('[a-zA-Z]{0,10}', line.replace('\n', '')): self.configs.tags.append(line.replace("\n", '')) Logger.info( __file__, 'Loading tags from file {} finished'.format( self.configs.settings[Values.GENERAL_SECTION_NAME][ Values.TAGS_FILE_PATH])) Logger.debug( __file__, 'Loaded {} tags: {}'.format(len(self.configs.tags), ', '.join(self.configs.tags))) self.configs.settings[Values.TAGS_GENERATOR_1][ LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5) self.configs.settings[Values.TAGS_GENERATOR_2][ LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5) self.configs.settings[Values.TAGS_GENERATOR_3][ LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5) self.configs.settings[Values.TAGS_GENERATOR_4][ LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5) self.configs.settings[Values.TAGS_GENERATOR_5][ LCGParams.MODULUS.value] = int(len(self.configs.tags) * 1.5) except: Logger.error( __file__, "Loading tags from file {} failed".format( self.configs.settings[Values.GENERAL_SECTION_NAME][ Values.TAGS_FILE_PATH])) if len(self.configs.tags) == 0: exit(-2)
def prepare_configurations_for_generation(self): ''' Call all function for prepare to generation ordera ''' Logger.info(__file__, "Execute preparing to execution") self.__load_currency_pairs_from_file() self.__load_tags_from_file() self.__register_lcg_generators() self.__calculate_orders_period_volumes() self.__calculate_orders_volumes_for_generations() self.__calculate_first_generation_period_start_date() self.__calculate_avg_values_of_id() self.init_rmq() Logger.info(__file__, "Execute preparing finished")
def consume(self): Logger.info(__file__, 'Configuration consumer...') rmq_settings = self.configs.settings[Values.RMQ_SECTION_NAME] self.rmq = RmqService() while not self.rmq.open_connection( host=rmq_settings[Values.RMQ_HOST], port=rmq_settings[Values.RMQ_PORT], virtual_host=rmq_settings[Values.RMQ_VIRTUAL_HOST], user=rmq_settings[Values.RMQ_USER], password=rmq_settings[Values.RMQ_PASSWORD]): self.rmq.reconfig() self.rmq.consume(queue_name=Zone.RED.value, on_consume_callback=self.__msg_consumer) self.rmq.consume(queue_name=Zone.BLUE.value, on_consume_callback=self.__msg_consumer) self.rmq.consume(queue_name=Zone.GREEN.value, on_consume_callback=self.__msg_consumer) Logger.info(__file__, 'Consumer configurated') self.configurate_db_service() self.previous_time = datetime.datetime.now() Logger.info(__file__, 'Start consuming') self.stop_messages_count = 0 self.rmq.start_consuming()
def __execute(self): self.generator_and_publisher_event = threading.Event() self.history_maker = OrderHistoryMaker( self.generator_and_publisher_event) self.history_maker.prepare_configurations_for_generation() self.consumer_thread = threading.Thread(target=self.start_consumer) self.generator_and_publisher_thread = threading.Thread( target=self.start_orders_history_generation) self.generator_and_publisher_thread.start() self.consumer_thread.start() sec = self.configs.settings[Values.GENERAL_SECTION_NAME][ Values.REPORTER_DELAY] while not self.generator_and_publisher_event.is_set( ) or not self.consumer_event.is_set(): time.sleep(sec) self.report() self.report() Logger.info(__file__, 'Program finished')
def __calculate_orders_period_volumes(self): ''' Calculate orders volumes list than equal to percentage of zones: red = 15%, green = 60% and blue = 25% ''' red_zone_percent = self.configs.settings[Values.GENERAL_SECTION_NAME][ Values.RED_ZONE_ORDERS_PERCENT] green_zone_percent = self.configs.settings[ Values.GENERAL_SECTION_NAME][Values.GREEN_ZONE_ORDERS_PERCENT] blue_zone_percent = self.configs.settings[Values.GENERAL_SECTION_NAME][ Values.BLUE_ZONE_ORDERS_PERCENT] Logger.info( __file__, 'Started calculating of orders volumes to period with zones percentes: red = {}, green ={}, blue ={}' .format(red_zone_percent, green_zone_percent, blue_zone_percent)) for i in range( 1, self.configs.settings[Values.GENERAL_SECTION_NAME][ Values.ORDERS_AMOUNT] + 1): first_percent = Utils.calculate_percent_from_value( i, red_zone_percent) second_percent = Utils.calculate_percent_from_value( i, green_zone_percent) third_percent = Utils.calculate_percent_from_value( i, blue_zone_percent) if Utils.is_int(first_percent) and Utils.is_int( second_percent) and Utils.is_int(third_percent): self.configs.min_orders_volumes = { Values.RED_ZONE_VOLUME: first_percent, Values.GREEN_ZONE_VOLUME: second_percent, Values.BLUE_ZONE_VOLUME: third_percent } break
def configurate_db_service(self): Logger.info(__file__, 'Configuration database service...') mysql_settings = self.configs.settings[Values.MYSQL_SECTION_NAME] self.mysql = MySqlService( user=mysql_settings[Values.MYSQL_USER], password=mysql_settings[Values.MYSQL_PASSWORD], host=mysql_settings[Values.MYSQL_HOST], port=mysql_settings[Values.MYSQL_PORT], database=mysql_settings[Values.MYSQL_DB_NAME]) try: self.mysql.open_connection() self.mysql.execute( f'TRUNCATE `{self.configs.settings[Values.MYSQL_SECTION_NAME][Values.MYSQL_DB_NAME]}`.`history`' ) Logger.info(__file__, 'Database service configurated') except AttributeError as er: Logger.error(__file__, er.args) Logger.info(__file__, 'Sending records to MySQL aborted')
def start_orders_history_generation(self): self.history_maker.execute_generation() Logger.info(__file__, 'Order history generation finished')
def execute(self): data = dict() Logger.info(__file__, f"Opening {self.__PAGE_URL} page") self.driver.get(self.__PAGE_URL) Logger.info(__file__, "Finding login input web element") login_web_element = self.get_element( by=By.XPATH, value=self.__EMAIL_INPUT_XPATH, wait=self.configs.settings[Values.SETTINGS][ Values.ELEMENT_WAIT_TIME]) Logger.info(__file__, "Send email to login input web element}") login_web_element.send_keys( self.configs.settings[Values.SETTINGS][Values.FACEBOOK_LOGIN]) Logger.info(__file__, "Finding password input web element") password_web_element = self.get_element( by=By.XPATH, value=self.__PASSWORD_INPUT_XPATH, wait=self.configs.settings[Values.SETTINGS][ Values.ELEMENT_WAIT_TIME]) Logger.info(__file__, "Send password to password input web element}") password_web_element.send_keys( self.configs.settings[Values.SETTINGS][Values.FACEBOOK_PASSWORD]) Logger.info(__file__, "Send Enter key pressing to password input web element}") password_web_element.send_keys(Keys.ENTER) Logger.info(__file__, "Finding account link") account_button_web_element = self.get_element( by=By.XPATH, value=self.__ACCOUNT_LINK_XPATH, wait=self.configs.settings[Values.SETTINGS][ Values.ELEMENT_WAIT_TIME]) Logger.info(__file__, "Click on account link") account_button_web_element.click() Logger.info(__file__, "Extracting total friends amount value") total_friends_amount = int( self.get_element(by=By.XPATH, value=self.__FRIENDS_AMOUNT_XPATH, wait=self.configs.settings[Values.SETTINGS][ Values.ELEMENT_WAIT_TIME]).text) Logger.debug(__file__, f"Extracted {total_friends_amount} value") Logger.info(__file__, "Finding friends page link") friends = self.get_element(by=By.XPATH, value=self.__FRIENDS_LINK_XPATH, wait=self.configs.settings[Values.SETTINGS][ Values.ELEMENT_WAIT_TIME]) Logger.info(__file__, "Click on friends page link") friends.click() Logger.info(__file__, "Extracting account name") name = self.get_element(by=By.XPATH, value=self.__ACCOUNT_NAME, wait=self.configs.settings[Values.SETTINGS][ Values.ELEMENT_WAIT_TIME]).text friends = self.__get_friends_list() previous_friend_amount = 0 current_friends_amount = len(friends) while previous_friend_amount != current_friends_amount: Logger.info(__file__, "Friends list can be scroll") previous_friend_amount = current_friends_amount Logger.info(__file__, "Scroll to last friends list element") ActionChains(self.driver).move_to_element(friends[-1]).perform() time.sleep(1) friends = self.__get_friends_list() current_friends_amount = len(friends) Logger.info(__file__, "Full friends list already loaded") Logger.info(__file__, "Extract data from friends web elements list") data["Account name: "] = name for friend in friends: link = friend.get_attribute("href") data[link[:friend.get_attribute('href').find('fref') - 1]] = friend.text Logger.info(__file__, "Closing web driver") Logger.info(__file__, "Closing selenium web driver") self.driver.quit() data["Total friends amount"] = total_friends_amount data["Scanned friends amount"] = current_friends_amount return data