def iteration(self): """ Run single iteration, entire logic of executor should be specified in this method, unless there is an additional logic between iterations. Iteration is cancelled, if executor is terminated. :return: boolean flag, True - run next iteration, False - terminate """ # we process special case of terminated executor in case someone would launch it again. if self._terminated: logger.warning("Executor %s has been terminated", self.name) return False logger.debug("%s - Run iteration, timeout=%s", self.name, self.timeout) try: # send reponse to the scheduler that this executor is up and processing tasks self._respond_is_alive() # check if there are any messages in connection, process one message per iteration if self.conn.poll(): self._process_message(self.conn.recv()) # check if there is any outstanding task to run, otherwise poll data for current task self._process_task() except ExecutorInterruptedException: logger.info("%s - Requested termination of executor", self.name) self._terminated = True # cancel task that is currently running and clean up state self._cancel_active_task() return False # pylint: disable=W0703,broad-except except Exception as e: logger.exception("%s - Unrecoverable error %s, terminating", self.name, e) self._terminated = True return False # pylint: enable=W0703,broad-except else: return True
def split_odds(result, regex=re.compile(r'[0-9.]+')): result = regex.findall(result) # -> ['2.05', '1.95'] try: if len(result) == 2: result = {'open_odd': result[0], 'close_odd': result[1]} else: result = {'open_odd': result[0], 'close_odd': result[0]} except IndexError as e: logger.exception('ERROR func split_odds {}'.format(e)) result = {'open_odd': 'not_data', 'close_odd': 'not_data'} return result
def load_script_data(self): error = None xpath = ( '/html/body/div[3]/div[2]/div/div[2]/div[1]/div[7]/table/tbody/tr/td/a' ) res = self.driver.find_element_by_xpath(xpath) while not error: try: res.click() sleep(1) except BaseException: logger.exception('i know about it!') error = True logger.debug('load script data')
def main(bot): NUM = bot.queue LEN_NUM = len(NUM) logger.debug('all pair_urls: {}'.format(LEN_NUM)) try: for i, _id in enumerate(NUM): if all([i % BOT_LIVES == 0, i != 0]): logger.debug(bot) bot.turn_off_bot() logger.debug('DIE BOT') bot = Bot.new_bot() logger.debug('NEW {}'.format(bot)) bot_soups = new_bot_soups(bot, _id) data_to_db = union_parse_pages(*bot_soups) data_to_db.update({'myscore_id': _id}) append_data_to_file(bot.crawled_file, _id) logger.info('crawl_url: \n{}, \n{}'.format(*url_mask(_id))) logger.info('complite on: {} %'.format((i + 1) / LEN_NUM * 100)) # write data to db db = get_db() insert_to_football_db(db, data_to_db) logger.info('data write to db') bot.turn_off_bot() except AttributeError as e: logger.exception(e) logger.info('error match_id:', _id) create_restore_file( path_queue_file=bot.queue_file, path_crawled_file=bot.crawled_file, path_restore_file=bot.restore_file ) restore_main(bot) finally: create_restore_file( path_queue_file=bot.queue_file, path_crawled_file=bot.crawled_file, path_restore_file=bot.restore_file )
def connect_server(self): self.ssh_client = SSHClient() try: self.ssh_client.connect_server( conf.SSH_SERVER_IP_ADDRESS, conf.SSH_SERVER_PORT, conf.SSH_SERVER_USERNAME, conf.SSH_SERVER_PASSWORD) logger.info('Connect to SSH Server') logger.debug('SSH Server IP Address -> %s' % conf.SSH_SERVER_IP_ADDRESS) logger.debug('SSH Server Port -> %s' % conf.SSH_SERVER_PORT) logger.debug('SSH Connect Username -> %s' % conf.SSH_SERVER_USERNAME) return self.is_connected_server() except SSHException: logger.error( "can't connect netserial server, please contact the administrator.") raise SSHException except Exception: logger.exception("something wrong with the netserial server.") raise Exception