def iteration(self):
"""
Run single iteration, entire logic of executor should be specified in this method, unless
there is an additional logic between iterations. Iteration is cancelled, if executor is
terminated.
:return: boolean flag, True - run next iteration, False - terminate
"""
# we process special case of terminated executor in case someone would launch it again.
if self._terminated:
logger.warning("Executor %s has been terminated", self.name)
return False
logger.debug("%s - Run iteration, timeout=%s", self.name, self.timeout)
try:
# send reponse to the scheduler that this executor is up and processing tasks
self._respond_is_alive()
# check if there are any messages in connection, process one message per iteration
if self.conn.poll():
self._process_message(self.conn.recv())
# check if there is any outstanding task to run, otherwise poll data for current task
self._process_task()
except ExecutorInterruptedException:
logger.info("%s - Requested termination of executor", self.name)
self._terminated = True
# cancel task that is currently running and clean up state
self._cancel_active_task()
return False
# pylint: disable=W0703,broad-except
except Exception as e:
logger.exception("%s - Unrecoverable error %s, terminating", self.name, e)
self._terminated = True
return False
# pylint: enable=W0703,broad-except
else:
return True
def split_odds(result, regex=re.compile(r'[0-9.]+')):
result = regex.findall(result) # -> ['2.05', '1.95']
try:
if len(result) == 2:
result = {'open_odd': result[0], 'close_odd': result[1]}
else:
result = {'open_odd': result[0], 'close_odd': result[0]}
except IndexError as e:
logger.exception('ERROR func split_odds {}'.format(e))
result = {'open_odd': 'not_data', 'close_odd': 'not_data'}
return result
def load_script_data(self):
error = None
xpath = (
'/html/body/div[3]/div[2]/div/div[2]/div[1]/div[7]/table/tbody/tr/td/a'
)
res = self.driver.find_element_by_xpath(xpath)
while not error:
try:
res.click()
sleep(1)
except BaseException:
logger.exception('i know about it!')
error = True
logger.debug('load script data')
def main(bot):
NUM = bot.queue
LEN_NUM = len(NUM)
logger.debug('all pair_urls: {}'.format(LEN_NUM))
try:
for i, _id in enumerate(NUM):
if all([i % BOT_LIVES == 0, i != 0]):
logger.debug(bot)
bot.turn_off_bot()
logger.debug('DIE BOT')
bot = Bot.new_bot()
logger.debug('NEW {}'.format(bot))
bot_soups = new_bot_soups(bot, _id)
data_to_db = union_parse_pages(*bot_soups)
data_to_db.update({'myscore_id': _id})
append_data_to_file(bot.crawled_file, _id)
logger.info('crawl_url: \n{}, \n{}'.format(*url_mask(_id)))
logger.info('complite on: {} %'.format((i + 1) / LEN_NUM * 100))
# write data to db
db = get_db()
insert_to_football_db(db, data_to_db)
logger.info('data write to db')
bot.turn_off_bot()
except AttributeError as e:
logger.exception(e)
logger.info('error match_id:', _id)
create_restore_file(
path_queue_file=bot.queue_file,
path_crawled_file=bot.crawled_file,
path_restore_file=bot.restore_file
)
restore_main(bot)
finally:
create_restore_file(
path_queue_file=bot.queue_file,
path_crawled_file=bot.crawled_file,
path_restore_file=bot.restore_file
)
def connect_server(self):
self.ssh_client = SSHClient()
try:
self.ssh_client.connect_server(
conf.SSH_SERVER_IP_ADDRESS, conf.SSH_SERVER_PORT, conf.SSH_SERVER_USERNAME, conf.SSH_SERVER_PASSWORD)
logger.info('Connect to SSH Server')
logger.debug('SSH Server IP Address -> %s' %
conf.SSH_SERVER_IP_ADDRESS)
logger.debug('SSH Server Port -> %s' % conf.SSH_SERVER_PORT)
logger.debug('SSH Connect Username -> %s' %
conf.SSH_SERVER_USERNAME)
return self.is_connected_server()
except SSHException:
logger.error(
"can't connect netserial server, please contact the administrator.")
raise SSHException
except Exception:
logger.exception("something wrong with the netserial server.")
raise Exception
def iteration(self):
"""
Run single iteration, entire logic of executor should be specified in this method, unless
there is an additional logic between iterations. Iteration is cancelled, if executor is
terminated.
:return: boolean flag, True - run next iteration, False - terminate
"""
# we process special case of terminated executor in case someone would launch it again.
if self._terminated:
logger.warning("Executor %s has been terminated", self.name)
return False
logger.debug("%s - Run iteration, timeout=%s", self.name, self.timeout)
try:
# send reponse to the scheduler that this executor is up and processing tasks
self._respond_is_alive()
# check if there are any messages in connection, process one message per iteration
if self.conn.poll():
self._process_message(self.conn.recv())
# check if there is any outstanding task to run, otherwise poll data for current task
self._process_task()
except ExecutorInterruptedException:
logger.info("%s - Requested termination of executor", self.name)
self._terminated = True
# cancel task that is currently running and clean up state
self._cancel_active_task()
return False
# pylint: disable=W0703,broad-except
except Exception as e:
logger.exception("%s - Unrecoverable error %s, terminating",
self.name, e)
self._terminated = True
return False
# pylint: enable=W0703,broad-except
else:
return True