def start(self):
self.logger.info('Bot start processing')
self.source_pipeline = None
self.destination_pipeline = None
local_retry_delay = 0
while True:
try:
if self.parameters.source_pipeline_enable and not self.source_pipeline:
time.sleep(local_retry_delay)
self.logger.info("Connecting to source pipeline")
self.source_pipeline = Pipeline(
self.parameters.source_pipeline_host,
self.parameters.source_pipeline_port,
self.parameters.source_pipeline_db
)
self.source_pipeline.source_queues(self.source_queue)
self.logger.info("Connected to source pipeline")
if self.parameters.destination_pipeline_enable and not self.destination_pipeline:
time.sleep(local_retry_delay)
self.logger.info("Connecting to destination pipeline")
self.destination_pipeline = Pipeline(
self.parameters.destination_pipeline_host,
self.parameters.destination_pipeline_port,
self.parameters.destination_pipeline_db
)
self.destination_pipeline.destination_queues(self.destination_queues)
self.logger.info("Connected to destination pipeline")
self.logger.info("Start processing")
self.process()
if self.source_pipeline:
self.source_pipeline.sleep(self.parameters.rate_limit)
except Exception, ex:
local_retry_delay = self.parameters.retry_delay
self.logger.info("Last Correct Message(event): %r" % self.last_message)
self.logger.info("Current Message(event): %r" % self.current_message)
self.logger.exception("Check the following exception:")
self.logger.error('Pipeline connection failed (%r)' % ex)
self.logger.info('Pipeline will reconnect in %s seconds' % local_retry_delay)
self.source_pipeline = None
self.destination_pipeline = None
except KeyboardInterrupt as e:
if self.source_pipeline:
self.source_pipeline.disconnect()
self.logger.info("Disconnecting from source pipeline")
if self.destination_pipeline:
self.destination_pipeline.disconnect()
self.logger.info("Disconnecting from destination pipeline")
self.logger.info("Bot is shutting down")
break
def start(self):
self.source_pipeline = None
self.destination_pipeline = None
local_retry_delay = 0
self.parameters.retry_delay = 30 # Temporary fix. Need to add to BOTS conf
self.logger.info('Bot start processing')
while True:
try:
if not self.source_pipeline:
time.sleep(local_retry_delay)
self.logger.info("Connecting to source pipeline")
self.source_pipeline = Pipeline()
self.source_pipeline.source_queues(self.source_queues)
self.logger.info("Connected to source pipeline")
if not self.destination_pipeline:
time.sleep(local_retry_delay)
self.logger.info("Connecting to destination pipeline")
self.destination_pipeline = Pipeline()
self.destination_pipeline.destination_queues(self.destination_queues)
self.logger.info("Connected to destination pipeline")
self.logger.info("Start processing")
self.process()
self.source_pipeline.sleep(self.parameters.rate_limit)
except IntelMQPipeline, ex:
# NAO LOGAR A MENSAGEM PQ O PROBLEMA É NA PIPELINE
# na excepção em baixo deve estar o caso de não ser um erro de pipeline e assim
# deve loggar consoante o parametro do bot.
except Exception, ex:
local_retry_delay = self.parameters.retry_delay
self.logger.info("Last Correct Message(event): %r" % self.last_message)
self.logger.info("Current Message(event): %r" % self.current_message)
self.logger.exception("Check the following exception:")
self.logger.error('Pipeline connection failed (%r)' % ex)
self.logger.info('Pipeline will reconnect in %s seconds' % local_retry_delay)
self.source_pipeline = None
self.destination_pipeline = None
except KeyboardInterrupt as e:
if self.source_pipeline:
self.source_pipeline.disconnect()
self.logger.info("Disconnecting from source pipeline")
if self.destination_pipeline:
self.destination_pipeline.disconnect()
self.logger.info("Disconnecting from destination pipeline")
self.logger.info("Bot is shutting down")
break
def internal_start(self, thread_identifier):
self.source_pipeline = None
self.destination_pipeline = None
local_retry_delay = 0
self.logger.info("Bot start processing")
while True:
try:
if not self.source_pipeline:
time.sleep(local_retry_delay)
self.logger.info("Connecting to source pipeline")
self.source_pipeline = Pipeline()
# == NEW CODE
self.source_pipeline.source_queues(self.source_queues, thread_identifier)
# self.source_pipeline.source_queues(self.source_queues)
# == NEW CODE
self.logger.info("Connected to source pipeline")
if not self.destination_pipeline:
time.sleep(local_retry_delay)
self.logger.info("Connecting to destination pipeline")
self.destination_pipeline = Pipeline()
self.destination_pipeline.destination_queues(self.destination_queues)
self.logger.info("Connected to destination pipeline")
self.logger.info("Start processing")
self.process()
self.source_pipeline.sleep(self.parameters.rate_limit)
except Exception, ex:
local_retry_delay = self.parameters.retry_delay
self.logger.info("Last Correct Message(event): %r" % self.last_message)
self.logger.info("Current Message(event): %r" % self.current_message)
self.logger.exception("Check the following exception:")
self.logger.error("Pipeline connection failed (%r)" % ex)
self.logger.info("Pipeline will reconnect in %s seconds" % local_retry_delay)
self.source_pipeline = None
self.destination_pipeline = None
except KeyboardInterrupt as e:
if self.source_pipeline:
self.source_pipeline.disconnect()
self.logger.info("Disconnecting from source pipeline")
if self.destination_pipeline:
self.destination_pipeline.disconnect()
self.logger.info("Disconnecting from destination pipeline")
self.logger.info("Bot is shutting down")
break
def start(self):
"""Starts a bot"""
self.source_pipeline = None
self.destination_pipeline = None
local_retry_delay = 0
self.parameters.retry_delay = 30 # Temporary fix. Need to add to BOTS conf
self.logger.info('Bot start processing')
while True:
try:
if not self.source_pipeline:
time.sleep(local_retry_delay)
self.logger.info("Connecting to source pipeline")
self.source_pipeline = Pipeline()
self.source_pipeline.source_queues(self.source_queues)
self.logger.info("Connected to source pipeline")
if not self.destination_pipeline:
time.sleep(local_retry_delay)
self.logger.info("Connecting to destination pipeline")
self.destination_pipeline = Pipeline()
self.destination_pipeline.destination_queues(self.destination_queues)
self.logger.info("Connected to destination pipeline")
self.process()
self.logger.info("Bot stops processing. Sleeps for 'rate_limit' = %ds" % self.parameters.rate_limit)
self.source_pipeline.sleep(self.parameters.rate_limit)
except Exception, ex:
local_retry_delay = self.parameters.retry_delay
self.logger.info("Last Correct Message(event): %r" % self.last_message)
self.logger.info("Current Message(event): %r" % self.current_message)
self.logger.exception("Check the following exception:")
self.logger.error('Pipeline connection failed (%r)' % ex)
self.logger.info('Pipeline will reconnect in %s seconds' % local_retry_delay)
self.source_pipeline = None
self.destination_pipeline = None
except KeyboardInterrupt as e:
if self.source_pipeline:
self.source_pipeline.disconnect()
self.logger.info("Disconnecting from source pipeline")
if self.destination_pipeline:
self.destination_pipeline.disconnect()
self.logger.info("Disconnecting from destination pipeline")
self.logger.info("Bot is shutting down")
break
def start(self):
self.logger.info('Bot start processing')
self.pipeline = None
retry_delay = 0
while True:
try:
if not self.pipeline:
time.sleep(retry_delay)
self.logger.info("Connecting to pipeline queues")
self.pipeline = Pipeline()
self.pipeline.queues(self.src_queue, self.dest_queues)
self.logger.info("Connected to pipeline queues. Start processing")
self.process()
self.pipeline.sleep(self.parameters.processing_interval)
except Exception, ex:
retry_delay = 30
self.logger.info("Last Correct Message(event): %r" % self.last_message)
self.logger.info("Current Message(event): %r" % self.current_message)
self.logger.exception("Check the following exception:")
self.logger.error('Pipeline connection failed (%r)' % ex)
self.logger.info('Pipeline will reconnect in %s seconds' % retry_delay)
#self.pipeline.disconnect() # caused problems
self.pipeline = None
except KeyboardInterrupt as e:
if self.pipeline:
self.pipeline.disconnect()
self.logger.info("Disconnecting from pipeline")
self.logger.info("Bot is shutting down")
break
def _process(self, dryrun, msg, show):
if msg:
msg = MessageFactory.serialize(self.arg2msg(msg))
if not self.instance._Bot__source_pipeline:
# is None if source pipeline does not exist
self.instance._Bot__source_pipeline = Pipeline(None)
self.instance._Bot__source_pipeline.receive = lambda *args, **kwargs: msg
self.instance._Bot__source_pipeline.acknowledge = lambda *args, **kwargs: None
self.instance.logger.info(
" * Message from cli will be used when processing.")
if dryrun:
self.instance.send_message = lambda *args, **kwargs: self.instance.logger.info(
"DRYRUN: Message would be sent now to %r!",
kwargs.get('path', "_default"))
self.instance.acknowledge_message = lambda *args, **kwargs: self.instance.logger.info(
"DRYRUN: Message would be acknowledged now!")
self.instance.logger.info(
" * Dryrun only, no message will be really sent through.")
if show:
fn = self.instance.send_message
self.instance.send_message = lambda *args, **kwargs: [
self.pprint(args or "No message generated"),
fn(*args, **kwargs)
]
self.instance.logger.info("Processing...")
self.instance.process()
def _process(self, dryrun, msg, show):
if msg:
msg = MessageFactory.serialize(self.arg2msg(msg))
if not self.instance._Bot__source_pipeline:
# is None if source pipeline does not exist
self.instance._Bot__source_pipeline = Pipeline(None)
self.instance._Bot__source_pipeline.receive = lambda: msg
self.instance.logger.info(
" * Message from cli will be used when processing.")
if dryrun:
self.instance.send_message = lambda msg, path="_default": self.instance.logger.info(
"DRYRUN: Message would be sent now{}!".format(
" to the {} path".format(path)
if (path != "_default") else ""))
self.instance.acknowledge_message = lambda: self.instance.logger.info(
"DRYRUN: Message would be acknowledged now!")
self.instance.logger.info(
" * Dryrun only, no message will be really sent through.")
if show:
fn = self.instance.send_message
self.instance.send_message = lambda msg, path="_default": [
self.pprint(msg), fn(msg, path=path)
]
self.instance.logger.info("Processing...")
self.instance.process()
class Bot(object):
def __init__(self, bot_id):
self.parameters = Parameters()
self.current_message = None
self.last_message = None
self.message_counter = 0
self.check_bot_id(bot_id)
self.bot_id = bot_id
self.load_system_configurations()
self.logger = log(self.parameters.logging_path, self.bot_id, self.parameters.logging_level)
self.logger.info('Bot is starting')
self.load_runtime_configurations()
self.source_queue, self.destination_queues = self.load_pipeline()
self.parameters.rate_limit = float(self.parameters.rate_limit)
self.parameters.retry_delay = int(self.parameters.retry_delay)
self.init()
def init(self):
pass
def start(self):
self.logger.info('Bot start processing')
self.source_pipeline = None
self.destination_pipeline = None
local_retry_delay = 0
while True:
try:
if self.parameters.source_pipeline_enable and not self.source_pipeline:
time.sleep(local_retry_delay)
self.logger.info("Connecting to source pipeline")
self.source_pipeline = Pipeline(
self.parameters.source_pipeline_host,
self.parameters.source_pipeline_port,
self.parameters.source_pipeline_db
)
self.source_pipeline.source_queues(self.source_queue)
self.logger.info("Connected to source pipeline")
if self.parameters.destination_pipeline_enable and not self.destination_pipeline:
time.sleep(local_retry_delay)
self.logger.info("Connecting to destination pipeline")
self.destination_pipeline = Pipeline(
self.parameters.destination_pipeline_host,
self.parameters.destination_pipeline_port,
self.parameters.destination_pipeline_db
)
self.destination_pipeline.destination_queues(self.destination_queues)
self.logger.info("Connected to destination pipeline")
self.logger.info("Start processing")
self.process()
if self.source_pipeline:
self.source_pipeline.sleep(self.parameters.rate_limit)
except Exception, ex:
local_retry_delay = self.parameters.retry_delay
self.logger.info("Last Correct Message(event): %r" % self.last_message)
self.logger.info("Current Message(event): %r" % self.current_message)
self.logger.exception("Check the following exception:")
self.logger.error('Pipeline connection failed (%r)' % ex)
self.logger.info('Pipeline will reconnect in %s seconds' % local_retry_delay)
self.source_pipeline = None
self.destination_pipeline = None
except KeyboardInterrupt as e:
if self.source_pipeline:
self.source_pipeline.disconnect()
self.logger.info("Disconnecting from source pipeline")
if self.destination_pipeline:
self.destination_pipeline.disconnect()
self.logger.info("Disconnecting from destination pipeline")
self.logger.info("Bot is shutting down")
break
class Bot(object):
def __init__(self, bot_id):
self.parameters = Parameters()
self.current_message = None
self.last_message = None
self.message_counter = 0
self.check_bot_id(bot_id)
self.bot_id = bot_id
self.load_system_configurations()
self.logger = log(self.parameters.logging_path, self.bot_id, self.parameters.logging_level)
self.logger.info('Bot is starting')
self.load_runtime_configurations()
self.src_queue, self.dest_queues = self.load_pipeline()
self.parameters.processing_interval = float(self.parameters.processing_interval)
self.init()
def init(self):
pass
def start(self):
self.logger.info('Bot start processing')
self.pipeline = None
retry_delay = 0
while True:
try:
if not self.pipeline:
time.sleep(retry_delay)
self.logger.info("Connecting to pipeline queues")
self.pipeline = Pipeline()
self.pipeline.queues(self.src_queue, self.dest_queues)
self.logger.info("Connected to pipeline queues. Start processing")
self.process()
self.pipeline.sleep(self.parameters.processing_interval)
except Exception, ex:
retry_delay = 30
self.logger.info("Last Correct Message(event): %r" % self.last_message)
self.logger.info("Current Message(event): %r" % self.current_message)
self.logger.exception("Check the following exception:")
self.logger.error('Pipeline connection failed (%r)' % ex)
self.logger.info('Pipeline will reconnect in %s seconds' % retry_delay)
#self.pipeline.disconnect() # caused problems
self.pipeline = None
except KeyboardInterrupt as e:
if self.pipeline:
self.pipeline.disconnect()
self.logger.info("Disconnecting from pipeline")
self.logger.info("Bot is shutting down")
break
class Bot(object):
def __init__(self, bot_id):
self.parameters = Parameters()
self.current_message = None
self.last_message = None
self.message_counter = 0
self.check_bot_id(bot_id)
self.bot_id = bot_id
self.load_system_configurations()
self.logger = log(self.parameters.logging_path, self.bot_id, self.parameters.logging_level)
self.logger.info("Bot is starting")
self.load_runtime_configurations()
self.load_pipeline_configurations()
# == NEW CODE
self.parameters.threads_number = 1
# == NEW CODE
self.init()
def init(self):
pass
# == NEW CODE
def start(self):
for thread_identifier in range(0, self.parameters.threads_number):
thread = threading.Thread(target=self.internal_start, args=(thread_identifier,))
thread.start()
# == NEW CODE
def internal_start(self, thread_identifier):
self.source_pipeline = None
self.destination_pipeline = None
local_retry_delay = 0
self.logger.info("Bot start processing")
while True:
try:
if not self.source_pipeline:
time.sleep(local_retry_delay)
self.logger.info("Connecting to source pipeline")
self.source_pipeline = Pipeline()
# == NEW CODE
self.source_pipeline.source_queues(self.source_queues, thread_identifier)
# self.source_pipeline.source_queues(self.source_queues)
# == NEW CODE
self.logger.info("Connected to source pipeline")
if not self.destination_pipeline:
time.sleep(local_retry_delay)
self.logger.info("Connecting to destination pipeline")
self.destination_pipeline = Pipeline()
self.destination_pipeline.destination_queues(self.destination_queues)
self.logger.info("Connected to destination pipeline")
self.logger.info("Start processing")
self.process()
self.source_pipeline.sleep(self.parameters.rate_limit)
except Exception, ex:
local_retry_delay = self.parameters.retry_delay
self.logger.info("Last Correct Message(event): %r" % self.last_message)
self.logger.info("Current Message(event): %r" % self.current_message)
self.logger.exception("Check the following exception:")
self.logger.error("Pipeline connection failed (%r)" % ex)
self.logger.info("Pipeline will reconnect in %s seconds" % local_retry_delay)
self.source_pipeline = None
self.destination_pipeline = None
except KeyboardInterrupt as e:
if self.source_pipeline:
self.source_pipeline.disconnect()
self.logger.info("Disconnecting from source pipeline")
if self.destination_pipeline:
self.destination_pipeline.disconnect()
self.logger.info("Disconnecting from destination pipeline")
self.logger.info("Bot is shutting down")
break