def start(self): self.logger.info('Bot start processing') self.source_pipeline = None self.destination_pipeline = None local_retry_delay = 0 while True: try: if self.parameters.source_pipeline_enable and not self.source_pipeline: time.sleep(local_retry_delay) self.logger.info("Connecting to source pipeline") self.source_pipeline = Pipeline( self.parameters.source_pipeline_host, self.parameters.source_pipeline_port, self.parameters.source_pipeline_db ) self.source_pipeline.source_queues(self.source_queue) self.logger.info("Connected to source pipeline") if self.parameters.destination_pipeline_enable and not self.destination_pipeline: time.sleep(local_retry_delay) self.logger.info("Connecting to destination pipeline") self.destination_pipeline = Pipeline( self.parameters.destination_pipeline_host, self.parameters.destination_pipeline_port, self.parameters.destination_pipeline_db ) self.destination_pipeline.destination_queues(self.destination_queues) self.logger.info("Connected to destination pipeline") self.logger.info("Start processing") self.process() if self.source_pipeline: self.source_pipeline.sleep(self.parameters.rate_limit) except Exception, ex: local_retry_delay = self.parameters.retry_delay self.logger.info("Last Correct Message(event): %r" % self.last_message) self.logger.info("Current Message(event): %r" % self.current_message) self.logger.exception("Check the following exception:") self.logger.error('Pipeline connection failed (%r)' % ex) self.logger.info('Pipeline will reconnect in %s seconds' % local_retry_delay) self.source_pipeline = None self.destination_pipeline = None except KeyboardInterrupt as e: if self.source_pipeline: self.source_pipeline.disconnect() self.logger.info("Disconnecting from source pipeline") if self.destination_pipeline: self.destination_pipeline.disconnect() self.logger.info("Disconnecting from destination pipeline") self.logger.info("Bot is shutting down") break
def start(self): self.source_pipeline = None self.destination_pipeline = None local_retry_delay = 0 self.parameters.retry_delay = 30 # Temporary fix. Need to add to BOTS conf self.logger.info('Bot start processing') while True: try: if not self.source_pipeline: time.sleep(local_retry_delay) self.logger.info("Connecting to source pipeline") self.source_pipeline = Pipeline() self.source_pipeline.source_queues(self.source_queues) self.logger.info("Connected to source pipeline") if not self.destination_pipeline: time.sleep(local_retry_delay) self.logger.info("Connecting to destination pipeline") self.destination_pipeline = Pipeline() self.destination_pipeline.destination_queues(self.destination_queues) self.logger.info("Connected to destination pipeline") self.logger.info("Start processing") self.process() self.source_pipeline.sleep(self.parameters.rate_limit) except IntelMQPipeline, ex: # NAO LOGAR A MENSAGEM PQ O PROBLEMA É NA PIPELINE # na excepção em baixo deve estar o caso de não ser um erro de pipeline e assim # deve loggar consoante o parametro do bot. except Exception, ex: local_retry_delay = self.parameters.retry_delay self.logger.info("Last Correct Message(event): %r" % self.last_message) self.logger.info("Current Message(event): %r" % self.current_message) self.logger.exception("Check the following exception:") self.logger.error('Pipeline connection failed (%r)' % ex) self.logger.info('Pipeline will reconnect in %s seconds' % local_retry_delay) self.source_pipeline = None self.destination_pipeline = None except KeyboardInterrupt as e: if self.source_pipeline: self.source_pipeline.disconnect() self.logger.info("Disconnecting from source pipeline") if self.destination_pipeline: self.destination_pipeline.disconnect() self.logger.info("Disconnecting from destination pipeline") self.logger.info("Bot is shutting down") break
def internal_start(self, thread_identifier): self.source_pipeline = None self.destination_pipeline = None local_retry_delay = 0 self.logger.info("Bot start processing") while True: try: if not self.source_pipeline: time.sleep(local_retry_delay) self.logger.info("Connecting to source pipeline") self.source_pipeline = Pipeline() # == NEW CODE self.source_pipeline.source_queues(self.source_queues, thread_identifier) # self.source_pipeline.source_queues(self.source_queues) # == NEW CODE self.logger.info("Connected to source pipeline") if not self.destination_pipeline: time.sleep(local_retry_delay) self.logger.info("Connecting to destination pipeline") self.destination_pipeline = Pipeline() self.destination_pipeline.destination_queues(self.destination_queues) self.logger.info("Connected to destination pipeline") self.logger.info("Start processing") self.process() self.source_pipeline.sleep(self.parameters.rate_limit) except Exception, ex: local_retry_delay = self.parameters.retry_delay self.logger.info("Last Correct Message(event): %r" % self.last_message) self.logger.info("Current Message(event): %r" % self.current_message) self.logger.exception("Check the following exception:") self.logger.error("Pipeline connection failed (%r)" % ex) self.logger.info("Pipeline will reconnect in %s seconds" % local_retry_delay) self.source_pipeline = None self.destination_pipeline = None except KeyboardInterrupt as e: if self.source_pipeline: self.source_pipeline.disconnect() self.logger.info("Disconnecting from source pipeline") if self.destination_pipeline: self.destination_pipeline.disconnect() self.logger.info("Disconnecting from destination pipeline") self.logger.info("Bot is shutting down") break
def start(self): """Starts a bot""" self.source_pipeline = None self.destination_pipeline = None local_retry_delay = 0 self.parameters.retry_delay = 30 # Temporary fix. Need to add to BOTS conf self.logger.info('Bot start processing') while True: try: if not self.source_pipeline: time.sleep(local_retry_delay) self.logger.info("Connecting to source pipeline") self.source_pipeline = Pipeline() self.source_pipeline.source_queues(self.source_queues) self.logger.info("Connected to source pipeline") if not self.destination_pipeline: time.sleep(local_retry_delay) self.logger.info("Connecting to destination pipeline") self.destination_pipeline = Pipeline() self.destination_pipeline.destination_queues(self.destination_queues) self.logger.info("Connected to destination pipeline") self.process() self.logger.info("Bot stops processing. Sleeps for 'rate_limit' = %ds" % self.parameters.rate_limit) self.source_pipeline.sleep(self.parameters.rate_limit) except Exception, ex: local_retry_delay = self.parameters.retry_delay self.logger.info("Last Correct Message(event): %r" % self.last_message) self.logger.info("Current Message(event): %r" % self.current_message) self.logger.exception("Check the following exception:") self.logger.error('Pipeline connection failed (%r)' % ex) self.logger.info('Pipeline will reconnect in %s seconds' % local_retry_delay) self.source_pipeline = None self.destination_pipeline = None except KeyboardInterrupt as e: if self.source_pipeline: self.source_pipeline.disconnect() self.logger.info("Disconnecting from source pipeline") if self.destination_pipeline: self.destination_pipeline.disconnect() self.logger.info("Disconnecting from destination pipeline") self.logger.info("Bot is shutting down") break
def start(self): self.logger.info('Bot start processing') self.pipeline = None retry_delay = 0 while True: try: if not self.pipeline: time.sleep(retry_delay) self.logger.info("Connecting to pipeline queues") self.pipeline = Pipeline() self.pipeline.queues(self.src_queue, self.dest_queues) self.logger.info("Connected to pipeline queues. Start processing") self.process() self.pipeline.sleep(self.parameters.processing_interval) except Exception, ex: retry_delay = 30 self.logger.info("Last Correct Message(event): %r" % self.last_message) self.logger.info("Current Message(event): %r" % self.current_message) self.logger.exception("Check the following exception:") self.logger.error('Pipeline connection failed (%r)' % ex) self.logger.info('Pipeline will reconnect in %s seconds' % retry_delay) #self.pipeline.disconnect() # caused problems self.pipeline = None except KeyboardInterrupt as e: if self.pipeline: self.pipeline.disconnect() self.logger.info("Disconnecting from pipeline") self.logger.info("Bot is shutting down") break
def _process(self, dryrun, msg, show): if msg: msg = MessageFactory.serialize(self.arg2msg(msg)) if not self.instance._Bot__source_pipeline: # is None if source pipeline does not exist self.instance._Bot__source_pipeline = Pipeline(None) self.instance._Bot__source_pipeline.receive = lambda *args, **kwargs: msg self.instance._Bot__source_pipeline.acknowledge = lambda *args, **kwargs: None self.instance.logger.info( " * Message from cli will be used when processing.") if dryrun: self.instance.send_message = lambda *args, **kwargs: self.instance.logger.info( "DRYRUN: Message would be sent now to %r!", kwargs.get('path', "_default")) self.instance.acknowledge_message = lambda *args, **kwargs: self.instance.logger.info( "DRYRUN: Message would be acknowledged now!") self.instance.logger.info( " * Dryrun only, no message will be really sent through.") if show: fn = self.instance.send_message self.instance.send_message = lambda *args, **kwargs: [ self.pprint(args or "No message generated"), fn(*args, **kwargs) ] self.instance.logger.info("Processing...") self.instance.process()
def _process(self, dryrun, msg, show): if msg: msg = MessageFactory.serialize(self.arg2msg(msg)) if not self.instance._Bot__source_pipeline: # is None if source pipeline does not exist self.instance._Bot__source_pipeline = Pipeline(None) self.instance._Bot__source_pipeline.receive = lambda: msg self.instance.logger.info( " * Message from cli will be used when processing.") if dryrun: self.instance.send_message = lambda msg, path="_default": self.instance.logger.info( "DRYRUN: Message would be sent now{}!".format( " to the {} path".format(path) if (path != "_default") else "")) self.instance.acknowledge_message = lambda: self.instance.logger.info( "DRYRUN: Message would be acknowledged now!") self.instance.logger.info( " * Dryrun only, no message will be really sent through.") if show: fn = self.instance.send_message self.instance.send_message = lambda msg, path="_default": [ self.pprint(msg), fn(msg, path=path) ] self.instance.logger.info("Processing...") self.instance.process()
class Bot(object): def __init__(self, bot_id): self.parameters = Parameters() self.current_message = None self.last_message = None self.message_counter = 0 self.check_bot_id(bot_id) self.bot_id = bot_id self.load_system_configurations() self.logger = log(self.parameters.logging_path, self.bot_id, self.parameters.logging_level) self.logger.info('Bot is starting') self.load_runtime_configurations() self.source_queue, self.destination_queues = self.load_pipeline() self.parameters.rate_limit = float(self.parameters.rate_limit) self.parameters.retry_delay = int(self.parameters.retry_delay) self.init() def init(self): pass def start(self): self.logger.info('Bot start processing') self.source_pipeline = None self.destination_pipeline = None local_retry_delay = 0 while True: try: if self.parameters.source_pipeline_enable and not self.source_pipeline: time.sleep(local_retry_delay) self.logger.info("Connecting to source pipeline") self.source_pipeline = Pipeline( self.parameters.source_pipeline_host, self.parameters.source_pipeline_port, self.parameters.source_pipeline_db ) self.source_pipeline.source_queues(self.source_queue) self.logger.info("Connected to source pipeline") if self.parameters.destination_pipeline_enable and not self.destination_pipeline: time.sleep(local_retry_delay) self.logger.info("Connecting to destination pipeline") self.destination_pipeline = Pipeline( self.parameters.destination_pipeline_host, self.parameters.destination_pipeline_port, self.parameters.destination_pipeline_db ) self.destination_pipeline.destination_queues(self.destination_queues) self.logger.info("Connected to destination pipeline") self.logger.info("Start processing") self.process() if self.source_pipeline: self.source_pipeline.sleep(self.parameters.rate_limit) except Exception, ex: local_retry_delay = self.parameters.retry_delay self.logger.info("Last Correct Message(event): %r" % self.last_message) self.logger.info("Current Message(event): %r" % self.current_message) self.logger.exception("Check the following exception:") self.logger.error('Pipeline connection failed (%r)' % ex) self.logger.info('Pipeline will reconnect in %s seconds' % local_retry_delay) self.source_pipeline = None self.destination_pipeline = None except KeyboardInterrupt as e: if self.source_pipeline: self.source_pipeline.disconnect() self.logger.info("Disconnecting from source pipeline") if self.destination_pipeline: self.destination_pipeline.disconnect() self.logger.info("Disconnecting from destination pipeline") self.logger.info("Bot is shutting down") break
class Bot(object): def __init__(self, bot_id): self.parameters = Parameters() self.current_message = None self.last_message = None self.message_counter = 0 self.check_bot_id(bot_id) self.bot_id = bot_id self.load_system_configurations() self.logger = log(self.parameters.logging_path, self.bot_id, self.parameters.logging_level) self.logger.info('Bot is starting') self.load_runtime_configurations() self.src_queue, self.dest_queues = self.load_pipeline() self.parameters.processing_interval = float(self.parameters.processing_interval) self.init() def init(self): pass def start(self): self.logger.info('Bot start processing') self.pipeline = None retry_delay = 0 while True: try: if not self.pipeline: time.sleep(retry_delay) self.logger.info("Connecting to pipeline queues") self.pipeline = Pipeline() self.pipeline.queues(self.src_queue, self.dest_queues) self.logger.info("Connected to pipeline queues. Start processing") self.process() self.pipeline.sleep(self.parameters.processing_interval) except Exception, ex: retry_delay = 30 self.logger.info("Last Correct Message(event): %r" % self.last_message) self.logger.info("Current Message(event): %r" % self.current_message) self.logger.exception("Check the following exception:") self.logger.error('Pipeline connection failed (%r)' % ex) self.logger.info('Pipeline will reconnect in %s seconds' % retry_delay) #self.pipeline.disconnect() # caused problems self.pipeline = None except KeyboardInterrupt as e: if self.pipeline: self.pipeline.disconnect() self.logger.info("Disconnecting from pipeline") self.logger.info("Bot is shutting down") break
class Bot(object): def __init__(self, bot_id): self.parameters = Parameters() self.current_message = None self.last_message = None self.message_counter = 0 self.check_bot_id(bot_id) self.bot_id = bot_id self.load_system_configurations() self.logger = log(self.parameters.logging_path, self.bot_id, self.parameters.logging_level) self.logger.info("Bot is starting") self.load_runtime_configurations() self.load_pipeline_configurations() # == NEW CODE self.parameters.threads_number = 1 # == NEW CODE self.init() def init(self): pass # == NEW CODE def start(self): for thread_identifier in range(0, self.parameters.threads_number): thread = threading.Thread(target=self.internal_start, args=(thread_identifier,)) thread.start() # == NEW CODE def internal_start(self, thread_identifier): self.source_pipeline = None self.destination_pipeline = None local_retry_delay = 0 self.logger.info("Bot start processing") while True: try: if not self.source_pipeline: time.sleep(local_retry_delay) self.logger.info("Connecting to source pipeline") self.source_pipeline = Pipeline() # == NEW CODE self.source_pipeline.source_queues(self.source_queues, thread_identifier) # self.source_pipeline.source_queues(self.source_queues) # == NEW CODE self.logger.info("Connected to source pipeline") if not self.destination_pipeline: time.sleep(local_retry_delay) self.logger.info("Connecting to destination pipeline") self.destination_pipeline = Pipeline() self.destination_pipeline.destination_queues(self.destination_queues) self.logger.info("Connected to destination pipeline") self.logger.info("Start processing") self.process() self.source_pipeline.sleep(self.parameters.rate_limit) except Exception, ex: local_retry_delay = self.parameters.retry_delay self.logger.info("Last Correct Message(event): %r" % self.last_message) self.logger.info("Current Message(event): %r" % self.current_message) self.logger.exception("Check the following exception:") self.logger.error("Pipeline connection failed (%r)" % ex) self.logger.info("Pipeline will reconnect in %s seconds" % local_retry_delay) self.source_pipeline = None self.destination_pipeline = None except KeyboardInterrupt as e: if self.source_pipeline: self.source_pipeline.disconnect() self.logger.info("Disconnecting from source pipeline") if self.destination_pipeline: self.destination_pipeline.disconnect() self.logger.info("Disconnecting from destination pipeline") self.logger.info("Bot is shutting down") break