Exemplo n.º 1
0
    def start(self):
        self.logger.info('Bot start processing')
        self.source_pipeline = None
        self.destination_pipeline = None
        local_retry_delay = 0
 
        while True:
            try:
                if self.parameters.source_pipeline_enable and not self.source_pipeline:
                    time.sleep(local_retry_delay)
                    self.logger.info("Connecting to source pipeline")
                    self.source_pipeline = Pipeline(
                                                     self.parameters.source_pipeline_host,
                                                     self.parameters.source_pipeline_port,
                                                     self.parameters.source_pipeline_db
                                                   )
                    self.source_pipeline.source_queues(self.source_queue)
                    self.logger.info("Connected to source pipeline")

                if self.parameters.destination_pipeline_enable and not self.destination_pipeline:
                    time.sleep(local_retry_delay)
                    self.logger.info("Connecting to destination pipeline")
                    self.destination_pipeline = Pipeline(
                                                          self.parameters.destination_pipeline_host,
                                                          self.parameters.destination_pipeline_port,
                                                          self.parameters.destination_pipeline_db
                                                        )
                    self.destination_pipeline.destination_queues(self.destination_queues)
                    self.logger.info("Connected to destination pipeline")

                self.logger.info("Start processing")
                self.process()
                if self.source_pipeline:
                    self.source_pipeline.sleep(self.parameters.rate_limit)
                
            except Exception, ex:
                local_retry_delay = self.parameters.retry_delay
                self.logger.info("Last Correct Message(event): %r" % self.last_message)
                self.logger.info("Current Message(event): %r" % self.current_message)
                self.logger.exception("Check the following exception:")
                self.logger.error('Pipeline connection failed (%r)' % ex)
                self.logger.info('Pipeline will reconnect in %s seconds' % local_retry_delay)
                self.source_pipeline = None
                self.destination_pipeline = None
                
            except KeyboardInterrupt as e:
                if self.source_pipeline:
                    self.source_pipeline.disconnect()
                    self.logger.info("Disconnecting from source pipeline")
                if self.destination_pipeline:
                    self.destination_pipeline.disconnect()
                    self.logger.info("Disconnecting from destination pipeline")

                self.logger.info("Bot is shutting down")
                break
Exemplo n.º 2
0
    def start(self):
        self.source_pipeline = None
        self.destination_pipeline = None
        local_retry_delay = 0
        self.parameters.retry_delay = 30 # Temporary fix. Need to add to BOTS conf
 
        self.logger.info('Bot start processing')

        while True:
            try:
                if not self.source_pipeline:
                    time.sleep(local_retry_delay)
                    self.logger.info("Connecting to source pipeline")
                    self.source_pipeline = Pipeline()
                    self.source_pipeline.source_queues(self.source_queues)
                    self.logger.info("Connected to source pipeline")

                if not self.destination_pipeline:
                    time.sleep(local_retry_delay)
                    self.logger.info("Connecting to destination pipeline")
                    self.destination_pipeline = Pipeline()
                    self.destination_pipeline.destination_queues(self.destination_queues)
                    self.logger.info("Connected to destination pipeline")

                self.logger.info("Start processing")
                self.process()
                self.source_pipeline.sleep(self.parameters.rate_limit)

            except IntelMQPipeline, ex:
                # NAO LOGAR A MENSAGEM PQ O PROBLEMA É NA PIPELINE
                # na excepção em baixo deve estar o caso de não ser um erro de pipeline e assim
                # deve loggar consoante o parametro do bot.
                
            except Exception, ex:
                local_retry_delay = self.parameters.retry_delay
                self.logger.info("Last Correct Message(event): %r" % self.last_message)
                self.logger.info("Current Message(event): %r" % self.current_message)
                self.logger.exception("Check the following exception:")
                self.logger.error('Pipeline connection failed (%r)' % ex)
                self.logger.info('Pipeline will reconnect in %s seconds' % local_retry_delay)
                self.source_pipeline = None
                self.destination_pipeline = None
                
            except KeyboardInterrupt as e:
                if self.source_pipeline:
                    self.source_pipeline.disconnect()
                    self.logger.info("Disconnecting from source pipeline")
                if self.destination_pipeline:
                    self.destination_pipeline.disconnect()
                    self.logger.info("Disconnecting from destination pipeline")

                self.logger.info("Bot is shutting down")
                break
    def internal_start(self, thread_identifier):
        self.source_pipeline = None
        self.destination_pipeline = None
        local_retry_delay = 0

        self.logger.info("Bot start processing")

        while True:
            try:
                if not self.source_pipeline:
                    time.sleep(local_retry_delay)
                    self.logger.info("Connecting to source pipeline")
                    self.source_pipeline = Pipeline()

                    # == NEW CODE
                    self.source_pipeline.source_queues(self.source_queues, thread_identifier)
                    #                    self.source_pipeline.source_queues(self.source_queues)
                    # == NEW CODE

                    self.logger.info("Connected to source pipeline")

                if not self.destination_pipeline:
                    time.sleep(local_retry_delay)
                    self.logger.info("Connecting to destination pipeline")
                    self.destination_pipeline = Pipeline()
                    self.destination_pipeline.destination_queues(self.destination_queues)
                    self.logger.info("Connected to destination pipeline")

                self.logger.info("Start processing")
                self.process()
                self.source_pipeline.sleep(self.parameters.rate_limit)

            except Exception, ex:
                local_retry_delay = self.parameters.retry_delay
                self.logger.info("Last Correct Message(event): %r" % self.last_message)
                self.logger.info("Current Message(event): %r" % self.current_message)
                self.logger.exception("Check the following exception:")
                self.logger.error("Pipeline connection failed (%r)" % ex)
                self.logger.info("Pipeline will reconnect in %s seconds" % local_retry_delay)
                self.source_pipeline = None
                self.destination_pipeline = None

            except KeyboardInterrupt as e:
                if self.source_pipeline:
                    self.source_pipeline.disconnect()
                    self.logger.info("Disconnecting from source pipeline")
                if self.destination_pipeline:
                    self.destination_pipeline.disconnect()
                    self.logger.info("Disconnecting from destination pipeline")

                self.logger.info("Bot is shutting down")
                break
Exemplo n.º 4
0
    def start(self):
        """Starts a bot"""
        self.source_pipeline = None
        self.destination_pipeline = None
        local_retry_delay = 0
        self.parameters.retry_delay = 30 # Temporary fix. Need to add to BOTS conf

        self.logger.info('Bot start processing')

        while True:
            try:
                if not self.source_pipeline:
                    time.sleep(local_retry_delay)
                    self.logger.info("Connecting to source pipeline")
                    self.source_pipeline = Pipeline()
                    self.source_pipeline.source_queues(self.source_queues)
                    self.logger.info("Connected to source pipeline")

                if not self.destination_pipeline:
                    time.sleep(local_retry_delay)
                    self.logger.info("Connecting to destination pipeline")
                    self.destination_pipeline = Pipeline()
                    self.destination_pipeline.destination_queues(self.destination_queues)
                    self.logger.info("Connected to destination pipeline")

                self.process()
                self.logger.info("Bot stops processing. Sleeps for 'rate_limit' = %ds" % self.parameters.rate_limit)
                self.source_pipeline.sleep(self.parameters.rate_limit)

            except Exception, ex:
                local_retry_delay = self.parameters.retry_delay
                self.logger.info("Last Correct Message(event): %r" % self.last_message)
                self.logger.info("Current Message(event): %r" % self.current_message)
                self.logger.exception("Check the following exception:")
                self.logger.error('Pipeline connection failed (%r)' % ex)
                self.logger.info('Pipeline will reconnect in %s seconds' % local_retry_delay)
                self.source_pipeline = None
                self.destination_pipeline = None

            except KeyboardInterrupt as e:
                if self.source_pipeline:
                    self.source_pipeline.disconnect()
                    self.logger.info("Disconnecting from source pipeline")
                if self.destination_pipeline:
                    self.destination_pipeline.disconnect()
                    self.logger.info("Disconnecting from destination pipeline")

                self.logger.info("Bot is shutting down")
                break
Exemplo n.º 5
0
    def start(self):
        self.logger.info('Bot start processing')
        self.pipeline = None
        retry_delay = 0
 
        while True:
            try:
                if not self.pipeline:
                    time.sleep(retry_delay)
                    self.logger.info("Connecting to pipeline queues")
                    self.pipeline = Pipeline()
                    self.pipeline.queues(self.src_queue, self.dest_queues)
                    self.logger.info("Connected to pipeline queues. Start processing")
                self.process()
                self.pipeline.sleep(self.parameters.processing_interval)
                
            except Exception, ex:
                retry_delay = 30
                self.logger.info("Last Correct Message(event): %r" % self.last_message)
                self.logger.info("Current Message(event): %r" % self.current_message)
                self.logger.exception("Check the following exception:")
                self.logger.error('Pipeline connection failed (%r)' % ex)
                self.logger.info('Pipeline will reconnect in %s seconds' % retry_delay)
                #self.pipeline.disconnect() # caused problems
                self.pipeline = None
                
            except KeyboardInterrupt as e:
                if self.pipeline:
                    self.pipeline.disconnect()
                    self.logger.info("Disconnecting from pipeline")
                self.logger.info("Bot is shutting down")
                break
Exemplo n.º 6
0
    def _process(self, dryrun, msg, show):
        if msg:
            msg = MessageFactory.serialize(self.arg2msg(msg))
            if not self.instance._Bot__source_pipeline:
                # is None if source pipeline does not exist
                self.instance._Bot__source_pipeline = Pipeline(None)
            self.instance._Bot__source_pipeline.receive = lambda *args, **kwargs: msg
            self.instance._Bot__source_pipeline.acknowledge = lambda *args, **kwargs: None
            self.instance.logger.info(
                " * Message from cli will be used when processing.")

        if dryrun:
            self.instance.send_message = lambda *args, **kwargs: self.instance.logger.info(
                "DRYRUN: Message would be sent now to %r!",
                kwargs.get('path', "_default"))
            self.instance.acknowledge_message = lambda *args, **kwargs: self.instance.logger.info(
                "DRYRUN: Message would be acknowledged now!")
            self.instance.logger.info(
                " * Dryrun only, no message will be really sent through.")

        if show:
            fn = self.instance.send_message
            self.instance.send_message = lambda *args, **kwargs: [
                self.pprint(args or "No message generated"),
                fn(*args, **kwargs)
            ]

        self.instance.logger.info("Processing...")
        self.instance.process()
Exemplo n.º 7
0
    def _process(self, dryrun, msg, show):
        if msg:
            msg = MessageFactory.serialize(self.arg2msg(msg))
            if not self.instance._Bot__source_pipeline:
                # is None if source pipeline does not exist
                self.instance._Bot__source_pipeline = Pipeline(None)
            self.instance._Bot__source_pipeline.receive = lambda: msg
            self.instance.logger.info(
                " * Message from cli will be used when processing.")

        if dryrun:
            self.instance.send_message = lambda msg, path="_default": self.instance.logger.info(
                "DRYRUN: Message would be sent now{}!".format(
                    " to the {} path".format(path)
                    if (path != "_default") else ""))
            self.instance.acknowledge_message = lambda: self.instance.logger.info(
                "DRYRUN: Message would be acknowledged now!")
            self.instance.logger.info(
                " * Dryrun only, no message will be really sent through.")

        if show:
            fn = self.instance.send_message
            self.instance.send_message = lambda msg, path="_default": [
                self.pprint(msg), fn(msg, path=path)
            ]

        self.instance.logger.info("Processing...")
        self.instance.process()
Exemplo n.º 8
0
class Bot(object):

    def __init__(self, bot_id):
        self.parameters = Parameters()
        
        self.current_message = None
        self.last_message = None
        self.message_counter = 0

        self.check_bot_id(bot_id)
        self.bot_id = bot_id

        self.load_system_configurations()
        
        self.logger = log(self.parameters.logging_path, self.bot_id, self.parameters.logging_level)
        self.logger.info('Bot is starting')

        self.load_runtime_configurations()

        self.source_queue, self.destination_queues = self.load_pipeline()
        self.parameters.rate_limit = float(self.parameters.rate_limit)
        self.parameters.retry_delay = int(self.parameters.retry_delay)
        
        self.init()


    def init(self):
        pass


    def start(self):
        self.logger.info('Bot start processing')
        self.source_pipeline = None
        self.destination_pipeline = None
        local_retry_delay = 0
 
        while True:
            try:
                if self.parameters.source_pipeline_enable and not self.source_pipeline:
                    time.sleep(local_retry_delay)
                    self.logger.info("Connecting to source pipeline")
                    self.source_pipeline = Pipeline(
                                                     self.parameters.source_pipeline_host,
                                                     self.parameters.source_pipeline_port,
                                                     self.parameters.source_pipeline_db
                                                   )
                    self.source_pipeline.source_queues(self.source_queue)
                    self.logger.info("Connected to source pipeline")

                if self.parameters.destination_pipeline_enable and not self.destination_pipeline:
                    time.sleep(local_retry_delay)
                    self.logger.info("Connecting to destination pipeline")
                    self.destination_pipeline = Pipeline(
                                                          self.parameters.destination_pipeline_host,
                                                          self.parameters.destination_pipeline_port,
                                                          self.parameters.destination_pipeline_db
                                                        )
                    self.destination_pipeline.destination_queues(self.destination_queues)
                    self.logger.info("Connected to destination pipeline")

                self.logger.info("Start processing")
                self.process()
                if self.source_pipeline:
                    self.source_pipeline.sleep(self.parameters.rate_limit)
                
            except Exception, ex:
                local_retry_delay = self.parameters.retry_delay
                self.logger.info("Last Correct Message(event): %r" % self.last_message)
                self.logger.info("Current Message(event): %r" % self.current_message)
                self.logger.exception("Check the following exception:")
                self.logger.error('Pipeline connection failed (%r)' % ex)
                self.logger.info('Pipeline will reconnect in %s seconds' % local_retry_delay)
                self.source_pipeline = None
                self.destination_pipeline = None
                
            except KeyboardInterrupt as e:
                if self.source_pipeline:
                    self.source_pipeline.disconnect()
                    self.logger.info("Disconnecting from source pipeline")
                if self.destination_pipeline:
                    self.destination_pipeline.disconnect()
                    self.logger.info("Disconnecting from destination pipeline")

                self.logger.info("Bot is shutting down")
                break
Exemplo n.º 9
0
class Bot(object):

    def __init__(self, bot_id):
        self.parameters = Parameters()
        
        self.current_message = None
        self.last_message = None
        self.message_counter = 0

        self.check_bot_id(bot_id)
        self.bot_id = bot_id

        self.load_system_configurations()
        
        self.logger = log(self.parameters.logging_path, self.bot_id, self.parameters.logging_level)
        self.logger.info('Bot is starting')

        self.load_runtime_configurations()

        self.src_queue, self.dest_queues = self.load_pipeline()
        self.parameters.processing_interval = float(self.parameters.processing_interval)
        
        self.init()


    def init(self):
        pass


    def start(self):
        self.logger.info('Bot start processing')
        self.pipeline = None
        retry_delay = 0
 
        while True:
            try:
                if not self.pipeline:
                    time.sleep(retry_delay)
                    self.logger.info("Connecting to pipeline queues")
                    self.pipeline = Pipeline()
                    self.pipeline.queues(self.src_queue, self.dest_queues)
                    self.logger.info("Connected to pipeline queues. Start processing")
                self.process()
                self.pipeline.sleep(self.parameters.processing_interval)
                
            except Exception, ex:
                retry_delay = 30
                self.logger.info("Last Correct Message(event): %r" % self.last_message)
                self.logger.info("Current Message(event): %r" % self.current_message)
                self.logger.exception("Check the following exception:")
                self.logger.error('Pipeline connection failed (%r)' % ex)
                self.logger.info('Pipeline will reconnect in %s seconds' % retry_delay)
                #self.pipeline.disconnect() # caused problems
                self.pipeline = None
                
            except KeyboardInterrupt as e:
                if self.pipeline:
                    self.pipeline.disconnect()
                    self.logger.info("Disconnecting from pipeline")
                self.logger.info("Bot is shutting down")
                break
class Bot(object):
    def __init__(self, bot_id):
        self.parameters = Parameters()

        self.current_message = None
        self.last_message = None
        self.message_counter = 0

        self.check_bot_id(bot_id)
        self.bot_id = bot_id

        self.load_system_configurations()

        self.logger = log(self.parameters.logging_path, self.bot_id, self.parameters.logging_level)
        self.logger.info("Bot is starting")

        self.load_runtime_configurations()
        self.load_pipeline_configurations()

        # == NEW CODE
        self.parameters.threads_number = 1
        # == NEW CODE

        self.init()

    def init(self):
        pass

    # == NEW CODE
    def start(self):
        for thread_identifier in range(0, self.parameters.threads_number):
            thread = threading.Thread(target=self.internal_start, args=(thread_identifier,))
            thread.start()

    # == NEW CODE

    def internal_start(self, thread_identifier):
        self.source_pipeline = None
        self.destination_pipeline = None
        local_retry_delay = 0

        self.logger.info("Bot start processing")

        while True:
            try:
                if not self.source_pipeline:
                    time.sleep(local_retry_delay)
                    self.logger.info("Connecting to source pipeline")
                    self.source_pipeline = Pipeline()

                    # == NEW CODE
                    self.source_pipeline.source_queues(self.source_queues, thread_identifier)
                    #                    self.source_pipeline.source_queues(self.source_queues)
                    # == NEW CODE

                    self.logger.info("Connected to source pipeline")

                if not self.destination_pipeline:
                    time.sleep(local_retry_delay)
                    self.logger.info("Connecting to destination pipeline")
                    self.destination_pipeline = Pipeline()
                    self.destination_pipeline.destination_queues(self.destination_queues)
                    self.logger.info("Connected to destination pipeline")

                self.logger.info("Start processing")
                self.process()
                self.source_pipeline.sleep(self.parameters.rate_limit)

            except Exception, ex:
                local_retry_delay = self.parameters.retry_delay
                self.logger.info("Last Correct Message(event): %r" % self.last_message)
                self.logger.info("Current Message(event): %r" % self.current_message)
                self.logger.exception("Check the following exception:")
                self.logger.error("Pipeline connection failed (%r)" % ex)
                self.logger.info("Pipeline will reconnect in %s seconds" % local_retry_delay)
                self.source_pipeline = None
                self.destination_pipeline = None

            except KeyboardInterrupt as e:
                if self.source_pipeline:
                    self.source_pipeline.disconnect()
                    self.logger.info("Disconnecting from source pipeline")
                if self.destination_pipeline:
                    self.destination_pipeline.disconnect()
                    self.logger.info("Disconnecting from destination pipeline")

                self.logger.info("Bot is shutting down")
                break