Exemple #1
0
 def _start_consuming(self):
     errors = (self._connection.connection_errors +
               self._connection.channel_errors)
     while self._running:
         try:
             self._ensure_connection()
             for consumer in self._removed_consumers:
                 bound_queue = consumer.queue(self._channel)
                 if bound_queue is not None:
                     bound_queue.delete()
             consumers = [
                 kombu.Consumer(
                     self._channel,
                     queues=c.queue,
                     callbacks=[c.callback] if c.callback else None)
                 for c in self._consumers.values()
             ]
             self._consumers_changed = False
             if len(consumers) == 0:
                 gevent.sleep(0.1)
             else:
                 with nested(*consumers):
                     while self._running and not self._consumers_changed and\
                         self._connection.connected:
                         try:
                             self._connection.drain_events(timeout=1)
                         except socket.timeout:
                             pass
         except errors as e:
             msg = 'Connection error in Kombu amqp consumer greenlet: %s' % str(
                 e)
             self._logger(msg, level=SandeshLevel.SYS_WARN)
         except Exception as e:
             msg = 'Error in Kombu amqp consumer greenlet: %s' % str(e)
             self._logger(msg, level=SandeshLevel.SYS_ERR)
def create_mix_data_list(folder_path_wifi: str, folder_path_beacon: str,
                         dates: List[str], times: List[str]):
    for date in dates:
        for time in times:

            sub_folder_wifi = "{}/{}/{}/Center of Zone/".format(
                folder_path_wifi, date, time)

            data_files_wifi = [
                f for f in listdir(sub_folder_wifi)
                if isfile(join(sub_folder_wifi, f))
            ]

            sub_folder_beacon = "{}/{}/{}/Center of Zone/".format(
                folder_path_beacon, date, time)

            data_files_beacon = [
                f for f in listdir(sub_folder_beacon)
                if isfile(join(sub_folder_beacon, f))
            ]

            for file in data_files_wifi:
                if file in data_files_beacon:
                    with nested(
                            open(sub_folder_wifi + file, mode='a', newline=""),
                            open(sub_folder_beacon + file,
                                 mode='r',
                                 newline="")) as (new_file, old_file):
                        old_file_reader = csv.reader(old_file)
                        new_file_writer = csv.writer(new_file)
                        for scan in old_file_reader:
                            new_file_writer.writerow(scan)
 def _start_consuming(self):
     errors = (self._connection.connection_errors +
               self._connection.channel_errors)
     while self._running:
         try:
             self._ensure_connection()
             for consumer in self._removed_consumers:
                 bound_queue = consumer.queue(self._channel)
                 if bound_queue is not None:
                     bound_queue.delete()
             consumers = [kombu.Consumer(self._channel, queues=c.queue,
                          callbacks=[c.callback])
                          for c in self._consumers.values()]
             self._consumers_changed = False
             if len(consumers) == 0:
                 gevent.sleep(0.1)
             else:
                 with nested(*consumers):
                     while self._running and not self._consumers_changed and\
                         self._connection.connected:
                         try:
                             self._connection.drain_events(timeout=1)
                         except socket.timeout:
                             pass
         except errors as e:
             msg = 'Connection error in Kombu amqp consumer greenlet: %s' % str(e)
             self._logger(msg, level=SandeshLevel.SYS_WARN)
         except Exception as e:
             msg = 'Error in Kombu amqp consumer greenlet: %s' % str(e)
             self._logger(msg, level=SandeshLevel.SYS_ERR)
Exemple #4
0
    def _start_consuming(self):
        errors = (self._connection.connection_errors +
                  self._connection.channel_errors)
        removed_consumer = None
        msg = 'KombuAmqpClient: Starting consumer greenlet'
        self._logger(msg, level=SandeshLevel.SYS_DEBUG)
        while self._running:
            try:
                self._ensure_connection(self._connection, "Consumer")
                self._connected = True

                while len(self._removed_consumers) > 0 or removed_consumer:
                    if removed_consumer is None:
                        removed_consumer = self._removed_consumers.pop(0)
                    if removed_consumer.queue.is_bound:
                        removed_consumer.queue.delete()
                    removed_consumer = None

                if len(self._consumers.values()) == 0:
                    msg = 'KombuAmqpClient: Waiting for consumer'
                    self._logger(msg, level=SandeshLevel.SYS_DEBUG)
                    self._consumer_event.wait()
                    self._consumer_event.clear()

                if not self._running or len(self._consumers.values()) == 0:
                    msg = 'KombuAmqpClient: No consumers found'
                    self._logger(msg, level=SandeshLevel.SYS_DEBUG)
                    continue

                consumers = [
                    kombu.Consumer(
                        self._connection,
                        queues=c.queue,
                        callbacks=[c.callback] if c.callback else None)
                    for c in self._consumers.values()
                ]
                msg = 'KombuAmqpClient: Created consumers %s' % str(
                    self._consumers.keys())
                self._logger(msg, level=SandeshLevel.SYS_DEBUG)
                self._consumers_changed = False
                with nested(*consumers):
                    while self._running and not self._consumers_changed:
                        try:
                            self._connection.drain_events(timeout=1)
                        except socket.timeout:
                            pass
            except errors as e:
                msg = 'KombuAmqpClient: Connection error in Kombu amqp consumer greenlet: %s' % str(
                    e)
                self._logger(msg, level=SandeshLevel.SYS_WARN)
                self._connected = False
                gevent.sleep(0.1)
            except Exception as e:
                msg = 'KombuAmqpClient: Error in Kombu amqp consumer greenlet: %s' % str(
                    e)
                self._logger(msg, level=SandeshLevel.SYS_ERR)
                self._connected = False
                gevent.sleep(0.1)
        msg = 'KombuAmqpClient: Exited consumer greenlet'
        self._logger(msg, level=SandeshLevel.SYS_DEBUG)
Exemple #5
0
 def drain_events(self, infinity=True):
     with nested(*self.consumers):
         while True:
             try:
                 self.connection.drain_events(timeout=self.timeout)
             except socket.timeout:
                 if not infinity:
                     return
             except Exception as e:
                 self.logger.exception(e)
Exemple #6
0
    def test_fanout(self, name='test_mongodb_fanout'):
        if not self.verify_alive():
            return
        c = self.connection
        self.e = Exchange(name, type='fanout')
        self.q = Queue(name, exchange=self.e, routing_key=name)
        self.q2 = Queue(name + '2', exchange=self.e, routing_key=name + '2')

        channel = c.default_channel
        producer = Producer(channel, self.e)
        consumer1 = Consumer(channel, self.q)
        consumer2 = Consumer(channel, self.q2)
        self.q2(channel).declare()

        for i in range(10):
            producer.publish({'foo': i}, routing_key=name)
        for i in range(10):
            producer.publish({'foo': i}, routing_key=name + '2')

        _received1 = []
        _received2 = []

        def callback1(message_data, message):
            _received1.append(message)
            message.ack()

        def callback2(message_data, message):
            _received2.append(message)
            message.ack()

        consumer1.register_callback(callback1)
        consumer2.register_callback(callback2)

        with nested(consumer1, consumer2):

            while 1:
                if len(_received1) + len(_received2) == 20:
                    break
                c.drain_events(timeout=60)
        self.assertEqual(len(_received1) + len(_received2), 20)

        # queue.delete
        for i in range(10):
            producer.publish({'foo': i}, routing_key=name)
        self.assertTrue(self.q(channel).get())
        self.q(channel).delete()
        self.q(channel).declare()
        self.assertIsNone(self.q(channel).get())

        # queue.purge
        for i in range(10):
            producer.publish({'foo': i}, routing_key=name + '2')
        self.assertTrue(self.q2(channel).get())
        self.q2(channel).purge()
        self.assertIsNone(self.q2(channel).get())
Exemple #7
0
    def test_fanout(self, name='test_mongodb_fanout'):
        if not self.verify_alive():
            return
        c = self.connection
        self.e = Exchange(name, type='fanout')
        self.q = Queue(name, exchange=self.e, routing_key=name)
        self.q2 = Queue(name + '2', exchange=self.e, routing_key=name + '2')

        channel = c.default_channel
        producer = Producer(channel, self.e)
        consumer1 = Consumer(channel, self.q)
        consumer2 = Consumer(channel, self.q2)
        self.q2(channel).declare()

        for i in range(10):
            producer.publish({'foo': i}, routing_key=name)
        for i in range(10):
            producer.publish({'foo': i}, routing_key=name + '2')

        _received1 = []
        _received2 = []

        def callback1(message_data, message):
            _received1.append(message)
            message.ack()

        def callback2(message_data, message):
            _received2.append(message)
            message.ack()

        consumer1.register_callback(callback1)
        consumer2.register_callback(callback2)

        with nested(consumer1, consumer2):

            while 1:
                if len(_received1) + len(_received2) == 20:
                    break
                c.drain_events(timeout=60)
        self.assertEqual(len(_received1) + len(_received2), 20)

        # queue.delete
        for i in range(10):
            producer.publish({'foo': i}, routing_key=name)
        self.assertTrue(self.q(channel).get())
        self.q(channel).delete()
        self.q(channel).declare()
        self.assertIsNone(self.q(channel).get())

        # queue.purge
        for i in range(10):
            producer.publish({'foo': i}, routing_key=name + '2')
        self.assertTrue(self.q2(channel).get())
        self.q2(channel).purge()
        self.assertIsNone(self.q2(channel).get())
    def _start_consuming(self):
        errors = (self._connection.connection_errors +
                  self._connection.channel_errors)
        removed_consumer = None
        msg = 'KombuAmqpClient: Starting consumer greenlet'
        self._logger(msg, level=SandeshLevel.SYS_DEBUG)
        while self._running:
            try:
                self._ensure_connection(self._connection, "Consumer")
                self._connected = True

                while len(self._removed_consumers) > 0 or removed_consumer:
                    if removed_consumer is None:
                        removed_consumer = self._removed_consumers.pop(0)
                    self._delete_consumer(removed_consumer)
                    removed_consumer = None

                if len(list(self._consumers.values())) == 0:
                    msg = 'KombuAmqpClient: Waiting for consumer'
                    self._logger(msg, level=SandeshLevel.SYS_DEBUG)
                    self._consumer_event.wait()
                    self._consumer_event.clear()
                    continue

                consumers = self._create_consumer_list()

                msg = 'KombuAmqpClient: Created consumers %s' % str(
                    list(self._consumers.keys()))
                self._logger(msg, level=SandeshLevel.SYS_DEBUG)
                with nested(*consumers):
                    self._consumers_created_event.set()
                    if self._consumer_lock.locked():
                        self._consumer_lock.release()
                    while self._running and not self._consumers_changed:
                        try:
                            self._connection.drain_events(timeout=1)
                        except socket.timeout:
                            pass
                    self._consumers_changed = False
                    self._consumer_lock.acquire()
            except errors as e:
                msg = 'KombuAmqpClient: Connection error in Kombu amqp consumer greenlet: %s' % str(
                    e)
                self._logger(msg, level=SandeshLevel.SYS_WARN)
                self._connected = False
                gevent.sleep(0.1)
            except Exception as e:
                msg = 'KombuAmqpClient: Error in Kombu amqp consumer greenlet: %s' % str(
                    e)
                self._logger(msg, level=SandeshLevel.SYS_ERR)
                self._connected = False
                gevent.sleep(0.1)
        msg = 'KombuAmqpClient: Exited consumer greenlet'
        self._logger(msg, level=SandeshLevel.SYS_DEBUG)
Exemple #9
0
 def poll(self, queues, timeout=None, **data):
     log.debug('Starting Poller for %s, %s' % (queues.keys(), data))
     with AMQPConnection(self.url) as conn:
         consumers = [Consumer(conn.channel(), queues=[queue],
                               callbacks=[processor(name, **data)])
                      for name, queue in queues.items()]
         with nested(*consumers):
             while True:
                 try:
                     conn.drain_events(timeout=timeout)
                 except socket.timeout:
                     print "Timeout."
                     break
Exemple #10
0
    def drain_events(self, infinity=True):

        with nested(*self.consumers):
            while not self._stop:
                try:
                    self.connection.drain_events(timeout=self.timeout)
                except socket.timeout:
                    if not infinity:
                        break
                except ConnectionError as e:  # pragma no cover
                    self.logger.error(
                        'Connection to mq has broken off because: %s. Try to reconnect, %s',
                        e)
                    self.connect()
                    self.revive()
                    break
                except HandlerError:
                    self.logger.exception('Handler error')
                except Exception as e:  # pragma no cover
                    if not self._stop:
                        self.logger.exception(
                            'Something wrong! Try to restart the loop')
                        self.revive()
                        break
                    else:  # pragma: no cover
                        self.logger.exception(
                            'Something wrong! And stopping...')
                        break
                if self.with_pool:
                    try:
                        self.drain_results()
                    except Exception:  # pragma no cover
                        self.logger.exception('Unknown error when '
                                              'draining results')
        if self._stop:
            if self.with_pool:
                try:
                    self.pool.join()
                    self.drain_results()  # pragma: no cover
                except AssertionError:
                    pass
                except Exception: # pragma: no cover
                    self.logger.exception(
                        'Unknown error when '
                        'draining results'
                    )
            self._stopped = True
            self.logger.info('Stopped draining events.')
    def _start_consuming(self):
        errors = (self._connection.connection_errors +
                  self._connection.channel_errors)
        removed_consumer = None
        msg = 'KombuAmqpClient: Starting consumer greenlet'
        self._logger(msg, level=SandeshLevel.SYS_DEBUG)
        while self._running:
            try:
                self._ensure_connection(self._connection, "Consumer")
                self._connected = True

                while len(self._removed_consumers) > 0 or removed_consumer:
                    if removed_consumer is None:
                        removed_consumer = self._removed_consumers.pop(0)
                    self._delete_consumer(removed_consumer)
                    removed_consumer = None

                if len(self._consumers.values()) == 0:
                    msg = 'KombuAmqpClient: Waiting for consumer'
                    self._logger(msg, level=SandeshLevel.SYS_DEBUG)
                    self._consumer_event.wait()
                    self._consumer_event.clear()
                    continue

                consumers = [kombu.Consumer(self._connection, queues=c.queue,
                             callbacks=[c.callback] if c.callback else None)
                             for c in self._consumers.values()]
                msg = 'KombuAmqpClient: Created consumers %s' % str(self._consumers.keys())
                self._logger(msg, level=SandeshLevel.SYS_DEBUG)
                self._consumers_changed = False
                with nested(*consumers):
                    while self._running and not self._consumers_changed:
                        try:
                            self._connection.drain_events(timeout=1)
                        except socket.timeout:
                            pass
            except errors as e:
                msg = 'KombuAmqpClient: Connection error in Kombu amqp consumer greenlet: %s' % str(e)
                self._logger(msg, level=SandeshLevel.SYS_WARN)
                self._connected = False
                gevent.sleep(0.1)
            except Exception as e:
                msg = 'KombuAmqpClient: Error in Kombu amqp consumer greenlet: %s' % str(e)
                self._logger(msg, level=SandeshLevel.SYS_ERR)
                self._connected = False
                gevent.sleep(0.1)
        msg = 'KombuAmqpClient: Exited consumer greenlet'
        self._logger(msg, level=SandeshLevel.SYS_DEBUG)
    def drain_events(self, infinity=True):

        with nested(*self.consumers):
            while not self._stop:
                try:
                    self.connection.drain_events(timeout=self.timeout)
                except socket.timeout:
                    if not infinity:
                        break
                except ConnectionError as e:  # pragma no cover
                    self.logger.error(
                        'Connection to mq has broken off because: %s. Try to reconnect, %s',
                        e)
                    self.connect()
                    self.revive()
                    break
                except HandlerError:
                    self.logger.exception('Handler error')
                except Exception as e:  # pragma no cover
                    if not self._stop:
                        self.logger.exception(
                            'Something wrong! Try to restart the loop')
                        self.revive()
                        break
                    else:  # pragma: no cover
                        self.logger.exception(
                            'Something wrong! And stopping...')
                        break
                if self.with_pool:
                    try:
                        self.drain_results()
                    except Exception:  # pragma no cover
                        self.logger.exception('Unknown error when '
                                              'draining results')
        if self._stop:
            if self.with_pool:
                try:
                    self.pool.join()
                    self.drain_results()  # pragma: no cover
                except AssertionError:
                    pass
                except Exception:  # pragma: no cover
                    self.logger.exception('Unknown error when '
                                          'draining results')
            self._stopped = True
            self.logger.info('Stopped draining events.')
Exemple #13
0
def patch_many(*targets):
    return nested(*[patch(target) for target in targets])
Exemple #14
0
def patch_many(*targets):
    return nested(*[mock.patch(target) for target in targets])