def _start_consuming(self): errors = (self._connection.connection_errors + self._connection.channel_errors) while self._running: try: self._ensure_connection() for consumer in self._removed_consumers: bound_queue = consumer.queue(self._channel) if bound_queue is not None: bound_queue.delete() consumers = [ kombu.Consumer( self._channel, queues=c.queue, callbacks=[c.callback] if c.callback else None) for c in self._consumers.values() ] self._consumers_changed = False if len(consumers) == 0: gevent.sleep(0.1) else: with nested(*consumers): while self._running and not self._consumers_changed and\ self._connection.connected: try: self._connection.drain_events(timeout=1) except socket.timeout: pass except errors as e: msg = 'Connection error in Kombu amqp consumer greenlet: %s' % str( e) self._logger(msg, level=SandeshLevel.SYS_WARN) except Exception as e: msg = 'Error in Kombu amqp consumer greenlet: %s' % str(e) self._logger(msg, level=SandeshLevel.SYS_ERR)
def create_mix_data_list(folder_path_wifi: str, folder_path_beacon: str, dates: List[str], times: List[str]): for date in dates: for time in times: sub_folder_wifi = "{}/{}/{}/Center of Zone/".format( folder_path_wifi, date, time) data_files_wifi = [ f for f in listdir(sub_folder_wifi) if isfile(join(sub_folder_wifi, f)) ] sub_folder_beacon = "{}/{}/{}/Center of Zone/".format( folder_path_beacon, date, time) data_files_beacon = [ f for f in listdir(sub_folder_beacon) if isfile(join(sub_folder_beacon, f)) ] for file in data_files_wifi: if file in data_files_beacon: with nested( open(sub_folder_wifi + file, mode='a', newline=""), open(sub_folder_beacon + file, mode='r', newline="")) as (new_file, old_file): old_file_reader = csv.reader(old_file) new_file_writer = csv.writer(new_file) for scan in old_file_reader: new_file_writer.writerow(scan)
def _start_consuming(self): errors = (self._connection.connection_errors + self._connection.channel_errors) while self._running: try: self._ensure_connection() for consumer in self._removed_consumers: bound_queue = consumer.queue(self._channel) if bound_queue is not None: bound_queue.delete() consumers = [kombu.Consumer(self._channel, queues=c.queue, callbacks=[c.callback]) for c in self._consumers.values()] self._consumers_changed = False if len(consumers) == 0: gevent.sleep(0.1) else: with nested(*consumers): while self._running and not self._consumers_changed and\ self._connection.connected: try: self._connection.drain_events(timeout=1) except socket.timeout: pass except errors as e: msg = 'Connection error in Kombu amqp consumer greenlet: %s' % str(e) self._logger(msg, level=SandeshLevel.SYS_WARN) except Exception as e: msg = 'Error in Kombu amqp consumer greenlet: %s' % str(e) self._logger(msg, level=SandeshLevel.SYS_ERR)
def _start_consuming(self): errors = (self._connection.connection_errors + self._connection.channel_errors) removed_consumer = None msg = 'KombuAmqpClient: Starting consumer greenlet' self._logger(msg, level=SandeshLevel.SYS_DEBUG) while self._running: try: self._ensure_connection(self._connection, "Consumer") self._connected = True while len(self._removed_consumers) > 0 or removed_consumer: if removed_consumer is None: removed_consumer = self._removed_consumers.pop(0) if removed_consumer.queue.is_bound: removed_consumer.queue.delete() removed_consumer = None if len(self._consumers.values()) == 0: msg = 'KombuAmqpClient: Waiting for consumer' self._logger(msg, level=SandeshLevel.SYS_DEBUG) self._consumer_event.wait() self._consumer_event.clear() if not self._running or len(self._consumers.values()) == 0: msg = 'KombuAmqpClient: No consumers found' self._logger(msg, level=SandeshLevel.SYS_DEBUG) continue consumers = [ kombu.Consumer( self._connection, queues=c.queue, callbacks=[c.callback] if c.callback else None) for c in self._consumers.values() ] msg = 'KombuAmqpClient: Created consumers %s' % str( self._consumers.keys()) self._logger(msg, level=SandeshLevel.SYS_DEBUG) self._consumers_changed = False with nested(*consumers): while self._running and not self._consumers_changed: try: self._connection.drain_events(timeout=1) except socket.timeout: pass except errors as e: msg = 'KombuAmqpClient: Connection error in Kombu amqp consumer greenlet: %s' % str( e) self._logger(msg, level=SandeshLevel.SYS_WARN) self._connected = False gevent.sleep(0.1) except Exception as e: msg = 'KombuAmqpClient: Error in Kombu amqp consumer greenlet: %s' % str( e) self._logger(msg, level=SandeshLevel.SYS_ERR) self._connected = False gevent.sleep(0.1) msg = 'KombuAmqpClient: Exited consumer greenlet' self._logger(msg, level=SandeshLevel.SYS_DEBUG)
def drain_events(self, infinity=True): with nested(*self.consumers): while True: try: self.connection.drain_events(timeout=self.timeout) except socket.timeout: if not infinity: return except Exception as e: self.logger.exception(e)
def test_fanout(self, name='test_mongodb_fanout'): if not self.verify_alive(): return c = self.connection self.e = Exchange(name, type='fanout') self.q = Queue(name, exchange=self.e, routing_key=name) self.q2 = Queue(name + '2', exchange=self.e, routing_key=name + '2') channel = c.default_channel producer = Producer(channel, self.e) consumer1 = Consumer(channel, self.q) consumer2 = Consumer(channel, self.q2) self.q2(channel).declare() for i in range(10): producer.publish({'foo': i}, routing_key=name) for i in range(10): producer.publish({'foo': i}, routing_key=name + '2') _received1 = [] _received2 = [] def callback1(message_data, message): _received1.append(message) message.ack() def callback2(message_data, message): _received2.append(message) message.ack() consumer1.register_callback(callback1) consumer2.register_callback(callback2) with nested(consumer1, consumer2): while 1: if len(_received1) + len(_received2) == 20: break c.drain_events(timeout=60) self.assertEqual(len(_received1) + len(_received2), 20) # queue.delete for i in range(10): producer.publish({'foo': i}, routing_key=name) self.assertTrue(self.q(channel).get()) self.q(channel).delete() self.q(channel).declare() self.assertIsNone(self.q(channel).get()) # queue.purge for i in range(10): producer.publish({'foo': i}, routing_key=name + '2') self.assertTrue(self.q2(channel).get()) self.q2(channel).purge() self.assertIsNone(self.q2(channel).get())
def _start_consuming(self): errors = (self._connection.connection_errors + self._connection.channel_errors) removed_consumer = None msg = 'KombuAmqpClient: Starting consumer greenlet' self._logger(msg, level=SandeshLevel.SYS_DEBUG) while self._running: try: self._ensure_connection(self._connection, "Consumer") self._connected = True while len(self._removed_consumers) > 0 or removed_consumer: if removed_consumer is None: removed_consumer = self._removed_consumers.pop(0) self._delete_consumer(removed_consumer) removed_consumer = None if len(list(self._consumers.values())) == 0: msg = 'KombuAmqpClient: Waiting for consumer' self._logger(msg, level=SandeshLevel.SYS_DEBUG) self._consumer_event.wait() self._consumer_event.clear() continue consumers = self._create_consumer_list() msg = 'KombuAmqpClient: Created consumers %s' % str( list(self._consumers.keys())) self._logger(msg, level=SandeshLevel.SYS_DEBUG) with nested(*consumers): self._consumers_created_event.set() if self._consumer_lock.locked(): self._consumer_lock.release() while self._running and not self._consumers_changed: try: self._connection.drain_events(timeout=1) except socket.timeout: pass self._consumers_changed = False self._consumer_lock.acquire() except errors as e: msg = 'KombuAmqpClient: Connection error in Kombu amqp consumer greenlet: %s' % str( e) self._logger(msg, level=SandeshLevel.SYS_WARN) self._connected = False gevent.sleep(0.1) except Exception as e: msg = 'KombuAmqpClient: Error in Kombu amqp consumer greenlet: %s' % str( e) self._logger(msg, level=SandeshLevel.SYS_ERR) self._connected = False gevent.sleep(0.1) msg = 'KombuAmqpClient: Exited consumer greenlet' self._logger(msg, level=SandeshLevel.SYS_DEBUG)
def poll(self, queues, timeout=None, **data): log.debug('Starting Poller for %s, %s' % (queues.keys(), data)) with AMQPConnection(self.url) as conn: consumers = [Consumer(conn.channel(), queues=[queue], callbacks=[processor(name, **data)]) for name, queue in queues.items()] with nested(*consumers): while True: try: conn.drain_events(timeout=timeout) except socket.timeout: print "Timeout." break
def drain_events(self, infinity=True): with nested(*self.consumers): while not self._stop: try: self.connection.drain_events(timeout=self.timeout) except socket.timeout: if not infinity: break except ConnectionError as e: # pragma no cover self.logger.error( 'Connection to mq has broken off because: %s. Try to reconnect, %s', e) self.connect() self.revive() break except HandlerError: self.logger.exception('Handler error') except Exception as e: # pragma no cover if not self._stop: self.logger.exception( 'Something wrong! Try to restart the loop') self.revive() break else: # pragma: no cover self.logger.exception( 'Something wrong! And stopping...') break if self.with_pool: try: self.drain_results() except Exception: # pragma no cover self.logger.exception('Unknown error when ' 'draining results') if self._stop: if self.with_pool: try: self.pool.join() self.drain_results() # pragma: no cover except AssertionError: pass except Exception: # pragma: no cover self.logger.exception( 'Unknown error when ' 'draining results' ) self._stopped = True self.logger.info('Stopped draining events.')
def _start_consuming(self): errors = (self._connection.connection_errors + self._connection.channel_errors) removed_consumer = None msg = 'KombuAmqpClient: Starting consumer greenlet' self._logger(msg, level=SandeshLevel.SYS_DEBUG) while self._running: try: self._ensure_connection(self._connection, "Consumer") self._connected = True while len(self._removed_consumers) > 0 or removed_consumer: if removed_consumer is None: removed_consumer = self._removed_consumers.pop(0) self._delete_consumer(removed_consumer) removed_consumer = None if len(self._consumers.values()) == 0: msg = 'KombuAmqpClient: Waiting for consumer' self._logger(msg, level=SandeshLevel.SYS_DEBUG) self._consumer_event.wait() self._consumer_event.clear() continue consumers = [kombu.Consumer(self._connection, queues=c.queue, callbacks=[c.callback] if c.callback else None) for c in self._consumers.values()] msg = 'KombuAmqpClient: Created consumers %s' % str(self._consumers.keys()) self._logger(msg, level=SandeshLevel.SYS_DEBUG) self._consumers_changed = False with nested(*consumers): while self._running and not self._consumers_changed: try: self._connection.drain_events(timeout=1) except socket.timeout: pass except errors as e: msg = 'KombuAmqpClient: Connection error in Kombu amqp consumer greenlet: %s' % str(e) self._logger(msg, level=SandeshLevel.SYS_WARN) self._connected = False gevent.sleep(0.1) except Exception as e: msg = 'KombuAmqpClient: Error in Kombu amqp consumer greenlet: %s' % str(e) self._logger(msg, level=SandeshLevel.SYS_ERR) self._connected = False gevent.sleep(0.1) msg = 'KombuAmqpClient: Exited consumer greenlet' self._logger(msg, level=SandeshLevel.SYS_DEBUG)
def drain_events(self, infinity=True): with nested(*self.consumers): while not self._stop: try: self.connection.drain_events(timeout=self.timeout) except socket.timeout: if not infinity: break except ConnectionError as e: # pragma no cover self.logger.error( 'Connection to mq has broken off because: %s. Try to reconnect, %s', e) self.connect() self.revive() break except HandlerError: self.logger.exception('Handler error') except Exception as e: # pragma no cover if not self._stop: self.logger.exception( 'Something wrong! Try to restart the loop') self.revive() break else: # pragma: no cover self.logger.exception( 'Something wrong! And stopping...') break if self.with_pool: try: self.drain_results() except Exception: # pragma no cover self.logger.exception('Unknown error when ' 'draining results') if self._stop: if self.with_pool: try: self.pool.join() self.drain_results() # pragma: no cover except AssertionError: pass except Exception: # pragma: no cover self.logger.exception('Unknown error when ' 'draining results') self._stopped = True self.logger.info('Stopped draining events.')
def patch_many(*targets): return nested(*[patch(target) for target in targets])
def patch_many(*targets): return nested(*[mock.patch(target) for target in targets])