def _start_consuming(self):
errors = (self._connection.connection_errors +
self._connection.channel_errors)
while self._running:
try:
self._ensure_connection()
for consumer in self._removed_consumers:
bound_queue = consumer.queue(self._channel)
if bound_queue is not None:
bound_queue.delete()
consumers = [
kombu.Consumer(
self._channel,
queues=c.queue,
callbacks=[c.callback] if c.callback else None)
for c in self._consumers.values()
]
self._consumers_changed = False
if len(consumers) == 0:
gevent.sleep(0.1)
else:
with nested(*consumers):
while self._running and not self._consumers_changed and\
self._connection.connected:
try:
self._connection.drain_events(timeout=1)
except socket.timeout:
pass
except errors as e:
msg = 'Connection error in Kombu amqp consumer greenlet: %s' % str(
e)
self._logger(msg, level=SandeshLevel.SYS_WARN)
except Exception as e:
msg = 'Error in Kombu amqp consumer greenlet: %s' % str(e)
self._logger(msg, level=SandeshLevel.SYS_ERR)
def create_mix_data_list(folder_path_wifi: str, folder_path_beacon: str,
dates: List[str], times: List[str]):
for date in dates:
for time in times:
sub_folder_wifi = "{}/{}/{}/Center of Zone/".format(
folder_path_wifi, date, time)
data_files_wifi = [
f for f in listdir(sub_folder_wifi)
if isfile(join(sub_folder_wifi, f))
]
sub_folder_beacon = "{}/{}/{}/Center of Zone/".format(
folder_path_beacon, date, time)
data_files_beacon = [
f for f in listdir(sub_folder_beacon)
if isfile(join(sub_folder_beacon, f))
]
for file in data_files_wifi:
if file in data_files_beacon:
with nested(
open(sub_folder_wifi + file, mode='a', newline=""),
open(sub_folder_beacon + file,
mode='r',
newline="")) as (new_file, old_file):
old_file_reader = csv.reader(old_file)
new_file_writer = csv.writer(new_file)
for scan in old_file_reader:
new_file_writer.writerow(scan)
def _start_consuming(self):
errors = (self._connection.connection_errors +
self._connection.channel_errors)
while self._running:
try:
self._ensure_connection()
for consumer in self._removed_consumers:
bound_queue = consumer.queue(self._channel)
if bound_queue is not None:
bound_queue.delete()
consumers = [kombu.Consumer(self._channel, queues=c.queue,
callbacks=[c.callback])
for c in self._consumers.values()]
self._consumers_changed = False
if len(consumers) == 0:
gevent.sleep(0.1)
else:
with nested(*consumers):
while self._running and not self._consumers_changed and\
self._connection.connected:
try:
self._connection.drain_events(timeout=1)
except socket.timeout:
pass
except errors as e:
msg = 'Connection error in Kombu amqp consumer greenlet: %s' % str(e)
self._logger(msg, level=SandeshLevel.SYS_WARN)
except Exception as e:
msg = 'Error in Kombu amqp consumer greenlet: %s' % str(e)
self._logger(msg, level=SandeshLevel.SYS_ERR)
def _start_consuming(self):
errors = (self._connection.connection_errors +
self._connection.channel_errors)
removed_consumer = None
msg = 'KombuAmqpClient: Starting consumer greenlet'
self._logger(msg, level=SandeshLevel.SYS_DEBUG)
while self._running:
try:
self._ensure_connection(self._connection, "Consumer")
self._connected = True
while len(self._removed_consumers) > 0 or removed_consumer:
if removed_consumer is None:
removed_consumer = self._removed_consumers.pop(0)
if removed_consumer.queue.is_bound:
removed_consumer.queue.delete()
removed_consumer = None
if len(self._consumers.values()) == 0:
msg = 'KombuAmqpClient: Waiting for consumer'
self._logger(msg, level=SandeshLevel.SYS_DEBUG)
self._consumer_event.wait()
self._consumer_event.clear()
if not self._running or len(self._consumers.values()) == 0:
msg = 'KombuAmqpClient: No consumers found'
self._logger(msg, level=SandeshLevel.SYS_DEBUG)
continue
consumers = [
kombu.Consumer(
self._connection,
queues=c.queue,
callbacks=[c.callback] if c.callback else None)
for c in self._consumers.values()
]
msg = 'KombuAmqpClient: Created consumers %s' % str(
self._consumers.keys())
self._logger(msg, level=SandeshLevel.SYS_DEBUG)
self._consumers_changed = False
with nested(*consumers):
while self._running and not self._consumers_changed:
try:
self._connection.drain_events(timeout=1)
except socket.timeout:
pass
except errors as e:
msg = 'KombuAmqpClient: Connection error in Kombu amqp consumer greenlet: %s' % str(
e)
self._logger(msg, level=SandeshLevel.SYS_WARN)
self._connected = False
gevent.sleep(0.1)
except Exception as e:
msg = 'KombuAmqpClient: Error in Kombu amqp consumer greenlet: %s' % str(
e)
self._logger(msg, level=SandeshLevel.SYS_ERR)
self._connected = False
gevent.sleep(0.1)
msg = 'KombuAmqpClient: Exited consumer greenlet'
self._logger(msg, level=SandeshLevel.SYS_DEBUG)
def drain_events(self, infinity=True):
with nested(*self.consumers):
while True:
try:
self.connection.drain_events(timeout=self.timeout)
except socket.timeout:
if not infinity:
return
except Exception as e:
self.logger.exception(e)
def test_fanout(self, name='test_mongodb_fanout'):
if not self.verify_alive():
return
c = self.connection
self.e = Exchange(name, type='fanout')
self.q = Queue(name, exchange=self.e, routing_key=name)
self.q2 = Queue(name + '2', exchange=self.e, routing_key=name + '2')
channel = c.default_channel
producer = Producer(channel, self.e)
consumer1 = Consumer(channel, self.q)
consumer2 = Consumer(channel, self.q2)
self.q2(channel).declare()
for i in range(10):
producer.publish({'foo': i}, routing_key=name)
for i in range(10):
producer.publish({'foo': i}, routing_key=name + '2')
_received1 = []
_received2 = []
def callback1(message_data, message):
_received1.append(message)
message.ack()
def callback2(message_data, message):
_received2.append(message)
message.ack()
consumer1.register_callback(callback1)
consumer2.register_callback(callback2)
with nested(consumer1, consumer2):
while 1:
if len(_received1) + len(_received2) == 20:
break
c.drain_events(timeout=60)
self.assertEqual(len(_received1) + len(_received2), 20)
# queue.delete
for i in range(10):
producer.publish({'foo': i}, routing_key=name)
self.assertTrue(self.q(channel).get())
self.q(channel).delete()
self.q(channel).declare()
self.assertIsNone(self.q(channel).get())
# queue.purge
for i in range(10):
producer.publish({'foo': i}, routing_key=name + '2')
self.assertTrue(self.q2(channel).get())
self.q2(channel).purge()
self.assertIsNone(self.q2(channel).get())
def test_fanout(self, name='test_mongodb_fanout'):
if not self.verify_alive():
return
c = self.connection
self.e = Exchange(name, type='fanout')
self.q = Queue(name, exchange=self.e, routing_key=name)
self.q2 = Queue(name + '2', exchange=self.e, routing_key=name + '2')
channel = c.default_channel
producer = Producer(channel, self.e)
consumer1 = Consumer(channel, self.q)
consumer2 = Consumer(channel, self.q2)
self.q2(channel).declare()
for i in range(10):
producer.publish({'foo': i}, routing_key=name)
for i in range(10):
producer.publish({'foo': i}, routing_key=name + '2')
_received1 = []
_received2 = []
def callback1(message_data, message):
_received1.append(message)
message.ack()
def callback2(message_data, message):
_received2.append(message)
message.ack()
consumer1.register_callback(callback1)
consumer2.register_callback(callback2)
with nested(consumer1, consumer2):
while 1:
if len(_received1) + len(_received2) == 20:
break
c.drain_events(timeout=60)
self.assertEqual(len(_received1) + len(_received2), 20)
# queue.delete
for i in range(10):
producer.publish({'foo': i}, routing_key=name)
self.assertTrue(self.q(channel).get())
self.q(channel).delete()
self.q(channel).declare()
self.assertIsNone(self.q(channel).get())
# queue.purge
for i in range(10):
producer.publish({'foo': i}, routing_key=name + '2')
self.assertTrue(self.q2(channel).get())
self.q2(channel).purge()
self.assertIsNone(self.q2(channel).get())
def _start_consuming(self):
errors = (self._connection.connection_errors +
self._connection.channel_errors)
removed_consumer = None
msg = 'KombuAmqpClient: Starting consumer greenlet'
self._logger(msg, level=SandeshLevel.SYS_DEBUG)
while self._running:
try:
self._ensure_connection(self._connection, "Consumer")
self._connected = True
while len(self._removed_consumers) > 0 or removed_consumer:
if removed_consumer is None:
removed_consumer = self._removed_consumers.pop(0)
self._delete_consumer(removed_consumer)
removed_consumer = None
if len(list(self._consumers.values())) == 0:
msg = 'KombuAmqpClient: Waiting for consumer'
self._logger(msg, level=SandeshLevel.SYS_DEBUG)
self._consumer_event.wait()
self._consumer_event.clear()
continue
consumers = self._create_consumer_list()
msg = 'KombuAmqpClient: Created consumers %s' % str(
list(self._consumers.keys()))
self._logger(msg, level=SandeshLevel.SYS_DEBUG)
with nested(*consumers):
self._consumers_created_event.set()
if self._consumer_lock.locked():
self._consumer_lock.release()
while self._running and not self._consumers_changed:
try:
self._connection.drain_events(timeout=1)
except socket.timeout:
pass
self._consumers_changed = False
self._consumer_lock.acquire()
except errors as e:
msg = 'KombuAmqpClient: Connection error in Kombu amqp consumer greenlet: %s' % str(
e)
self._logger(msg, level=SandeshLevel.SYS_WARN)
self._connected = False
gevent.sleep(0.1)
except Exception as e:
msg = 'KombuAmqpClient: Error in Kombu amqp consumer greenlet: %s' % str(
e)
self._logger(msg, level=SandeshLevel.SYS_ERR)
self._connected = False
gevent.sleep(0.1)
msg = 'KombuAmqpClient: Exited consumer greenlet'
self._logger(msg, level=SandeshLevel.SYS_DEBUG)
def poll(self, queues, timeout=None, **data):
log.debug('Starting Poller for %s, %s' % (queues.keys(), data))
with AMQPConnection(self.url) as conn:
consumers = [Consumer(conn.channel(), queues=[queue],
callbacks=[processor(name, **data)])
for name, queue in queues.items()]
with nested(*consumers):
while True:
try:
conn.drain_events(timeout=timeout)
except socket.timeout:
print "Timeout."
break
def drain_events(self, infinity=True):
with nested(*self.consumers):
while not self._stop:
try:
self.connection.drain_events(timeout=self.timeout)
except socket.timeout:
if not infinity:
break
except ConnectionError as e: # pragma no cover
self.logger.error(
'Connection to mq has broken off because: %s. Try to reconnect, %s',
e)
self.connect()
self.revive()
break
except HandlerError:
self.logger.exception('Handler error')
except Exception as e: # pragma no cover
if not self._stop:
self.logger.exception(
'Something wrong! Try to restart the loop')
self.revive()
break
else: # pragma: no cover
self.logger.exception(
'Something wrong! And stopping...')
break
if self.with_pool:
try:
self.drain_results()
except Exception: # pragma no cover
self.logger.exception('Unknown error when '
'draining results')
if self._stop:
if self.with_pool:
try:
self.pool.join()
self.drain_results() # pragma: no cover
except AssertionError:
pass
except Exception: # pragma: no cover
self.logger.exception(
'Unknown error when '
'draining results'
)
self._stopped = True
self.logger.info('Stopped draining events.')
def _start_consuming(self):
errors = (self._connection.connection_errors +
self._connection.channel_errors)
removed_consumer = None
msg = 'KombuAmqpClient: Starting consumer greenlet'
self._logger(msg, level=SandeshLevel.SYS_DEBUG)
while self._running:
try:
self._ensure_connection(self._connection, "Consumer")
self._connected = True
while len(self._removed_consumers) > 0 or removed_consumer:
if removed_consumer is None:
removed_consumer = self._removed_consumers.pop(0)
self._delete_consumer(removed_consumer)
removed_consumer = None
if len(self._consumers.values()) == 0:
msg = 'KombuAmqpClient: Waiting for consumer'
self._logger(msg, level=SandeshLevel.SYS_DEBUG)
self._consumer_event.wait()
self._consumer_event.clear()
continue
consumers = [kombu.Consumer(self._connection, queues=c.queue,
callbacks=[c.callback] if c.callback else None)
for c in self._consumers.values()]
msg = 'KombuAmqpClient: Created consumers %s' % str(self._consumers.keys())
self._logger(msg, level=SandeshLevel.SYS_DEBUG)
self._consumers_changed = False
with nested(*consumers):
while self._running and not self._consumers_changed:
try:
self._connection.drain_events(timeout=1)
except socket.timeout:
pass
except errors as e:
msg = 'KombuAmqpClient: Connection error in Kombu amqp consumer greenlet: %s' % str(e)
self._logger(msg, level=SandeshLevel.SYS_WARN)
self._connected = False
gevent.sleep(0.1)
except Exception as e:
msg = 'KombuAmqpClient: Error in Kombu amqp consumer greenlet: %s' % str(e)
self._logger(msg, level=SandeshLevel.SYS_ERR)
self._connected = False
gevent.sleep(0.1)
msg = 'KombuAmqpClient: Exited consumer greenlet'
self._logger(msg, level=SandeshLevel.SYS_DEBUG)
def drain_events(self, infinity=True):
with nested(*self.consumers):
while not self._stop:
try:
self.connection.drain_events(timeout=self.timeout)
except socket.timeout:
if not infinity:
break
except ConnectionError as e: # pragma no cover
self.logger.error(
'Connection to mq has broken off because: %s. Try to reconnect, %s',
e)
self.connect()
self.revive()
break
except HandlerError:
self.logger.exception('Handler error')
except Exception as e: # pragma no cover
if not self._stop:
self.logger.exception(
'Something wrong! Try to restart the loop')
self.revive()
break
else: # pragma: no cover
self.logger.exception(
'Something wrong! And stopping...')
break
if self.with_pool:
try:
self.drain_results()
except Exception: # pragma no cover
self.logger.exception('Unknown error when '
'draining results')
if self._stop:
if self.with_pool:
try:
self.pool.join()
self.drain_results() # pragma: no cover
except AssertionError:
pass
except Exception: # pragma: no cover
self.logger.exception('Unknown error when '
'draining results')
self._stopped = True
self.logger.info('Stopped draining events.')
def patch_many(*targets):
return nested(*[patch(target) for target in targets])
def patch_many(*targets):
return nested(*[mock.patch(target) for target in targets])
