def __init__(self, url):
self.url = url
self.message_defaults = {
"source": "hub",
"version": pkg_resources.require("ulearnhub")[0].version,
}
client_properties = {
"product":
"hub",
"version":
pkg_resources.require("ulearnhub")[0].version,
"platform":
'Python {0.major}.{0.minor}.{0.micro}'.format(sys.version_info),
}
self.enabled = True
try:
self.client = RabbitClient(self.url,
client_properties=client_properties,
transport='gevent')
except AttributeError:
self.enabled = False
except socket_error:
raise ConnectionError("Could not connect to rabbitmq broker")
class RabbitNotifications(object):
"""
Wrapper to access notification methods, and catch possible exceptions
"""
def __init__(self, url):
self.url = url
self.message_defaults = {
"source": "hub",
"version": pkg_resources.require("ulearnhub")[0].version,
}
client_properties = {
"product":
"hub",
"version":
pkg_resources.require("ulearnhub")[0].version,
"platform":
'Python {0.major}.{0.minor}.{0.micro}'.format(sys.version_info),
}
self.enabled = True
try:
self.client = RabbitClient(self.url,
client_properties=client_properties,
transport='gevent')
except AttributeError:
self.enabled = False
except socket_error:
raise ConnectionError("Could not connect to rabbitmq broker")
def sync_acl(self, domain, context, username, tasks):
"""
Sends a Carrot (TM) notification of a new sync acl task
"""
# Send a conversation creation notification to rabbit
message = RabbitMessage()
message.prepare(self.message_defaults)
message.update({
"user": {
'username': username,
},
"domain": domain,
"action": "modify",
"object": "context",
"data": {
'context': context,
'tasks': tasks
}
})
self.client.send('syncacl', json.dumps(message.packed), routing_key='')
def setUp(self):
conf_dir = os.path.dirname(__file__)
self.app = loadapp('config:tests.ini', relative_to=conf_dir)
self.testapp = UlearnhubTestApp(self)
self.rabbit = RabbitClient(TEST_VHOST_URL)
self.rabbit.management.cleanup(delete_all=True)
self.rabbit.declare()
httpretty.enable()
http_mock_info()
http_mock_checktoken()
create_defaults(self.testapp.testapp.app.registry,
BASE_DOMAIN,
quiet=True)
self.initialize_test_deployment()
self.initialize_test_domain()
self.patches = []
self.clients = {}
class RabbitTests(unittest.TestCase):
def setUp(self):
self.server = RabbitClient(TEST_VHOST_URL)
self.server.management.cleanup(delete_all=True)
self.server.declare()
self.clients = {}
def tearDown(self):
self.server.disconnect()
for user_clients in self.clients.values():
for client in user_clients:
client.disconnect()
def getClient(self, username, reuse=True):
self.clients.setdefault(username, [])
if not self.clients[username] or not reuse:
self.clients[username].append(RabbitClient(TEST_VHOST_URL, user=username))
if reuse:
return self.clients[username][0]
else:
return self.clients[username][-1]
def setUp(self):
conf_dir = os.path.dirname(__file__)
self.app = loadapp('config:rabbitmq.ini', relative_to=conf_dir)
self.reset_database(self.app)
self.app.registry.max_store.security.insert(test_default_security)
self.patched_post = patch('requests.post', new=partial(mock_post, self))
self.patched_post.start()
self.testapp = MaxTestApp(self)
self.create_user(test_manager)
# Rabbitmq test client initialization
rabbitmq_url = self.app.registry.settings['max.rabbitmq']
self.server = RabbitClient(rabbitmq_url)
self.server.management.cleanup(delete_all=True)
self.server.declare()
def setUp(self):
# Resets the global that holds the mocked stmp sent messages
import maxbunny.tests
maxbunny.tests.sent = []
self.log_patch = patch('maxbunny.consumer.BunnyConsumer.configure_logger', new=get_storing_logger)
self.log_patch.start()
self.smtp_patch = patch('smtplib.SMTP', new=MockSMTP)
self.smtp_patch.start()
self.server = RabbitClient(TEST_VHOST_URL)
self.server.management.cleanup(delete_all=True)
self.server.declare()
self.server.ch.queue.declare(
queue='tests',
durable=True,
auto_delete=False
)
self.process = None
def __init__(self, request):
self.request = request
settings = getMAXSettings(request)
self.url = settings.get('max_rabbitmq', '')
self.message_defaults = settings.get('max_message_defaults', {})
self.enabled = True
client_properties = {
"product": "max",
"version": pkg_resources.require('max')[0].version,
"platform": 'Python {0.major}.{0.minor}.{0.micro}'.format(sys.version_info),
"server": settings.get('max_server', '')
}
try:
self.client = RabbitClient(self.url, client_properties=client_properties)
except AttributeError:
self.enabled = False
except socket_error:
raise ConnectionError("Could not connect to rabbitmq broker")
def run(self):
# Create client without declaring anything
self.server = RabbitClient(self.rabbitmq_url)
# Clear all non-native exchanges and queues
print '> Cleaning up rabbitmq'
self.server.management.cleanup(delete_all=self.options.deleteall)
# Create all exchanges and queues defined in spec
print '> Creating default exchanges, queues and bindings'
self.server.declare()
# Keep track of all bindings created on any maxserver batch during this
# script execution, in order to avoid deleting valid bindings
self.global_valid_bindings = []
# Refresh
print '> Loading current exchanges and queues'
self.server.management.load_exchanges()
self.server.management.load_queues()
self.exchanges_by_name = self.server.management.exchanges_by_name
print ' Found {} exchanges'.format(len(self.server.management.exchanges))
print ' Found {} queues'.format(len(self.server.management.queues))
print '> Loading current conversation bindings'
bindings = self.server.management.load_exchange_bindings('conversations')
self.conversation_bindings = ['{source}_{routing_key}_{destination}'.format(**binding) for binding in bindings]
print ' Found {} conversation bindings'.format(len(self.conversation_bindings))
print '> Loading current context bindings'
bindings = self.context_bindings = self.server.management.load_exchange_bindings('activity')
self.context_bindings = ['{source}_{routing_key}_{destination}'.format(**binding) for binding in bindings]
print ' Found {} context bindings'.format(len(self.context_bindings))
# Mongodb connection initialization
cluster_enabled = self.cluster
auth_enabled = self.mongo_auth
mongodb_uri = self.clustermembers if cluster_enabled else self.standaloneserver
conn = mongodb.get_connection(
mongodb_uri,
use_greenlets=True,
cluster=self.replicaset if cluster_enabled else None)
for maxname in self.maxserver_names:
dbname = 'max_{}'.format(maxname)
if dbname not in conn.database_names():
print ('Skipping "{}" max instance, no database found, maybe is an alias?'.format(maxname))
continue
print('')
print('============================================================================')
print(' Processing "{}" max instance'.format(maxname))
print('============================================================================')
print('')
db = mongodb.get_database(
conn,
dbname,
username=self.mongo_username if auth_enabled else None,
password=self.mongo_password if auth_enabled else None,
authdb=self.mongo_authdb if auth_enabled else None)
print "> Getting users list from database"
# Get all users to create their rabbit exchange and bindings
query = {} if not self.options.usernamefilter else {'username': self.options.usernamefilter}
all_users = db.users.find(query, {'_id': 0, 'username': 1, 'talkingIn': 1, 'subscribedTo': 1})
contexts_with_notifications = db.contexts.find({'notifications': {'$nin': [False], '$exists': 1}}, {'hash': 1})
contexts_with_notifications_hashs = [a['hash'] for a in contexts_with_notifications]
# unpack lazy results
all_users = [a for a in all_users]
print '> Got {} users'.format(len(all_users))
print
# Extract the list of conversations and context subscriptions from all users
conversations = {}
contexts = {}
for user in all_users:
for conversation in user.get('talkingIn', []):
conv = conversations.setdefault(conversation['id'], [])
conv.append(user['username'])
for context in user.get('subscribedTo', []):
if context['hash'] in contexts_with_notifications_hashs:
ctxt = contexts.setdefault(context['hash'], [])
ctxt.append(user['username'])
print '> Starting Batch: Create user exchanges and bindings'
self.do_batch(self.add_users, all_users)
print '> Starting Batch: Create conversation bindings'
self.do_batch(self.add_conversation_bindings, conversations.items())
print '> Starting Batch: Create context bindings'
self.do_batch(self.add_context_bindings, contexts.items())
self.server.disconnect()
class ConsumerTests(MaxBunnyTestCase):
def setUp(self):
# Resets the global that holds the mocked stmp sent messages
import maxbunny.tests
maxbunny.tests.sent = []
self.log_patch = patch('maxbunny.consumer.BunnyConsumer.configure_logger', new=get_storing_logger)
self.log_patch.start()
self.smtp_patch = patch('smtplib.SMTP', new=MockSMTP)
self.smtp_patch.start()
self.server = RabbitClient(TEST_VHOST_URL)
self.server.management.cleanup(delete_all=True)
self.server.declare()
self.server.ch.queue.declare(
queue='tests',
durable=True,
auto_delete=False
)
self.process = None
def tearDown(self):
self.log_patch.stop()
self.smtp_patch.stop()
self.server.get_all('tests')
self.server.disconnect()
try:
self.process.terminate()
except:
pass # pragma: no cover
def test_consumer_drop_no_uuid(self):
"""
Given a invalid non-json message
When the consumer loop processes the message
And the message triggers a Cancel exception
Then the message is dropped
And a warning is logged
And the channel remains Open
"""
runner = MockRunner('tests', 'maxbunny.ini', 'instances.ini')
consumer = TestConsumer(runner, BunnyMessageCancel('Testing message drop'))
self.process = ConsumerThread(consumer)
self.server.send('', '', routing_key='tests')
self.process.start()
sleep(0.3) # give a minum time to mail to be sent
from maxbunny.tests import sent # MUST import sent here to get current sent mails,
self.assertEqual(len(sent), 1)
self.assertEqual(len(consumer.logger.infos), 1)
self.assertEqual(len(consumer.logger.warnings), 1)
self.assertTrue(self.process.isAlive())
self.assertEqual(consumer.logger.warnings[0], 'Message dropped (NO_UUID), reason: Testing message drop')
self.server.management.force_close(consumer.remote())
sleep(0.2) # Leave a minimum time to rabbitmq to release messages
queued = self.server.get_all('tests')
self.assertEqual(len(queued), 0)
def test_consumer_drops_on_requeue_exception_without_uuid(self):
"""
Given a message without UUID field
When the consumer loop processes the message
And the message triggers a Requeue exception
Then the message is requeued
And a warning is logged
And the channel remains Open
And a mail notification is sent
"""
runner = MockRunner('tests', 'maxbunny.ini', 'instances.ini')
consumer = TestConsumer(runner, BunnyMessageRequeue('Test requeueing'))
self.process = ConsumerThread(consumer)
self.server.send('', '{}', routing_key='tests')
self.process.start()
sleep(0.3) # give a minum time to mail to be sent
from maxbunny.tests import sent # MUST import sent here to get current sent mails,
self.assertEqual(len(sent), 1)
self.assertEqual(len(consumer.logger.infos), 1)
self.assertEqual(len(consumer.logger.warnings), 1)
self.assertTrue(self.process.isAlive())
self.assertEqual(consumer.logger.warnings[0], 'Message dropped (NO_UUID), reason: Test requeueing')
self.server.management.force_close(consumer.remote())
sleep(0.2) # Leave a minimum time to rabbitmq to release messages
queued = self.server.get_all('tests')
self.assertEqual(len(queued), 0)
def test_consumer_drop_with_uuid(self):
"""
Given a message with UUID field
When the consumer loop processes the message
And the message triggers a Cancel exception
Then the message is dropped
And a warning is logged
And the channel remains Open
And no mail notification is sent
"""
runner = MockRunner('tests', 'maxbunny.ini', 'instances.ini')
consumer = TestConsumer(runner, BunnyMessageCancel('Testing message drop', notify=False))
self.process = ConsumerThread(consumer)
self.server.send('', '{"g": "0123456789"}', routing_key='tests')
self.process.start()
sleep(0.3) # give a minum time to mail to be sent
from maxbunny.tests import sent # MUST import sent here to get current sent mails,
self.assertEqual(len(sent), 0)
self.assertEqual(len(consumer.logger.infos), 1)
self.assertEqual(len(consumer.logger.warnings), 1)
self.assertTrue(self.process.isAlive())
self.assertEqual(consumer.logger.warnings[0], 'Message dropped, reason: Testing message drop')
self.server.management.force_close(consumer.remote())
sleep(0.2) # Leave a minimum time to rabbitmq to release messages
queued = self.server.get_all('tests')
self.assertEqual(len(queued), 0)
def test_consumer_drop_with_notification(self):
"""
Given a message with UUID field
When the consumer loop processes the message
And the message triggers a Cancel exception
Then the message is dropped
And a warning is logged
And the channel remains Open
And a mail notification is sent
"""
runner = MockRunner('tests', 'maxbunny.ini', 'instances.ini')
consumer = TestConsumer(runner, BunnyMessageCancel('Testing message drop', notify=True))
self.process = ConsumerThread(consumer)
self.server.send('', '{"g": "0123456789"}', routing_key='tests')
self.process.start()
sleep(0.3) # give a minum time to mail to be sent
from maxbunny.tests import sent # MUST import sent here to get current sent mails,
self.assertEqual(len(sent), 1)
self.assertEqual(len(consumer.logger.infos), 1)
self.assertEqual(len(consumer.logger.warnings), 1)
self.assertTrue(self.process.isAlive())
self.assertEqual(consumer.logger.warnings[0], 'Message dropped, reason: Testing message drop')
self.server.management.force_close(consumer.remote())
sleep(0.2) # Leave a minimum time to rabbitmq to release messages
queued = self.server.get_all('tests')
self.assertEqual(len(queued), 0)
def test_consumer_drop_no_recipient(self):
"""
Given a message with UUID field
And a missing recipients smtp setting
When the consumer loop processes the message
And the message triggers a Cancel exception
Then the message is dropped
And a warning is logged
And the channel remains Open
And a mail notification is not sent
"""
runner = MockRunner('tests', 'maxbunny-norecipients.ini', 'instances.ini')
consumer = TestConsumer(runner, BunnyMessageCancel('Testing message drop', notify=True))
self.process = ConsumerThread(consumer)
self.server.send('', '{"g": "0123456789"}', routing_key='tests')
self.process.start()
sleep(0.6) # give a minum time to mail to be sent
from maxbunny.tests import sent # MUST import sent here to get current sent mails,
self.assertEqual(len(sent), 0)
self.assertEqual(len(consumer.logger.infos), 1)
self.assertEqual(len(consumer.logger.warnings), 1)
self.assertTrue(self.process.isAlive())
self.assertEqual(consumer.logger.warnings[0], 'Message dropped, reason: Testing message drop')
self.server.management.force_close(consumer.remote())
sleep(0.2) # Leave a minimum time to rabbitmq to release messages
queued = self.server.get_all('tests')
self.assertEqual(len(queued), 0)
def test_consumer_drop_on_max_non_5xx_error(self):
"""
Given a message with UUID field
When the consumer loop processes the message
And the message triggers a RequestError with status code different from 5xx
Then the message is dropped
And a warning is logged
And the channel remains Open
And a mail notification is sent
"""
from maxclient.client import RequestError
runner = MockRunner('tests', 'maxbunny.ini', 'instances.ini')
consumer = TestConsumer(runner, RequestError(401, 'Unauthorized'))
self.process = ConsumerThread(consumer)
self.server.send('', '{"g": "0123456789"}', routing_key='tests')
self.process.start()
sleep(0.3) # give a minum time to mail to be sent
from maxbunny.tests import sent # MUST import sent here to get current sent mails,
self.assertEqual(len(sent), 1)
self.assertEqual(len(consumer.logger.infos), 1)
self.assertEqual(len(consumer.logger.warnings), 1)
self.assertTrue(self.process.isAlive())
self.assertEqual(consumer.logger.warnings[0], 'Message dropped, reason: Max server error: Unauthorized')
self.server.management.force_close(consumer.remote())
sleep(0.2) # Leave a minimum time to rabbitmq to release messages
queued = self.server.get_all('tests')
self.assertEqual(len(queued), 0)
def test_consumer_drop_on_maxcarrot_exception(self):
"""
Given a message with UUID field
When the consumer loop processes the message
And the message triggers a MaxCarrotParsingError
Then the message is dropped
And a warning is logged
And the channel remains Open
And a mail notification is sent
"""
from maxcarrot.message import MaxCarrotParsingError
runner = MockRunner('tests', 'maxbunny.ini', 'instances.ini')
consumer = TestConsumer(runner, MaxCarrotParsingError())
self.process = ConsumerThread(consumer)
self.server.send('', '{"g": "0123456789"}', routing_key='tests')
self.process.start()
sleep(0.3) # give a minum time to mail to be sent
from maxbunny.tests import sent # MUST import sent here to get current sent mails,
self.assertEqual(len(sent), 1)
self.assertEqual(len(consumer.logger.infos), 1)
self.assertEqual(len(consumer.logger.warnings), 1)
self.assertTrue(self.process.isAlive())
self.assertEqual(consumer.logger.warnings[0], 'Message dropped, reason: MaxCarrot Parsing error')
self.server.management.force_close(consumer.remote())
sleep(0.2) # Leave a minimum time to rabbitmq to release messages
queued = self.server.get_all('tests')
self.assertEqual(len(queued), 0)
def test_consumer_requeues_on_max_5xx_error(self):
"""
Given a message with UUID field
When the consumer loop processes the message
And the message triggers a RequestError with status code different from 5xx
Then the message is requeued
And a warning is logged
And the channel remains Open
And a mail notification is sent
"""
from maxclient.client import RequestError
runner = MockRunner('tests', 'maxbunny.ini', 'instances.ini')
consumer = TestConsumer(runner, RequestError(500, 'Internal Server Error'))
self.process = ConsumerThread(consumer)
self.server.send('', '{"g": "0123456789"}', routing_key='tests')
self.process.start()
sleep(0.3) # give a minum time to mail to be sent
from maxbunny.tests import sent # MUST import sent here to get current sent mails,
self.assertEqual(len(sent), 1)
self.assertEqual(len(consumer.logger.infos), 1)
self.assertEqual(len(consumer.logger.warnings), 1)
self.assertTrue(self.process.isAlive())
self.assertEqual(consumer.logger.warnings[0], 'Message 0123456789 reueued, reason: Max server error: Internal Server Error')
self.server.management.force_close(consumer.remote())
sleep(0.2) # Leave a minimum time to rabbitmq to release messages
queued = self.server.get_all('tests')
self.assertEqual(len(queued), 1)
def test_consumer_requeues_on_requeue_exception(self):
"""
Given a message with UUID field
When the consumer loop processes the message
And the message triggers a Requeue exception
Then the message is requeued
And a warning is logged
And the channel remains Open
And a mail notification is sent
And the id of the queued message is stored
"""
runner = MockRunner('tests', 'maxbunny.ini', 'instances.ini')
consumer = TestConsumer(runner, BunnyMessageRequeue('Test requeueing'))
self.process = ConsumerThread(consumer)
self.server.send('', '{"g": "0123456789"}', routing_key='tests')
self.process.start()
sleep(0.3) # give a minum time to mail to be sent
from maxbunny.tests import sent # MUST import sent here to get current sent mails,
self.assertEqual(len(sent), 1)
self.assertEqual(len(consumer.logger.infos), 1)
self.assertEqual(len(consumer.logger.warnings), 1)
self.assertTrue(self.process.isAlive())
self.assertEqual(consumer.logger.warnings[0], 'Message 0123456789 reueued, reason: Test requeueing')
self.server.management.force_close(consumer.remote())
sleep(0.2) # Leave a minimum time to rabbitmq to release messages
queued = self.server.get_all('tests')
self.assertEqual(len(queued), 1)
self.assertEqual(len(consumer.requeued), 1)
self.assertIn('0123456789', consumer.requeued)
def test_consumer_requeues_on_requeue_exception_unqueues_after(self):
"""
Given a message with UUID field
When the consumer loop processes the message
And the first time the message triggers a Requeue exception
And the second time the message is succesfully processed
Then the message is unqueued the second time
And a warning is logged
And the channel remains Open
And a mail notification is sent
And the id is removed from queued ones
"""
runner = MockRunner('tests', 'maxbunny.ini', 'instances.ini')
consumer = TestConsumer(runner, BunnyMessageRequeue('Test requeueing'), after=None)
self.process = ConsumerThread(consumer)
self.server.send('', '{"g": "0123456789"}', routing_key='tests')
self.process.start()
sleep(0.3) # give a minum time to mail to be sent
from maxbunny.tests import sent # MUST import sent here to get current sent mails,
self.assertEqual(len(sent), 1)
self.assertEqual(len(consumer.logger.infos), 1)
self.assertEqual(len(consumer.logger.warnings), 1)
self.assertTrue(self.process.isAlive())
self.assertEqual(consumer.logger.warnings[0], 'Message 0123456789 reueued, reason: Test requeueing')
self.server.management.force_close(consumer.remote())
sleep(0.2) # Leave a minimum time to rabbitmq to release messages
queued = self.server.get_all('tests')
self.assertEqual(len(queued), 0)
self.assertEqual(len(consumer.requeued), 0)
def test_consumer_requeues_on_requeue_exception_drops_after(self):
"""
Given a message with UUID field
When the consumer loop processes the message
And the first time the message triggers a Requeue exception
And the second time the message is triggers a Cancel exception
Then the message is unqueued the second time
And a warning is logged
And the channel remains Open
And a mail notification is sent
And the id is removed from queued ones
"""
runner = MockRunner('tests', 'maxbunny.ini', 'instances.ini')
consumer = TestConsumer(runner, BunnyMessageRequeue('Test requeueing'), after=BunnyMessageCancel('Testing message drop'))
self.process = ConsumerThread(consumer)
self.server.send('', '{"g": "0123456789"}', routing_key='tests')
self.process.start()
sleep(0.3) # give a minum time to mail to be sent
from maxbunny.tests import sent # MUST import sent here to get current sent mails,
self.assertEqual(len(sent), 2)
self.assertEqual(len(consumer.logger.infos), 1)
self.assertEqual(len(consumer.logger.warnings), 2)
self.assertTrue(self.process.isAlive())
self.assertEqual(consumer.logger.warnings[0], 'Message 0123456789 reueued, reason: Test requeueing')
self.assertEqual(consumer.logger.warnings[1], 'Message dropped, reason: Testing message drop')
self.server.management.force_close(consumer.remote())
sleep(0.2) # Leave a minimum time to rabbitmq to release messages
queued = self.server.get_all('tests')
self.assertEqual(len(queued), 0)
self.assertEqual(len(consumer.requeued), 0)
def test_consumer_requeues_on_unknown_exception(self):
"""
Given a message with UUID field
When the consumer loop processes the message
And the message triggers an unknown Exception
Then the message is requeued
And a warning is logged
And the channel remains Open
And a mail notification is sent
"""
runner = MockRunner('tests', 'maxbunny.ini', 'instances.ini')
consumer = TestConsumer(runner, Exception('Unknown exception'))
self.process = ConsumerThread(consumer)
self.server.send('', '{"g": "0123456789"}', routing_key='tests')
self.process.start()
sleep(0.3) # give a minum time to mail to be sent
from maxbunny.tests import sent # MUST import sent here to get current sent mails,
self.assertEqual(len(sent), 1)
self.assertEqual(len(consumer.logger.infos), 1)
self.assertEqual(len(consumer.logger.warnings), 1)
self.assertTrue(self.process.isAlive())
self.assertEqual(consumer.logger.warnings[0], 'Message 0123456789 reueued, reason: Consumer failure: Unknown exception')
self.server.management.force_close(consumer.remote())
sleep(0.2) # Leave a minimum time to rabbitmq to release messages
queued = self.server.get_all('tests')
self.assertEqual(len(queued), 1)
def test_consumer_stops_on_force_stop_connection(self):
"""
Given a running consumer
When the rabbitmq connection is closed remotely
Then the channel closes
And a warning is logged
"""
runner = MockRunner('tests', 'maxbunny.ini', 'instances.ini')
consumer = TestConsumer(runner)
self.process = ConsumerThread(consumer)
self.process.start()
sleep(0.2) # Leave a minimum life time to consumer
self.server.management.force_close(consumer.remote(), 'Closed via ')
sleep(1) # Leave a minimum time to consumer to stop
self.assertEqual(len(consumer.logger.infos), 1)
self.assertEqual(len(consumer.logger.errors), 1)
self.assertFalse(self.process.isAlive())
self.assertEqual(consumer.logger.errors[0], 'CONNECTION_FORCED - Closed via management plugin')
sleep(0.2) # Leave a minimum time to rabbitmq to release messages
queued = self.server.get_all('tests')
self.assertEqual(len(queued), 0)
def test_consumer_success(self):
"""
Given a message with UUID field
When the consumer loop processes the message
And the message suceeds
Then the message is acks
And nothing is logged
And the channel remains Open
And a no mail notification is sent
"""
runner = MockRunner('tests', 'maxbunny.ini', 'instances.ini')
consumer = TestConsumer(runner)
self.process = ConsumerThread(consumer)
self.server.send('', '{"g": "0123456789"}', routing_key='tests')
self.process.start()
sleep(0.3) # give a minum time to mail to be sent
from maxbunny.tests import sent # MUST import sent here to get current sent mails,
self.assertEqual(len(sent), 0)
self.assertEqual(len(consumer.logger.infos), 1)
self.assertEqual(len(consumer.logger.warnings), 0)
self.assertTrue(self.process.isAlive())
self.server.management.force_close(consumer.remote())
sleep(0.2) # Leave a minimum time to rabbitmq to release messages
queued = self.server.get_all('tests')
self.assertEqual(len(queued), 0)
class RabbitNotifications(object):
"""
Wrapper to access notification methods, and catch possible exceptions
"""
def __init__(self, request):
self.request = request
settings = getMAXSettings(request)
self.url = settings.get('max_rabbitmq', '')
self.message_defaults = settings.get('max_message_defaults', {})
self.enabled = True
client_properties = {
"product": "max",
"version": pkg_resources.require('max')[0].version,
"platform": 'Python {0.major}.{0.minor}.{0.micro}'.format(sys.version_info),
"server": settings.get('max_server', '')
}
try:
self.client = RabbitClient(self.url, client_properties=client_properties)
except AttributeError:
self.enabled = False
except socket_error:
raise ConnectionError("Could not connect to rabbitmq broker")
def __getattribute__(self, name):
"""
Returns the requested method if notifier is enabled, otherwise
performs a noop
"""
enabled = object.__getattribute__(self, 'enabled')
if enabled or name in ['enabled', 'url', 'request', 'client', 'message_defaults']:
return object.__getattribute__(self, name)
else:
return noop
def restart_tweety(self):
"""
Sends a timestamp to tweety_restart queue, trough the default exchange
(binding to tweety_restart queue is implicit in this special exchange)
"""
default_exchange = ''
restart_request_time = datetime.datetime.now().strftime('%s.%f')
self.client.send(default_exchange, restart_request_time, 'tweety_restart')
#self.client.disconnect()
def add_user(self, username):
"""
Creates the specified user exchange and bindings
"""
self.client.create_user(username)
#self.client.disconnect()
def delete_user(self, username):
"""
Deletes the specified user exchange and bindings
"""
self.client.delete_user(username)
#self.client.disconnect()
def bind_user_to_context(self, context, username):
"""
Creates a binding between user exchanges and a context
"""
context_id = context.getIdentifier()
self.client.activity.bind_user(context_id, username)
#self.client.disconnect()
def unbind_user_from_context(self, context, username):
"""
Destroys a binding between user exchanges and a context
"""
context_id = context.getIdentifier()
self.client.activity.unbind_user(context_id, username)
#self.client.disconnect()
def unbind_context(self, context):
"""
Destroys all bindings between a context and any user
"""
context_id = context.getIdentifier()
self.client.activity.delete(context_id)
#self.client.disconnect()
def bind_user_to_conversation(self, conversation, username):
"""
Creates a binding between user exchanges and a conversation
"""
context_id = conversation.getIdentifier()
self.client.conversations.bind_user(context_id, username)
#self.client.disconnect()
def unbind_user_from_conversation(self, conversation, username):
"""
Destroys a binding between user exchanges and a conversation
"""
context_id = conversation.getIdentifier()
self.client.conversations.unbind_user(context_id, username)
#self.client.disconnect()
def unbind_conversation(self, conversation):
"""
Destroys all bindings between a conversation and any user
"""
context_id = conversation.getIdentifier()
self.client.conversations.delete(context_id)
#self.client.disconnect()
def notify_context_activity(self, activity):
"""
Sends a Carrot (TM) notification of a new post on a context
"""
message = RabbitMessage()
message.prepare(self.message_defaults)
message.update({
"user": {
'username': self.request.actor['username'],
'displayname': self.request.actor['displayName'],
},
"action": "add",
"object": "activity",
"data": {
'text': activity['object'].get('content', ''),
'activityid': str(activity['_id'])
}
})
self.client.send('activity', json.dumps(message.packed), activity['contexts'][0]['hash'])
#self.client.disconnect()
def notify_context_activity_comment(self, activity, comment):
"""
Sends a Carrot (TM) notification of a new post on a context
"""
message = RabbitMessage()
message.prepare(self.message_defaults)
message.update({
"user": {
'username': self.request.actor['username'],
'displayname': self.request.actor['displayName'],
},
"action": "add",
"object": "comment",
"data": {
'text': comment['content'],
'activityid': str(activity['_id']),
'commentid': comment['id']
}
})
self.client.send('activity', json.dumps(message.packed), activity['contexts'][0]['hash'])
#self.client.disconnect()
def add_conversation(self, conversation):
"""
Sends a Carrot (TM) notification of a new conversation creation
"""
conversation_id = conversation.getIdentifier()
participants_usernames = [user['username'] for user in conversation['participants']]
self.client.conversations.create(conversation_id, users=participants_usernames)
# Send a conversation creation notification to rabbit
message = RabbitMessage()
message.prepare(self.message_defaults)
message.update({
"user": {
'username': self.request.actor['username'],
'displayname': self.request.actor['displayName'],
},
"action": "add",
"object": "conversation",
"data": {}
})
self.client.send('conversations', json.dumps(message.packed), routing_key='{}.notifications'.format(conversation_id))
class UlearnhubSyncLDAPGroupFunctionalTests(UlearnHUBBaseTestCase):
def setUp(self):
conf_dir = os.path.dirname(__file__)
self.app = loadapp('config:tests.ini', relative_to=conf_dir)
self.testapp = UlearnhubTestApp(self)
self.rabbit = RabbitClient(TEST_VHOST_URL)
self.rabbit.management.cleanup(delete_all=True)
self.rabbit.declare()
httpretty.enable()
http_mock_info()
http_mock_checktoken()
create_defaults(self.testapp.testapp.app.registry,
BASE_DOMAIN,
quiet=True)
self.initialize_test_deployment()
self.initialize_test_domain()
self.patches = []
self.clients = {}
def tearDown(self):
# Make sure httpretty is disabled
httpretty.disable()
httpretty.reset()
for testpatch in self.patches:
testpatch.stop()
self.rabbit.get_all('syncacl')
self.rabbit.disconnect()
for user_clients in self.clients.values():
for client in user_clients:
client.disconnect()
def assertMessagesInQueue(self, queue, retries=0, expected=None):
messages = self.rabbit.get_all(queue)
expected_message_count = expected if expected is not None else len(
messages)
for retry in xrange(retries):
if len(messages) < expected_message_count:
messages += self.rabbit.get_all(queue)
else:
break
time.sleep(1)
if len(messages) < expected_message_count:
raise AssertionError(
'Missing messages on queue, expected {}, received {}, retryied {}'
.format(expected_message_count, len(messages), retries))
return {item[0]['u']['u']: item[0] for item in messages}
def test_syncldapgroup(self):
"""
"""
from .mockers.syncgroup import update_group_request
from .mockers.syncacl import context
from .mockers.syncacl import initial_subscriptions as subscriptions
from .mockers.syncacl import ldap_test_group4
http_mock_group_communities([{
'url': context['url'],
'groups': [],
'users': ['testuser1.creator']
}])
http_mock_get_context(context)
http_mock_get_context_subscriptions(subscriptions)
self.add_patch(ldap_patch_connect())
self.add_patch(ldap_patch_disconnect())
self.add_patch(ldap_patch_group_search({
'group4': ldap_test_group4,
}))
self.testapp.post(
'/api/deployments/{deployment}/components/{component}/services/{service}'
.format(deployment='test',
component='testldap',
service='syncldapgroup'),
json.dumps(update_group_request),
headers=oauth2Header(test_user),
status=200)
# Index by username to be able to make asserts
# This is mandatory, as we cannot assume the order of the queue
messages = self.assertMessagesInQueue('syncacl', retries=3, expected=1)
# Test subscribed user revoke permission, preserves most important role
self.assertItemsEqual(messages['groupuser1']['d']['tasks'],
['subscribe'])
self.assertIn('context', messages['groupuser1']['d'])
class UlearnhubSyncaclFunctionalTests(UlearnHUBBaseTestCase):
def setUp(self):
conf_dir = os.path.dirname(__file__)
self.app = loadapp('config:tests.ini', relative_to=conf_dir)
self.testapp = UlearnhubTestApp(self)
self.rabbit = RabbitClient(TEST_VHOST_URL)
self.rabbit.management.cleanup(delete_all=True)
self.rabbit.declare()
httpretty.enable()
http_mock_info()
http_mock_checktoken()
create_defaults(self.testapp.testapp.app.registry,
BASE_DOMAIN,
quiet=True)
self.initialize_test_deployment()
self.initialize_test_domain()
self.patches = []
self.clients = {}
def tearDown(self):
# Make sure httpretty is disabled
httpretty.disable()
httpretty.reset()
for testpatch in self.patches:
testpatch.stop()
self.rabbit.get_all('syncacl')
self.rabbit.disconnect()
for user_clients in self.clients.values():
for client in user_clients:
client.disconnect()
def assertMessagesInQueue(self, queue, retries=0, expected=None):
messages = self.rabbit.get_all(queue)
expected_message_count = expected if expected is not None else len(
messages)
for retry in xrange(retries):
if len(messages) < expected_message_count:
messages += self.rabbit.get_all(queue)
else:
break
time.sleep(1)
if len(messages) < expected_message_count:
raise AssertionError(
'Missing messages on queue, expected {}, received {}, retryied {}'
.format(expected_message_count, len(messages), retries))
return {item[0]['u']['u']: item[0] for item in messages}
def test_domain_syncacl_bad_subscription_permissions(self):
"""
Given I'm a user without enough subscription permissions on max
When I try to execute the service
I get a Forbidden exception
"""
from .mockers.syncacl import batch_subscribe_request
from .mockers.syncacl import context as context
from .mockers.syncacl import initial_subscriptions as subscriptions
aclless_context = context.copy()
aclless_context['acls'] = []
http_mock_get_context(aclless_context)
http_mock_get_context_subscriptions(subscriptions)
self.testapp.post('/api/domains/test/services/syncacl'.format(),
json.dumps(batch_subscribe_request),
headers=oauth2Header(test_user),
status=403)
def test_domain_syncacl_bad_context_permissions(self):
"""
Given I'm a user without enough context permissions on max
When I try to execute the service
I get a Forbidden exception
"""
from .mockers.syncacl import batch_subscribe_request
from .mockers.syncacl import context as context
from .mockers.syncacl import initial_subscriptions as subscriptions
http_mock_get_context(context, status=403)
http_mock_get_context_subscriptions(subscriptions)
self.testapp.post('/api/domains/test/services/syncacl'.format(),
json.dumps(batch_subscribe_request),
headers=oauth2Header(test_user),
status=403)
def test_domain_syncacl_initial_subscriptions(self):
"""
Given a newly created context
When a bunch of users and groups acls are synced
Then a set of actions is generated to generate needed subscriptions grants and revokes for new subscriptors
"""
from .mockers.syncacl import batch_subscribe_request
from .mockers.syncacl import context as context
from .mockers.syncacl import initial_subscriptions as subscriptions
from .mockers.syncacl import ldap_test_group, ldap_test_group2, ldap_test_group3
http_mock_get_context(context)
http_mock_get_context_subscriptions(subscriptions)
self.add_patch(ldap_patch_connect())
self.add_patch(ldap_patch_disconnect())
self.add_patch(
ldap_patch_group_search({
'TestGroup': ldap_test_group,
'TestGroup2': ldap_test_group2,
'TestGroup3': ldap_test_group3
}))
self.testapp.post('/api/domains/test/services/syncacl'.format(),
json.dumps(batch_subscribe_request),
headers=oauth2Header(test_user),
status=200)
# Index by username to be able to make asserts
# This is mandatory, as we cannot assume the order of the queue
messages = self.assertMessagesInQueue('syncacl', retries=3, expected=9)
# Test group users new subscription without grants
self.assertItemsEqual(messages['groupuser1']['d']['tasks'],
['subscribe'])
self.assertIn('context', messages['groupuser1']['d'])
self.assertItemsEqual(messages['groupuser2']['d']['tasks'],
['subscribe'])
self.assertIn('context', messages['groupuser2']['d'])
# Test group users new subscription with single grant
self.assertItemsEqual(messages['groupuser3']['d']['tasks'],
['subscribe', 'grant'])
self.assertItemsEqual(messages['groupuser3']['d']['tasks']['grant'],
['write'])
self.assertIn('context', messages['groupuser3']['d'])
self.assertItemsEqual(messages['groupuser4']['d']['tasks'],
['subscribe', 'grant'])
self.assertItemsEqual(messages['groupuser4']['d']['tasks']['grant'],
['write'])
self.assertIn('context', messages['groupuser4']['d'])
# Test group users new subscription with single revoke
self.assertItemsEqual(messages['groupuser5']['d']['tasks'],
['subscribe', 'revoke'])
self.assertItemsEqual(messages['groupuser5']['d']['tasks']['revoke'],
['unsubscribe'])
self.assertIn('context', messages['groupuser5']['d'])
self.assertItemsEqual(messages['groupuser6']['d']['tasks'],
['subscribe', 'revoke'])
self.assertItemsEqual(messages['groupuser6']['d']['tasks']['revoke'],
['unsubscribe'])
self.assertIn('context', messages['groupuser6']['d'])
# Test single user new subscription with single grant
self.assertItemsEqual(messages['testuser1']['d']['tasks'],
['subscribe', 'grant'])
self.assertItemsEqual(messages['testuser1']['d']['tasks']['grant'],
['write'])
self.assertIn('context', messages['testuser1']['d'])
# Test single user new subscription with multiple grant
self.assertItemsEqual(messages['testowner']['d']['tasks'],
['subscribe', 'grant'])
self.assertItemsEqual(messages['testowner']['d']['tasks']['grant'],
['write', 'flag'])
self.assertIn('context', messages['testowner']['d'])
# Test cretor grant
self.assertItemsEqual(messages['testuser.creator']['d']['tasks'],
['grant'])
self.assertItemsEqual(
messages['testuser.creator']['d']['tasks']['grant'], ['flag'])
self.assertIn('context', messages['testuser1']['d'])
def test_domain_syncacl_change_acls(self):
"""
Given a existing context with subcriptions
When a bunch of users and groups acls are synced
Then a set of actions is generated to update thouse users subscriptions
And the users that have been removed from acl are unsubscribed
"""
from .mockers.syncacl import batch_subscribe_request2
from .mockers.syncacl import context as context
from .mockers.syncacl import existing_subscriptions as subscriptions
from .mockers.syncacl import ldap_test_group, ldap_test_group2, ldap_test_group3
http_mock_get_context(context)
http_mock_get_context_subscriptions(subscriptions)
self.add_patch(ldap_patch_connect())
self.add_patch(ldap_patch_disconnect())
self.add_patch(
ldap_patch_group_search({
'TestGroup': ldap_test_group,
'TestGroup2': ldap_test_group2,
'TestGroup3': ldap_test_group3
}))
self.testapp.post('/api/domains/test/services/syncacl'.format(),
json.dumps(batch_subscribe_request2),
headers=oauth2Header(test_user),
status=200)
# Index by username to be able to make asserts
# This is mandatory, as we cannot assume the order of the queue
messages = self.assertMessagesInQueue('syncacl', retries=3, expected=6)
# Testuser1 remains untouched
self.assertNotIn('testuser1', messages)
# Users from gropu 2 remains untouched
self.assertNotIn('groupuser3', messages)
self.assertNotIn('groupuser4', messages)
# Test subscribed group users revoke permission
self.assertItemsEqual(messages['groupuser1']['d']['tasks'], ['revoke'])
self.assertItemsEqual(messages['groupuser1']['d']['tasks']['revoke'],
['write'])
self.assertIn('context', messages['groupuser1']['d'])
self.assertItemsEqual(messages['groupuser2']['d']['tasks'], ['revoke'])
self.assertItemsEqual(messages['groupuser2']['d']['tasks']['revoke'],
['write'])
self.assertIn('context', messages['groupuser2']['d'])
# Test subscribed group users unsubscribe
self.assertItemsEqual(messages['groupuser5']['d']['tasks'],
['unsubscribe'])
self.assertIn('context', messages['groupuser5']['d'])
self.assertItemsEqual(messages['groupuser6']['d']['tasks'],
['unsubscribe'])
self.assertIn('context', messages['groupuser6']['d'])
# Test subscribed single user unsubscribe
self.assertItemsEqual(messages['testowner']['d']['tasks'],
['unsubscribe'])
self.assertIn('context', messages['testowner']['d'])
# Test subscribed user revoke permission
self.assertItemsEqual(messages['testuser.creator']['d']['tasks'],
['revoke'])
self.assertItemsEqual(
messages['testuser.creator']['d']['tasks']['revoke'], ['flag'])
self.assertIn('context', messages['testuser.creator']['d'])
def test_domain_syncacl_user_overwrites_group_permissions(self):
"""
Given a existing context with subcriptions
When a bunch of users and groups acls are synced
And a user from a group acl is also in users acl
And both group and user has the same role
Then the same and only action is generated
"""
from .mockers.syncacl import batch_subscribe_request3
from .mockers.syncacl import context as context
from .mockers.syncacl import initial_subscriptions as subscriptions
from .mockers.syncacl import ldap_test_group4
http_mock_get_context(context)
http_mock_get_context_subscriptions(subscriptions)
self.add_patch(ldap_patch_connect())
self.add_patch(ldap_patch_disconnect())
self.add_patch(
ldap_patch_group_search({'TestGroup4': ldap_test_group4}))
self.testapp.post('/api/domains/test/services/syncacl'.format(),
json.dumps(batch_subscribe_request3),
headers=oauth2Header(test_user),
status=200)
# Index by username to be able to make asserts
# This is mandatory, as we cannot assume the order of the queue
messages = self.assertMessagesInQueue('syncacl', retries=3, expected=2)
self.assertItemsEqual(messages['groupuser1']['d']['tasks'],
['grant', 'subscribe'])
self.assertItemsEqual(messages['groupuser1']['d']['tasks']['grant'],
['write', 'flag'])
self.assertIn('context', messages['groupuser1']['d'])
# Test subscribed user revoke permission
self.assertItemsEqual(messages['testuser.creator']['d']['tasks'],
['grant'])
self.assertItemsEqual(
messages['testuser.creator']['d']['tasks']['grant'], ['flag'])
self.assertIn('context', messages['testuser.creator']['d'])
def test_domain_syncacl_user_overwrites_user_permissions(self):
"""
Given a existing context with subcriptions
When a bunch of users and groups acls are synced
And a user from a group acl is also in users acl
And both group and user has the same role
Then the action with more permissions is preserved
"""
from .mockers.syncacl import batch_subscribe_request4
from .mockers.syncacl import context as context
from .mockers.syncacl import initial_subscriptions as subscriptions
http_mock_get_context(context)
http_mock_get_context_subscriptions(subscriptions)
self.testapp.post('/api/domains/test/services/syncacl'.format(),
json.dumps(batch_subscribe_request4),
headers=oauth2Header(test_user),
status=200)
# Index by username to be able to make asserts
# This is mandatory, as we cannot assume the order of the queue
messages = self.assertMessagesInQueue('syncacl', retries=3, expected=1)
# Test subscribed user revoke permission, preserves most important role
self.assertItemsEqual(messages['testuser.creator']['d']['tasks'],
['grant'])
self.assertItemsEqual(
messages['testuser.creator']['d']['tasks']['grant'], ['flag'])
self.assertIn('context', messages['testuser.creator']['d'])
def set_server(self, message, mid):
self.server = RabbitClient(TEST_VHOST_URL)
self.server.management.cleanup(delete_all=True)
self.server.declare()
self.server.create_users(CONVERSATION_0.users)
self.server.conversations.create(CONVERSATION_0.id, users=CONVERSATION_0.users)
class ConversationTests(MaxBunnyTestCase):
def setUp(self):
self.log_patch = patch('maxbunny.consumer.BunnyConsumer.configure_logger', new=get_storing_logger)
self.log_patch.start()
def tearDown(self):
self.log_patch.stop()
self.server.delete_user('testuser1')
self.server.get_all('push')
self.server.disconnect()
# Make sure httpretty is disabled
httpretty.disable()
httpretty.reset()
def set_server(self, message, mid):
self.server = RabbitClient(TEST_VHOST_URL)
self.server.management.cleanup(delete_all=True)
self.server.declare()
self.server.create_users(CONVERSATION_0.users)
self.server.conversations.create(CONVERSATION_0.id, users=CONVERSATION_0.users)
# ===========================
# TESTS FOR FAILING SCENARIOS
# ===========================
def test_invalid_message_empty_message(self):
"""
Given a message with missing routing_key
When the message is processed
Then an exception is raised
And the push message is not queued
"""
from maxbunny.consumers.conversations import __consumer__
from maxbunny.tests.mockers import BAD_MESSAGE as message
self.set_server({}, None)
httpretty.enable()
http_mock_info()
runner = MockRunner('conversations', 'maxbunny.ini', 'instances.ini')
consumer = __consumer__(runner)
self.assertRaisesWithMessage(
BunnyMessageCancel,
'Conversation id missing on routing_key ""',
consumer.process,
message
)
httpretty.disable()
httpretty.reset()
sleep(0.1) # Leave a minimum time to message to reach rabbitmq
messages = self.server.get_all('push')
self.assertEqual(len(messages), 0)
def test_invalid_message_missing_username(self):
"""
Given a message with missing username
When the message is processed
Then an exception is raised
And the push message is not queued
"""
from maxbunny.consumers.conversations import __consumer__
from maxbunny.tests.mockers.conversations import MISSING_USERNAME_MESSAGE as message
self.set_server({}, None)
httpretty.enable()
http_mock_info()
runner = MockRunner('conversations', 'maxbunny.ini', 'instances2.ini')
consumer = __consumer__(runner)
self.assertRaisesWithMessage(
BunnyMessageCancel,
'Missing username in message',
consumer.process,
message
)
httpretty.disable()
httpretty.reset()
sleep(0.1) # Leave a minimum time to message to reach rabbitmq
messages = self.server.get_all('push')
self.assertEqual(len(messages), 0)
def test_invalid_message_unknown_domain(self):
"""
Given a message with a domain specified
And that domain doesn't match any of the known domains
When the message is processed
Then an exception is raised
And the push message is not queued
"""
from maxbunny.consumers.conversations import __consumer__
from maxbunny.tests.mockers.conversations import UNKNOWN_DOMAIN_MESSAGE as message
self.set_server({}, None)
httpretty.enable()
http_mock_info()
runner = MockRunner('conversations', 'maxbunny.ini', 'instances2.ini')
consumer = __consumer__(runner)
self.assertRaisesWithMessage(
BunnyMessageCancel,
'Unknown domain "unknown"',
consumer.process,
message
)
httpretty.disable()
httpretty.reset()
sleep(0.1) # Leave a minimum time to message to reach rabbitmq
messages = self.server.get_all('push')
self.assertEqual(len(messages), 0)
def test_missing_domain_missing_default(self):
"""
Given a message with no domain specified
And there is no default domain specified
When the message is processed
Then an exception is raised
And the push message is not queued
"""
from maxbunny.consumers.conversations import __consumer__
from maxbunny.tests.mockers.conversations import MISSING_DOMAIN_MESSAGE as message
self.set_server({}, None)
httpretty.enable()
http_mock_info()
runner = MockRunner('conversations', 'maxbunny.ini', 'instances.ini')
consumer = __consumer__(runner)
self.assertRaisesWithMessage(
BunnyMessageCancel,
'Missing domain, and default could not be loaded',
consumer.process,
message
)
httpretty.disable()
httpretty.reset()
sleep(0.1) # Leave a minimum time to message to reach rabbitmq
messages = self.server.get_all('push')
self.assertEqual(len(messages), 0)
def test_message_with_domain_not_found_from_max(self):
"""
Given a message with a domain specified
And that domain exists in the list of known domains
When the message is processed
And the conversation or user was not found on max
Then the message is not posted
And the push message is not queued
"""
from maxbunny.consumers.conversations import __consumer__
from maxbunny.tests.mockers.conversations import CONVERSATION_MESSAGE as message
message_id = '00000000001'
self.set_server(message, message_id)
httpretty.enable()
http_mock_info()
http_mock_post_user_message(uri='tests.local', message_id=message_id, status=404)
runner = MockRunner('conversations', 'maxbunny.ini', 'instances2.ini')
consumer = __consumer__(runner)
self.assertRaisesWithMessage(
BunnyMessageCancel,
"User or conversation not found",
consumer.process,
message
)
httpretty.disable()
httpretty.reset()
sleep(0.1) # Leave a minimum time to message to reach rabbitmq
messages = self.server.get_all('push')
self.assertEqual(len(messages), 0)
# ==============================
# TESTS FOR SUCCESFULL SCENARIOS
# ==============================
def test_message_without_domain_to_default(self):
"""
Given a message with no domain specified
And there is a default domain specified
When the message is processed
Then the message is posted
And the push message is queued
"""
from maxbunny.consumers.conversations import __consumer__
from maxbunny.tests.mockers.conversations import MISSING_DOMAIN_MESSAGE as message
message_id = '00000000001'
self.set_server(message, message_id)
httpretty.enable()
http_mock_info()
http_mock_post_user_message(uri='tests.default', message_id=message_id)
runner = MockRunner('conversations', 'maxbunny.ini', 'instances2.ini')
consumer = __consumer__(runner)
consumer.process(message)
httpretty.disable()
httpretty.reset()
sleep(0.1) # Leave a minimum time to message to reach rabbitmq
messages = self.server.get_all('push')
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0][0]['a'], 'k')
self.assertEqual(messages[0][0]['o'], 'm')
self.assertEqual(messages[0][0]['s'], 'b')
self.assertEqual(messages[0][0]['d']['id'], '00000000001')
def test_message_with_domain(self):
"""
Given a message with a domain specified
And that domain exists in the list of known domains
When the message is processed
Then the message is posted
And the push message is queued
"""
from maxbunny.consumers.conversations import __consumer__
from maxbunny.tests.mockers.conversations import CONVERSATION_MESSAGE as message
message_id = '00000000001'
self.set_server(message, message_id)
httpretty.enable()
http_mock_info()
http_mock_post_user_message(uri='tests.local', message_id=message_id)
runner = MockRunner('conversations', 'maxbunny.ini', 'instances2.ini')
consumer = __consumer__(runner)
consumer.process(message)
httpretty.disable()
httpretty.reset()
sleep(0.1) # Leave a minimum time to message to reach rabbitmq
messages = self.server.get_all('push')
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0][0]['a'], 'k')
self.assertEqual(messages[0][0]['o'], 'm')
self.assertEqual(messages[0][0]['s'], 'b')
self.assertEqual(messages[0][0]['d']['id'], '00000000001')
class InitAndPurgeRabbitServer(object): # pragma: no cover
def log(self, message):
if self.verbose:
print message
def __init__(self, options, quiet=False):
self.quiet = quiet
self.options = options
self.exchanges_by_name = {}
self.queues_by_name = {}
self.common = ConfigParser.ConfigParser()
self.common.read(self.options.commonfile)
self.instances = ConfigParser.ConfigParser()
self.instances.read(self.options.instancesfile)
self.workers = self.options.workers
self.verbose = self.options.verbose
self.lograte = self.options.lograte
try:
self.cluster = asbool(self.common.get('mongodb', 'cluster'))
self.standaloneserver = self.common.get('mongodb', 'url')
self.clustermembers = self.common.get('mongodb', 'hosts')
self.replicaset = self.common.get('mongodb', 'replica_set')
self.mongo_auth = asbool(self.common.get('mongodb', 'auth'))
self.mongo_authdb = self.common.get('mongodb', 'authdb')
self.mongo_username = self.common.get('mongodb', 'username')
self.mongo_password = self.common.get('mongodb', 'password')
self.rabbitmq_url = self.common.get('rabbitmq', 'server')
self.rabbitmq_manage_url = self.common.get('rabbitmq', 'manage')
self.maxserver_names = [maxserver for maxserver in self.instances.sections()]
except:
print('You must provide a valid configuration .ini file.')
sys.exit()
def add_users(self, server, users, task=''):
"""
Add users exchanges and internal binding
"""
for count, user in enumerate(users):
if server.user_publish_exchange(user['username']) in self.exchanges_by_name and \
server.user_subscribe_exchange(user['username']) in self.exchanges_by_name:
server.create_user(user['username'], create_exchanges=False)
self.log('Created internal bindings for {}'.format(user['username']))
else:
self.log('Created exchanges and bindings for {}'.format(user['username']))
server.create_user(user['username'])
if count % self.lograte == 0:
print '{}Progress: {} / {}'.format(task, count, len(users))
print '{}Done!'.format(task)
def add_conversation_bindings(self, server, conversations, task=''):
"""
Create bindings between "pub" and "sub" exchanges of users in
existing conversations
"""
expected_bindings = []
for count, conversation in enumerate(conversations):
cid, members = conversation
for member in members:
pub_binding_key = '{}_{}.*_conversations'.format(server.user_publish_exchange(member), cid)
sub_binding_key = 'conversations_{}.*_{}'.format(cid, server.user_subscribe_exchange(member))
expected_bindings.append(pub_binding_key)
expected_bindings.append(sub_binding_key)
if pub_binding_key in self.conversation_bindings and \
sub_binding_key in self.conversation_bindings:
pass
else:
self.log('Create pub/sub bindings for user {} on conversation {}'.format(member, cid))
server.conversations.bind_user(cid, member)
if count % self.lograte == 0:
print '{}Progress: {} / {}'.format(task, count, len(conversations))
self.global_valid_bindings += expected_bindings
default_bindings = set(['conversations_*.messages_messages', 'conversations_*.notifications_push'])
non_expected = (set(self.conversation_bindings) - set(self.global_valid_bindings)) - default_bindings
for binding in non_expected:
source, routing_key, destination = binding.split('_')
self.server.management.delete_binding(source, destination, routing_key)
print "Removing unknown binding: {}".format(binding)
print '{}Done!'.format(task)
def add_context_bindings(self, server, contexts, task=''):
"""
Create bindings between "sub" exchange of users on contexts with
notifications enabled and the "activity" exchange
"""
expected_bindings = []
for count, context in enumerate(contexts):
cid, members = context
for member in members:
sub_binding_key = 'activity_{}_{}'.format(cid, server.user_subscribe_exchange(member))
expected_bindings.append(sub_binding_key)
if sub_binding_key in self.context_bindings:
pass
else:
self.log('Create pub/sub bindings for user {} on context {}'.format(member, cid))
server.activity.bind_user(cid, member)
if count % self.lograte == 0:
print '{}Progress: {} / {}'.format(task, count, len(contexts))
default_bindings = set(['activity_#_push'])
self.global_valid_bindings += expected_bindings
non_expected = (set(self.context_bindings) - set(self.global_valid_bindings)) - default_bindings
for binding in non_expected:
source, routing_key, destination = binding.split('_')
self.server.management.delete_binding(source, destination, routing_key)
print "Removing unknown binding: {}".format(binding)
print '{}Done!'.format(task)
def do_batch(self, method, items):
batch = []
tasks = self.workers if len(items) > self.workers else 1
if tasks > 1:
users_per_task = len(items) / tasks
for task_index in range(tasks + 1):
start = task_index * users_per_task
end = (task_index + 1) * users_per_task
if start < len(items):
batch.append(items[start:end])
start = time.time()
if tasks == 1:
method(self.server, items)
else:
threads = [gevent.spawn(method, RabbitClient(self.rabbitmq_url), batch_items, 'Task #{} '.format(taskid)) for taskid, batch_items in enumerate(batch)]
gevent.joinall(threads)
end = time.time()
total_seconds = end - start
print
print ' | Total items: {}'.format(len(items))
print ' | Total seconds: {:.2f}'.format(total_seconds)
print ' | Items/second: {:.2f}'.format(len(items) / total_seconds)
print
def run(self):
# Create client without declaring anything
self.server = RabbitClient(self.rabbitmq_url)
# Clear all non-native exchanges and queues
print '> Cleaning up rabbitmq'
self.server.management.cleanup(delete_all=self.options.deleteall)
# Create all exchanges and queues defined in spec
print '> Creating default exchanges, queues and bindings'
self.server.declare()
# Keep track of all bindings created on any maxserver batch during this
# script execution, in order to avoid deleting valid bindings
self.global_valid_bindings = []
# Refresh
print '> Loading current exchanges and queues'
self.server.management.load_exchanges()
self.server.management.load_queues()
self.exchanges_by_name = self.server.management.exchanges_by_name
print ' Found {} exchanges'.format(len(self.server.management.exchanges))
print ' Found {} queues'.format(len(self.server.management.queues))
print '> Loading current conversation bindings'
bindings = self.server.management.load_exchange_bindings('conversations')
self.conversation_bindings = ['{source}_{routing_key}_{destination}'.format(**binding) for binding in bindings]
print ' Found {} conversation bindings'.format(len(self.conversation_bindings))
print '> Loading current context bindings'
bindings = self.context_bindings = self.server.management.load_exchange_bindings('activity')
self.context_bindings = ['{source}_{routing_key}_{destination}'.format(**binding) for binding in bindings]
print ' Found {} context bindings'.format(len(self.context_bindings))
# Mongodb connection initialization
cluster_enabled = self.cluster
auth_enabled = self.mongo_auth
mongodb_uri = self.clustermembers if cluster_enabled else self.standaloneserver
conn = mongodb.get_connection(
mongodb_uri,
use_greenlets=True,
cluster=self.replicaset if cluster_enabled else None)
for maxname in self.maxserver_names:
dbname = 'max_{}'.format(maxname)
if dbname not in conn.database_names():
print ('Skipping "{}" max instance, no database found, maybe is an alias?'.format(maxname))
continue
print('')
print('============================================================================')
print(' Processing "{}" max instance'.format(maxname))
print('============================================================================')
print('')
db = mongodb.get_database(
conn,
dbname,
username=self.mongo_username if auth_enabled else None,
password=self.mongo_password if auth_enabled else None,
authdb=self.mongo_authdb if auth_enabled else None)
print "> Getting users list from database"
# Get all users to create their rabbit exchange and bindings
query = {} if not self.options.usernamefilter else {'username': self.options.usernamefilter}
all_users = db.users.find(query, {'_id': 0, 'username': 1, 'talkingIn': 1, 'subscribedTo': 1})
contexts_with_notifications = db.contexts.find({'notifications': {'$nin': [False], '$exists': 1}}, {'hash': 1})
contexts_with_notifications_hashs = [a['hash'] for a in contexts_with_notifications]
# unpack lazy results
all_users = [a for a in all_users]
print '> Got {} users'.format(len(all_users))
print
# Extract the list of conversations and context subscriptions from all users
conversations = {}
contexts = {}
for user in all_users:
for conversation in user.get('talkingIn', []):
conv = conversations.setdefault(conversation['id'], [])
conv.append(user['username'])
for context in user.get('subscribedTo', []):
if context['hash'] in contexts_with_notifications_hashs:
ctxt = contexts.setdefault(context['hash'], [])
ctxt.append(user['username'])
print '> Starting Batch: Create user exchanges and bindings'
self.do_batch(self.add_users, all_users)
print '> Starting Batch: Create conversation bindings'
self.do_batch(self.add_conversation_bindings, conversations.items())
print '> Starting Batch: Create context bindings'
self.do_batch(self.add_context_bindings, contexts.items())
self.server.disconnect()
def setUp(self):
self.server = RabbitClient(TEST_VHOST_URL)
self.server.management.cleanup(delete_all=True)
self.server.declare()
self.clients = {}
class FunctionalTests(unittest.TestCase, MaxTestBase):
def setUp(self):
conf_dir = os.path.dirname(__file__)
self.app = loadapp('config:rabbitmq.ini', relative_to=conf_dir)
self.reset_database(self.app)
self.app.registry.max_store.security.insert(test_default_security)
self.patched_post = patch('requests.post', new=partial(mock_post, self))
self.patched_post.start()
self.testapp = MaxTestApp(self)
self.create_user(test_manager)
# Rabbitmq test client initialization
rabbitmq_url = self.app.registry.settings['max.rabbitmq']
self.server = RabbitClient(rabbitmq_url)
self.server.management.cleanup(delete_all=True)
self.server.declare()
def tearDown(self):
import pyramid.testing
pyramid.testing.tearDown()
self.server.disconnect()
def run_test(self, test_module_name, test_name):
"""
Runs a test method from another module in a dirty (but awesome) way
"""
method_dotted_name = '{}.FunctionalTests.{}'.format(test_module_name, test_name)
test_method = import_object('max.tests', method_dotted_name)
# Create a new function sharing code, name and current globals
# plus other imports needed by other modules
current_globals = globals()
current_globals.update({
'json': get_module('json'),
'oauth2Header': import_object('max.tests', 'base.oauth2Header'),
'test_manager': import_object('max.tests', 'test_manager')
})
wrapped_test_method = new.function(test_method.func_code, current_globals, test_method.func_name)
# execute the new method and return result (if any)
return wrapped_test_method(self)
# All tests within this module executes code from tests on other modules
# As this module has rabbitmq activated in the ini, each test should produce
# rabbitmq associated actions, coded with each tested codebase, so we just
# execute the code, and check for the existence of the desired structures.
#
# NOTE that we may expect return values from the test, to know which values
# the test produced. Please modify used tests to return those values if needed.
@skipRabbitTest()
def test_create_user_bindings(self):
username = self.run_test('test_people', 'test_create_user')
self.server.management.load_exchanges()
self.assertIn('{}.publish'.format(username), self.server.management.exchanges_by_name)
self.assertIn('{}.subscribe'.format(username), self.server.management.exchanges_by_name)
@skipRabbitTest()
def test_create_user_without_bindings(self):
username = '******'
self.create_user(username, qs_params={"notifications": False})
self.server.management.load_exchanges()
self.assertNotIn('{}.publish'.format(username), self.server.management.exchanges_by_name)
self.assertNotIn('{}.subscribe'.format(username), self.server.management.exchanges_by_name)
@skipRabbitTest()
def test_create_existing_user_dont_create_bindings(self):
"""
Given a created user without rabbitmq exchanges
When i try to create the user again
The exchanges won't be created
"""
username = '******'
self.create_user(username, qs_params={"notifications": False})
self.create_user(username, expect=200)
self.server.management.load_exchanges()
self.assertNotIn('{}.publish'.format(username), self.server.management.exchanges_by_name)
self.assertNotIn('{}.subscribe'.format(username), self.server.management.exchanges_by_name)
@skipRabbitTest()
def test_create_existing_user_create_bindings(self):
"""
Given a created user without rabbitmq exchanges
When i try to create the user again
And i explicitly request to create exchanges for notifications
The exchanges will be created
"""
username = '******'
self.create_user(username, qs_params={"notifications": False})
self.create_user(username, qs_params={"notifications": True}, expect=200)
self.server.management.load_exchanges()
self.assertIn('{}.publish'.format(username), self.server.management.exchanges_by_name)
self.assertIn('{}.subscribe'.format(username), self.server.management.exchanges_by_name)
@skipRabbitTest()
def test_self_create_existing_user_dont_create_bindings(self):
"""
Given a created user without rabbitmq exchanges
When i try to create the user again
The exchanges won't be created
"""
username = '******'
self.create_user(username, qs_params={"notifications": False}, creator=username)
self.create_user(username, expect=200, creator=username)
self.server.management.load_exchanges()
self.assertNotIn('{}.publish'.format(username), self.server.management.exchanges_by_name)
self.assertNotIn('{}.subscribe'.format(username), self.server.management.exchanges_by_name)
@skipRabbitTest()
def test_self_create_existing_user_create_bindings(self):
"""
Given a created user without rabbitmq exchanges
When i try to create the user again
And i explicitly request to create exchanges for notifications
The exchanges will be created
"""
username = '******'
self.create_user(username, qs_params={"notifications": False}, creator=username)
self.create_user(username, qs_params={"notifications": True}, expect=200, creator=username)
self.server.management.load_exchanges()
self.assertIn('{}.publish'.format(username), self.server.management.exchanges_by_name)
self.assertIn('{}.subscribe'.format(username), self.server.management.exchanges_by_name)
@skipRabbitTest()
def test_create_conversation_bindings(self):
cid, creator = self.run_test('test_conversations', 'test_post_message_to_conversation_check_conversation')
# Find defined bindings for this conversation
bindings = self.server.management.load_exchange_bindings('conversations')
bindings = [bind for bind in bindings if ".*".format(cid) in bind['routing_key']]
self.assertEqual(len(bindings), 4)
@skipRabbitTest()
def test_create_conversation_check_notification(self):
cid, creator = self.run_test('test_conversations', 'test_post_message_to_conversation_check_conversation')
sleep(0.1)
messages_to_push_queue = self.server.get_all('push')
self.assertEqual(len(messages_to_push_queue), 1)
carrot_message, haigha_message = messages_to_push_queue[0]
self.assertEqual(haigha_message.delivery_info['routing_key'], '{}.notifications'.format(cid))
self.assertEqual(carrot_message['a'], 'a')
self.assertEqual(carrot_message['o'], 'c')
self.assertEqual(carrot_message['u']['u'], creator)
self.assertEqual(carrot_message['u']['d'], creator)
@skipRabbitTest()
def test_delete_conversation_bindings(self):
cid = self.run_test('test_conversations', 'test_conversation_owner_deletes_conversation')
# Find defined bindings for this conversation
bindings = self.server.management.load_exchange_bindings('conversations')
bindings = [bind for bind in bindings if ".*".format(cid) in bind['routing_key']]
self.assertEqual(len(bindings), 0)
@skipRabbitTest()
def test_remove_user_from_conversation_bindings(self):
cid, userin, userout = self.run_test('test_conversations', 'test_user_leaves_two_people_conversation')
# Find defined bindings for this conversation
bindings = self.server.management.load_exchange_bindings('conversations')
bindings = [bind for bind in bindings if ".*".format(cid) in bind['routing_key']]
# Search for the bindings of the user still on the conversation
userin_bindings = [
bind for bind in bindings
if "{}.publish".format(userin) == bind['source']
or "{}.subscribe".format(userin) == bind['destination']
]
# search for the bindings of the user that left the conversation
userout_bindings = [
bind for bind in bindings
if "{}.publish".format(userout) == bind['source']
or "{}.subscribe".format(userout) == bind['destination']
]
self.assertEqual(len(bindings), 2)
self.assertEqual(len(userin_bindings), 2)
self.assertEqual(len(userout_bindings), 0)
@skipRabbitTest()
def test_add_new_user_to_conversation_bindings(self):
cid, newuser = self.run_test('test_conversations', 'test_add_participant_to_conversation')
# Find defined bindings for this conversation
bindings = self.server.management.load_exchange_bindings('conversations')
bindings = [bind for bind in bindings if ".*".format(cid) in bind['routing_key']]
# Search for the bindings of the user still on the conversation
newuser_bindings = [
bind for bind in bindings
if "{}.publish".format(newuser) == bind['source']
or "{}.subscribe".format(newuser) == bind['destination']
]
self.assertEqual(len(bindings), 8)
self.assertEqual(len(newuser_bindings), 2)
@skipRabbitTest()
def test_delete_context_bindings(self):
context_hash = self.run_test('test_contexts_notifications', 'test_delete_context_with_notifications_removes_subscriptions')
# Find defined bindings for this context
bindings = self.server.management.load_exchange_bindings('activity')
bindings = [bind for bind in bindings if ".*".format(context_hash) in bind['routing_key']]
self.assertEqual(len(bindings), 0)
@skipRabbitTest()
def test_add_user_subscription_bindings(self):
context_hash, subscribed_user = self.run_test('test_contexts_notifications', 'test_subscribe_user_to_context_with_notifications')
# Find defined bindings for this context
bindings = self.server.management.load_exchange_bindings('activity')
bindings = [bind for bind in bindings if context_hash in bind['routing_key']]
# Search for the bindings of the user still on the conversation
subscribed_bindings = [
bind for bind in bindings
if "{}.subscribe".format(subscribed_user) == bind['destination']
]
self.assertEqual(len(bindings), 1)
self.assertEqual(len(subscribed_bindings), 1)
@skipRabbitTest()
def test_remove_user_subscription_bindings(self):
context_hash, unsubscribed_user = self.run_test('test_contexts_notifications', 'test_unsubscribe_user_from_context_with_notifications')
# Find defined bindings for this context
bindings = self.server.management.load_exchange_bindings('activity')
bindings = [bind for bind in bindings if context_hash in bind['routing_key']]
# Search for the bindings of the user still on the conversation
unsubscribed_bindings = [
bind for bind in bindings
if "{}.subscribe".format(unsubscribed_user) == bind['destination']
]
self.assertEqual(len(bindings), 0)
self.assertEqual(len(unsubscribed_bindings), 0)
@skipRabbitTest()
def test_post_message_check_notification(self):
cid, creator, activity = self.run_test('test_contexts_notifications', 'test_post_activity_on_context_with_notifications')
sleep(0.1)
messages_to_push_queue = self.server.get_all('push')
carrot_message, haigha_message = messages_to_push_queue[0]
self.assertEqual(len(messages_to_push_queue), 1)
self.assertEqual(haigha_message.delivery_info['routing_key'], '{}'.format(cid))
self.assertEqual(carrot_message['a'], 'a')
self.assertEqual(carrot_message['o'], 'a')
self.assertEqual(carrot_message['u']['u'], creator)
self.assertEqual(carrot_message['u']['d'], creator)
self.assertEqual(carrot_message['d']['text'].encode('utf-8'), activity['object']['content'])
@skipRabbitTest()
def test_post_message_check_no_notification(self):
cid, creator, activity = self.run_test('test_contexts', 'test_post_activity_with_private_read_write_context')
messages_to_push_queue = self.server.get_all('push')
self.assertEqual(len(messages_to_push_queue), 0)
@skipRabbitTest()
def test_post_comment_check_notification(self):
cid, creator, activity, comment = self.run_test('test_contexts_notifications', 'test_post_comment_with_comments_notification')
sleep(0.1)
messages_to_push_queue = self.server.get_all('push')
self.assertEqual(len(messages_to_push_queue), 2)
carrot_message, haigha_message = messages_to_push_queue[0]
self.assertEqual(haigha_message.delivery_info['routing_key'], '{}'.format(cid))
self.assertEqual(carrot_message['a'], 'a')
self.assertEqual(carrot_message['o'], 'a')
self.assertEqual(carrot_message['u']['u'], creator)
self.assertEqual(carrot_message['u']['d'], creator)
self.assertEqual(carrot_message['d']['text'], activity['object']['content'])
carrot_message, haigha_message = messages_to_push_queue[1]
self.assertEqual(haigha_message.delivery_info['routing_key'], '{}'.format(cid))
self.assertEqual(carrot_message['a'], 'a')
self.assertEqual(carrot_message['o'], 't')
self.assertEqual(carrot_message['u']['u'], creator)
self.assertEqual(carrot_message['u']['d'], creator)
self.assertEqual(carrot_message['d']['text'], comment['object']['content'])