def delete_rabbitmq_account(rabbitmq_username):
rabbitmq_account = RabbitMQAccount.query.filter(
RabbitMQAccount.username == rabbitmq_username).first()
if rabbitmq_account and (g.user.admin or g.user in
rabbitmq_account.owners):
details = {
'username': g.user.email,
'rabbitmqusername': rabbitmq_username,
}
try:
pulse_management.delete_user(rabbitmq_account.username)
except pulse_management.PulseManagementException as e:
details['message'] = str(e)
mozdef.log(
mozdef.ERROR,
mozdef.OTHER,
'Error deleting RabbitMQ account',
details=details,
)
return jsonify(ok=False)
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'RabbitMQ account deleted',
details=details,
)
db_session.delete(rabbitmq_account)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def clear_deleted_queues(self, queues, all_bindings):
db_queues = Queue.query.all()
# Filter queues that are in the database but no longer on RabbitMQ.
alive_queues_names = {q['name'] for q in queues}
deleted_queues = {
q
for q in db_queues if q.name not in alive_queues_names
}
# Delete those queues.
for queue in deleted_queues:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Queue no longer exists.',
details={'queuename': queue.name},
tags=['queue'],
)
db_session.delete(queue)
# Clean up bindings on queues that are not deleted.
for queue_name in alive_queues_names:
bindings = self.get_queue_bindings(all_bindings, queue_name)
self.clear_deleted_bindings(queue_name, bindings)
db_session.commit()
def clear_deleted_queues(self, queues, all_bindings):
db_queues = Queue.query.all()
# Filter queues that are in the database but no longer on RabbitMQ.
alive_queues_names = {q['name'] for q in queues}
deleted_queues = {q for q in db_queues
if q.name not in alive_queues_names}
# Delete those queues.
for queue in deleted_queues:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Queue no longer exists.',
details={'queuename': queue.name},
tags=['queue'],
)
db_session.delete(queue)
# Clean up bindings on queues that are not deleted.
for queue_name in alive_queues_names:
bindings = self.get_queue_bindings(all_bindings, queue_name)
self.clear_deleted_bindings(queue_name, bindings)
db_session.commit()
def delete_rabbitmq_account(rabbitmq_username):
rabbitmq_account = RabbitMQAccount.query.filter(
RabbitMQAccount.username == rabbitmq_username).first()
if rabbitmq_account and (g.user.admin
or g.user in rabbitmq_account.owners):
details = {
'username': g.user.email,
'rabbitmqusername': rabbitmq_username,
}
try:
pulse_management.delete_user(rabbitmq_account.username)
except pulse_management.PulseManagementException as e:
details['message'] = str(e)
mozdef.log(
mozdef.ERROR,
mozdef.OTHER,
'Error deleting RabbitMQ account',
details=details,
)
return jsonify(ok=False)
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'RabbitMQ account deleted',
details=details,
)
db_session.delete(rabbitmq_account)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def setUp(self):
global pulse_cfg
self.proc = None
self.publisher = None
self.guardian = PulseGuardian(warn_queue_size=TEST_WARN_SIZE,
del_queue_size=TEST_DELETE_SIZE,
emails=False)
dbinit.init_and_clear_db()
self.consumer_cfg = pulse_cfg.copy()
self.consumer_cfg['applabel'] = str(uuid.uuid1())
# Configure/create the test user to be used for message consumption.
self.consumer_cfg['user'] = CONSUMER_USER
self.consumer_cfg['password'] = CONSUMER_PASSWORD
self.user = User.new_user(email=CONSUMER_EMAIL, admin=False)
# As a default owner for rabbitmq_account in some tests where an owner
# is not provided.
self.admin = User.new_user(email=ADMIN_EMAIL, admin=True)
db_session.add(self.user)
db_session.commit()
self.rabbitmq_account = RabbitMQAccount.new_user(
username=CONSUMER_USER,
password=CONSUMER_PASSWORD,
owners=self.user)
db_session.add(self.rabbitmq_account)
db_session.commit()
def monitor_queues(self, queues, all_bindings):
for queue_data in queues:
# Updating the queue's information in the database (owner, size).
queue = self.update_queue_information(queue_data, all_bindings)
if not queue:
continue
# If a queue is over the deletion size and ``unbounded`` is
# False (the default), then delete it regardless of it having
# an owner or not
# If ``unbounded`` is True, then let it grow indefinitely.
if queue.size > self.del_queue_size and not queue.unbounded:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Deleting queue.',
details=self._queue_details_dict(queue),
tags=['queue'],
)
if queue.owner and queue.owner.owners:
self.deletion_email(queue.owner.owners, queue_data)
if self.on_delete:
self.on_delete(queue.name)
pulse_management.delete_queue(vhost=queue_data['vhost'],
queue=queue.name)
db_session.delete(queue)
db_session.commit()
continue
if queue.owner is None or not queue.owner.owners:
continue
if queue.size > self.warn_queue_size and not queue.warned:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Queue-size warning.',
details=self._queue_details_dict(queue),
tags=['queue'],
)
queue.warned = True
if self.on_warn:
self.on_warn(queue.name)
self.warning_email(queue.owner.owners, queue_data)
elif queue.size <= self.warn_queue_size and queue.warned:
# A previously warned queue got out of the warning threshold;
# its owner should not be warned again.
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Queue-size recovered.',
details=self._queue_details_dict(queue),
tags=['queue'],
)
queue.warned = False
self.back_to_normal_email(queue.owner.owners, queue_data)
# Commit any changes to the queue.
db_session.add(queue)
db_session.commit()
def update_info():
rabbitmq_username = request.form['rabbitmq-username']
new_password = request.form['new-password']
password_verification = request.form['new-password-verification']
new_owners = _clean_owners_str(request.form['owners-list'])
try:
rabbitmq_account = RabbitMQAccount.query.filter(
RabbitMQAccount.username == rabbitmq_username).one()
except sqlalchemy.orm.exc.NoResultFound:
return rabbitmq_accounts(messages=[
"RabbitMQ account {} not found.".format(rabbitmq_username)
])
if g.user not in rabbitmq_account.owners:
return rabbitmq_accounts(messages=[
"Invalid user: {} is not an owner.".format(g.user.email)
])
messages = []
error = None
if new_password:
if new_password != password_verification:
return rabbitmq_accounts(
error="Password verification doesn't match the password.")
if not RabbitMQAccount.strong_password(new_password):
return rabbitmq_accounts(
error="Your password must contain a mix of letters and "
"numerical characters and be at least 6 characters long.")
rabbitmq_account.change_password(new_password)
messages.append(
"Password updated for user {0}.".format(rabbitmq_username))
# Update the owners list, if needed.
old_owners = {user.email for user in rabbitmq_account.owners}
if new_owners and new_owners != old_owners:
# The list was changed. Do an update.
new_owner_users = list(User.query.filter(User.email.in_(new_owners)))
if new_owner_users:
# At least some of the new owners are real users in the db.
rabbitmq_account.owners = new_owner_users
db_session.commit()
updated_owners = {user.email for user in new_owner_users}
invalid_owners = sorted(new_owners - updated_owners)
if invalid_owners:
error = "Some user emails not found: {}".format(
', '.join(invalid_owners))
else:
messages = ["Email list updated."]
else:
error = ("Invalid owners: "
"Must be a comma-delimited list of existing user emails.")
if not error and not messages:
messages = ["No info updated."]
return rabbitmq_accounts(messages=messages, error=error)
def update_queue_information(self, queue_data):
if not 'messages' in queue_data:
# FIXME: We should do something here, probably delete the queue,
# as it's in a weird state. More investigation is required.
# See bug 1066338.
return None
q_size, q_name, q_durable = (queue_data['messages'],
queue_data['name'], queue_data['durable'])
queue = Queue.query.filter(Queue.name == q_name).first()
# If the queue doesn't exist in the db, create it.
if queue is None:
m = re.match('queue/([^/]+)/', q_name)
logging.info("New queue '{0}' encountered. "
"Adding to the database.".format(q_name))
if m:
owner_name = m.group(1)
owner = PulseUser.query.filter(
PulseUser.username == owner_name).first()
# If the queue was created by a user that isn't in the
# pulseguardian database, skip the queue.
if owner is None:
logging.info(
"Queue '{0}' owner, {1}, isn't in the db. Creating "
"the user.".format(q_name, owner_name))
owner = PulseUser.new_user(owner_name)
# Assign the user to the queue.
logging.info("Assigning queue '{0}' to user "
"{1}.".format(q_name, owner))
else:
logging.warn(
"'{0}' is not a standard queue name.".format(q_name))
owner = None
queue = Queue(name=q_name, owner=owner)
# add the queue bindings to the db.
bindings = pulse_management.queue_bindings(config.rabbit_vhost,
queue.name)
for binding in bindings:
db_binding = Binding.query.filter(
Binding.exchange == binding["source"],
Binding.routing_key == binding["routing_key"],
Binding.queue_name == queue.name).first()
if not db_binding:
# need to create the binding in the DB
binding = Binding(exchange=binding["source"],
routing_key=binding["routing_key"],
queue_name=queue.name)
db_session.add(binding)
# Update the saved queue size.
queue.size = q_size
queue.durable = q_durable
db_session.add(queue)
db_session.commit()
return queue
def setUp(self):
global pulse_cfg
self.proc = None
self.publisher = None
self.guardian = PulseGuardian(warn_queue_size=TEST_WARN_SIZE,
del_queue_size=TEST_DELETE_SIZE,
emails=False)
dbinit.init_and_clear_db()
self.consumer_cfg = pulse_cfg.copy()
self.consumer_cfg['applabel'] = str(uuid.uuid1())
# Configure/create the test user to be used for message consumption.
self.consumer_cfg['user'] = CONSUMER_USER
self.consumer_cfg['password'] = CONSUMER_PASSWORD
self.user = User.new_user(email=CONSUMER_EMAIL, admin=False)
db_session.add(self.user)
db_session.commit()
self.pulse_user = PulseUser.new_user(username=CONSUMER_USER,
password=CONSUMER_PASSWORD,
owner=self.user)
db_session.add(self.pulse_user)
db_session.commit()
def test_user(self):
user = User.new_user(email='*****@*****.**', admin=False)
db_session.add(user)
db_session.commit()
rabbitmq_account = RabbitMQAccount.new_user(username='******',
password='******',
owners=user,
create_rabbitmq_user=False)
db_session.add(rabbitmq_account)
db_session.commit()
self.assertTrue(user in User.query.all())
# Emails are normalized by putting them lower-case
self.assertEqual(
User.query.filter(User.email == '*****@*****.**').first(), user)
self.assertEqual(
RabbitMQAccount.query.filter(
RabbitMQAccount.username == 'dummy').first(),
rabbitmq_account)
self.assertEqual(
RabbitMQAccount.query.filter(
RabbitMQAccount.username == 'DUMMY').first(),
None)
def setUp(self):
global pulse_cfg
self.proc = None
self.publisher = None
self.guardian = PulseGuardian(warn_queue_size=TEST_WARN_SIZE,
del_queue_size=TEST_DELETE_SIZE,
emails=False)
dbinit.init_and_clear_db()
self.consumer_cfg = pulse_cfg.copy()
self.consumer_cfg['applabel'] = str(uuid.uuid1())
# Configure/create the test user to be used for message consumption.
self.consumer_cfg['user'] = CONSUMER_USER
self.consumer_cfg['password'] = CONSUMER_PASSWORD
self.user = User.new_user(email=CONSUMER_EMAIL, admin=False)
# As a default owner for pulse_user in some tests where an owner is not
# provided.
self.admin = User.new_user(email=ADMIN_EMAIL, admin=True)
db_session.add(self.user)
db_session.commit()
self.pulse_user = PulseUser.new_user(
username=CONSUMER_USER,
password=CONSUMER_PASSWORD,
owners=self.user)
db_session.add(self.pulse_user)
db_session.commit()
def delete_queue(queue_name):
queue = Queue.query.get(queue_name)
if queue and (g.user.admin or
(queue.owner and g.user in queue.owner.owners)):
details = {
'queuename': queue_name,
'username': g.user.email,
}
try:
pulse_management.delete_queue(vhost='/', queue=queue.name)
except pulse_management.PulseManagementException as e:
details['message'] = str(e)
mozdef.log(
mozdef.ERROR,
mozdef.OTHER,
'Error deleting queue',
details=details,
tags=['queue'],
)
return jsonify(ok=False)
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Deleting queue',
details=details,
tags=['queue'],
)
db_session.delete(queue)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def delete_pulse_user(pulse_username):
pulse_user = PulseUser.query.filter(
PulseUser.username == pulse_username).first()
if pulse_user and (g.user.admin or g.user in pulse_user.owners):
details = {
'username': g.user.email,
'pulseusername': pulse_username,
}
try:
pulse_management.delete_user(pulse_user.username)
except pulse_management.PulseManagementException as e:
details['message'] = str(e)
mozdef.log(
mozdef.ERROR,
mozdef.OTHER,
'Error deleting Pulse user',
details=details,
)
return jsonify(ok=False)
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Pulse user deleted',
details=details,
)
db_session.delete(pulse_user)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def delete_queue(queue_name):
queue = Queue.query.get(queue_name)
if queue and (g.user.admin or
(queue.owner and g.user in queue.owner.owners)):
details = {
'queuename': queue_name,
'username': g.user.email,
}
try:
pulse_management.delete_queue(vhost='/', queue=queue.name)
except pulse_management.PulseManagementException as e:
details['message'] = str(e)
mozdef.log(
mozdef.ERROR,
mozdef.OTHER,
'Error deleting queue',
details=details,
tags=['queue'],
)
return jsonify(ok=False)
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Deleting queue',
details=details,
tags=['queue'],
)
db_session.delete(queue)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def monitor_queues(self, queues, all_bindings):
for queue_data in queues:
# Updating the queue's information in the database (owner, size).
queue = self.update_queue_information(queue_data, all_bindings)
if not queue:
continue
# If a queue is over the deletion size and ``unbounded`` is
# False (the default), then delete it regardless of it having
# an owner or not
# If ``unbounded`` is True, then let it grow indefinitely.
if queue.size > self.del_queue_size and not queue.unbounded:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Deleting queue.',
details=self._queue_details_dict(queue),
tags=['queue'],
)
if queue.owner and queue.owner.owners:
self.deletion_email(queue.owner.owners, queue_data)
if self.on_delete:
self.on_delete(queue.name)
pulse_management.delete_queue(vhost=queue_data['vhost'],
queue=queue.name)
db_session.delete(queue)
db_session.commit()
continue
if queue.owner is None or not queue.owner.owners:
continue
if queue.size > self.warn_queue_size and not queue.warned:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Queue-size warning.',
details=self._queue_details_dict(queue),
tags=['queue'],
)
queue.warned = True
if self.on_warn:
self.on_warn(queue.name)
self.warning_email(queue.owner.owners, queue_data)
elif queue.size <= self.warn_queue_size and queue.warned:
# A previously warned queue got out of the warning threshold;
# its owner should not be warned again.
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Queue-size recovered.',
details=self._queue_details_dict(queue),
tags=['queue'],
)
queue.warned = False
self.back_to_normal_email(queue.owner.owners, queue_data)
# Commit any changes to the queue.
db_session.add(queue)
db_session.commit()
def update_info():
pulse_username = request.form['pulse-user']
new_password = request.form['new-password']
password_verification = request.form['new-password-verification']
new_owners = _clean_owners_str(request.form['owners-list'])
try:
pulse_user = PulseUser.query.filter(
PulseUser.username == pulse_username).one()
except sqlalchemy.orm.exc.NoResultFound:
return profile(
messages=["Pulse user {} not found.".format(pulse_username)])
if g.user not in pulse_user.owners:
return profile(
messages=["Invalid user: {} is not an owner.".format(g.user.email)])
messages = []
error = None
if new_password:
if new_password != password_verification:
return profile(error="Password verification doesn't match the "
"password.")
if not PulseUser.strong_password(new_password):
return profile(error="Your password must contain a mix of "
"letters and numerical characters and be at "
"least 6 characters long.")
pulse_user.change_password(new_password)
messages.append("Password updated for user {0}.".format(
pulse_username))
# Update the owners list, if needed.
old_owners = {user.email for user in pulse_user.owners}
if new_owners and new_owners != old_owners:
# The list was changed. Do an update.
new_owner_users = list(User.query.filter(User.email.in_(new_owners)))
if new_owner_users:
# At least some of the new owners are real users in the db.
pulse_user.owners = new_owner_users
db_session.commit()
updated_owners = {user.email for user in new_owner_users}
invalid_owners = sorted(new_owners - updated_owners)
if invalid_owners:
error = "Some user emails not found: {}".format(
', '.join(invalid_owners))
else:
messages = ["Email list updated."]
else:
error = ("Invalid owners: "
"Must be a comma-delimited list of existing user emails.")
if not error and not messages:
messages = ["No info updated."]
return profile(messages=messages, error=error)
def new_user(email_address, admin=False):
"""Initializes a new user, generating a salt and encrypting
his password. Then creates a RabbitMQ user if needed and sets
permissions.
"""
email = Email.get_email(email_address)
user = User(email=email, admin=admin)
db_session.add(user)
db_session.commit()
return user
def test_user_set_admin(self):
user = User.new_user(email='*****@*****.**', admin=False)
db_session.add(user)
db_session.commit()
user = User.query.filter(User.email == '*****@*****.**').first()
user.set_admin(True)
userDb = User.query.filter(User.email == '*****@*****.**').first()
self.assertTrue(userDb.admin)
def test_user_set_admin(self):
user = User.new_user(email='*****@*****.**', admin=False)
db_session.add(user)
db_session.commit()
user = User.query.filter(User.email == '*****@*****.**').first()
user.set_admin(True)
userDb = User.query.filter(User.email == '*****@*****.**').first()
self.assertTrue(userDb.admin)
def notification_delete(queue, email):
queue_obj = Queue.query.filter(Queue.id==queue).first()
queue_obj.notifications.remove(Email.get_email(email))
db_session.commit()
user = User.get_by_email(email)
notification = Queue.query.filter(
Queue.notifications.any(Email.address==email)).count()
if not user and not notification:
db_session.delete(Email.get_email(email))
db_session.commit()
def new_user(email, admin=False):
"""Initializes a new user, generating a salt and encrypting
his password. Then creates a RabbitMQ user if needed and sets
permissions.
"""
email = email.lower()
user = User(email=email, admin=admin)
db_session.add(user)
db_session.commit()
return user
def clear_deleted_queues(self, queues):
db_queues = Queue.query.all()
# Filter queues that are in the database but no longer on RabbitMQ.
alive_queues_names = set(q["name"] for q in queues)
deleted_queues = set(q for q in db_queues if q.name not in alive_queues_names)
# Delete those queues.
for queue in deleted_queues:
logging.info("Queue '{0}' has been deleted.".format(queue))
db_session.delete(queue)
db_session.commit()
def dummy_data():
# Dummy test users
User.new_user(email='*****@*****.**')
users = User.query.all()
for i in xrange(4):
PulseUser.new_user(
username='******'.format(i),
password='******',
owner=users[0])
pulse_users = PulseUser.query.all()
# And some dummy queues
dummy_queue = Queue(name='dummy-empty-queue', size=0, owner=pulse_users[0])
db_session.add(dummy_queue)
db_session.commit()
dummy_queue = Queue(name='dummy-non-empty-queue', size=config.warn_queue_size/5, owner=pulse_users[0])
db_session.add(dummy_queue)
db_session.commit()
dummy_queue = Queue(name='dummy-warning-queue', size=config.warn_queue_size + 1, owner=pulse_users[1])
db_session.add(dummy_queue)
db_session.commit()
dummy_queue = Queue(name='dummy-deletion-queue', size=int(config.del_queue_size * 1.2), owner=pulse_users[2])
db_session.add(dummy_queue)
db_session.commit()
# Test admin user
User.new_user(email='*****@*****.**', admin=True)
logger.info('Finished generating dummy data.')
def dummy_data():
# Dummy test users
User.new_user(email='*****@*****.**')
users = User.query.all()
for i in xrange(4):
RabbitMQAccount.new_user(
username='******'.format(i),
password='******',
owners=users[0])
rabbitmq_accounts = RabbitMQAccount.query.all()
# And some dummy queues
dummy_queue = Queue(name='dummy-empty-queue', size=0, owner=rabbitmq_accounts[0])
db_session.add(dummy_queue)
db_session.commit()
dummy_queue = Queue(name='dummy-non-empty-queue', size=config.warn_queue_size/5, owner=rabbitmq_accounts[0])
db_session.add(dummy_queue)
db_session.commit()
dummy_queue = Queue(name='dummy-warning-queue', size=config.warn_queue_size + 1, owner=rabbitmq_accounts[1])
db_session.add(dummy_queue)
db_session.commit()
dummy_queue = Queue(name='dummy-deletion-queue', size=int(config.del_queue_size * 1.2), owner=rabbitmq_accounts[2])
db_session.add(dummy_queue)
db_session.commit()
# Test admin user
User.new_user(email='*****@*****.**', admin=True)
logger.info('Finished generating dummy data.')
def change_password(self, new_password):
""""Changes" a user's password by deleting his rabbitmq account
and recreating it with the new password.
"""
try:
pulse_management.delete_user(self.username)
except pulse_management.PulseManagementException:
pass
self._create_user(new_password)
self._set_permissions()
db_session.add(self)
db_session.commit()
def dummy_data():
# Dummy test users
User.new_user('*****@*****.**')
for i in xrange(4):
PulseUser.new_user(
username='******'.format(i),
password='******',
owner=User.query.first(),
management_api=pulse_management)
pulse_users = PulseUser.query.all()
# And some dummy queues
dummy_queue = Queue(name='dummy-empty-queue', size=0, owner=pulse_users[0])
db_session.add(dummy_queue)
db_session.commit()
dummy_queue = Queue(name='dummy-non-empty-queue', size=config.warn_queue_size/5, owner=pulse_users[0])
db_session.add(dummy_queue)
db_session.commit()
dummy_queue = Queue(name='dummy-warning-queue', size=config.warn_queue_size + 1, owner=pulse_users[1])
db_session.add(dummy_queue)
db_session.commit()
dummy_queue = Queue(name='dummy-deletion-queue', size=int(config.del_queue_size * 1.2), owner=pulse_users[2])
db_session.add(dummy_queue)
db_session.commit()
# Test admin user
User.new_user('*****@*****.**', admin=True)
logger.info('Finished generating dummy data.')
def change_password(self, new_password):
""""Changes" a user's password by deleting his rabbitmq account
and recreating it with the new password.
"""
try:
pulse_management.delete_user(self.username)
except pulse_management.PulseManagementException:
pass
self._create_user(new_password)
self._set_permissions()
db_session.add(self)
db_session.commit()
def change_password(self, new_password, management_api):
""""Changes" a user's password by deleting his rabbitmq account
and recreating it with the new password.
"""
try:
management_api.delete_user(self.username)
except management_api.exception:
pass
self._create_user(management_api, new_password)
self._set_permissions(management_api)
db_session.add(self)
db_session.commit()
def monitor_queues(self, queues, all_bindings):
for queue_data in queues:
# Updating the queue's information in the database (owner, size).
queue = self.update_queue_information(queue_data, all_bindings)
if not queue:
continue
# If a queue is over the deletion size and ``unbounded`` is
# False (the default), then delete it regardless of it having
# an owner or not
# If ``unbounded`` is True, then let it grow indefinitely.
if queue.size > self.del_queue_size and not queue.unbounded:
logging.warning("Queue '{0}' deleted. Queue size = {1}; "
"del_queue_size = {2}".format(
queue.name, queue.size, self.del_queue_size))
if queue.owner and queue.owner.owners:
self.deletion_email(queue.owner.owners, queue_data)
if self.on_delete:
self.on_delete(queue.name)
pulse_management.delete_queue(vhost=queue_data['vhost'],
queue=queue.name)
db_session.delete(queue)
db_session.commit()
continue
if queue.owner is None or not queue.owner.owners:
continue
if queue.size > self.warn_queue_size and not queue.warned:
logging.warning("Warning queue '{0}' owner. Queue size = "
"{1}; warn_queue_size = {2}".format(
queue.name, queue.size,
self.warn_queue_size))
queue.warned = True
if self.on_warn:
self.on_warn(queue.name)
self.warning_email(queue.owner.owners, queue_data)
elif queue.size <= self.warn_queue_size and queue.warned:
# A previously warned queue got out of the warning threshold;
# its owner should not be warned again.
logging.warning("Queue '{0}' was in warning zone but is OK "
"now".format(queue.name, queue.size,
self.del_queue_size))
queue.warned = False
self.back_to_normal_email(queue.owner.owners, queue_data)
# Commit any changes to the queue.
db_session.add(queue)
db_session.commit()
def new_user(username, password='', owner=None, management_api=None):
"""Initializes a new user, generating a salt and encrypting
his password. Then creates a RabbitMQ user if needed and sets
permissions.
"""
pulse_user = PulseUser(owner=owner, username=username)
if management_api is not None:
pulse_user._create_user(management_api, password)
pulse_user._set_permissions(management_api)
db_session.add(pulse_user)
db_session.commit()
return pulse_user
def monitor_queues(self, queues):
for queue_data in queues:
# Updating the queue's information in the database (owner, size).
queue = self.update_queue_information(queue_data)
if not queue:
continue
# If a queue is over the deletion size, regardless of it having an
# owner or not, delete it.
if queue.size > self.del_queue_size:
logging.warning(
"Queue '{0}' deleted. Queue size = {1}; "
"del_queue_size = {2}".format(queue.name, queue.size, self.del_queue_size)
)
if queue.owner and queue.owner.owner:
self.deletion_email(queue.owner.owner, queue_data)
if self.on_delete:
self.on_delete(queue.name)
self.api.delete_queue(vhost=queue_data["vhost"], queue=queue.name)
db_session.delete(queue)
db_session.commit()
continue
if queue.owner is None or queue.owner.owner is None:
continue
if queue.size > self.warn_queue_size and not queue.warned:
logging.warning(
"Warning queue '{0}' owner. Queue size = "
"{1}; warn_queue_size = {2}".format(queue.name, queue.size, self.warn_queue_size)
)
queue.warned = True
if self.on_warn:
self.on_warn(queue.name)
self.warning_email(queue.owner.owner, queue_data)
elif queue.size <= self.warn_queue_size and queue.warned:
# A previously warned queue got out of the warning threshold;
# its owner should not be warned again.
logging.warning(
"Queue '{0}' was in warning zone but is OK "
"now".format(queue.name, queue.size, self.del_queue_size)
)
queue.warned = False
self.back_to_normal_email(queue.owner.owner, queue_data)
# Commit any changes to the queue.
db_session.add(queue)
db_session.commit()
def monitor_queues(self, queues):
for queue_data in queues:
# Updating the queue's information in the database (owner, size).
queue = self.update_queue_information(queue_data)
if not queue:
continue
# If a queue is over the deletion size, regardless of it having an
# owner or not, delete it.
if queue.size > self.del_queue_size:
logging.warning("Queue '{0}' deleted. Queue size = {1}; "
"del_queue_size = {2}".format(
queue.name, queue.size,
self.del_queue_size))
if queue.owner and queue.owner.owner:
self.deletion_email(queue.owner.owner, queue_data)
if self.on_delete:
self.on_delete(queue.name)
pulse_management.delete_queue(vhost=queue_data['vhost'],
queue=queue.name)
db_session.delete(queue)
db_session.commit()
continue
if queue.owner is None or queue.owner.owner is None:
continue
if queue.size > self.warn_queue_size and not queue.warned:
logging.warning("Warning queue '{0}' owner. Queue size = "
"{1}; warn_queue_size = {2}".format(
queue.name, queue.size,
self.warn_queue_size))
queue.warned = True
if self.on_warn:
self.on_warn(queue.name)
self.warning_email(queue.owner.owner, queue_data)
elif queue.size <= self.warn_queue_size and queue.warned:
# A previously warned queue got out of the warning threshold;
# its owner should not be warned again.
logging.warning("Queue '{0}' was in warning zone but is OK "
"now".format(queue.name, queue.size,
self.del_queue_size))
queue.warned = False
self.back_to_normal_email(queue.owner.owner, queue_data)
# Commit any changes to the queue.
db_session.add(queue)
db_session.commit()
def delete_queue(queue_name):
queue = Queue.query.get(queue_name)
if queue and (g.user.admin or
(queue.owner and queue.owner.owner == g.user)):
try:
pulse_management.delete_queue(vhost='/', queue=queue.name)
except pulse_management.PulseManagementException as e:
logging.warning("Couldn't delete the queue '{0}' on "
"rabbitmq: {1}".format(queue_name, e))
return jsonify(ok=False)
db_session.delete(queue)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def delete_queue(queue_name):
queue = Queue.query.get(queue_name)
if queue and (g.user.admin or
(queue.owner and queue.owner.owner == g.user)):
try:
pulse_management.delete_queue(vhost='/', queue=queue.name)
except PulseManagementException as e:
logging.warning("Couldn't delete the queue '{0}' on "
"rabbitmq: {1}".format(queue_name, e))
return jsonify(ok=False)
db_session.delete(queue)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def delete_pulse_user(pulse_username):
logging.info('Request to delete Pulse user "{0}".'.format(pulse_username))
pulse_user = PulseUser.query.filter(PulseUser.username == pulse_username).first()
if pulse_user and (g.user.admin or pulse_user.owner == g.user):
try:
pulse_management.delete_user(pulse_user.username)
except pulse_management.PulseManagementException as e:
logging.warning("Couldn't delete user '{0}' on "
"rabbitmq: {1}".format(pulse_username, e))
return jsonify(ok=False)
logging.info('Pulse user "{0}" deleted.'.format(pulse_username))
db_session.delete(pulse_user)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def new_user(username, password='', owner=None, create_rabbitmq_user=True):
"""Initializes a new user, generating a salt and encrypting
his password. Then creates a RabbitMQ user if needed and sets
permissions.
:param create_rabbitmq_user: Whether to add this user to the rabbitmq
server via the management plugin. Used by tests.
"""
pulse_user = PulseUser(owner=owner, username=username)
if create_rabbitmq_user:
pulse_user._create_user(password)
pulse_user._set_permissions()
db_session.add(pulse_user)
db_session.commit()
return pulse_user
def delete_pulse_user(pulse_username):
logging.info('Request to delete Pulse user "{0}".'.format(pulse_username))
pulse_user = PulseUser.query.filter(
PulseUser.username == pulse_username).first()
if pulse_user and (g.user.admin or pulse_user.owner == g.user):
try:
pulse_management.delete_user(pulse_user.username)
except pulse_management.PulseManagementException as e:
logging.warning("Couldn't delete user '{0}' on "
"rabbitmq: {1}".format(pulse_username, e))
return jsonify(ok=False)
logging.info('Pulse user "{0}" deleted.'.format(pulse_username))
db_session.delete(pulse_user)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def new_user(username, password='', owner=None, create_rabbitmq_user=True):
"""Initializes a new user, generating a salt and encrypting
his password. Then creates a RabbitMQ user if needed and sets
permissions.
:param create_rabbitmq_user: Whether to add this user to the rabbitmq
server via the management plugin. Used by tests.
"""
pulse_user = PulseUser(owner=owner, username=username)
if create_rabbitmq_user:
pulse_user._create_user(password)
pulse_user._set_permissions()
db_session.add(pulse_user)
db_session.commit()
return pulse_user
def clear_deleted_queues(self, queues, all_bindings):
db_queues = Queue.query.all()
# Filter queues that are in the database but no longer on RabbitMQ.
alive_queues_names = {q['name'] for q in queues}
deleted_queues = {q for q in db_queues
if q.name not in alive_queues_names}
# Delete those queues.
for queue in deleted_queues:
logging.debug("Queue '{0}' has been deleted.".format(queue))
db_session.delete(queue)
# Clean up bindings on queues that are not deleted.
for queue_name in alive_queues_names:
bindings = self.get_queue_bindings(all_bindings, queue_name)
self.clear_deleted_bindings(queue_name, bindings)
db_session.commit()
def clear_deleted_queues(self, queues):
db_queues = Queue.query.all()
# Filter queues that are in the database but no longer on RabbitMQ.
alive_queues_names = {q['name'] for q in queues}
deleted_queues = {
q
for q in db_queues if q.name not in alive_queues_names
}
# Delete those queues.
for queue in deleted_queues:
logging.info("Queue '{0}' has been deleted.".format(queue))
db_session.delete(queue)
# Clean up bindings on queues that are not deleted.
for queue_name in alive_queues_names:
self.clear_deleted_bindings(queue_name)
db_session.commit()
def new_user(username, password='', owners=None, create_rabbitmq_user=True):
"""Initializes a new account object, generating a salt and encrypting
its password. Then creates a RabbitMQ user if needed and sets
permissions.
:param create_rabbitmq_user: Whether to add this user to the rabbitmq
server via the management plugin. Used by tests.
"""
# Ensure that ``owners`` is a list.
if owners and not isinstance(owners, list):
owners = [owners]
rabbitmq_account = RabbitMQAccount(owners=owners, username=username)
if create_rabbitmq_user:
rabbitmq_account._create_user(password)
rabbitmq_account._set_permissions()
db_session.add(rabbitmq_account)
db_session.commit()
return rabbitmq_account
def setUp(self):
global pulse_cfg
self.proc = None
self.publisher = None
self.management_api = PulseManagementAPI(
management_url='http://{}:{}/api'.format(
pulse_cfg['host'], pulse_cfg['management_port']),
user=pulse_cfg['user'],
password=pulse_cfg['password']
)
self.guardian = PulseGuardian(self.management_api,
warn_queue_size=TEST_WARN_SIZE,
del_queue_size=TEST_DELETE_SIZE,
emails=False)
# Hack in a test config.
dbinit.pulse_management = self.management_api
dbinit.init_and_clear_db()
self.consumer_cfg = pulse_cfg.copy()
self.consumer_cfg['applabel'] = str(uuid.uuid1())
# Configure/create the test user to be used for message consumption.
self.consumer_cfg['user'] = CONSUMER_USER
self.consumer_cfg['password'] = CONSUMER_PASSWORD
self.user = User.new_user(email=CONSUMER_EMAIL, admin=False)
db_session.add(self.user)
db_session.commit()
self.pulse_user = PulseUser.new_user(
username=CONSUMER_USER,
password=CONSUMER_PASSWORD,
owner=self.user,
management_api=self.management_api)
db_session.add(self.pulse_user)
db_session.commit()
def update_queue_information(self, queue_data):
if not "messages" in queue_data:
# FIXME: We should do something here, probably delete the queue,
# as it's in a weird state. More investigation is required.
# See bug 1066338.
return None
q_size, q_name, q_durable = (queue_data["messages"], queue_data["name"], queue_data["durable"])
queue = Queue.query.filter(Queue.name == q_name).first()
# If the queue doesn't exist in the db, create it.
if queue is None:
m = re.match("queue/([^/]+)/", q_name)
logging.info("New queue '{0}' encountered. " "Adding to the database.".format(q_name))
if m:
owner_name = m.group(1)
owner = PulseUser.query.filter(PulseUser.username == owner_name).first()
# If the queue was created by a user that isn't in the
# pulseguardian database, skip the queue.
if owner is None:
logging.info(
"Queue '{0}' owner, {1}, isn't in the db. Creating " "the user.".format(q_name, owner_name)
)
owner = PulseUser.new_user(owner_name)
# Assign the user to the queue.
logging.info("Assigning queue '{0}' to user " "{1}.".format(q_name, owner))
else:
logging.warn("'{0}' is not a standard queue name.".format(q_name))
owner = None
queue = Queue(name=q_name, owner=owner)
# Update the saved queue size.
queue.size = q_size
queue.durable = q_durable
db_session.add(queue)
db_session.commit()
return queue
def init_and_clear_db():
# Initialize the database schema.
init_db()
# Remove all users and pulse users created by the web app.
for pulse_user in PulseUser.query.all():
try:
pulse_management.delete_user(pulse_user.username)
except PulseManagementException:
pass
# Clear the database of old data.
for queue in Queue.query.all():
db_session.delete(queue)
for pulse_user in PulseUser.query.all():
db_session.delete(pulse_user)
for user in User.query.all():
db_session.delete(user)
db_session.commit()
logger.info('Finished initializing database.')
def test_user(self):
user = User.new_user(email='*****@*****.**', admin=False)
db_session.add(user)
db_session.commit()
pulse_user = PulseUser.new_user(username='******',
password='******',
owner=user,
management_api=None)
db_session.add(pulse_user)
db_session.commit()
self.assertTrue(user in User.query.all())
# Emails are normalized by putting them lower-case
self.assertEqual(
User.query.filter(User.email == '*****@*****.**').first(), user)
self.assertEqual(
PulseUser.query.filter(PulseUser.username == 'dummy').first(),
pulse_user)
self.assertEqual(
PulseUser.query.filter(PulseUser.username == 'DUMMY').first(),
None)
def test_user(self):
user = User.new_user(email='*****@*****.**', admin=False)
db_session.add(user)
db_session.commit()
rabbitmq_account = RabbitMQAccount.new_user(username='******',
password='******',
owners=user,
create_rabbitmq_user=False)
db_session.add(rabbitmq_account)
db_session.commit()
self.assertTrue(user in User.query.all())
# Emails are normalized by putting them lower-case
self.assertEqual(
User.query.filter(User.email == '*****@*****.**').first(), user)
self.assertEqual(
RabbitMQAccount.query.filter(
RabbitMQAccount.username == 'dummy').first(), rabbitmq_account)
self.assertEqual(
RabbitMQAccount.query.filter(
RabbitMQAccount.username == 'DUMMY').first(), None)
def test_abnormal_queue_name(self):
self.consumer_class = AbnormalQueueConsumer
# Use account with full permissions.
self.consumer_cfg['user'] = pulse_cfg['user']
self.consumer_cfg['password'] = pulse_cfg['password']
self._create_publisher()
self._create_consumer_proc()
self._wait_for_queue()
self._wait_for_queue_record()
queue = Queue.query.filter(
Queue.name == AbnormalQueueConsumer.QUEUE_NAME).first()
owner = queue.owner
# Queue is not durable and will be cleaned up when consumer process
# exits; delete it from the queue to avoid assertion failure in
# tearDown().
self._terminate_consumer_proc()
self._wait_for_queue(False)
db_session.delete(queue)
db_session.commit()
self.assertEqual(owner, None)
def test_abnormal_queue_name(self):
self.consumer_class = AbnormalQueueConsumer
# Use account with full permissions.
self.consumer_cfg['user'] = pulse_cfg['user']
self.consumer_cfg['password'] = pulse_cfg['password']
self._create_publisher()
self._create_consumer_proc()
self._wait_for_queue()
self._wait_for_queue_record()
queue = Queue.query.filter(Queue.name ==
AbnormalQueueConsumer.QUEUE_NAME).first()
owner = queue.owner
# Queue is not durable and will be cleaned up when consumer process
# exits; delete it from the queue to avoid assertion failure in
# tearDown().
self._terminate_consumer_proc()
self._wait_for_queue(False)
db_session.delete(queue)
db_session.commit()
self.assertEqual(owner, None)
def init_and_clear_db():
# Initialize the database schema.
init_db()
# Remove all users and pulse users created by the web app.
for rabbitmq_account in RabbitMQAccount.query.all():
try:
pulse_management.delete_user(rabbitmq_account.username)
except pulse_management.PulseManagementException:
pass
# Clear the database of old data.
for queue in Queue.query.all():
db_session.delete(queue)
for binding in Binding.query.all():
db_session.delete(binding)
for rabbitmq_account in RabbitMQAccount.query.all():
db_session.delete(rabbitmq_account)
for user in User.query.all():
db_session.delete(user)
db_session.commit()
logger.info('Finished initializing database.')
def init_and_clear_db():
# Initialize the database schema.
init_db()
# Remove all users and pulse users created by the web app.
for rabbitmq_account in RabbitMQAccount.query.all():
try:
pulse_management.delete_user(rabbitmq_account.username)
except pulse_management.PulseManagementException:
pass
# Clear the database of old data.
for queue in Queue.query.all():
db_session.delete(queue)
for binding in Binding.query.all():
db_session.delete(binding)
for rabbitmq_account in RabbitMQAccount.query.all():
db_session.delete(rabbitmq_account)
for user in User.query.all():
db_session.delete(user)
db_session.commit()
logger.info('Finished initializing database.')
def set_admin(self, is_admin):
self.admin = is_admin
db_session.commit()
def update_queue_information(self, queue_data, all_bindings):
if 'messages' not in queue_data:
# FIXME: We should do something here, probably delete the queue,
# as it's in a weird state. More investigation is required.
# See bug 1066338.
return None
q_size, q_name, q_durable = (queue_data['messages'],
queue_data['name'], queue_data['durable'])
queue = Queue.query.filter(Queue.name == q_name).first()
# If the queue doesn't exist in the db, create it.
if queue is None:
log_details = {
'queuename': q_name,
'queuesize': q_size,
'queuedurable': q_durable,
}
m = re.match('queue/([^/]+)/', q_name)
if not m:
log_details['valid'] = False
owner = None
elif (config.reserved_users_regex
and re.match(config.reserved_users_regex, m.group(1))):
# Ignore this queue entirely as we will see it again on the
# next iteration.
return None
else:
log_details['valid'] = True
owner_name = m.group(1)
owner = RabbitMQAccount.query.filter(
RabbitMQAccount.username == owner_name).first()
log_details['ownername'] = owner_name
log_details['newowner'] = not owner
# If the queue belongs to a pulse user that isn't in the
# pulseguardian database, add the user to the DB, owned by an
# admin.
if owner is None:
# RabbitMQAccount needs at least one owner as well, but
# since we have no way of knowing who really owns it, find
# the first admin, and set it to that.
user = User.query.filter(User.admin == True).first()
owner = RabbitMQAccount.new_user(owner_name, owners=user)
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'New queue.',
details=log_details,
tags=['queue'],
)
queue = Queue(name=q_name, owner=owner)
# add the queue bindings to the db.
bindings = self.get_queue_bindings(all_bindings, queue.name)
for binding in bindings:
db_binding = Binding.query.filter(
Binding.exchange == binding["source"],
Binding.routing_key == binding["routing_key"],
Binding.queue_name == queue.name).first()
if not db_binding:
# need to create the binding in the DB
binding = Binding(exchange=binding["source"],
routing_key=binding["routing_key"],
queue_name=queue.name)
db_session.add(binding)
# Update the saved queue size.
queue.size = q_size
queue.durable = q_durable
db_session.add(queue)
db_session.commit()
return queue