def clear_deleted_bindings(self, queue_name, queue_bindings):
db_bindings = Binding.query.filter(Binding.queue_name == queue_name)
# Filter bindings that are in the database but no longer on RabbitMQ.
alive_bindings_names = {
Binding.as_string(b['source'], b['routing_key'])
for b in queue_bindings
}
deleted_bindings = {
b
for b in db_bindings if b.name not in alive_bindings_names
}
# Delete those bindings.
for binding in deleted_bindings:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Binding no longer exists.',
details={
'queuename': queue_name,
'binding': binding.name,
},
tags=['queue'],
)
db_session.delete(binding)
def clear_deleted_queues(self, queues, all_bindings):
db_queues = Queue.query.all()
# Filter queues that are in the database but no longer on RabbitMQ.
alive_queues_names = {q['name'] for q in queues}
deleted_queues = {q for q in db_queues
if q.name not in alive_queues_names}
# Delete those queues.
for queue in deleted_queues:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Queue no longer exists.',
details={'queuename': queue.name},
tags=['queue'],
)
db_session.delete(queue)
# Clean up bindings on queues that are not deleted.
for queue_name in alive_queues_names:
bindings = self.get_queue_bindings(all_bindings, queue_name)
self.clear_deleted_bindings(queue_name, bindings)
db_session.commit()
def delete_rabbitmq_account(rabbitmq_username):
rabbitmq_account = RabbitMQAccount.query.filter(
RabbitMQAccount.username == rabbitmq_username).first()
if rabbitmq_account and (g.user.admin
or g.user in rabbitmq_account.owners):
details = {
'username': g.user.email,
'rabbitmqusername': rabbitmq_username,
}
try:
pulse_management.delete_user(rabbitmq_account.username)
except pulse_management.PulseManagementException as e:
details['message'] = str(e)
mozdef.log(
mozdef.ERROR,
mozdef.OTHER,
'Error deleting RabbitMQ account',
details=details,
)
return jsonify(ok=False)
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'RabbitMQ account deleted',
details=details,
)
db_session.delete(rabbitmq_account)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def clear_deleted_queues(self, queues, all_bindings):
db_queues = Queue.query.all()
# Filter queues that are in the database but no longer on RabbitMQ.
alive_queues_names = {q['name'] for q in queues}
deleted_queues = {
q
for q in db_queues if q.name not in alive_queues_names
}
# Delete those queues.
for queue in deleted_queues:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Queue no longer exists.',
details={'queuename': queue.name},
tags=['queue'],
)
db_session.delete(queue)
# Clean up bindings on queues that are not deleted.
for queue_name in alive_queues_names:
bindings = self.get_queue_bindings(all_bindings, queue_name)
self.clear_deleted_bindings(queue_name, bindings)
db_session.commit()
def delete_rabbitmq_account(rabbitmq_username):
rabbitmq_account = RabbitMQAccount.query.filter(
RabbitMQAccount.username == rabbitmq_username).first()
if rabbitmq_account and (g.user.admin or g.user in
rabbitmq_account.owners):
details = {
'username': g.user.email,
'rabbitmqusername': rabbitmq_username,
}
try:
pulse_management.delete_user(rabbitmq_account.username)
except pulse_management.PulseManagementException as e:
details['message'] = str(e)
mozdef.log(
mozdef.ERROR,
mozdef.OTHER,
'Error deleting RabbitMQ account',
details=details,
)
return jsonify(ok=False)
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'RabbitMQ account deleted',
details=details,
)
db_session.delete(rabbitmq_account)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def delete_queue(queue_name):
queue = Queue.query.get(queue_name)
if queue and (g.user.admin or
(queue.owner and g.user in queue.owner.owners)):
details = {
'queuename': queue_name,
'username': g.user.email,
}
try:
pulse_management.delete_queue(vhost='/', queue=queue.name)
except pulse_management.PulseManagementException as e:
details['message'] = str(e)
mozdef.log(
mozdef.ERROR,
mozdef.OTHER,
'Error deleting queue',
details=details,
tags=['queue'],
)
return jsonify(ok=False)
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Deleting queue',
details=details,
tags=['queue'],
)
db_session.delete(queue)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def delete_pulse_user(pulse_username):
pulse_user = PulseUser.query.filter(
PulseUser.username == pulse_username).first()
if pulse_user and (g.user.admin or g.user in pulse_user.owners):
details = {
'username': g.user.email,
'pulseusername': pulse_username,
}
try:
pulse_management.delete_user(pulse_user.username)
except pulse_management.PulseManagementException as e:
details['message'] = str(e)
mozdef.log(
mozdef.ERROR,
mozdef.OTHER,
'Error deleting Pulse user',
details=details,
)
return jsonify(ok=False)
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Pulse user deleted',
details=details,
)
db_session.delete(pulse_user)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def monitor_queues(self, queues, all_bindings):
for queue_data in queues:
# Updating the queue's information in the database (owner, size).
queue = self.update_queue_information(queue_data, all_bindings)
if not queue:
continue
# If a queue is over the deletion size and ``unbounded`` is
# False (the default), then delete it regardless of it having
# an owner or not
# If ``unbounded`` is True, then let it grow indefinitely.
if queue.size > self.del_queue_size and not queue.unbounded:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Deleting queue.',
details=self._queue_details_dict(queue),
tags=['queue'],
)
if queue.owner and queue.owner.owners:
self.deletion_email(queue.owner.owners, queue_data)
if self.on_delete:
self.on_delete(queue.name)
pulse_management.delete_queue(vhost=queue_data['vhost'],
queue=queue.name)
db_session.delete(queue)
db_session.commit()
continue
if queue.owner is None or not queue.owner.owners:
continue
if queue.size > self.warn_queue_size and not queue.warned:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Queue-size warning.',
details=self._queue_details_dict(queue),
tags=['queue'],
)
queue.warned = True
if self.on_warn:
self.on_warn(queue.name)
self.warning_email(queue.owner.owners, queue_data)
elif queue.size <= self.warn_queue_size and queue.warned:
# A previously warned queue got out of the warning threshold;
# its owner should not be warned again.
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Queue-size recovered.',
details=self._queue_details_dict(queue),
tags=['queue'],
)
queue.warned = False
self.back_to_normal_email(queue.owner.owners, queue_data)
# Commit any changes to the queue.
db_session.add(queue)
db_session.commit()
def delete_queue(queue_name):
queue = Queue.query.get(queue_name)
if queue and (g.user.admin or
(queue.owner and g.user in queue.owner.owners)):
details = {
'queuename': queue_name,
'username': g.user.email,
}
try:
pulse_management.delete_queue(vhost='/', queue=queue.name)
except pulse_management.PulseManagementException as e:
details['message'] = str(e)
mozdef.log(
mozdef.ERROR,
mozdef.OTHER,
'Error deleting queue',
details=details,
tags=['queue'],
)
return jsonify(ok=False)
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Deleting queue',
details=details,
tags=['queue'],
)
db_session.delete(queue)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def monitor_queues(self, queues, all_bindings):
for queue_data in queues:
# Updating the queue's information in the database (owner, size).
queue = self.update_queue_information(queue_data, all_bindings)
if not queue:
continue
# If a queue is over the deletion size and ``unbounded`` is
# False (the default), then delete it regardless of it having
# an owner or not
# If ``unbounded`` is True, then let it grow indefinitely.
if queue.size > self.del_queue_size and not queue.unbounded:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Deleting queue.',
details=self._queue_details_dict(queue),
tags=['queue'],
)
if queue.owner and queue.owner.owners:
self.deletion_email(queue.owner.owners, queue_data)
if self.on_delete:
self.on_delete(queue.name)
pulse_management.delete_queue(vhost=queue_data['vhost'],
queue=queue.name)
db_session.delete(queue)
db_session.commit()
continue
if queue.owner is None or not queue.owner.owners:
continue
if queue.size > self.warn_queue_size and not queue.warned:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Queue-size warning.',
details=self._queue_details_dict(queue),
tags=['queue'],
)
queue.warned = True
if self.on_warn:
self.on_warn(queue.name)
self.warning_email(queue.owner.owners, queue_data)
elif queue.size <= self.warn_queue_size and queue.warned:
# A previously warned queue got out of the warning threshold;
# its owner should not be warned again.
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Queue-size recovered.',
details=self._queue_details_dict(queue),
tags=['queue'],
)
queue.warned = False
self.back_to_normal_email(queue.owner.owners, queue_data)
# Commit any changes to the queue.
db_session.add(queue)
db_session.commit()
def clear_deleted_queues(self, queues):
db_queues = Queue.query.all()
# Filter queues that are in the database but no longer on RabbitMQ.
alive_queues_names = set(q["name"] for q in queues)
deleted_queues = set(q for q in db_queues if q.name not in alive_queues_names)
# Delete those queues.
for queue in deleted_queues:
logging.info("Queue '{0}' has been deleted.".format(queue))
db_session.delete(queue)
db_session.commit()
def notification_delete(queue, email):
queue_obj = Queue.query.filter(Queue.id==queue).first()
queue_obj.notifications.remove(Email.get_email(email))
db_session.commit()
user = User.get_by_email(email)
notification = Queue.query.filter(
Queue.notifications.any(Email.address==email)).count()
if not user and not notification:
db_session.delete(Email.get_email(email))
db_session.commit()
def clear_deleted_bindings(self, queue_name, queue_bindings):
db_bindings = Binding.query.filter(Binding.queue_name == queue_name)
# Filter bindings that are in the database but no longer on RabbitMQ.
alive_bindings_names = {Binding.as_string(b['source'], b['routing_key'])
for b in queue_bindings}
deleted_bindings = {b for b in db_bindings
if b.name not in alive_bindings_names}
# Delete those bindings.
for binding in deleted_bindings:
logging.debug("Binding '{}' for queue '{}' has been deleted.".format(
binding, queue_name))
db_session.delete(binding)
def monitor_queues(self, queues, all_bindings):
for queue_data in queues:
# Updating the queue's information in the database (owner, size).
queue = self.update_queue_information(queue_data, all_bindings)
if not queue:
continue
# If a queue is over the deletion size and ``unbounded`` is
# False (the default), then delete it regardless of it having
# an owner or not
# If ``unbounded`` is True, then let it grow indefinitely.
if queue.size > self.del_queue_size and not queue.unbounded:
logging.warning("Queue '{0}' deleted. Queue size = {1}; "
"del_queue_size = {2}".format(
queue.name, queue.size, self.del_queue_size))
if queue.owner and queue.owner.owners:
self.deletion_email(queue.owner.owners, queue_data)
if self.on_delete:
self.on_delete(queue.name)
pulse_management.delete_queue(vhost=queue_data['vhost'],
queue=queue.name)
db_session.delete(queue)
db_session.commit()
continue
if queue.owner is None or not queue.owner.owners:
continue
if queue.size > self.warn_queue_size and not queue.warned:
logging.warning("Warning queue '{0}' owner. Queue size = "
"{1}; warn_queue_size = {2}".format(
queue.name, queue.size,
self.warn_queue_size))
queue.warned = True
if self.on_warn:
self.on_warn(queue.name)
self.warning_email(queue.owner.owners, queue_data)
elif queue.size <= self.warn_queue_size and queue.warned:
# A previously warned queue got out of the warning threshold;
# its owner should not be warned again.
logging.warning("Queue '{0}' was in warning zone but is OK "
"now".format(queue.name, queue.size,
self.del_queue_size))
queue.warned = False
self.back_to_normal_email(queue.owner.owners, queue_data)
# Commit any changes to the queue.
db_session.add(queue)
db_session.commit()
def monitor_queues(self, queues):
for queue_data in queues:
# Updating the queue's information in the database (owner, size).
queue = self.update_queue_information(queue_data)
if not queue:
continue
# If a queue is over the deletion size, regardless of it having an
# owner or not, delete it.
if queue.size > self.del_queue_size:
logging.warning(
"Queue '{0}' deleted. Queue size = {1}; "
"del_queue_size = {2}".format(queue.name, queue.size, self.del_queue_size)
)
if queue.owner and queue.owner.owner:
self.deletion_email(queue.owner.owner, queue_data)
if self.on_delete:
self.on_delete(queue.name)
self.api.delete_queue(vhost=queue_data["vhost"], queue=queue.name)
db_session.delete(queue)
db_session.commit()
continue
if queue.owner is None or queue.owner.owner is None:
continue
if queue.size > self.warn_queue_size and not queue.warned:
logging.warning(
"Warning queue '{0}' owner. Queue size = "
"{1}; warn_queue_size = {2}".format(queue.name, queue.size, self.warn_queue_size)
)
queue.warned = True
if self.on_warn:
self.on_warn(queue.name)
self.warning_email(queue.owner.owner, queue_data)
elif queue.size <= self.warn_queue_size and queue.warned:
# A previously warned queue got out of the warning threshold;
# its owner should not be warned again.
logging.warning(
"Queue '{0}' was in warning zone but is OK "
"now".format(queue.name, queue.size, self.del_queue_size)
)
queue.warned = False
self.back_to_normal_email(queue.owner.owner, queue_data)
# Commit any changes to the queue.
db_session.add(queue)
db_session.commit()
def monitor_queues(self, queues):
for queue_data in queues:
# Updating the queue's information in the database (owner, size).
queue = self.update_queue_information(queue_data)
if not queue:
continue
# If a queue is over the deletion size, regardless of it having an
# owner or not, delete it.
if queue.size > self.del_queue_size:
logging.warning("Queue '{0}' deleted. Queue size = {1}; "
"del_queue_size = {2}".format(
queue.name, queue.size,
self.del_queue_size))
if queue.owner and queue.owner.owner:
self.deletion_email(queue.owner.owner, queue_data)
if self.on_delete:
self.on_delete(queue.name)
pulse_management.delete_queue(vhost=queue_data['vhost'],
queue=queue.name)
db_session.delete(queue)
db_session.commit()
continue
if queue.owner is None or queue.owner.owner is None:
continue
if queue.size > self.warn_queue_size and not queue.warned:
logging.warning("Warning queue '{0}' owner. Queue size = "
"{1}; warn_queue_size = {2}".format(
queue.name, queue.size,
self.warn_queue_size))
queue.warned = True
if self.on_warn:
self.on_warn(queue.name)
self.warning_email(queue.owner.owner, queue_data)
elif queue.size <= self.warn_queue_size and queue.warned:
# A previously warned queue got out of the warning threshold;
# its owner should not be warned again.
logging.warning("Queue '{0}' was in warning zone but is OK "
"now".format(queue.name, queue.size,
self.del_queue_size))
queue.warned = False
self.back_to_normal_email(queue.owner.owner, queue_data)
# Commit any changes to the queue.
db_session.add(queue)
db_session.commit()
def delete_queue(queue_name):
queue = Queue.query.get(queue_name)
if queue and (g.user.admin or
(queue.owner and queue.owner.owner == g.user)):
try:
pulse_management.delete_queue(vhost='/', queue=queue.name)
except pulse_management.PulseManagementException as e:
logging.warning("Couldn't delete the queue '{0}' on "
"rabbitmq: {1}".format(queue_name, e))
return jsonify(ok=False)
db_session.delete(queue)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def delete_queue(queue_name):
queue = Queue.query.get(queue_name)
if queue and (g.user.admin or
(queue.owner and queue.owner.owner == g.user)):
try:
pulse_management.delete_queue(vhost='/', queue=queue.name)
except PulseManagementException as e:
logging.warning("Couldn't delete the queue '{0}' on "
"rabbitmq: {1}".format(queue_name, e))
return jsonify(ok=False)
db_session.delete(queue)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def delete_pulse_user(pulse_username):
logging.info('Request to delete Pulse user "{0}".'.format(pulse_username))
pulse_user = PulseUser.query.filter(PulseUser.username == pulse_username).first()
if pulse_user and (g.user.admin or pulse_user.owner == g.user):
try:
pulse_management.delete_user(pulse_user.username)
except pulse_management.PulseManagementException as e:
logging.warning("Couldn't delete user '{0}' on "
"rabbitmq: {1}".format(pulse_username, e))
return jsonify(ok=False)
logging.info('Pulse user "{0}" deleted.'.format(pulse_username))
db_session.delete(pulse_user)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def delete_pulse_user(pulse_username):
logging.info('Request to delete Pulse user "{0}".'.format(pulse_username))
pulse_user = PulseUser.query.filter(
PulseUser.username == pulse_username).first()
if pulse_user and (g.user.admin or pulse_user.owner == g.user):
try:
pulse_management.delete_user(pulse_user.username)
except pulse_management.PulseManagementException as e:
logging.warning("Couldn't delete user '{0}' on "
"rabbitmq: {1}".format(pulse_username, e))
return jsonify(ok=False)
logging.info('Pulse user "{0}" deleted.'.format(pulse_username))
db_session.delete(pulse_user)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def clear_deleted_queues(self, queues, all_bindings):
db_queues = Queue.query.all()
# Filter queues that are in the database but no longer on RabbitMQ.
alive_queues_names = {q['name'] for q in queues}
deleted_queues = {q for q in db_queues
if q.name not in alive_queues_names}
# Delete those queues.
for queue in deleted_queues:
logging.debug("Queue '{0}' has been deleted.".format(queue))
db_session.delete(queue)
# Clean up bindings on queues that are not deleted.
for queue_name in alive_queues_names:
bindings = self.get_queue_bindings(all_bindings, queue_name)
self.clear_deleted_bindings(queue_name, bindings)
db_session.commit()
def clear_deleted_queues(self, queues):
db_queues = Queue.query.all()
# Filter queues that are in the database but no longer on RabbitMQ.
alive_queues_names = {q['name'] for q in queues}
deleted_queues = {
q
for q in db_queues if q.name not in alive_queues_names
}
# Delete those queues.
for queue in deleted_queues:
logging.info("Queue '{0}' has been deleted.".format(queue))
db_session.delete(queue)
# Clean up bindings on queues that are not deleted.
for queue_name in alive_queues_names:
self.clear_deleted_bindings(queue_name)
db_session.commit()
def clear_deleted_bindings(self, queue_name):
rabbit_bindings = pulse_management.queue_bindings(
config.rabbit_vhost, queue_name)
db_bindings = Binding.query.filter(Binding.queue_name == queue_name)
# Filter bindings that are in the database but no longer on RabbitMQ.
alive_bindings_names = {
Binding.as_string(b['source'], b['routing_key'])
for b in rabbit_bindings
}
deleted_bindings = {
b
for b in db_bindings if b.name not in alive_bindings_names
}
# Delete those bindings.
for binding in deleted_bindings:
logging.info(
"Binding '{}' for queue '{}' has been deleted.".format(
binding, queue_name))
db_session.delete(binding)
def clear_deleted_bindings(self, queue_name, queue_bindings):
db_bindings = Binding.query.filter(Binding.queue_name == queue_name)
# Filter bindings that are in the database but no longer on RabbitMQ.
alive_bindings_names = {Binding.as_string(b['source'],
b['routing_key'])
for b in queue_bindings}
deleted_bindings = {b for b in db_bindings
if b.name not in alive_bindings_names}
# Delete those bindings.
for binding in deleted_bindings:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Binding no longer exists.',
details={
'queuename': queue_name,
'binding': binding.name,
},
tags=['queue'],
)
db_session.delete(binding)
def test_abnormal_queue_name(self):
self.consumer_class = AbnormalQueueConsumer
# Use account with full permissions.
self.consumer_cfg['user'] = pulse_cfg['user']
self.consumer_cfg['password'] = pulse_cfg['password']
self._create_publisher()
self._create_consumer_proc()
self._wait_for_queue()
self._wait_for_queue_record()
queue = Queue.query.filter(Queue.name ==
AbnormalQueueConsumer.QUEUE_NAME).first()
owner = queue.owner
# Queue is not durable and will be cleaned up when consumer process
# exits; delete it from the queue to avoid assertion failure in
# tearDown().
self._terminate_consumer_proc()
self._wait_for_queue(False)
db_session.delete(queue)
db_session.commit()
self.assertEqual(owner, None)
def test_abnormal_queue_name(self):
self.consumer_class = AbnormalQueueConsumer
# Use account with full permissions.
self.consumer_cfg['user'] = pulse_cfg['user']
self.consumer_cfg['password'] = pulse_cfg['password']
self._create_publisher()
self._create_consumer_proc()
self._wait_for_queue()
self._wait_for_queue_record()
queue = Queue.query.filter(
Queue.name == AbnormalQueueConsumer.QUEUE_NAME).first()
owner = queue.owner
# Queue is not durable and will be cleaned up when consumer process
# exits; delete it from the queue to avoid assertion failure in
# tearDown().
self._terminate_consumer_proc()
self._wait_for_queue(False)
db_session.delete(queue)
db_session.commit()
self.assertEqual(owner, None)
def init_and_clear_db():
# Initialize the database schema.
init_db()
# Remove all users and pulse users created by the web app.
for rabbitmq_account in RabbitMQAccount.query.all():
try:
pulse_management.delete_user(rabbitmq_account.username)
except pulse_management.PulseManagementException:
pass
# Clear the database of old data.
for queue in Queue.query.all():
db_session.delete(queue)
for binding in Binding.query.all():
db_session.delete(binding)
for rabbitmq_account in RabbitMQAccount.query.all():
db_session.delete(rabbitmq_account)
for user in User.query.all():
db_session.delete(user)
db_session.commit()
logger.info('Finished initializing database.')
def init_and_clear_db():
# Initialize the database schema.
init_db()
# Remove all users and pulse users created by the web app.
for rabbitmq_account in RabbitMQAccount.query.all():
try:
pulse_management.delete_user(rabbitmq_account.username)
except pulse_management.PulseManagementException:
pass
# Clear the database of old data.
for queue in Queue.query.all():
db_session.delete(queue)
for binding in Binding.query.all():
db_session.delete(binding)
for rabbitmq_account in RabbitMQAccount.query.all():
db_session.delete(rabbitmq_account)
for user in User.query.all():
db_session.delete(user)
db_session.commit()
logger.info('Finished initializing database.')
def init_and_clear_db():
# Initialize the database schema.
init_db()
# Remove all users and pulse users created by the web app.
for pulse_user in PulseUser.query.all():
try:
pulse_management.delete_user(pulse_user.username)
except PulseManagementException:
pass
# Clear the database of old data.
for queue in Queue.query.all():
db_session.delete(queue)
for pulse_user in PulseUser.query.all():
db_session.delete(pulse_user)
for user in User.query.all():
db_session.delete(user)
db_session.commit()
logger.info('Finished initializing database.')
