def delete_rabbitmq_account(rabbitmq_username):
rabbitmq_account = RabbitMQAccount.query.filter(
RabbitMQAccount.username == rabbitmq_username).first()
if rabbitmq_account and (g.user.admin
or g.user in rabbitmq_account.owners):
details = {
'username': g.user.email,
'rabbitmqusername': rabbitmq_username,
}
try:
pulse_management.delete_user(rabbitmq_account.username)
except pulse_management.PulseManagementException as e:
details['message'] = str(e)
mozdef.log(
mozdef.ERROR,
mozdef.OTHER,
'Error deleting RabbitMQ account',
details=details,
)
return jsonify(ok=False)
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'RabbitMQ account deleted',
details=details,
)
db_session.delete(rabbitmq_account)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def set_user_admin(user_id):
if 'isAdmin' not in request.json:
abort(400)
user = User.query.get(user_id)
if not user:
abort(400)
is_admin = request.json['isAdmin']
details = {
'username': g.user.email,
'newadminvalue': is_admin,
'targetusername': user.email,
}
try:
user.set_admin(is_admin)
mozdef.log(
mozdef.NOTICE,
mozdef.ACCOUNT_UPDATE,
'Admin role changed',
details=details,
)
except Exception as e:
details['message'] = str(e)
mozdef.log(mozdef.ERROR,
mozdef.ACCOUNT_UPDATE,
'Admin role update failed.',
details=details)
return jsonify(ok=False)
return jsonify(ok=True)
def set_user_admin(user_id):
if 'isAdmin' not in request.json:
abort(400)
user = User.query.get(user_id)
if not user:
abort(400)
is_admin = request.json['isAdmin']
details = {
'username': g.user.email,
'newadminvalue': is_admin,
'targetusername': user.email,
}
try:
user.set_admin(is_admin)
mozdef.log(
mozdef.NOTICE,
mozdef.ACCOUNT_UPDATE,
'Admin role changed',
details=details,
)
except Exception as e:
details['message'] = str(e)
mozdef.log(
mozdef.ERROR,
mozdef.ACCOUNT_UPDATE,
'Admin role update failed.',
details=details
)
return jsonify(ok=False)
return jsonify(ok=True)
def delete_pulse_user(pulse_username):
pulse_user = PulseUser.query.filter(
PulseUser.username == pulse_username).first()
if pulse_user and (g.user.admin or g.user in pulse_user.owners):
details = {
'username': g.user.email,
'pulseusername': pulse_username,
}
try:
pulse_management.delete_user(pulse_user.username)
except pulse_management.PulseManagementException as e:
details['message'] = str(e)
mozdef.log(
mozdef.ERROR,
mozdef.OTHER,
'Error deleting Pulse user',
details=details,
)
return jsonify(ok=False)
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Pulse user deleted',
details=details,
)
db_session.delete(pulse_user)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def delete_queue(queue_name):
queue = Queue.query.get(queue_name)
if queue and (g.user.admin or
(queue.owner and g.user in queue.owner.owners)):
details = {
'queuename': queue_name,
'username': g.user.email,
}
try:
pulse_management.delete_queue(vhost='/', queue=queue.name)
except pulse_management.PulseManagementException as e:
details['message'] = str(e)
mozdef.log(
mozdef.ERROR,
mozdef.OTHER,
'Error deleting queue',
details=details,
tags=['queue'],
)
return jsonify(ok=False)
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Deleting queue',
details=details,
tags=['queue'],
)
db_session.delete(queue)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def clear_deleted_bindings(self, queue_name, queue_bindings):
db_bindings = Binding.query.filter(Binding.queue_name == queue_name)
# Filter bindings that are in the database but no longer on RabbitMQ.
alive_bindings_names = {
Binding.as_string(b['source'], b['routing_key'])
for b in queue_bindings
}
deleted_bindings = {
b
for b in db_bindings if b.name not in alive_bindings_names
}
# Delete those bindings.
for binding in deleted_bindings:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Binding no longer exists.',
details={
'queuename': queue_name,
'binding': binding.name,
},
tags=['queue'],
)
db_session.delete(binding)
def notify_unknown_error(self):
"""Log and email to admin(s) that an unexpected error occurred.
Note that this function expects to be called from within an
exception handler.
"""
ex_details = traceback.format_exc()
errmsg = '''Unknown error occurred.
Rabbit URL: {0}
Rabbit user: {1}
Error:
{2}
'''.format(config.rabbit_management_url, config.rabbit_user, ex_details)
mozdef.log(
mozdef.ERROR,
mozdef.OTHER,
'Unknown error occurred.',
details={
'managementurl': config.rabbit_management_url,
'managementuser': config.rabbit_user,
'message': ex_details,
},
)
if self.emails and not self._unknown_error_notified:
admins = list(User.query.filter_by(admin=True))
subject = "PulseGuardian error: Unknown error"
self._sendemail(subject=subject, to_users=admins, text_data=errmsg)
self._unknown_error_notified = True
def delete_rabbitmq_account(rabbitmq_username):
rabbitmq_account = RabbitMQAccount.query.filter(
RabbitMQAccount.username == rabbitmq_username).first()
if rabbitmq_account and (g.user.admin or g.user in
rabbitmq_account.owners):
details = {
'username': g.user.email,
'rabbitmqusername': rabbitmq_username,
}
try:
pulse_management.delete_user(rabbitmq_account.username)
except pulse_management.PulseManagementException as e:
details['message'] = str(e)
mozdef.log(
mozdef.ERROR,
mozdef.OTHER,
'Error deleting RabbitMQ account',
details=details,
)
return jsonify(ok=False)
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'RabbitMQ account deleted',
details=details,
)
db_session.delete(rabbitmq_account)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def clear_deleted_queues(self, queues, all_bindings):
db_queues = Queue.query.all()
# Filter queues that are in the database but no longer on RabbitMQ.
alive_queues_names = {q['name'] for q in queues}
deleted_queues = {q for q in db_queues
if q.name not in alive_queues_names}
# Delete those queues.
for queue in deleted_queues:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Queue no longer exists.',
details={'queuename': queue.name},
tags=['queue'],
)
db_session.delete(queue)
# Clean up bindings on queues that are not deleted.
for queue_name in alive_queues_names:
bindings = self.get_queue_bindings(all_bindings, queue_name)
self.clear_deleted_bindings(queue_name, bindings)
db_session.commit()
def notify_unknown_error(self):
"""Log and email to admin(s) that an unexpected error occurred.
Note that this function expects to be called from within an
exception handler.
"""
ex_details = traceback.format_exc()
errmsg = '''Unknown error occurred.
Rabbit URL: {0}
Rabbit user: {1}
Error:
{2}
'''.format(config.rabbit_management_url, config.rabbit_user, ex_details)
mozdef.log(
mozdef.ERROR,
mozdef.OTHER,
'Unknown error occurred.',
details={
'managementurl': config.rabbit_management_url,
'managementuser': config.rabbit_user,
'message': ex_details,
},
)
if self.emails and not self._unknown_error_notified:
admins = list(User.query.filter_by(admin=True))
subject = "PulseGuardian error: Unknown error"
self._sendemail(
subject=subject, to_users=admins, text_data=errmsg)
self._unknown_error_notified = True
def clear_deleted_queues(self, queues, all_bindings):
db_queues = Queue.query.all()
# Filter queues that are in the database but no longer on RabbitMQ.
alive_queues_names = {q['name'] for q in queues}
deleted_queues = {
q
for q in db_queues if q.name not in alive_queues_names
}
# Delete those queues.
for queue in deleted_queues:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Queue no longer exists.',
details={'queuename': queue.name},
tags=['queue'],
)
db_session.delete(queue)
# Clean up bindings on queues that are not deleted.
for queue_name in alive_queues_names:
bindings = self.get_queue_bindings(all_bindings, queue_name)
self.clear_deleted_bindings(queue_name, bindings)
db_session.commit()
def delete_queue(queue_name):
queue = Queue.query.get(queue_name)
if queue and (g.user.admin or
(queue.owner and g.user in queue.owner.owners)):
details = {
'queuename': queue_name,
'username': g.user.email,
}
try:
pulse_management.delete_queue(vhost='/', queue=queue.name)
except pulse_management.PulseManagementException as e:
details['message'] = str(e)
mozdef.log(
mozdef.ERROR,
mozdef.OTHER,
'Error deleting queue',
details=details,
tags=['queue'],
)
return jsonify(ok=False)
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Deleting queue',
details=details,
tags=['queue'],
)
db_session.delete(queue)
db_session.commit()
return jsonify(ok=True)
return jsonify(ok=False)
def init_db():
while True:
try:
Base.metadata.create_all(bind=engine)
except OperationalError as e:
mozdef.log(
mozdef.NOTICE,
mozdef.STARTUP,
'Failed to connect to database. Retrying...',
details={
'error': str(e),
},
)
time.sleep(5)
else:
break
def init_db():
while True:
try:
Base.metadata.create_all(bind=engine)
except OperationalError as e:
mozdef.log(
mozdef.NOTICE,
mozdef.STARTUP,
'Failed to connect to database. Retrying...',
details={
'error': str(e),
},
)
time.sleep(5)
else:
break
def monitor_queues(self, queues, all_bindings):
for queue_data in queues:
# Updating the queue's information in the database (owner, size).
queue = self.update_queue_information(queue_data, all_bindings)
if not queue:
continue
# If a queue is over the deletion size and ``unbounded`` is
# False (the default), then delete it regardless of it having
# an owner or not
# If ``unbounded`` is True, then let it grow indefinitely.
if queue.size > self.del_queue_size and not queue.unbounded:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Deleting queue.',
details=self._queue_details_dict(queue),
tags=['queue'],
)
if queue.owner and queue.owner.owners:
self.deletion_email(queue.owner.owners, queue_data)
if self.on_delete:
self.on_delete(queue.name)
pulse_management.delete_queue(vhost=queue_data['vhost'],
queue=queue.name)
db_session.delete(queue)
db_session.commit()
continue
if queue.owner is None or not queue.owner.owners:
continue
if queue.size > self.warn_queue_size and not queue.warned:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Queue-size warning.',
details=self._queue_details_dict(queue),
tags=['queue'],
)
queue.warned = True
if self.on_warn:
self.on_warn(queue.name)
self.warning_email(queue.owner.owners, queue_data)
elif queue.size <= self.warn_queue_size and queue.warned:
# A previously warned queue got out of the warning threshold;
# its owner should not be warned again.
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Queue-size recovered.',
details=self._queue_details_dict(queue),
tags=['queue'],
)
queue.warned = False
self.back_to_normal_email(queue.owner.owners, queue_data)
# Commit any changes to the queue.
db_session.add(queue)
db_session.commit()
def monitor_queues(self, queues, all_bindings):
for queue_data in queues:
# Updating the queue's information in the database (owner, size).
queue = self.update_queue_information(queue_data, all_bindings)
if not queue:
continue
# If a queue is over the deletion size and ``unbounded`` is
# False (the default), then delete it regardless of it having
# an owner or not
# If ``unbounded`` is True, then let it grow indefinitely.
if queue.size > self.del_queue_size and not queue.unbounded:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Deleting queue.',
details=self._queue_details_dict(queue),
tags=['queue'],
)
if queue.owner and queue.owner.owners:
self.deletion_email(queue.owner.owners, queue_data)
if self.on_delete:
self.on_delete(queue.name)
pulse_management.delete_queue(vhost=queue_data['vhost'],
queue=queue.name)
db_session.delete(queue)
db_session.commit()
continue
if queue.owner is None or not queue.owner.owners:
continue
if queue.size > self.warn_queue_size and not queue.warned:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Queue-size warning.',
details=self._queue_details_dict(queue),
tags=['queue'],
)
queue.warned = True
if self.on_warn:
self.on_warn(queue.name)
self.warning_email(queue.owner.owners, queue_data)
elif queue.size <= self.warn_queue_size and queue.warned:
# A previously warned queue got out of the warning threshold;
# its owner should not be warned again.
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Queue-size recovered.',
details=self._queue_details_dict(queue),
tags=['queue'],
)
queue.warned = False
self.back_to_normal_email(queue.owner.owners, queue_data)
# Commit any changes to the queue.
db_session.add(queue)
db_session.commit()
def clear_deleted_bindings(self, queue_name, queue_bindings):
db_bindings = Binding.query.filter(Binding.queue_name == queue_name)
# Filter bindings that are in the database but no longer on RabbitMQ.
alive_bindings_names = {Binding.as_string(b['source'],
b['routing_key'])
for b in queue_bindings}
deleted_bindings = {b for b in db_bindings
if b.name not in alive_bindings_names}
# Delete those bindings.
for binding in deleted_bindings:
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'Binding no longer exists.',
details={
'queuename': queue_name,
'binding': binding.name,
},
tags=['queue'],
)
db_session.delete(binding)
def guard(self):
mozdef.log(
mozdef.NOTICE,
mozdef.STARTUP,
'PulseGuardian started.',
)
while True:
mozdef.log(
mozdef.DEBUG,
mozdef.OTHER,
'Guard loop starting.',
)
try:
queues = pulse_management.queues()
bindings = pulse_management.bindings()
mozdef.log(
mozdef.DEBUG,
mozdef.OTHER,
'Fetched queue and binding data.',
)
if queues:
mozdef.log(
mozdef.DEBUG,
mozdef.OTHER,
'Monitoring queues.',
)
self.monitor_queues(queues, bindings)
mozdef.log(
mozdef.DEBUG,
mozdef.OTHER,
'Clearing deleted queues.',
)
self.clear_deleted_queues(queues, bindings)
if (self._connection_error_notified or
self._unknown_error_notified):
self._reset_notification_error_params()
except (requests.ConnectionError, socket.error):
self.notify_connection_error()
self._increase_interval()
except KeyboardInterrupt:
break
except Exception:
self.notify_unknown_error()
self._increase_interval()
mozdef.log(
mozdef.DEBUG,
mozdef.OTHER,
'Sleeping for {} seconds'.format(self._polling_interval),
)
time.sleep(self._polling_interval)
def update_queue_information(self, queue_data, all_bindings):
if 'messages' not in queue_data:
# FIXME: We should do something here, probably delete the queue,
# as it's in a weird state. More investigation is required.
# See bug 1066338.
return None
q_size, q_name, q_durable = (queue_data['messages'],
queue_data['name'],
queue_data['durable'])
queue = Queue.query.filter(Queue.name == q_name).first()
# If the queue doesn't exist in the db, create it.
if queue is None:
log_details = {
'queuename': q_name,
'queuesize': q_size,
'queuedurable': q_durable,
}
m = re.match('queue/([^/]+)/', q_name)
if not m:
log_details['valid'] = False
owner = None
elif (config.reserved_users_regex and
re.match(config.reserved_users_regex, m.group(1))):
# Ignore this queue entirely as we will see it again on the
# next iteration.
return None
else:
log_details['valid'] = True
owner_name = m.group(1)
owner = RabbitMQAccount.query.filter(
RabbitMQAccount.username == owner_name).first()
log_details['ownername'] = owner_name
log_details['newowner'] = not owner
# If the queue belongs to a pulse user that isn't in the
# pulseguardian database, add the user to the DB, owned by an
# admin.
if owner is None:
# RabbitMQAccount needs at least one owner as well, but
# since we have no way of knowing who really owns it, find
# the first admin, and set it to that.
user = User.query.filter(User.admin == True).first()
owner = RabbitMQAccount.new_user(owner_name, owners=user)
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'New queue.',
details=log_details,
tags=['queue'],
)
queue = Queue(name=q_name, owner=owner)
# add the queue bindings to the db.
bindings = self.get_queue_bindings(all_bindings, queue.name)
for binding in bindings:
db_binding = Binding.query.filter(
Binding.exchange == binding["source"],
Binding.routing_key == binding["routing_key"],
Binding.queue_name == queue.name
).first()
if not db_binding:
# need to create the binding in the DB
binding = Binding(exchange=binding["source"],
routing_key=binding["routing_key"],
queue_name=queue.name)
db_session.add(binding)
# Update the saved queue size.
queue.size = q_size
queue.durable = q_durable
db_session.add(queue)
db_session.commit()
return queue
def guard(self):
mozdef.log(
mozdef.NOTICE,
mozdef.STARTUP,
'PulseGuardian started.',
)
while True:
mozdef.log(
mozdef.DEBUG,
mozdef.OTHER,
'Guard loop starting.',
)
try:
queues = pulse_management.queues()
bindings = pulse_management.bindings()
mozdef.log(
mozdef.DEBUG,
mozdef.OTHER,
'Fetched queue and binding data.',
)
if queues:
mozdef.log(
mozdef.DEBUG,
mozdef.OTHER,
'Monitoring queues.',
)
self.monitor_queues(queues, bindings)
mozdef.log(
mozdef.DEBUG,
mozdef.OTHER,
'Clearing deleted queues.',
)
self.clear_deleted_queues(queues, bindings)
if (self._connection_error_notified
or self._unknown_error_notified):
self._reset_notification_error_params()
except (requests.ConnectionError, socket.error):
self.notify_connection_error()
self._increase_interval()
except KeyboardInterrupt:
break
except Exception:
self.notify_unknown_error()
self._increase_interval()
mozdef.log(
mozdef.DEBUG,
mozdef.OTHER,
'Sleeping for {} seconds'.format(self._polling_interval),
)
time.sleep(self._polling_interval)
def update_queue_information(self, queue_data, all_bindings):
if 'messages' not in queue_data:
# FIXME: We should do something here, probably delete the queue,
# as it's in a weird state. More investigation is required.
# See bug 1066338.
return None
q_size, q_name, q_durable = (queue_data['messages'],
queue_data['name'], queue_data['durable'])
queue = Queue.query.filter(Queue.name == q_name).first()
# If the queue doesn't exist in the db, create it.
if queue is None:
log_details = {
'queuename': q_name,
'queuesize': q_size,
'queuedurable': q_durable,
}
m = re.match('queue/([^/]+)/', q_name)
if not m:
log_details['valid'] = False
owner = None
elif (config.reserved_users_regex
and re.match(config.reserved_users_regex, m.group(1))):
# Ignore this queue entirely as we will see it again on the
# next iteration.
return None
else:
log_details['valid'] = True
owner_name = m.group(1)
owner = RabbitMQAccount.query.filter(
RabbitMQAccount.username == owner_name).first()
log_details['ownername'] = owner_name
log_details['newowner'] = not owner
# If the queue belongs to a pulse user that isn't in the
# pulseguardian database, add the user to the DB, owned by an
# admin.
if owner is None:
# RabbitMQAccount needs at least one owner as well, but
# since we have no way of knowing who really owns it, find
# the first admin, and set it to that.
user = User.query.filter(User.admin == True).first()
owner = RabbitMQAccount.new_user(owner_name, owners=user)
mozdef.log(
mozdef.NOTICE,
mozdef.OTHER,
'New queue.',
details=log_details,
tags=['queue'],
)
queue = Queue(name=q_name, owner=owner)
# add the queue bindings to the db.
bindings = self.get_queue_bindings(all_bindings, queue.name)
for binding in bindings:
db_binding = Binding.query.filter(
Binding.exchange == binding["source"],
Binding.routing_key == binding["routing_key"],
Binding.queue_name == queue.name).first()
if not db_binding:
# need to create the binding in the DB
binding = Binding(exchange=binding["source"],
routing_key=binding["routing_key"],
queue_name=queue.name)
db_session.add(binding)
# Update the saved queue size.
queue.size = q_size
queue.durable = q_durable
db_session.add(queue)
db_session.commit()
return queue
