def delegate_command(self, message_text):
response = ""
message_tokens = message_text.split()
command = message_tokens[0]
parameters = message_tokens[1:len(message_tokens)]
if command == '!help':
response = "\nHelp is on it's way...try these:\n"
for command_name, plugin in self.plugins.iteritems():
response += "\n{0} -- {1}".format(
command_name,
plugin['help_text']
)
else:
if command not in self.plugins:
response = "Unknown command: " + command + ". Try !help"
else:
plugin = self.plugins[command]
logger.info("Sending to {0}".format(plugin['plugin_class'].__module__))
try:
response = "\n" + plugin['plugin_class'].handle_command(parameters)
except Exception as e:
response = "\nReceived an error when processing your request."
logger.exception(e)
return response
def clearESCache():
es = esConnect(None)
indexes = es.get_open_indices()
# assums index names like events-YYYYMMDD etc.
# used to avoid operating on current indexes
dtNow = datetime.utcnow()
indexSuffix = date.strftime(dtNow, '%Y%m%d')
previousSuffix = date.strftime(dtNow - timedelta(days=1), '%Y%m%d')
for targetindex in sorted(indexes):
if indexSuffix not in targetindex and previousSuffix not in targetindex:
url = '{0}/{1}/_stats'.format(random.choice(options.esservers), targetindex)
r = requests.get(url)
if r.status_code == 200:
indexstats = json.loads(r.text)
if indexstats['_all']['total']['search']['query_current'] == 0:
fielddata = indexstats['_all']['total']['fielddata']['memory_size_in_bytes']
if fielddata > 0:
logger.info('target: {0}: field data {1}'.format(targetindex, indexstats['_all']['total']['fielddata']['memory_size_in_bytes']))
clearurl = '{0}/{1}/_cache/clear'.format(random.choice(options.esservers), targetindex)
clearRequest = requests.post(clearurl)
logger.info(clearRequest.text)
# stop at one?
if options.conservative:
return
else:
logger.debug('{0}: <ignoring due to current search > field data {1}'.format(targetindex, indexstats['_all']['total']['fielddata']['memory_size_in_bytes']))
else:
logger.error('{0} returned {1}'.format(url, r.status_code))
def run(self):
self.taskQueue.set_message_class(RawMessage)
while True:
try:
records = self.taskQueue.get_messages(self.options.prefetch)
for msg in records:
msg_body = msg.get_body()
try:
# get_body() should be json
message_json = json.loads(msg_body)
self.on_message(message_json)
# delete message from queue
self.taskQueue.delete_message(msg)
except ValueError:
logger.error('Invalid message, not JSON <dropping message and continuing>: %r' % msg_body)
self.taskQueue.delete_message(msg)
continue
time.sleep(.1)
except (SSLEOFError, SSLError, socket.error):
logger.info('Received network related error...reconnecting')
time.sleep(5)
self.connection, self.taskQueue = connect_sqs(
options.region,
options.accesskey,
options.secretkey,
options.taskexchange
)
self.taskQueue.set_message_class(RawMessage)
def main():
# connect and declare the message queue/kombu objects.
# only py-amqp supports ssl and doesn't recognize amqps
# so fix up the connection string accordingly
connString = 'amqp://{0}:{1}@{2}:{3}/{4}'.format(options.mquser, options.mqpassword, options.mqserver, options.mqport, options.mqvhost)
if options.mqprotocol == 'amqps':
mqSSL = True
else:
mqSSL = False
mqConn = Connection(connString, ssl=mqSSL)
# Task Exchange for events sent via http for us to normalize and post to elastic search
if options.mqack:
# conservative, store msgs to disk, ack each message
eventTaskExchange = Exchange(name=options.taskexchange, type='direct', durable=True, delivery_mode=2)
else:
# fast, transient delivery, store in memory only, auto-ack messages
eventTaskExchange = Exchange(name=options.taskexchange, type='direct', durable=True, delivery_mode=1)
eventTaskExchange(mqConn).declare()
# Queue for the exchange
if options.mqack:
eventTaskQueue = Queue(options.taskexchange, exchange=eventTaskExchange, routing_key=options.taskexchange, durable=True, no_ack=False)
else:
eventTaskQueue = Queue(options.taskexchange, exchange=eventTaskExchange, routing_key=options.taskexchange, durable=True, no_ack=True)
eventTaskQueue(mqConn).declare()
# topic exchange for anyone who wants to queue and listen for mozdef.event
eventTopicExchange = Exchange(name=options.eventexchange, type='topic', durable=False, delivery_mode=1)
eventTopicExchange(mqConn).declare()
if hasUWSGI:
logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
else:
logger.info('started without uwsgi')
# consume our queue and publish on the topic exchange
taskConsumer(mqConn, eventTaskQueue, eventTopicExchange, es).run()
def registerPlugins():
pluginList = list() # tuple of module,registration dict,priority
if os.path.exists('plugins'):
modules = pynsive.list_modules('plugins')
for mname in modules:
module = pynsive.import_module(mname)
importlib.reload(module)
if not module:
raise ImportError('Unable to load module {}'.format(mname))
else:
if 'message' in dir(module):
mclass = module.message()
mreg = mclass.registration
if type(mreg) != list:
raise ImportError(
'Plugin {0} registration needs to be a list'.
format(mname))
if 'priority' in dir(mclass):
mpriority = mclass.priority
else:
mpriority = 100
if isinstance(mreg, list):
logger.info(
'[*] plugin {0} registered to receive messages with {1}'
.format(mname, mreg))
pluginList.append((mclass, mreg, mpriority))
return pluginList
def main():
if options.checkjvmmemory:
if isJVMMemoryHigh():
logger.info('initiating cache clearing')
clearESCache()
else:
clearESCache()
def run(self):
while True:
try:
records = self.sqs_queue.receive_messages(
MaxNumberOfMessages=options.prefetch)
for msg in records:
msg_body = msg.body
try:
# get_body() should be json
message_json = json.loads(msg_body)
self.on_message(message_json)
# delete message from queue
msg.delete()
except ValueError:
logger.error(
'Invalid message, not JSON <dropping message and continuing>: %r'
% msg_body)
msg.delete()
continue
time.sleep(options.sleep_time)
except (SSLEOFError, SSLError, socket.error):
logger.info('Received network related error...reconnecting')
time.sleep(5)
self.sqs_queue = connect_sqs(options.region, options.accesskey,
options.secretkey,
options.taskexchange)
def delegate_command(self, message_text):
response = ""
message_tokens = message_text.split()
command = message_tokens[0]
parameters = message_tokens[1:len(message_tokens)]
if command == '!help':
response = "\nHelp is on it's way...try these:\n"
for command_name, plugin in self.plugins.iteritems():
response += "\n{0} -- {1}".format(command_name,
plugin['help_text'])
else:
if command not in self.plugins:
response = "Unknown command: " + command + ". Try !help"
else:
plugin = self.plugins[command]
logger.info("Sending to {0}".format(
plugin['plugin_class'].__module__))
try:
response = "\n" + plugin['plugin_class'].handle_command(
parameters)
except Exception as e:
response = "\nReceived an error when processing your request."
logger.exception(e)
return response
def identify_plugins(self, enabled_plugins):
if not os.path.exists(self.plugin_location):
return []
module_name = os.path.basename(self.plugin_location)
root_plugin_directory = self.plugin_location
plugin_manager = pynsive.PluginManager()
plugin_manager.plug_into(root_plugin_directory)
plugins = []
found_modules = pynsive.list_modules(module_name)
for found_module in found_modules:
module_filename, module_name = found_module.split('.')
if enabled_plugins is not None and module_name not in enabled_plugins:
# Skip this plugin since it's not listed as enabled plugins
# as long as we have specified some enabled plugins though
# this allows us to specify no specific plugins and get all of them
continue
module_obj = pynsive.import_module(found_module)
reload(module_obj)
plugin_class_obj = module_obj.Command()
logger.info('Plugin {0} registered to receive command with {1}'.format(module_name, plugin_class_obj.command_name))
plugins.append(
{
'plugin_class': plugin_class_obj,
'command_name': plugin_class_obj.command_name,
'help_text': plugin_class_obj.help_text
}
)
return plugins
def main():
# connect and declare the message queue/kombu objects.
# only py-amqp supports ssl and doesn't recognize amqps
# so fix up the connection string accordingly
connString = "amqp://{0}:{1}@{2}:{3}/{4}".format(options.mquser,
options.mqpassword,
options.mqserver,
options.mqport,
options.mqvhost)
if options.mqprotocol == "amqps":
mqSSL = True
else:
mqSSL = False
mqConn = Connection(connString, ssl=mqSSL)
# Task Exchange for events sent via http for us to normalize and post to elastic search
if options.mqack:
# conservative, store msgs to disk, ack each message
eventTaskExchange = Exchange(name=options.taskexchange,
type="direct",
durable=True,
delivery_mode=2)
else:
# fast, transient delivery, store in memory only, auto-ack messages
eventTaskExchange = Exchange(name=options.taskexchange,
type="direct",
durable=True,
delivery_mode=1)
eventTaskExchange(mqConn).declare()
# Queue for the exchange
if options.mqack:
eventTaskQueue = Queue(
options.taskexchange,
exchange=eventTaskExchange,
routing_key=options.taskexchange,
durable=True,
no_ack=False,
)
else:
eventTaskQueue = Queue(
options.taskexchange,
exchange=eventTaskExchange,
routing_key=options.taskexchange,
durable=True,
no_ack=True,
)
eventTaskQueue(mqConn).declare()
# topic exchange for anyone who wants to queue and listen for mozdef.event
eventTopicExchange = Exchange(name=options.eventexchange,
type="topic",
durable=False,
delivery_mode=1)
eventTopicExchange(mqConn).declare()
if hasUWSGI:
logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
else:
logger.info("started without uwsgi")
# consume our queue and publish on the topic exchange
taskConsumer(mqConn, eventTaskQueue, eventTopicExchange, es).run()
def isJVMMemoryHigh():
url = "{0}/_nodes/stats?pretty=true".format(
random.choice(options.esservers))
r = requests.get(url)
logger.debug(r)
if r.status_code == 200:
nodestats = r.json()
for node in nodestats['nodes']:
loadaverage = nodestats['nodes'][node]['os']['cpu']['load_average']
cpuusage = nodestats['nodes'][node]['os']['cpu']['percent']
nodename = nodestats['nodes'][node]['name']
jvmused = nodestats['nodes'][node]['jvm']['mem'][
'heap_used_percent']
logger.debug('{0}: cpu {1}% jvm {2}% load average: {3}'.format(
nodename, cpuusage, jvmused, loadaverage))
if jvmused > options.jvmlimit:
logger.info(
'{0}: cpu {1}% jvm {2}% load average: {3} recommending cache clear'
.format(nodename, cpuusage, jvmused, loadaverage))
return True
return False
else:
logger.error(r)
return False
def main():
if options.checkjvmmemory:
if isJVMMemoryHigh():
logger.info('initiating cache clearing')
clearESCache()
else:
clearESCache()
def clearESCache():
es = esConnect(None)
indexes = es.get_indices()
# assums index names like events-YYYYMMDD etc.
# used to avoid operating on current indexes
dtNow = datetime.utcnow()
indexSuffix = date.strftime(dtNow, '%Y%m%d')
previousSuffix = date.strftime(dtNow - timedelta(days=1), '%Y%m%d')
for targetindex in sorted(indexes):
if indexSuffix not in targetindex and previousSuffix not in targetindex:
url = '{0}/{1}/_stats'.format(random.choice(options.esservers), targetindex)
r = requests.get(url)
if r.status_code == 200:
indexstats = json.loads(r.text)
if indexstats['_all']['total']['search']['query_current'] == 0:
fielddata = indexstats['_all']['total']['fielddata']['memory_size_in_bytes']
if fielddata > 0:
logger.info('target: {0}: field data {1}'.format(targetindex, indexstats['_all']['total']['fielddata']['memory_size_in_bytes']))
clearurl = '{0}/{1}/_cache/clear'.format(random.choice(options.esservers), targetindex)
clearRequest = requests.post(clearurl)
logger.info(clearRequest.text)
# stop at one?
if options.conservative:
return
else:
logger.debug('{0}: <ignoring due to current search > field data {1}'.format(targetindex, indexstats['_all']['total']['fielddata']['memory_size_in_bytes']))
else:
logger.error('{0} returned {1}'.format(url, r.status_code))
def run(self):
if self.slack_client.rtm_connect():
logger.info("Bot connected to slack")
self.post_welcome_message(random.choice(greetings))
self.listen_for_messages()
else:
logger.error("Unable to connect to slack")
sys.exit(1)
def run(self):
if self.slack_client.rtm_connect():
logger.info("Bot connected to slack")
self.post_welcome_message(random.choice(greetings))
self.listen_for_messages()
else:
logger.error("Unable to connect to slack")
sys.exit(1)
def send_message_to_plugin(self, plugin_class, message, metadata=None):
if 'utctimestamp' in message and 'summary' in message:
message_log_str = '{0} received message: ({1}) {2}'.format(
plugin_class.__module__, message['utctimestamp'],
message['summary'])
logger.info(message_log_str)
return plugin_class.onMessage(message), metadata
def listen_for_messages(self):
while True:
for slack_message in self.slack_client.rtm_read():
message_type = slack_message.get('type')
if message_type == 'desktop_notification':
logger.info("Received message: {0}".format(slack_message['content']))
self.handle_message(slack_message)
time.sleep(1)
def listen_for_messages(self):
while True:
for slack_message in self.slack_client.rtm_read():
message_type = slack_message.get('type')
if message_type == 'desktop_notification':
logger.info("Received message: {0}".format(
slack_message['content']))
self.handle_message(slack_message)
time.sleep(1)
def run(self):
while True:
try:
records = self.sqs_queue.receive_messages(
MaxNumberOfMessages=options.prefetch)
for msg in records:
# msg.id is the id,
# get_body() should be json
# pre process the message a bit
tmp = msg.body
try:
msgbody = json.loads(tmp)
except ValueError:
# If Boto wrote to the queue, it might be base64 encoded, so let's decode that
try:
tmp = base64.b64decode(tmp)
msgbody = json.loads(tmp)
except Exception as e:
logger.error(
"Invalid message, not JSON <dropping message and continuing>: %r"
% msg.get_body())
msg.delete()
continue
# If this is still not a dict,
# let's just drop the message and move on
if type(msgbody) is not dict:
logger.debug(
"Message is not a dictionary, dropping message.")
msg.delete()
continue
event = dict()
event = msgbody
if "tags" in event:
event["tags"].extend([options.taskexchange])
else:
event["tags"] = [options.taskexchange]
# process message
self.on_message(event, msg)
# delete message from queue
msg.delete()
time.sleep(options.sleep_time)
except ValueError as e:
logger.exception("Exception while handling message: %r" % e)
msg.delete()
except (SSLEOFError, SSLError, socket.error):
logger.info("Received network related error...reconnecting")
time.sleep(5)
self.sqs_queue = connect_sqs(options.region, options.accesskey,
options.secretkey,
options.taskexchange)
def main():
if hasUWSGI:
logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
else:
logger.info('started without uwsgi')
# establish api interface with papertrail
ptRequestor = PTRequestor(options.ptapikey, evmax=options.ptquerymax)
# consume our queue
taskConsumer(ptRequestor, es).run()
def main():
if hasUWSGI:
logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
else:
logger.info('started without uwsgi')
# establish api interface with papertrail
ptRequestor = PTRequestor(options.ptapikey, evmax=options.ptquerymax)
# consume our queue
taskConsumer(ptRequestor, es).run()
def main():
if hasUWSGI:
logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
else:
logger.info("started without uwsgi")
if options.mqprotocol not in ("pubsub"):
logger.error("Can only process pubsub queues, terminating")
sys.exit(1)
# connect to GCP and consume our queue
PubSubtaskConsumer(es, options).run()
def listen_for_messages(self):
while True:
try:
for slack_message in self.slack_client.rtm_read():
message_type = slack_message.get('type')
if message_type == 'desktop_notification':
logger.info("Received message: {0}".format(slack_message['content']))
self.handle_message(slack_message)
except websocket.WebSocketConnectionClosedException:
logger.info("Received WebSocketConnectionClosedException exception...reconnecting")
time.sleep(3)
self.slack_client.rtm_connect()
time.sleep(1)
def listen_for_messages(self):
while True:
try:
for slack_message in self.slack_client.rtm_read():
message_type = slack_message.get('type')
if message_type == 'desktop_notification':
logger.info("Received message: {0}".format(slack_message['content']))
self.handle_message(slack_message)
except websocket.WebSocketConnectionClosedException:
logger.info("Received WebSocketConnectionClosedException exception...reconnecting")
time.sleep(3)
self.slack_client.rtm_connect()
time.sleep(1)
def on_message(self, body, message):
try:
# just to be safe..check what we were sent.
if isinstance(body, dict):
full_body = body
elif isinstance(body, str):
try:
full_body = json.loads(body)
except ValueError:
# not json..ack but log the message
logger.exception(
"mozdefbot_slack exception: unknown body type received %r"
% body)
return
else:
logger.exception(
"mozdefbot_slack exception: unknown body type received %r"
% body)
return
body_dict = full_body
# Handle messages that have full ES dict
if '_source' in full_body:
body_dict = full_body['_source']
if 'notify_mozdefbot' in body_dict and body_dict[
'notify_mozdefbot'] is False:
# If the alert tells us to not notify, then don't post message
message.ack()
return
# process valid message
# see where we send this alert
channel = options.default_alert_channel
if 'ircchannel' in body_dict:
if body_dict['ircchannel'] in options.channels:
channel = body_dict['ircchannel']
# see if we need to delay a bit before sending the alert, to avoid
# flooding the channel
if self.lastalert is not None:
delta = toUTC(datetime.now()) - self.lastalert
logger.info(
'new alert, delta since last is {}\n'.format(delta))
if delta.seconds < 2:
logger.info('throttling before writing next alert\n')
time.sleep(1)
self.lastalert = toUTC(datetime.now())
if len(body_dict['summary']) > 450:
logger.info('alert is more than 450 bytes, truncating\n')
body_dict[
'summary'] = body_dict['summary'][:450] + ' truncated...'
logger.info("Posting alert: {0}".format(body_dict['summary']))
self.bot.post_alert_message(body_dict, channel)
message.ack()
except ValueError as e:
logger.exception(
"mozdefbot_slack exception while processing events queue %r" %
e)
def run(self):
while True:
try:
records = self.sqs_queue.receive_messages(
MaxNumberOfMessages=options.prefetch)
for msg in records:
body_message = msg.body
event = json.loads(body_message)
if not event["Message"]:
logger.error(
"Invalid message format for cloudtrail SQS messages"
)
logger.error("Malformed Message: %r" % body_message)
continue
if event["Message"] == "CloudTrail validation message.":
# We don't care about these messages
continue
message_json = json.loads(event["Message"])
if "s3ObjectKey" not in message_json:
logger.error(
"Invalid message format, expecting an s3ObjectKey in Message"
)
logger.error("Malformed Message: %r" % body_message)
continue
s3_log_files = message_json["s3ObjectKey"]
for log_file in s3_log_files:
logger.debug("Downloading and parsing " + log_file)
s3_obj = self.s3_client.get_object(
Bucket=message_json["s3Bucket"], Key=log_file)
events = self.parse_s3_file(s3_obj)
for event in events:
self.on_message(event)
msg.delete()
except (SSLEOFError, SSLError, socket.error):
logger.info("Received network related error...reconnecting")
time.sleep(5)
self.sqs_queue = connect_sqs(
region_name=options.region,
aws_access_key_id=options.accesskey,
aws_secret_access_key=options.secretkey,
task_exchange=options.taskexchange,
)
time.sleep(options.sleep_time)
def run(self):
self.taskQueue.set_message_class(RawMessage)
while True:
try:
records = self.taskQueue.get_messages(options.prefetch)
for msg in records:
body_message = msg.get_body()
event = json.loads(body_message)
if not event['Message']:
logger.error(
'Invalid message format for cloudtrail SQS messages'
)
logger.error('Malformed Message: %r' % body_message)
continue
if event['Message'] == 'CloudTrail validation message.':
# We don't care about these messages
continue
message_json = json.loads(event['Message'])
if 's3ObjectKey' not in message_json:
logger.error(
'Invalid message format, expecting an s3ObjectKey in Message'
)
logger.error('Malformed Message: %r' % body_message)
continue
s3_log_files = message_json['s3ObjectKey']
for log_file in s3_log_files:
logger.debug('Downloading and parsing ' + log_file)
bucket = self.s3_connection.get_bucket(
message_json['s3Bucket'])
log_file_lookup = bucket.lookup(log_file)
events = self.process_file(log_file_lookup)
for event in events:
self.on_message(event)
self.taskQueue.delete_message(msg)
except (SSLEOFError, SSLError, socket.error):
logger.info('Received network related error...reconnecting')
time.sleep(5)
self.connection, self.taskQueue = connect_sqs(
task_exchange=options.taskexchange,
**get_aws_credentials(options.region, options.accesskey,
options.secretkey))
self.taskQueue.set_message_class(RawMessage)
def main():
if hasUWSGI:
logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
else:
logger.info('started without uwsgi')
if options.mqprotocol not in ('sqs'):
logger.error('Can only process SQS queues, terminating')
sys.exit(1)
mqConn, eventTaskQueue = connect_sqs(options.region, options.accesskey,
options.secretkey,
options.taskexchange)
# consume our queue
taskConsumer(mqConn, eventTaskQueue, es, options).run()
def main():
if hasUWSGI:
logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
else:
logger.info('started without uwsgi')
if options.mqprotocol not in ('sqs'):
logger.error('Can only process SQS queues, terminating')
sys.exit(1)
sqs_queue = connect_sqs(region_name=options.region,
aws_access_key_id=options.accesskey,
aws_secret_access_key=options.secretkey,
task_exchange=options.taskexchange)
# consume our queue
taskConsumer(sqs_queue, es, options).run()
def run(self):
while True:
try:
records = self.sqs_queue.receive_messages(
MaxNumberOfMessages=options.prefetch)
for msg in records:
body_message = msg.body
event = json.loads(body_message)
if not event['Message']:
logger.error(
'Invalid message format for cloudtrail SQS messages'
)
logger.error('Malformed Message: %r' % body_message)
continue
if event['Message'] == 'CloudTrail validation message.':
# We don't care about these messages
continue
message_json = json.loads(event['Message'])
if 's3ObjectKey' not in message_json:
logger.error(
'Invalid message format, expecting an s3ObjectKey in Message'
)
logger.error('Malformed Message: %r' % body_message)
continue
s3_log_files = message_json['s3ObjectKey']
for log_file in s3_log_files:
logger.debug('Downloading and parsing ' + log_file)
s3_obj = self.s3_client.get_object(
Bucket=message_json['s3Bucket'], Key=log_file)
events = self.parse_s3_file(s3_obj)
for event in events:
self.on_message(event)
msg.delete()
except (SSLEOFError, SSLError, socket.error):
logger.info('Received network related error...reconnecting')
time.sleep(5)
self.sqs_queue = connect_sqs(
task_exchange=options.taskexchange,
**get_aws_credentials(options.region, options.accesskey,
options.secretkey))
time.sleep(options.sleep_time)
def registerPlugins():
'''walk the plugins directory
and register modules in pluginList
as a tuple: (mfile, mname, mdescription, mreg, mpriority, mclass)
'''
plugin_location = os.path.join(os.path.dirname(__file__), "plugins")
module_name = os.path.basename(plugin_location)
root_plugin_directory = os.path.join(plugin_location, '..')
plugin_manager = pynsive.PluginManager()
plugin_manager.plug_into(root_plugin_directory)
if os.path.exists(plugin_location):
modules = pynsive.list_modules(module_name)
for mfile in modules:
module = pynsive.import_module(mfile)
importlib.reload(module)
if not module:
raise ImportError('Unable to load module {}'.format(mfile))
else:
if 'message' in dir(module):
mclass = module.message()
mreg = mclass.registration
mclass.restoptions = options.__dict__
if 'priority' in dir(mclass):
mpriority = mclass.priority
else:
mpriority = 100
if 'name' in dir(mclass):
mname = mclass.name
else:
mname = mfile
if 'description' in dir(mclass):
mdescription = mclass.description
else:
mdescription = mfile
if isinstance(mreg, list):
logger.info(
'[*] plugin {0} registered to receive messages from /{1}'
.format(mfile, mreg))
pluginList.append((mfile, mname, mdescription, mreg,
mpriority, mclass))
def run(self):
self.taskQueue.set_message_class(RawMessage)
while True:
try:
records = self.taskQueue.get_messages(options.prefetch)
for msg in records:
body_message = msg.get_body()
event = json.loads(body_message)
if not event['Message']:
logger.error('Invalid message format for cloudtrail SQS messages')
logger.error('Malformed Message: %r' % body_message)
continue
if event['Message'] == 'CloudTrail validation message.':
# We don't care about these messages
continue
message_json = json.loads(event['Message'])
if 's3ObjectKey' not in message_json:
logger.error('Invalid message format, expecting an s3ObjectKey in Message')
logger.error('Malformed Message: %r' % body_message)
continue
s3_log_files = message_json['s3ObjectKey']
for log_file in s3_log_files:
logger.debug('Downloading and parsing ' + log_file)
bucket = self.s3_connection.get_bucket(message_json['s3Bucket'])
log_file_lookup = bucket.lookup(log_file)
events = self.process_file(log_file_lookup)
for event in events:
self.on_message(event)
self.taskQueue.delete_message(msg)
except (SSLEOFError, SSLError, socket.error):
logger.info('Received network related error...reconnecting')
time.sleep(5)
self.connection, self.taskQueue = connect_sqs(
task_exchange=options.taskexchange,
**get_aws_credentials(
options.region,
options.accesskey,
options.secretkey))
self.taskQueue.set_message_class(RawMessage)
def main():
if hasUWSGI:
logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
else:
logger.info('started without uwsgi')
if options.mqprotocol not in ('sqs'):
logger.error('Can only process SQS queues, terminating')
sys.exit(1)
sqs_conn, eventTaskQueue = connect_sqs(
task_exchange=options.taskexchange,
**get_aws_credentials(
options.region,
options.accesskey,
options.secretkey))
# consume our queue
taskConsumer(sqs_conn, eventTaskQueue, es, options).run()
def main():
# meant only to talk to SQS using boto
# and process events as json.
if hasUWSGI:
logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
else:
logger.info('started without uwsgi')
if options.mqprotocol not in ('sqs'):
logger.error('Can only process SQS queues, terminating')
sys.exit(1)
sqs_conn, eventTaskQueue = connect_sqs(
task_exchange=options.taskexchange,
**get_aws_credentials(options.region, options.accesskey,
options.secretkey))
# consume our queue
taskConsumer(sqs_conn, eventTaskQueue, es).run()
def registerPlugins():
'''walk the plugins directory
and register modules in pluginList
as a tuple: (mfile, mname, mdescription, mreg, mpriority, mclass)
'''
plugin_location = os.path.join(os.path.dirname(__file__), "plugins")
module_name = os.path.basename(plugin_location)
root_plugin_directory = os.path.join(plugin_location, '..')
plugin_manager = pynsive.PluginManager()
plugin_manager.plug_into(root_plugin_directory)
if os.path.exists(plugin_location):
modules = pynsive.list_modules(module_name)
for mfile in modules:
module = pynsive.import_module(mfile)
reload(module)
if not module:
raise ImportError('Unable to load module {}'.format(mfile))
else:
if 'message' in dir(module):
mclass = module.message()
mreg = mclass.registration
mclass.restoptions = options.__dict__
if 'priority' in dir(mclass):
mpriority = mclass.priority
else:
mpriority = 100
if 'name' in dir(mclass):
mname = mclass.name
else:
mname = mfile
if 'description' in dir(mclass):
mdescription = mclass.description
else:
mdescription = mfile
if isinstance(mreg, list):
logger.info('[*] plugin {0} registered to receive messages from /{1}'.format(mfile, mreg))
pluginList.append((mfile, mname, mdescription, mreg, mpriority, mclass))
def onMessage(self, alert):
source = alert['_source']
# Find the self.option that contains one of the message tags
selected_option = self.identify_option(source['tags'])
if selected_option is None:
logger.error("Unable to find config option for alert tags: {0}".format(source['tags']))
if 'summary' in source:
headers = {
'Content-type': 'application/json',
}
payload = hjson.dumpsJSON({
"service_key": "{0}".format(selected_option['service_key']),
"event_type": "trigger",
"description": source['summary'],
"client": "MozDef",
"client_url": "{0}/alert/{1}".format(self.options['web_url'], alert['_id']),
"contexts": [
{
"type": "link",
"href": "{0}".format(selected_option['doc']),
"text": "View runbook on mana"
}
]
})
headers = {
'Content-type': 'application/json',
}
resp = requests.post(
'https://events.pagerduty.com/generic/2010-04-15/create_event.json',
headers=headers,
data=payload,
)
if not resp.ok:
logger.exception("Received invalid response from pagerduty: {0} - {1}".format(resp.status_code, resp.text))
else:
logger.info("Triggered pagerduty notification for alert - {0}".format(alert['_id']))
return message
def isJVMMemoryHigh():
url = "{0}/_nodes/stats?pretty=true".format(random.choice(options.esservers))
r = requests.get(url)
logger.debug(r)
if r.status_code == 200:
nodestats = r.json()
for node in nodestats['nodes']:
loadaverage = nodestats['nodes'][node]['os']['cpu']['load_average']
cpuusage = nodestats['nodes'][node]['os']['cpu']['percent']
nodename = nodestats['nodes'][node]['name']
jvmused = nodestats['nodes'][node]['jvm']['mem']['heap_used_percent']
logger.debug('{0}: cpu {1}% jvm {2}% load average: {3}'.format(nodename, cpuusage, jvmused, loadaverage))
if jvmused > options.jvmlimit:
logger.info('{0}: cpu {1}% jvm {2}% load average: {3} recommending cache clear'.format(nodename, cpuusage, jvmused, loadaverage))
return True
return False
else:
logger.error(r)
return False
def registerPlugins():
pluginList = list() # tuple of module,registration dict,priority
if os.path.exists('plugins'):
modules = pynsive.list_modules('plugins')
for mname in modules:
module = pynsive.import_module(mname)
reload(module)
if not module:
raise ImportError('Unable to load module {}'.format(mname))
else:
if 'message' in dir(module):
mclass = module.message()
mreg = mclass.registration
if 'priority' in dir(mclass):
mpriority = mclass.priority
else:
mpriority = 100
if isinstance(mreg, list):
logger.info('[*] plugin {0} registered to receive messages with {1}'.format(mname, mreg))
pluginList.append((mclass, mreg, mpriority))
return pluginList
def main():
# meant only to talk to SQS using boto
# and process events as json.
if hasUWSGI:
logger.info("started as uwsgi mule {0}".format(uwsgi.mule_id()))
else:
logger.info("started without uwsgi")
if options.mqprotocol not in ("sqs"):
logger.error("Can only process SQS queues, terminating")
sys.exit(1)
sqs_queue = connect_sqs(
region_name=options.region,
aws_access_key_id=options.accesskey,
aws_secret_access_key=options.secretkey,
task_exchange=options.taskexchange,
)
# consume our queue
taskConsumer(sqs_queue, es).run()
def onMessage(self, request, response):
'''
request: http://bottlepy.org/docs/dev/api.html#the-request-object
response: http://bottlepy.org/docs/dev/api.html#the-response-object
'''
# format/validate request.json:
ipaddress = None
sendToBHVPC = False
# loop through the fields of the form
# and fill in our values
try:
for field in request.json:
# were we checked?
if self.name in field:
sendToBHVPC = field[self.name]
if 'ipaddress' in field:
ipaddress = field['ipaddress']
# are we configured?
if self.multioptions is None:
logger.error(
"Customs server blockip requested but not configured\n")
sendToBHVPC = False
if sendToBHVPC and ipaddress is not None:
# figure out the CIDR mask
if isIPv4(ipaddress) or isIPv6(ipaddress):
ipcidr = netaddr.IPNetwork(ipaddress)
if not ipcidr.ip.is_loopback() \
and not ipcidr.ip.is_private() \
and not ipcidr.ip.is_reserved():
ipaddress = str(ipcidr.cidr)
self.addBlackholeEntry(ipaddress)
logger.info('Blackholed {0}\n'.format(ipaddress))
except Exception as e:
logger.error('Error handling request.json %r \n' % (e))
return (request, response)
def registerPlugins():
'''walk the ./plugins directory
and register modules in pluginList
as a tuple: (mfile, mname, mdescription, mreg, mpriority, mclass)
'''
plugin_manager = pynsive.PluginManager()
if os.path.exists('plugins'):
modules = pynsive.list_modules('plugins')
for mfile in modules:
module = pynsive.import_module(mfile)
reload(module)
if not module:
raise ImportError('Unable to load module {}'.format(mfile))
else:
if 'message' in dir(module):
mclass = module.message()
mreg = mclass.registration
mclass.restoptions = options
if 'priority' in dir(mclass):
mpriority = mclass.priority
else:
mpriority = 100
if 'name' in dir(mclass):
mname = mclass.name
else:
mname = mfile
if 'description' in dir(mclass):
mdescription = mclass.description
else:
mdescription = mfile
if isinstance(mreg, list):
logger.info('[*] plugin {0} registered to receive messages from /{1}'.format(mfile, mreg))
pluginList.append((mfile, mname, mdescription, mreg, mpriority, mclass))
def on_message(self, body, message):
try:
# just to be safe..check what we were sent.
if isinstance(body, dict):
body_dict = body
elif isinstance(body, str) or isinstance(body, unicode):
try:
body_dict = json.loads(body) # lets assume it's json
except ValueError as e:
# not json..ack but log the message
logger.exception("mozdefbot_slack exception: unknown body type received %r" % body)
return
else:
logger.exception("mozdefbot_slack exception: unknown body type received %r" % body)
return
if 'notify_mozdefbot' in body_dict and body_dict['notify_mozdefbot'] is False:
# If the alert tells us to not notify, then don't post message
message.ack()
return
# process valid message
# see where we send this alert
channel = options.default_alert_channel
if 'ircchannel' in body_dict:
if body_dict['ircchannel'] in options.channels:
channel = body_dict['ircchannel']
# see if we need to delay a bit before sending the alert, to avoid
# flooding the channel
if self.lastalert is not None:
delta = toUTC(datetime.now()) - self.lastalert
logger.info('new alert, delta since last is {}\n'.format(delta))
if delta.seconds < 2:
logger.info('throttling before writing next alert\n')
time.sleep(1)
self.lastalert = toUTC(datetime.now())
if len(body_dict['summary']) > 450:
logger.info('alert is more than 450 bytes, truncating\n')
body_dict['summary'] = body_dict['summary'][:450] + ' truncated...'
logger.info("Posting alert: {0}".format(body_dict['summary']))
self.bot.post_alert_message(body_dict, channel)
message.ack()
except ValueError as e:
logger.exception("mozdefbot_slack exception while processing events queue %r" % e)
def onMessage(self, request, response):
'''
request: http://bottlepy.org/docs/dev/api.html#the-request-object
response: http://bottlepy.org/docs/dev/api.html#the-response-object
'''
response.headers['X-PLUGIN'] = self.description
# Refresh the ip network list each time we get a message
self.options.ipwhitelist = self.parse_network_whitelist(
self.options.network_whitelist_file)
ipaddress = None
comment = None
duration = None
referenceID = None
userid = None
blockip = False
try:
# loop through the fields of the form
# and fill in our values
for field in request.json:
# were we checked?
if self.name in field:
blockip = field[self.name]
if 'ipaddress' in field:
ipaddress = field['ipaddress']
if 'duration' in field:
duration = field['duration']
if 'comment' in field:
comment = field['comment']
if 'referenceid' in field:
referenceID = field['referenceid']
if 'userid' in field:
userid = field['userid']
if blockip and ipaddress is not None:
# figure out the CIDR mask
if isIPv4(ipaddress) or isIPv6(ipaddress):
ipcidr = netaddr.IPNetwork(ipaddress)
if not ipcidr.ip.is_loopback() \
and not ipcidr.ip.is_private() \
and not ipcidr.ip.is_reserved():
whitelisted = False
for whitelist_range in self.options.ipwhitelist:
whitelist_network = netaddr.IPNetwork(
whitelist_range)
if ipcidr in whitelist_network:
whitelisted = True
logger.debug(
'{0} is whitelisted as part of {1}\n'.
format(ipcidr, whitelist_network))
if not whitelisted:
self.blockIP(str(ipcidr), comment, duration,
referenceID, userid)
logger.info(
'added {0} to blocklist\n'.format(ipaddress))
else:
logger.info(
'not adding {0} to blocklist, it was found in whitelist\n'
.format(ipaddress))
except Exception as e:
logger.error('Error handling request.json %r \n' % (e))
return (request, response)
def send_message_to_plugin(self, plugin_class, message, metadata=None):
if 'utctimestamp' in message and 'summary' in message:
message_log_str = u'{0} received message: ({1}) {2}'.format(plugin_class.__module__, message['utctimestamp'], message['summary'])
logger.info(message_log_str)
return plugin_class.onMessage(message), metadata
def run(self):
# Boto expects base64 encoded messages - but if the writer is not boto it's not necessarily base64 encoded
# Thus we've to detect that and decode or not decode accordingly
self.taskQueue.set_message_class(RawMessage)
while True:
try:
records = self.taskQueue.get_messages(options.prefetch)
for msg in records:
# msg.id is the id,
# get_body() should be json
# pre process the message a bit
tmp = msg.get_body()
try:
msgbody = json.loads(tmp)
except ValueError:
# If Boto wrote to the queue, it might be base64 encoded, so let's decode that
try:
tmp = base64.b64decode(tmp)
msgbody = json.loads(tmp)
except Exception as e:
logger.error('Invalid message, not JSON <dropping message and continuing>: %r' % msg.get_body())
self.taskQueue.delete_message(msg)
continue
# If this is still not a dict,
# let's just drop the message and move on
if type(msgbody) is not dict:
logger.debug("Message is not a dictionary, dropping message.")
self.taskQueue.delete_message(msg)
continue
event = dict()
event = msgbody
# Was this message sent by fluentd-sqs
fluentd_sqs_specific_fields = {
'az', 'instance_id', '__tag'}
if fluentd_sqs_specific_fields.issubset(
set(msgbody.keys())):
# Until we can influence fluentd-sqs to set the
# 'customendpoint' key before submitting to SQS, we'll
# need to do it here
# TODO : Change nubis fluentd output to include
# 'customendpoint'
event['customendpoint'] = True
if 'tags' in event:
event['tags'].extend([options.taskexchange])
else:
event['tags'] = [options.taskexchange]
# process message
self.on_message(event, msg)
# delete message from queue
self.taskQueue.delete_message(msg)
time.sleep(.1)
except ValueError as e:
logger.exception('Exception while handling message: %r' % e)
self.taskQueue.delete_message(msg)
except (SSLEOFError, SSLError, socket.error):
logger.info('Received network related error...reconnecting')
time.sleep(5)
self.connection, self.taskQueue = connect_sqs(
options.region,
options.accesskey,
options.secretkey,
options.taskexchange
)
self.taskQueue.set_message_class(RawMessage)
options.region = getConfig('region', '', options.configfile)
# How long to sleep between polling
options.sleep_time = getConfig('sleep_time', 0.1, options.configfile)
if __name__ == '__main__':
# configure ourselves
parser = OptionParser()
parser.add_option("-c",
dest='configfile',
default=sys.argv[0].replace('.py', '.conf'),
help="configuration file to use")
(options, args) = parser.parse_args()
initConfig()
initLogger(options)
# open ES connection globally so we don't waste time opening it per message
es = esConnect()
try:
main()
except KeyboardInterrupt as e:
logger.info("Exiting worker")
if options.esbulksize != 0:
es.finish_bulk()
except Exception as e:
if options.esbulksize != 0:
es.finish_bulk()
raise
def run(self):
while True:
try:
records = self.sqs_queue.receive_messages(
MaxNumberOfMessages=options.prefetch)
for msg in records:
# msg.id is the id,
# get_body() should be json
# pre process the message a bit
tmp = msg.body
try:
msgbody = json.loads(tmp)
except ValueError:
# If Boto wrote to the queue, it might be base64 encoded, so let's decode that
try:
tmp = base64.b64decode(tmp)
msgbody = json.loads(tmp)
except Exception as e:
logger.error(
'Invalid message, not JSON <dropping message and continuing>: %r'
% msg.get_body())
msg.delete()
continue
# If this is still not a dict,
# let's just drop the message and move on
if type(msgbody) is not dict:
logger.debug(
"Message is not a dictionary, dropping message.")
msg.delete()
continue
event = dict()
event = msgbody
# Was this message sent by fluentd-sqs
fluentd_sqs_specific_fields = {
'az', 'instance_id', '__tag'
}
if fluentd_sqs_specific_fields.issubset(set(
msgbody.keys())):
# Until we can influence fluentd-sqs to set the
# 'customendpoint' key before submitting to SQS, we'll
# need to do it here
# TODO : Change nubis fluentd output to include
# 'customendpoint'
event['customendpoint'] = True
if 'tags' in event:
event['tags'].extend([options.taskexchange])
else:
event['tags'] = [options.taskexchange]
# process message
self.on_message(event, msg)
# delete message from queue
msg.delete()
time.sleep(.1)
except ValueError as e:
logger.exception('Exception while handling message: %r' % e)
msg.delete()
except (SSLEOFError, SSLError, socket.error):
logger.info('Received network related error...reconnecting')
time.sleep(5)
self.sqs_queue = connect_sqs(options.region, options.accesskey,
options.secretkey,
options.taskexchange)
# secs to pass before checking for new/updated plugins
# seems to cause memory leaks..
# regular updates are disabled for now,
# though we set the frequency anyway.
options.plugincheckfrequency = getConfig('plugincheckfrequency', 120, options.configfile)
if __name__ == '__main__':
# configure ourselves
parser = OptionParser()
parser.add_option("-c", dest='configfile', default=sys.argv[0].replace('.py', '.conf'), help="configuration file to use")
(options, args) = parser.parse_args()
initConfig()
initLogger(options)
# open ES connection globally so we don't waste time opening it per message
es = esConnect()
pluginList = registerPlugins()
try:
main()
except KeyboardInterrupt as e:
logger.info("Exiting worker")
if options.esbulksize != 0:
es.finish_bulk()
except Exception as e:
if options.esbulksize != 0:
es.finish_bulk()
raise
