def setup(self):
sample_config = DotDict()
sample_config.configfile = os.path.join(os.path.dirname(__file__), 'index.conf')
OptionParser.parse_args = mock.Mock(return_value=(sample_config, {}))
from loginput import index
self.application = index.application
super(LoginputTestSuite, self).setup()
def onMessage(self, message, metadata):
if 'source' not in message:
return (message, metadata)
if not message['source'] == 'guardduty':
return (message, metadata)
# reformat the date fields to iosformat
for date_key in self.date_keys:
if key_exists(date_key, message):
message = self.convert_key_date_format(date_key, message)
# convert the dict to a dot dict for saner deep key/value processing
message = DotDict(message)
# pull out the likely source IP address
for ipaddress_key in self.ipaddress_keys:
if 'sourceipaddress' not in message['details'].keys():
if key_exists(ipaddress_key, message):
message.details.sourceipaddress = message.get(
ipaddress_key)
# if we still haven't found what we are looking for #U2
# sometimes it's in a list
if 'sourceipaddress' not in message['details'].keys():
if key_exists('details.finding.action.portProbeAction.portProbeDetails', message) \
and isinstance(message.details.finding.action.portProbeAction.portProbeDetails, list):
# inspect the first list entry and see if it contains an IP
portProbeDetails = DotDict(message.details.finding.action.
portProbeAction.portProbeDetails[0])
if key_exists('remoteIpDetails.ipAddressV4', portProbeDetails):
message.details.sourceipaddress = portProbeDetails.remoteIpDetails.ipAddressV4
# recovert the message back to a plain dict
return (dict(message), metadata)
def setup(self):
sample_config = DotDict()
sample_config.configfile = os.path.join(os.path.dirname(__file__),
'index.conf')
OptionParser.parse_args = mock.Mock(return_value=(sample_config, {}))
from loginput import index as loginput_index
self.application = loginput_index.application
super().setup()
def setup(self):
sample_config = DotDict()
sample_config.configfile = os.path.join(os.path.dirname(__file__),
'../../rest/index.conf')
OptionParser.parse_args = mock.Mock(return_value=(sample_config, {}))
from rest import index
self.application = index.application
super(RestTestSuite, self).setup()
def onAggregation(self, aggreg):
# aggreg['count']: number of items in the aggregation, ex: number of failed login attempts
# aggreg['value']: value of the aggregation field, ex: [email protected]
# aggreg['events']: list of events in the aggregation
category = aggreg['config']['alert_category']
tags = aggreg['config']['alert_tags']
severity = aggreg['config']['alert_severity']
url = aggreg['config']['alert_url']
# Find all affected hosts
# Normally, the hostname data is in e.hostname so try that first,
# but fall back to e.hostname if it is missing, or nothing at all if there's no hostname! ;-)
hostnames = []
for e in aggreg['events']:
event_source = e['_source']
if 'hostname' in event_source:
hostnames.append(event_source['hostname'])
summary = '{} ({}): {}'.format(
aggreg['config']['alert_summary'],
aggreg['count'],
aggreg['value'],
)
# If additional summary fields is defined, loop through each
# and pull out the value from each event (if it exists)
# and append key=value(s) to the summary field
if 'additional_summary_fields' in aggreg['config']:
for additional_field in aggreg['config'][
'additional_summary_fields']:
values_found = []
# If the field exists in each EVENT, include it in alert summary
for event in aggreg['events']:
dot_event = DotDict(event['_source'])
value = dot_event.get(additional_field)
if value:
values_found.append(value)
# Let's add the key=value(s) to summary
if len(values_found) != 0:
values_str = '{}'.format(', '.join(set(values_found)))
summary += " ({0}={1})".format(additional_field,
values_str)
if hostnames:
summary += ' [{}]'.format(', '.join(set(hostnames)))
return self.createAlertDict(summary, category, tags, aggreg['events'],
severity, url)
def setup(self):
sample_event = {
'_id': '123456789101112131415',
'auth0_client': {
'name': 'auth0.js',
'version': '1.23.4'
},
'client_id': 'abcd12345edfg678910',
'client_name': 'example.mozilla.org',
'date': '2019-08-01T19:58:58.912Z',
'description': 'None',
'details': {
'completedAt': '1564689538906',
'elapsedTime': 'None',
'initiatedAt': 'None',
'prompts': [],
'session_id': 'ABC12345defgHIJK'
},
'hostname': 'mozilla.org',
'ip': '1.2.3.4',
'isMobile': 'False',
'session_connection': 'Test-Connection',
'session_connection_id': 'None',
'type': 'ssa',
'user_agent': 'Firefox 68.0.0 / Linux 0.0.0',
'user_id': 'ad|Test-Connection|ttesterson',
'user_name': '*****@*****.**'
}
self.sample_event = DotDict(sample_event)
def main():
# Configuration loading
config_location = os.path.dirname(sys.argv[0]) + "/" + "auth02mozdef.json"
with open(config_location) as fd:
config = DotDict(hjson.load(fd))
if config is None:
logger.error("No configuration file 'auth02mozdef.json' found.")
sys.exit(1)
headers = {
"Authorization": "Bearer {}".format(config.auth0.token),
"Accept": "application/json"
}
fromid = load_state(config.state_file)
# Auth0 will interpret a 0 state as an error on our hosted instance, but will accept an empty parameter "as if it was 0"
if fromid == 0 or fromid == "0":
fromid = ""
totals = 1
start = 0
length = 0
# Fetch until we've gotten all messages
while totals > start + length:
(totals, start, length,
lastid) = fetch_auth0_logs(config, headers, fromid)
if totals == -1:
if fromid == lastid:
# We got everything, we're done!
break
fromid = lastid
save_state(config.state_file, lastid)
def fetch_auth0_logs(config, headers, fromid):
lastid = fromid
r = requests.get(
"{url}?take={reqnr}&sort=date:1&per_page={reqnr}&from={fromid}&include_totals=true".format(
url=config.auth0.url, reqnr=config.auth0.reqnr, fromid=fromid
),
headers=headers,
)
# If we fail here, auth0 is not responding to us the way we expected it
if not r.ok:
raise Exception(r.url, r.reason, r.status_code, r.json())
ret = r.json()
# Sometimes API give us the requested totals.. sometimes not.
# To be clear; totals are now only returned when using `page=..` and not using `from=..` parameters
# The issue is that when using `page`, auth0 internally splices the log by page, which is extremely slow and the
# call takes 10-20s to return each time.
# When using `from` auth0 queries the index for that location which is fast, so we use `from`
# this means we can't properly page results, so we have to "try to fetch" until no more logs are returned
# Finally note that when using `from` the `sort` ordering is not guaranteed to work according to the API docs
if type(ret) is dict and "logs" in ret:
have_totals = True
all_msgs = ret["logs"]
else:
have_totals = False
all_msgs = ret
# Process all new auth0 log msgs, normalize and send them to mozdef
for msg in all_msgs:
mozmsg = mozdef.MozDefEvent(config.mozdef.url)
if config.DEBUG == "True":
mozmsg.set_send_to_syslog(True, only_syslog=True)
mozmsg.hostname = config.auth0.url
mozmsg.tags = ["auth0"]
msg = byteify(msg)
msg = DotDict(msg)
lastid = msg._id
# Fill in mozdef msg fields from the auth0 msg
try:
mozmsg = process_msg(mozmsg, msg)
except KeyError as e:
# if this happens the msg was malformed in some way
mozmsg.details["error"] = "true"
mozmsg.details["errormsg"] = '"' + str(e) + '"'
mozmsg.summary = "Failed to parse auth0 message"
traceback.print_exc()
# Save raw initial message in final message
# in case we ran into parsing errors
mozmsg.details["raw"] = str(msg)
mozmsg.send()
if have_totals:
return (int(ret["total"]), int(ret["start"]), int(ret["length"]), lastid)
else:
return (-1, -1, -1, lastid)
def main():
# Configuration loading
config_location = os.path.dirname(sys.argv[0]) + '/' + 'auth02mozdef.json'
with open(config_location) as fd:
config = DotDict(hjson.load(fd))
if config is None:
print("No configuration file 'auth02mozdef.json' found.")
sys.exit(1)
headers = {
'Authorization': 'Bearer {}'.format(config.auth0.token),
'Accept': 'application/json'
}
fromid = load_state(config.state_file)
# Auth0 will interpret a 0 state as an error on our hosted instance, but will accept an empty parameter "as if it was 0"
if (fromid == 0 or fromid == "0"):
fromid = ""
totals = 1
start = 0
length = 0
# Fetch until we've gotten all messages
while (totals > start + length):
(totals, start, length,
lastid) = fetch_auth0_logs(config, headers, fromid)
fromid = lastid
save_state(config.state_file, lastid)
def setup(self):
super(TestEsworkerSNSSQS, self).setup()
mq_conn = 'abc'
task_queue = 'example-logs-mozdef'
es_connection = self.es_client
options = DotDict({
"esbulksize": 0,
"mozdefhostname": "unittest.hostname",
"taskexchange": task_queue,
'plugincheckfrequency': 120,
})
self.consumer = taskConsumer(mq_conn, es_connection, options)
def onMessage(self, message, metadata):
if 'source' not in message:
return (message, metadata)
if not message['source'] == 'guardduty':
return (message, metadata)
# reformat the date fields to iosformat
for date_key in self.date_keys:
if key_exists(date_key, message):
if message.get(date_key) is None:
continue
else:
message = self.convert_key_date_format(date_key, message)
# convert the dict to a dot dict for saner deep key/value processing
message = DotDict(message)
# pull out the likely source IP address
for ipaddress_key in self.ipaddress_keys:
if 'sourceipaddress' not in message['details']:
if key_exists(ipaddress_key, message):
message.details.sourceipaddress = message.get(
ipaddress_key)
# if we still haven't found what we are looking for #U2
# sometimes it's in a list
if 'sourceipaddress' not in message['details']:
if key_exists('details.finding.action.portprobeaction.portprobedetails', message) \
and isinstance(message.details.finding.action.portprobeaction.portprobedetails, list):
# inspect the first list entry and see if it contains an IP
portprobedetails = DotDict(
message.details.finding.action.portprobeaction.portprobedetails[0])
if key_exists('remoteipdetails.ipaddressv4', portprobedetails):
message.details.sourceipaddress = portprobedetails.remoteipdetails.ipaddressv4
# recovert the message back to a plain dict
return (dict(message), metadata)
def fetch_auth0_logs(config, headers, fromid):
lastid = fromid
r = requests.get(
'{url}?take={reqnr}&sort=date:1&per_page={reqnr}&include_totals=true&from={fromid}'
.format(url=config.auth0.url, reqnr=config.auth0.reqnr, fromid=fromid),
headers=headers)
# If we fail here, auth0 is not responding to us the way we expected it
if (not r.ok):
raise Exception(r.url, r.reason, r.status_code, r.json())
ret = r.json()
# Sometimes API give us the requested totals.. sometimes not.
if (type(ret) is dict) and ('logs' in ret.keys()):
have_totals = True
all_msgs = ret['logs']
else:
have_totals = False
all_msgs = ret
# Process all new auth0 log msgs, normalize and send them to mozdef
for msg in all_msgs:
mozmsg = mozdef.MozDefEvent(config.mozdef.url)
if config.DEBUG == 'True':
mozmsg.set_send_to_syslog(True, only_syslog=True)
mozmsg.hostname = config.auth0.url
mozmsg.tags = ['auth0']
msg = byteify(msg)
msg = DotDict(msg)
lastid = msg._id
# Fill in mozdef msg fields from the auth0 msg
try:
mozmsg = process_msg(mozmsg, msg)
except KeyError as e:
# if this happens the msg was malformed in some way
mozmsg.details['error'] = 'true'
mozmsg.details['errormsg'] = '"' + str(e) + '"'
mozmsg.summary = 'Failed to parse auth0 message'
if config.DEBUG == 'True':
traceback.print_exc()
mozmsg.send()
if have_totals:
return (int(ret['total']), int(ret['start']), int(ret['length']),
lastid)
else:
return (0, 0, 0, lastid)
def load_configs(self):
'''Load all configured rules'''
self.configs = []
rules_location = os.path.join(self.config.alert_data_location, "rules")
files = glob.glob(rules_location + "/*.json")
for f in files:
with open(f) as fd:
try:
cfg = DotDict(hjson.load(fd))
self.validate_alert(cfg)
# We set the alert name to the filename (excluding .json)
alert_name = basename(f).replace('.json', '')
cfg['custom_alert_name'] = alert_name
self.configs.append(cfg)
except Exception:
logger.error("Loading rule file {} failed".format(f))
def setup(self):
super().setup()
mq_conn = 'abc'
task_queue = 'example-logs-mozdef'
es_connection = self.es_client
options = DotDict({
"esbulksize": 0,
"mozdefhostname": "unittest.hostname",
"taskexchange": task_queue,
})
if 'lib' in sys.modules:
del sys.modules['lib']
self.mq_path = os.path.join(os.path.dirname(__file__), "../../mq/")
sys.path.insert(0, self.mq_path)
from mq import esworker_sns_sqs
self.consumer = esworker_sns_sqs.taskConsumer(mq_conn, es_connection,
options)
def test_complex_get(self):
original_dct = {
'details': {
'key1': 'value1',
'subkey': {
'subkey': 'subvalue'
}
}
}
dct = DotDict(original_dct)
assert dct.get('does.not.exist') is None
assert dct.get('details') == {'key1': 'value1','subkey': {'subkey': 'subvalue'}}
assert dct.get('details.key1') == 'value1'
assert dct.get('details.subkey') == {'subkey':'subvalue'}
assert dct.get('details.subkey.subkey') == 'subvalue'
def test_complex_get(self):
original_dct = {
'details': {
'key1': 'value1',
'subkey': {
'subkey': 'subvalue'
}
}
}
dct = DotDict(original_dct)
assert dct.get('does.not.exist') is None
assert dct.get('details') == {
'key1': 'value1',
'subkey': {
'subkey': 'subvalue'
}
}
assert dct.get('details.key1') == 'value1'
assert dct.get('details.subkey') == {'subkey': 'subvalue'}
assert dct.get('details.subkey.subkey') == 'subvalue'
def process_msg(mozmsg, msg):
"""Normalization function for auth0 msg.
@mozmsg: MozDefEvent (mozdef message as DotDict)
@msg: DotDict (json with auth0 raw message data).
All the try-except loops handle cases where the auth0 msg may or may not contain expected fields.
The msg structure is not garanteed.
See also https://auth0.com/docs/api/management/v2#!/Logs/get_logs
"""
details = DotDict({})
# defaults
details.username = "******"
details.userid = "UNKNOWN"
# key words used to set category and success/failure markers
authentication_words = ['Login', 'Logout', 'Auth']
authorization_words = ['Authorization', 'Access', 'Delegation']
success_words = ['Success']
failed_words = ['Failed']
# default category (might be modified below to be more specific)
mozmsg.set_category('iam')
mozmsg.source = 'auth0'
# fields that should always exist
mozmsg.timestamp = msg.date
details['messageid'] = msg._id
details['sourceipaddress'] = msg.ip
try:
details['userid'] = msg.user_id
except KeyError:
pass
try:
details['username'] = msg.user_name
except KeyError:
pass
try:
# the details.request/response exist for api calls
# but not for logins and other events
# check and prefer them if present.
details['username'] = msg.details.request.auth.user.name
details['action'] = msg.details.response.body.name
except KeyError:
pass
try:
details['useragent'] = msg.user_agent
except KeyError:
pass
try:
# auth0 calls these events with an acronym and name
details['eventname'] = log_types[msg.type].event
# determine the event category
if any(authword in details['eventname']
for authword in authentication_words):
mozmsg.set_category("authentication")
if any(authword in details['eventname']
for authword in authorization_words):
mozmsg.set_category("authorization")
# determine success/failure
if any(failword in details['eventname'] for failword in failed_words):
details.success = False
if any(successword in details['eventname']
for successword in success_words):
details.success = True
except KeyError:
# New message type, check https://manage-dev.mozilla.auth0.com/docs/api/management/v2#!/Logs/get_logs for ex.
debug('New auth0 message type, please add support: {}'.format(
msg.type))
details['eventname'] = msg.type
# determine severity level
if log_types[msg.type].level == 3:
mozmsg.set_severity(mozdef.MozDefEvent.SEVERITY_ERROR)
elif log_types[msg.type].level > 3:
mozmsg.set_severity(mozdef.MozDefEvent.SEVERITY_CRITICAL)
# default description
details['description'] = ""
try:
if 'description' in msg and msg.description is not None:
# use the detailed description of the operation sent from auth0
# Update a rule, add a site, update a site, etc
details['description'] = msg.description
except KeyError:
details['description'] = ""
# set the summary
if 'auth' in mozmsg._category:
# make summary be action/username (success login [email protected])
mozmsg.summary = "{event} {desc}".format(event=details.eventname,
desc=details.username)
else:
# default summary as action and description (if it exists)
mozmsg.summary = "{event} {desc}".format(event=details.eventname,
desc=details.description)
try:
details['clientname'] = msg.client_name
except KeyError:
pass
try:
details['connection'] = msg.connection
except KeyError:
pass
try:
details['clientid'] = msg.client_id
except KeyError:
pass
# Differenciate auto login (session cookie check validated) from logged in and had password verified
try:
for i in msg.details.prompt:
# Session cookie check
if i.get('name') == 'authenticate':
details[
'authtype'] = 'Login succeeded due to a valid session cookie being supplied'
elif i.get('name') == 'lock-password-authenticate':
details[
'authtype'] = 'Login succeeded due to a valid plaintext password being supplied'
except KeyError:
pass
mozmsg.details = details
mozmsg.details['raw'] = str(msg)
return mozmsg
log_types = DotDict({
's': {
"event": 'Success Login',
"level": 1
},
'slo': {
"event": 'Success Logout',
"level": 1
},
'flo': {
"event": 'Failed Logout',
"level": 3
},
'seacft': {
"event": 'Success Exchange (Authorization Code for Access Token)',
"level": 1
},
'feacft': {
"event": 'Failed Exchange (Authorization Code for Access Token)',
"level": 3
},
'f': {
"event": 'Failed Login',
"level": 3
},
'w': {
"event": 'Warnings During Login',
"level": 2
},
'du': {
"event": 'Deleted User',
"level": 1
},
'fu': {
"event": 'Failed Login (invalid email/username)',
"level": 3
},
'fp': {
"event": 'Failed Login (wrong password)',
"level": 3
},
'fc': {
"event": 'Failed by Connector',
"level": 3
},
'fco': {
"event": 'Failed by CORS',
"level": 3
},
'con': {
"event": 'Connector Online',
"level": 1
},
'coff': {
"event": 'Connector Offline',
"level": 3
},
'fcpro': {
"event": 'Failed Connector Provisioning',
"level": 4
},
'ss': {
"event": 'Success Signup',
"level": 1
},
'fs': {
"event": 'Failed Signup',
"level": 3
},
'cs': {
"event": 'Code Sent',
"level": 0
},
'cls': {
"event": 'Code/Link Sent',
"level": 0
},
'sv': {
"event": 'Success Verification Email',
"level": 0
},
'fv': {
"event": 'Failed Verification Email',
"level": 0
},
'scp': {
"event": 'Success Change Password',
"level": 1
},
'fcp': {
"event": 'Failed Change Password',
"level": 3
},
'sce': {
"event": 'Success Change Email',
"level": 1
},
'fce': {
"event": 'Failed Change Email',
"level": 3
},
'scu': {
"event": 'Success Change Username',
"level": 1
},
'fcu': {
"event": 'Failed Change Username',
"level": 3
},
'scpn': {
"event": 'Success Change Phone Number',
"level": 1
},
'fcpn': {
"event": 'Failed Change Phone Number',
"level": 3
},
'svr': {
"event": 'Success Verification Email Request',
"level": 0
},
'fvr': {
"event": 'Failed Verification Email Request',
"level": 3
},
'scpr': {
"event": 'Success Change Password Request',
"level": 0
},
'fcpr': {
"event": 'Failed Change Password Request',
"level": 3
},
'fn': {
"event": 'Failed Sending Notification',
"level": 3
},
'sapi': {
"event": 'API Operation',
"level": 1
},
'fapi': {
"event": 'Failed API Operation',
"level": 3
},
'limit_wc': {
"event": 'Blocked Account',
"level": 4
},
'limit_ui': {
"event": 'Too Many Calls to /userinfo',
"level": 4
},
'api_limit': {
"event": 'Rate Limit On API',
"level": 4
},
'sdu': {
"event": 'Successful User Deletion',
"level": 1
},
'fdu': {
"event": 'Failed User Deletion',
"level": 3
},
'sd': {
"event": 'Success Delegation',
"level": 3
},
'fd': {
"event": 'Failed Delegation',
"level": 3
},
'seccft': {
"event": "Success Exchange (Client Credentials for Access Token)",
"level": 1
},
'feccft': {
"event": "Failed Exchange (Client Credentials for Access Token)",
"level": 1
},
'fsa': {
"event": "Failed Silent Auth",
"level": 3
},
'ssa': {
"event": "Success Silent Auth",
"level": 1
},
'fepft': {
"event": "Failed Exchange (Password for Access Token)",
"level": 3
},
'limit_mu': {
"event": "Blocked IP Address",
"level": 3
},
'sepft': {
"event": "Success Exchange (Password for Access Token)",
"level": 1
},
'fcoa': {
"event": "Failed Cross Origin Authentication",
"level": 3
}
})
def test_nonexisting_key(self):
dct = DotDict()
with pytest.raises(KeyError):
dct.abcd
def test_basic_init(self):
dct = DotDict({'key1': 'value1', 'key2': 'value2'})
assert sorted(dct.keys()) == sorted(['key1', 'key2'])
assert dct.key1 == 'value1'
assert dct.key2 == 'value2'
def test_complex_init(self):
original_dct = {'details': {'key1': 'value1'}}
dct = DotDict(original_dct)
assert dct.details == {'key1': 'value1'}
assert dct.details.key1 == 'value1'
log_types = DotDict({
"s": {
"event": "Success Login",
"level": 1
},
"slo": {
"event": "Success Logout",
"level": 1
},
"flo": {
"event": "Failed Logout",
"level": 3
},
"seacft": {
"event": "Success Exchange (Authorization Code for Access Token)",
"level": 1
},
"feacft": {
"event": "Failed Exchange (Authorization Code for Access Token)",
"level": 3
},
"f": {
"event": "Failed Login",
"level": 3
},
"w": {
"event": "Warnings During Login",
"level": 2
},
"du": {
"event": "Deleted User",
"level": 1
},
"fu": {
"event": "Failed Login (invalid email/username)",
"level": 3
},
"fp": {
"event": "Failed Login (wrong password)",
"level": 3
},
"fc": {
"event": "Failed by Connector",
"level": 3
},
"fco": {
"event": "Failed by CORS",
"level": 3
},
"con": {
"event": "Connector Online",
"level": 1
},
"coff": {
"event": "Connector Offline",
"level": 3
},
"fcpro": {
"event": "Failed Connector Provisioning",
"level": 4
},
"ss": {
"event": "Success Signup",
"level": 1
},
"fs": {
"event": "Failed Signup",
"level": 3
},
"cs": {
"event": "Code Sent",
"level": 0
},
"cls": {
"event": "Code/Link Sent",
"level": 0
},
"sv": {
"event": "Success Verification Email",
"level": 0
},
"fv": {
"event": "Failed Verification Email",
"level": 0
},
"scp": {
"event": "Success Change Password",
"level": 1
},
"fcp": {
"event": "Failed Change Password",
"level": 3
},
"sce": {
"event": "Success Change Email",
"level": 1
},
"fce": {
"event": "Failed Change Email",
"level": 3
},
"scu": {
"event": "Success Change Username",
"level": 1
},
"fcu": {
"event": "Failed Change Username",
"level": 3
},
"scpn": {
"event": "Success Change Phone Number",
"level": 1
},
"fcpn": {
"event": "Failed Change Phone Number",
"level": 3
},
"svr": {
"event": "Success Verification Email Request",
"level": 0
},
"fvr": {
"event": "Failed Verification Email Request",
"level": 3
},
"scpr": {
"event": "Success Change Password Request",
"level": 0
},
"fcpr": {
"event": "Failed Change Password Request",
"level": 3
},
"fn": {
"event": "Failed Sending Notification",
"level": 3
},
"sapi": {
"event": "API Operation",
"level": 1
},
"fapi": {
"event": "Failed API Operation",
"level": 3
},
"limit_wc": {
"event": "Blocked Account",
"level": 4
},
"limit_ui": {
"event": "Too Many Calls to /userinfo",
"level": 4
},
"api_limit": {
"event": "Rate Limit On API",
"level": 4
},
"sdu": {
"event": "Successful User Deletion",
"level": 1
},
"fdu": {
"event": "Failed User Deletion",
"level": 3
},
"sd": {
"event": "Success Delegation",
"level": 3
},
"fd": {
"event": "Failed Delegation",
"level": 3
},
"seccft": {
"event": "Success Exchange (Client Credentials for Access Token)",
"level": 1
},
"feccft": {
"event": "Failed Exchange (Client Credentials for Access Token)",
"level": 1
},
"fsa": {
"event": "Failed Silent Auth",
"level": 3
},
"ssa": {
"event": "Success Silent Auth",
"level": 1
},
"fepft": {
"event": "Failed Exchange (Password for Access Token)",
"level": 3
},
"limit_mu": {
"event": "Blocked IP Address",
"level": 3
},
"sepft": {
"event": "Success Exchange (Password for Access Token)",
"level": 1
},
"fcoa": {
"event": "Failed Cross Origin Authentication",
"level": 3
},
"depnote": {
"event": "Deprecation Note",
"level": 1
},
})
def test_blank_init(self):
dct = DotDict()
assert list(dct.keys()) == []
def test_blank_init(self):
dct = DotDict()
assert dct.keys() == []
def test_basic_init(self):
dct = DotDict({'key1': 'value1', 'key2': 'value2'})
assert sorted(dct.keys()) == sorted(['key1', 'key2'])
assert dct.key1 == 'value1'
assert dct.key2 == 'value2'
def process_msg(mozmsg, msg):
"""Normalization function for auth0 msg.
@mozmsg: MozDefEvent (mozdef message as DotDict)
@msg: DotDict (json with auth0 raw message data).
All the try-except loops handle cases where the auth0 msg may or may not contain expected fields.
The msg structure is not garanteed.
See also https://auth0.com/docs/api/management/v2#!/Logs/get_logs
"""
details = DotDict({})
# defaults
details.username = "******"
details.userid = "UNKNOWN"
# key words used to set category and success/failure markers
authentication_words = ["Login", "Logout", "Auth"]
authorization_words = ["Authorization", "Access", "Delegation"]
success_words = ["Success"]
failed_words = ["Failed"]
# default category (might be modified below to be more specific)
mozmsg.set_category("iam")
mozmsg.source = "auth0"
# fields that should always exist
mozmsg.timestamp = msg.date
details["messageid"] = msg._id
details["sourceipaddress"] = msg.ip
try:
details["userid"] = msg.user_id
except KeyError:
pass
try:
details["username"] = msg.user_name
except KeyError:
pass
try:
# the details.request/response exist for api calls
# but not for logins and other events
# check and prefer them if present.
details["username"] = msg.details.request.auth.user.name
details["action"] = msg.details.response.body.name
except KeyError:
pass
try:
details["useragent"] = msg.user_agent
except KeyError:
pass
try:
# auth0 calls these events with an acronym and name
details["eventname"] = log_types[msg.type].event
# determine the event category
if any(authword in details["eventname"] for authword in authentication_words):
mozmsg.set_category("authentication")
if any(authword in details["eventname"] for authword in authorization_words):
mozmsg.set_category("authorization")
# determine success/failure
if any(failword in details["eventname"] for failword in failed_words):
details.success = False
if any(successword in details["eventname"] for successword in success_words):
details.success = True
except KeyError:
# New message type, check https://manage-dev.mozilla.auth0.com/docs/api/management/v2#!/Logs/get_logs for ex.
debug("New auth0 message type, please add support: {}".format(msg.type))
details["eventname"] = msg.type
# determine severity level
if log_types[msg.type].level == 3:
mozmsg.set_severity(mozdef.MozDefEvent.SEVERITY_ERROR)
elif log_types[msg.type].level > 3:
mozmsg.set_severity(mozdef.MozDefEvent.SEVERITY_CRITICAL)
# default description
details["description"] = ""
try:
if "description" in msg and msg.description is not None:
# use the detailed description of the operation sent from auth0
# Update a rule, add a site, update a site, etc
details["description"] = msg.description
except KeyError:
details["description"] = ""
# set the summary
if "auth" in mozmsg._category:
# make summary be action/username (success login [email protected])
mozmsg.summary = "{event} {desc}".format(event=details.eventname, desc=details.username)
else:
# default summary as action and description (if it exists)
mozmsg.summary = "{event} {desc}".format(event=details.eventname, desc=details.description)
try:
details["clientname"] = msg.client_name
except KeyError:
pass
try:
details["connection"] = msg.connection
except KeyError:
pass
try:
details["clientid"] = msg.client_id
except KeyError:
pass
# Differenciate auto login (session cookie check validated) from logged in and had password verified
try:
for i in msg.details.prompt:
# Session cookie check
if i.get("name") == "authenticate":
details["authtype"] = "Login succeeded due to a valid session cookie being supplied"
elif i.get("name") == "lock-password-authenticate":
details["authtype"] = "Login succeeded due to a valid plaintext password being supplied"
except KeyError:
pass
mozmsg.details = details
mozmsg.details["raw"] = str(msg)
return mozmsg
log_types = DotDict(
{
"admin_update_launch": {"event": "Auth0 Update Launched", "level": 1},
"api_limit": {"event": "Rate Limit On API", "level": 4},
"cls": {"event": "Code/Link Sent", "level": 0},
"coff": {"event": "Connector Offline", "level": 3},
"con": {"event": "Connector Online", "level": 1},
"cs": {"event": "Code Sent", "level": 0},
"depnote": {"event": "Deprecation Note", "level": 1},
"du": {"event": "Deleted User", "level": 1},
"f": {"event": "Failed Login", "level": 3},
"fapi": {"event": "Failed API Operation", "level": 3},
"fc": {"event": "Failed by Connector", "level": 3},
"fce": {"event": "Failed Change Email", "level": 3},
"fco": {"event": "Failed by CORS", "level": 3},
"fcoa": {"event": "Failed Cross Origin Authentication", "level": 3},
"fcp": {"event": "Failed Change Password", "level": 3},
"fcph": {"event": "Failed Post Change Password Hook", "level": 3},
"fcpn": {"event": "Failed Change Phone Number", "level": 3},
"fcpr": {"event": "Failed Change Password Request", "level": 3},
"fcpro": {"event": "Failed Connector Provisioning", "level": 4},
"fcu": {"event": "Failed Change Username", "level": 3},
"fd": {"event": "Failed Delegation", "level": 3},
"fdeac": {"event": "Failed Device Activation", "level": 3},
"fdeaz": {"event": "Failed Device Authorization Request", "level": 3},
"fdecc": {"event": "User Canceled Device Confirmation", "level": 2},
"fdu": {"event": "Failed User Deletion", "level": 3},
"feacft": {"event": "Failed Exchange (Authorization Code for Access Token)", "level": 3},
"feccft": {"event": "Failed Exchange (Client Credentials for Access Token)", "level": 1},
"fede": {"event": "Failed Exchange (Device Code for Access Token)", "level": 3},
"fens": {"event": "Failed Exchange (Native Social Login)", "level": 3},
"feoobft": {"event": "Failed Exchange (Password and OOB Challenge for Access Token)", "level": 3},
"feotpft": {"event": "Failed Exchange (Password and OTP Challenge for Access Token)", "level": 3},
"fepft": {"event": "Failed Exchange (Password for Access Token)", "level": 3},
"fercft": {"event": "Failed Exchange (Password and MFA Recovery code for Access Token)", "level": 3},
"fertft": {"event": "Failed Exchange (Refresh Token for Access Token)", "level": 3},
"flo": {"event": "Failed Logout", "level": 3},
"fn": {"event": "Failed Sending Notification", "level": 3},
"fp": {"event": "Failed Login (wrong password)", "level": 3},
"fs": {"event": "Failed Signup", "level": 3},
"fsa": {"event": "Failed Silent Auth", "level": 3},
"fu": {"event": "Failed Login (invalid email/username)", "level": 3},
"fui": {"event": "Failed users import", "level": 4},
"fv": {"event": "Failed Verification Email", "level": 0},
"fvr": {"event": "Failed Verification Email Request", "level": 3},
"gd_auth_failed": {"event": "OTP Auth failed", "level": 3},
"gd_auth_rejected": {"event": "OTP Auth rejected", "level": 3},
"gd_auth_succeed": {"event": "OTP Auth success", "level": 1},
"gd_enrollment_complete": {"event": "MFA Enrollment Complete", "level": 1},
"gd_module_switch": {"event": "Module switch", "level": 1},
"gd_otp_rate_limit_exceed": {"event": "Too many OTP failures", "level": 4},
"gd_recovery_failed": {"event": "Failed Authentication using Recovery code.", "level": 3},
"gd_recovery_rate_limit_exceed": {"event": "Failed Authentication using Recovery code too many times", "level": 4},
"gd_recovery_succeed": {"event": "Success Authentication using Recovery code", "level": 1},
"gd_send_pn": {"event": "Success Push notification for MFA sent", "level": 1},
"gd_send_sms": {"event": "Success SMS for MFA sent", "level": 1},
"gd_send_sms_failure": {"event": "Failed sending SMS for MFA", "level": 3},
"gd_start_auth": {"event": "Second factor authentication event started for MFA", "level": 1},
"gd_start_enroll": {"event": "Multi-factor authentication enroll has started", "level": 1},
"gd_tenant_update": {"event": "Guardian tenant update", "level": 3},
"gd_unenroll": {"event": "Device used for second factor authentication has been unenrolled", "level": 2},
"gd_update_device_account": {"event": "Device used for second factor authentication has been updated", "level": 2},
"gd_user_delete": {"event": "Deleted multi-factor user account", "level": 1},
"limit_delegation": {"event": "Rate limit exceeded to /delegation endpoint", "level": 4},
"limit_mu": {"event": "Blocked IP Address", "level": 3},
"limit_ui": {"event": "Too Many Calls to /userinfo", "level": 4},
"limit_wc": {"event": "Blocked Account", "level": 4},
"pwd_leak": {"event": "User attempted to login with a leaked password", "level": 4},
"s": {"event": "Success Login", "level": 1},
"sapi": {"event": "Success API Operation", "level": 1},
"sce": {"event": "Success Change Email", "level": 1},
"scoa": {"event": "Success cross-origin authentication", "level": 1},
"scp": {"event": "Success Change Password", "level": 1},
"scph": {"event": "Success Post Change Password Hook", "level": 1},
"scpn": {"event": "Success Change Phone Number", "level": 1},
"scpr": {"event": "Success Change Password Request", "level": 0},
"scu": {"event": "Success Change Username", "level": 1},
"sd": {"event": "Success Delegation", "level": 3},
"sdu": {"event": "Success User Deletion", "level": 1},
"seacft": {"event": "Success Exchange (Authorization Code for Access Token)", "level": 1},
"seccft": {"event": "Success Exchange (Client Credentials for Access Token)", "level": 1},
"sede": {"event": "Successful Exchange (Device Code for Access Token)", "level": 1},
"sens": {"event": "Success Exchange (Native Social Login)", "level": 1},
"seoobft": {"event": "Success Exchange (Password and OOB Challenge for Access Token)", "level": 1},
"seotpft": {"event": "Success Exchange (Password and OTP Challenge for Access Token)", "level": 1},
"sepft": {"event": "Success Exchange (Password for Access Token)", "level": 1},
"sercft": {"event": "Success Exchange (Password and MFA Recovery code for Access Token)", "level": 1},
"sertft": {"event": "Success Exchange (Refresh Token for Access Token)", "level": 1},
"slo": {"event": "Success Logout", "level": 1},
"ss": {"event": "Success Signup", "level": 1},
"ssa": {"event": "Success Silent Auth", "level": 1},
"sui": {"event": "Success User Import", "level": 1},
"sv": {"event": "Success Verification Email", "level": 0},
"svr": {"event": "Success Verification Email Request", "level": 0},
"sys_os_update_end": {"event": "Auth0 OS Update Ended", "level": 1},
"sys_os_update_start": {"event": "Auth0 OS Update Started", "level": 1},
"sys_update_end": {"event": "Auth0 Update Ended", "level": 1},
"sys_update_start": {"event": "Auth0 Update Started", "level": 1},
"ublkdu": {"event": "User block setup by anomaly detection has been released", "level": 3},
"w": {"event": "Warnings During Login", "level": 2},
}
)
def process_msg(mozmsg, msg):
"""Normalization function for auth0 msg.
@mozmsg: MozDefEvent (mozdef message as DotDict)
@msg: DotDict (json with auth0 raw message data).
All the try-except loops handle cases where the auth0 msg may or may not contain expected fields.
The msg structure is not garanteed.
See also https://auth0.com/docs/api/management/v2#!/Logs/get_logs
"""
details = DotDict({})
# key words used to set category and success/failure markers
authentication_words = ["Login", "Logout", "Silent", "Enrollment", "OTP", "Recovery", "Authentication", "Code", "Signup", "Push"]
authorization_words = ["Authorization", "Access", "Delegation"]
administration_words = ["API", "Operation", "Change", "Update", "Deleted", "unenrolled", "updated", "CORS", "Connector", "Blocked", "Breached", "Deletion", "block", "User", "released"]
success_words = ["Success"]
failed_words = ["Failed"]
# fields that should always exist
mozmsg.timestamp = msg.date
details["messageid"] = msg._id
details["sourceipaddress"] = msg.ip
try:
details["userid"] = msg.user_id
except KeyError:
pass
try:
if msg.user_name:
details["username"] = msg.user_name
except KeyError:
pass
try:
# the details.request/response exist for api calls
# but not for logins and other events
# check and prefer them if present.
if type(msg.details.response.body) is not list:
details["action"] = msg.details.response.body.name
except KeyError:
pass
try:
if "email" in msg.details.response.body and msg.details.response.body.email is not None:
details["email"] = msg.details.response.body.email
except KeyError:
pass
try:
details["useragent"] = msg.user_agent
except KeyError:
pass
try:
if msg.client_name:
details["clientname"] = msg.client_name
except KeyError:
pass
try:
if msg.connection:
details["connection"] = msg.connection
except KeyError:
pass
try:
if msg.client_id:
details["clientid"] = msg.client_id
except KeyError:
pass
try:
# auth0 calls these events with an acronym and name
details["eventname"] = log_types[msg.type].event
# determine the event category
if any(authword in details["eventname"] for authword in authentication_words):
mozmsg.set_category("authentication")
if any(authword in details["eventname"] for authword in authorization_words):
mozmsg.set_category("authorization")
if any(adminword in details["eventname"] for adminword in administration_words):
mozmsg.set_category("administration")
# determine success/failure
if any(failword in details["eventname"] for failword in failed_words):
details.success = False
if any(successword in details["eventname"] for successword in success_words):
details.success = True
except KeyError:
# New message type, check https://manage-dev.mozilla.auth0.com/docs/api/management/v2#!/Logs/get_logs for ex.
logger.error("New auth0 message type, please add support: {}".format(msg.type))
details["eventname"] = msg.type
# determine severity level
if log_types[msg.type].level == 3:
mozmsg.set_severity(mozdef.MozDefEvent.SEVERITY_ERROR)
elif log_types[msg.type].level > 3:
mozmsg.set_severity(mozdef.MozDefEvent.SEVERITY_CRITICAL)
# default description
details["description"] = ""
try:
if "description" in msg and msg.description is not None:
# use the detailed description of the operation sent from auth0
# Update a rule, add a site, update a site, etc
details["description"] = msg.description
except KeyError:
pass
# set the summary
# make summary be action/username (success login [email protected])
# if no details.username field exists we don't add it.
# Build summary if neither email, description, nor username exists
if 'eventname' in details:
mozmsg.summary = "{event}".format(event=details.eventname)
if 'description' in details and details['description'] != "None":
mozmsg.summary += " {description}".format(event=details.eventname, description=details.description)
if 'username' in details and details['username'] != "None":
mozmsg.summary += " by {username}".format(username=details.username)
if 'email' in details and details['email'] != "None":
mozmsg.summary += " account: {email}".format(email=details.email)
if 'clientname' in details and details['clientname'] != "None":
mozmsg.summary += " to: {clientname}".format(clientname=details.clientname)
# Get user data if present in response body
try:
if "multifactor" in msg.details.response.body and type(msg.details.response.body.multifactor) is list:
details.mfa_provider = msg.details.response.body.multifactor
except KeyError:
pass
try:
if "ldap_groups" in msg.details.response.body and type(msg.details.response.body.ldap_groups) is list:
details.ldap_groups = msg.details.response.body.ldap_groups
except KeyError:
pass
try:
if "last_ip" in msg.details.response.body and msg.details.response.body.last_ip is not None:
details.user_last_known_ip = msg.details.response.body.last_ip
except KeyError:
pass
try:
if "last_login" in msg.details.response.body and msg.details.response.body.last_login is not None:
details.user_last_login = msg.details.response.body.last_login
except KeyError:
pass
# Differentiate auto login (session cookie check validated) from logged in and had password verified
try:
for i in msg.details.prompt:
# Session cookie check
if i.get("name") == "authenticate":
details["authtype"] = "Login succeeded due to a valid session cookie being supplied"
elif i.get("name") == "lock-password-authenticate":
details["authtype"] = "Login succeeded due to a valid plaintext password being supplied"
except KeyError:
pass
mozmsg.details = details
return mozmsg
def parse_config_file():
global CONFIG_FILE_CONTENTS
try:
CONFIG_FILE_CONTENTS
except NameError:
default_config = os.path.join(os.path.dirname(__file__), "config.conf")
options = DotDict()
options.configfile = default_config
options.esservers = list(
getConfig('esservers', 'http://localhost:9200',
options.configfile).split(','))
options.alertExchange = getConfig('alertexchange', 'alerts',
options.configfile)
options.queueName = getConfig('alertqueuename', 'alertBot',
options.configfile)
options.alertqueue = getConfig('alertqueue', 'mozdef.alert',
options.configfile)
options.alerttopic = getConfig('alerttopic', 'mozdef.*',
options.configfile)
options.mquser = getConfig('mquser', 'guest', options.configfile)
options.mqalertserver = getConfig('mqalertserver', 'localhost',
options.configfile)
options.mqserver = getConfig('mqserver', 'localhost',
options.configfile)
options.mqpassword = getConfig('mqpassword', 'guest',
options.configfile)
options.mqport = getConfig('mqport', 5672, options.configfile)
options.mqack = getConfig('mqack', True, options.configfile)
CONFIG_FILE_CONTENTS = options
return CONFIG_FILE_CONTENTS
def parse_config_file():
global CONFIG_FILE_CONTENTS
try:
CONFIG_FILE_CONTENTS
except NameError:
default_config = os.path.join(os.path.dirname(__file__), "config.conf")
options = DotDict()
options.configfile = default_config
options.esservers = list(getConfig('esservers', 'http://localhost:9200', options.configfile).split(','))
options.alertExchange = getConfig('alertexchange', 'alerts', options.configfile)
options.queueName = getConfig('alertqueuename', 'alertBot', options.configfile)
options.alertqueue = getConfig('alertqueue', 'mozdef.alert', options.configfile)
options.alerttopic = getConfig('alerttopic', 'mozdef.*', options.configfile)
options.mquser = getConfig('mquser', 'guest', options.configfile)
options.mqalertserver = getConfig('mqalertserver', 'localhost', options.configfile)
options.mqserver = getConfig('mqserver', 'localhost', options.configfile)
options.mqpassword = getConfig('mqpassword', 'guest', options.configfile)
options.mqport = getConfig('mqport', 5672, options.configfile)
options.mqack = getConfig('mqack', True, options.configfile)
CONFIG_FILE_CONTENTS = options
return CONFIG_FILE_CONTENTS
