def publish_to_mozdef(summary='', details={}):
msg = mozdef_client.MozDefEvent('')
msg.summary = summary
msg.tags = ['asap']
msg.details = json.loads(json.dumps(details, default=json_serial))
region, account_id, queue_name = app.config.get('SQS_QUEUE_ARN').split(
':')[3:]
msg.set_send_to_sqs(True)
msg.set_sqs_queue_name(queue_name)
msg.set_sqs_region(region)
msg.set_sqs_aws_account_id(account_id)
# Note that unlike syslog this will NEVER send to MozDef HTTP (URL is
# ignored)
app.logger.debug(
"Alerter: Sending message to SQS queue {} in account {} in region {}".
format(queue_name, account_id, region))
try:
msg.send()
except (botocore.exceptions.ClientError,
botocore.parsers.ResponseParserError) as e:
app.logger.critical(
"Alerter: Attempt to send message to SQS queue {} in account {} "
"in region {} failed with error {}".format(queue_name, account_id,
region, e))
def fetch_auth0_logs(config, headers, fromid):
lastid = fromid
r = requests.get(
"{url}?take={reqnr}&sort=date:1&per_page={reqnr}&from={fromid}&include_totals=true".format(
url=config.auth0.url, reqnr=config.auth0.reqnr, fromid=fromid
),
headers=headers,
)
# If we fail here, auth0 is not responding to us the way we expected it
if not r.ok:
raise Exception(r.url, r.reason, r.status_code, r.json())
ret = r.json()
# Sometimes API give us the requested totals.. sometimes not.
# To be clear; totals are now only returned when using `page=..` and not using `from=..` parameters
# The issue is that when using `page`, auth0 internally splices the log by page, which is extremely slow and the
# call takes 10-20s to return each time.
# When using `from` auth0 queries the index for that location which is fast, so we use `from`
# this means we can't properly page results, so we have to "try to fetch" until no more logs are returned
# Finally note that when using `from` the `sort` ordering is not guaranteed to work according to the API docs
if type(ret) is dict and "logs" in ret:
have_totals = True
all_msgs = ret["logs"]
else:
have_totals = False
all_msgs = ret
# Process all new auth0 log msgs, normalize and send them to mozdef
for msg in all_msgs:
mozmsg = mozdef.MozDefEvent(config.mozdef.url)
if config.DEBUG == "True":
mozmsg.set_send_to_syslog(True, only_syslog=True)
mozmsg.hostname = config.auth0.url
mozmsg.tags = ["auth0"]
msg = byteify(msg)
msg = DotDict(msg)
lastid = msg._id
# Fill in mozdef msg fields from the auth0 msg
try:
mozmsg = process_msg(mozmsg, msg)
except KeyError as e:
# if this happens the msg was malformed in some way
mozmsg.details["error"] = "true"
mozmsg.details["errormsg"] = '"' + str(e) + '"'
mozmsg.summary = "Failed to parse auth0 message"
traceback.print_exc()
# Save raw initial message in final message
# in case we ran into parsing errors
mozmsg.details["raw"] = str(msg)
mozmsg.send()
if have_totals:
return (int(ret["total"]), int(ret["start"]), int(ret["length"]), lastid)
else:
return (-1, -1, -1, lastid)
def test_sample_event_username_nonexistent(self):
mozmsg = mozdef.MozDefEvent('http://localhost:9090')
del (self.sample_event['user_name'])
del (self.sample_event['user_id'])
process_msg(mozmsg, self.sample_event)
assert 'username' not in mozmsg.details
assert 'userid' not in mozmsg.details
def send(logger, method_name, event_dict):
# only send to mozdef if `mozdef` is set
if event_dict.pop('mozdef', False):
msg = mozdef_client.MozDefEvent(MOZDEF)
msg.summary = event_dict.get('event', '')
msg.tags = [
'mozilla/release-services/' + channel,
project_name,
]
if set(event_dict) - {'event'}:
msg.details = event_dict.copy()
msg.details.pop('event', None)
msg.source = logger.name
msg.set_severity(
sevirity_map.get(
method_name,
mozdef_client.MozDefEvent.SEVERITY_INFO,
), )
msg.send()
return event_dict
def main():
try:
state = pickle.load(open(options.statepath, 'rb'))
except IOError:
# Oh, you're new.
state = {'administration': 0, 'authentication': 0, 'telephony': 0}
duo = duo_client.Admin(ikey=options.IKEY,
skey=options.SKEY,
host=options.URL)
mozmsg = mozdef.MozDefEvent(options.MOZDEF_URL)
mozmsg.tags = ['duosecurity', 'logs']
if options.update_tags != '':
mozmsg.tags.append(options.update_tags)
mozmsg.category = 'Authentication'
mozmsg.source = 'DuoSecurity API'
if options.DEBUG:
mozmsg.debug = options.DEBUG
mozmsg.set_send_to_syslog(True, only_syslog=True)
# This will process events for all 3 log types and send them to MozDef. the state stores the last position in the
# log when this script was last called.
state = process_events(
mozmsg, duo.get_administrator_log(mintime=state['administration'] + 1),
'administration', state)
state = process_events(
mozmsg,
duo.get_authentication_log(mintime=state['authentication'] + 1),
'authentication', state)
state = process_events(
mozmsg, duo.get_telephony_log(mintime=state['telephony'] + 1),
'telephony', state)
pickle.dump(state, open(options.statepath, 'wb'))
def main():
try:
state = pickle.load(open(options.statepath, "rb"))
except IOError:
# Oh, you're new.
# Note API v2 expect full, correct and within range timestamps in millisec so we start recently
# API v1 uses normal timestamps in seconds instead
state = {
"administration": 0,
"administration_offset": None,
"authentication": 1547000000000,
"authentication_offset": None,
"telephony": 0,
"telephony_offset": None,
}
# Convert v1 (sec) timestamp to v2 (ms)...
if state["authentication"] < 1547000000000:
state["authentication"] = int(str(state["authentication"]) + "000")
duo = duo_client.Admin(ikey=options.IKEY,
skey=options.SKEY,
host=options.URL)
mozmsg = mozdef.MozDefEvent(options.MOZDEF_URL)
mozmsg.tags = ["duosecurity"]
if options.update_tags != "":
mozmsg.tags.append(options.update_tags)
mozmsg.set_category("authentication")
mozmsg.source = "DuoSecurityAPI"
if options.DEBUG:
mozmsg.debug = options.DEBUG
mozmsg.set_send_to_syslog(True, only_syslog=True)
# This will process events for all 3 log types and send them to MozDef. the state stores the last position in the
# log when this script was last called.
# NOTE: If administration and telephone logs support a "v2" API in the future it will most likely need to have the
# same code with `next_offset` as authentication uses.
state = process_events(
mozmsg, duo.get_administrator_log(mintime=state["administration"] + 1),
"administration", state)
state = process_events(
mozmsg,
duo.get_authentication_log(
api_version=2,
limit="1000",
sort="ts:asc",
mintime=state["authentication"] + 1,
next_offset=state["authentication_offset"],
),
"authentication",
state,
)
state = process_events(
mozmsg, duo.get_telephony_log(mintime=state["telephony"] + 1),
"telephony", state)
pickle.dump(state, open(options.statepath, "wb"))
def fetch_auth0_logs(config, headers, fromid):
lastid = fromid
r = requests.get(
'{url}?take={reqnr}&sort=date:1&per_page={reqnr}&include_totals=true&from={fromid}'
.format(url=config.auth0.url, reqnr=config.auth0.reqnr, fromid=fromid),
headers=headers)
#If we fail here, auth0 is not responding to us the way we expected it
if (not r.ok):
raise Exception(r.url, r.reason, r.status_code, r.json())
ret = r.json()
#Sometimes API give us the requested totals.. sometimes not.
if (type(ret) is dict) and ('logs' in ret.keys()):
have_totals = True
all_msgs = ret['logs']
else:
have_totals = False
all_msgs = ret
#Process all new auth0 log msgs, normalize and send them to mozdef
for msg in all_msgs:
mozmsg = mozdef.MozDefEvent(config.mozdef.url)
if config.DEBUG == 'True':
mozmsg.set_send_to_syslog(True, only_syslog=True)
mozmsg.hostname = config.auth0.url
mozmsg.tags = ['auth0']
msg = byteify(msg)
msg = DotDict(msg)
lastid = msg._id
#Fill in mozdef msg fields from the auth0 msg
try:
mozmsg = process_msg(mozmsg, msg)
except KeyError as e:
#if this happens the msg was malformed in some way
mozmsg.details['error'] = 'true'
mozmsg.details['errormsg'] = '"' + str(e) + '"'
mozmsg.summary = 'Failed to parse auth0 message'
if config.DEBUG == 'True':
traceback.print_exc()
mozmsg.send()
if have_totals:
return (int(ret['total']), int(ret['start']), int(ret['length']),
lastid)
else:
return (0, 0, 0, lastid)
def send(logger, method_name, event_dict):
# only send to mozdef if `mozdef` is set
if event_dict.pop('mozdef', False):
msg = mozdef_client.MozDefEvent(target)
msg.summary = event_dict.get('event', '')
msg.tags = ['relengapi']
if set(event_dict) - {'event'}:
msg.details = event_dict.copy()
msg.details.pop('event', None)
msg.source = logger.name
msg.set_severity(
sev_map.get(method_name,
mozdef_client.MozDefEvent.SEVERITY_INFO))
msg.send()
# return the message unchanged
return event_dict
def main():
mozmsg = mozdef.MozDefEvent(options.mozdef_url)
mozmsg.tags = ["uptycs"]
mozmsg.set_category("uptycs")
mozmsg.source = "UptycsAPI"
if options.debug:
mozmsg.debug = options.debug
mozmsg.set_send_to_syslog(True, only_syslog=True)
client = UptycsClient(options.uptycs_api_json_file)
# Adjust the filter as needed to set proper search window
uptycs_filter = UptycsFilter()
utc_now = toUTC(datetime.now())
# If an existing state file exists, set our start window to last run
if os.path.exists(options.statepath):
state = pickle.load(open(options.statepath, 'rb'))
uptycs_filter.start_time = state["last_run"]
last_alert_ids = state["last_alert_ids"]
# Otherwise, we pick from a boostrap time window
else:
last_alert_ids = []
uptycs_filter.start_time = utc_now - timedelta(
days=int(options.bootstrap_search_depth))
uptycs_filter.end_time = utc_now
# Query alerts that match the uptycs_filter
alerts = client.alerts(uptycs_filter)
alert_ids = []
if len(alerts) > 0:
for alert in alerts:
if alert['id'] in last_alert_ids:
continue
else:
alert_ids.append(alert['id'])
# Process all these alerts in MozDef
process_alerts(mozmsg, alerts)
state = {
"last_run": uptycs_filter.end_time,
"last_alert_ids": alert_ids,
}
pickle.dump(state, open(options.statepath, "wb"))
def main():
try:
state = pickle.load(open(options.statepath, 'rb'))
except IOError:
# Oh, you're new.
# Note API v2 expect full, correct and within range timestamps in millisec so we start recently
# API v1 uses normal timestamps in seconds instead
state = {
'administration': 0,
'authentication': 1547000000000,
'telephony': 0
}
# Convert v1 (sec) timestamp to v2 (ms)...
if state['authentication'] < 1547000000000:
state['authentication'] = int(str(state['authentication']) + '000')
duo = duo_client.Admin(ikey=options.IKEY,
skey=options.SKEY,
host=options.URL)
mozmsg = mozdef.MozDefEvent(options.MOZDEF_URL)
mozmsg.tags = ['duosecurity']
if options.update_tags != '':
mozmsg.tags.append(options.update_tags)
mozmsg.set_category('authentication')
mozmsg.source = 'DuoSecurityAPI'
if options.DEBUG:
mozmsg.debug = options.DEBUG
mozmsg.set_send_to_syslog(True, only_syslog=True)
# This will process events for all 3 log types and send them to MozDef. the state stores the last position in the
# log when this script was last called.
state = process_events(
mozmsg, duo.get_administrator_log(mintime=state['administration'] + 1),
'administration', state)
# TODO Should use `next_offset` instead of mintime in the future (for api v2) as its more efficient
state = process_events(
mozmsg,
duo.get_authentication_log(api_version=2,
mintime=state['authentication'] + 1),
'authentication', state)
state = process_events(
mozmsg, duo.get_telephony_log(mintime=state['telephony'] + 1),
'telephony', state)
pickle.dump(state, open(options.statepath, 'wb'))
def main():
#Configuration loading
with open('auth02mozdef.json') as fd:
config = DotDict(hjson.load(fd))
if config == None:
print("No configuration file 'auth02mozdef.json' found.")
sys.exit(1)
headers = {'Authorization': 'Bearer {}'.format(config.auth0.token),
'Accept': 'application/json'}
fromid = load_state(config.state_file)
r = requests.get('{url}?take={reqnr}&sort=date:1&per_page={reqnr}&include_totals=true&from={fromid}'.format(
url=config.auth0.url,
reqnr=config.auth0.reqnr,
fromid=fromid),
headers=headers)
#If we fail here, auth0 is not responding to us the way we expected it
if (not r.ok):
raise Exception(r.url, r.reason, r.status_code, r.json())
ret = r.json()
#Process all new auth0 log msgs, normalize and send them to mozdef
for msg in ret:
mozmsg = mozdef.MozDefEvent(config.mozdef.url)
if config.DEBUG:
mozmsg.set_send_to_syslog(True, only_syslog=True)
mozmsg.source = config.auth0.url
mozmsg.tags = ['auth0']
msg = DotDict(msg)
lastid = msg._id
#Fill in mozdef msg fields from the auth0 msg
try:
mozmsg = process_msg(mozmsg, msg)
except KeyError as e:
#if this happens the msg was malformed in some way
mozmsg.details['error'] = 'true'
mozmsg.details['errormsg'] = e
mozmsg.summary = 'Failed to parse auth0 message'
mozmsg.send()
save_state(config.state_file, lastid)
def test_sample_event_username(self):
mozmsg = mozdef.MozDefEvent('http://*****:*****@mozilla.com'
assert mozmsg.details.userid == 'ad|Test-Connection|ttesterson'
assert mozmsg.summary == 'Success Silent Auth [email protected]'
def main():
logger.debug('started')
state = State(options.state_file_name)
try:
# capture the time we start running so next time we catch any events
# created while we run.
lastrun = toUTC(datetime.now()).isoformat()
scope = [
'https://www.googleapis.com/auth/admin.reports.audit.readonly',
'https://www.googleapis.com/auth/admin.reports.usage.readonly'
]
# get our credentials
credentials = service_account.Credentials.from_service_account_file(
options.jsoncredentialfile,
scopes=scope,
subject=options.impersonate)
# build a request to the admin sdk
api = googleapiclient.discovery.build('admin',
'reports_v1',
credentials=credentials)
response = api.activities().list(
userKey='all',
applicationName='login',
startTime=toUTC(
state.data['lastrun']).strftime('%Y-%m-%dT%H:%M:%S.000Z'),
maxResults=options.recordlimit).execute()
# fix up the event craziness to a flatter format
events = []
if 'items' in response:
for i in response['items']:
# flatten the sub dict/lists to pull out the good parts
mozmsg = mozdef.MozDefEvent(options.url)
mozmsg.category = 'google'
mozmsg.tags = ['google', 'authentication']
mozmsg.severity = 'INFO'
mozmsg.summary = 'google authentication: '
details = dict()
for keyValue in flattenDict(i):
# change key/values like:
# [email protected]
# to actor_email=value
try:
key, value = keyValue.split('=')
except ValueError as e:
continue
key = key.replace('.', '_').lower()
details[key] = value
# find important keys
# and adjust their location/name
if 'ipaddress' in details:
# it's the source ip
details['sourceipaddress'] = details['ipaddress']
del details['ipaddress']
if 'id_time' in details:
mozmsg.timestamp = details['id_time']
mozmsg.utctimestamp = details['id_time']
if 'events_name' in details:
mozmsg.summary += details['events_name'] + ' '
if 'actor_email' in details:
mozmsg.summary += details['actor_email'] + ' '
mozmsg.details = details
events.append(mozmsg)
# post events to mozdef
logger.debug('posting {0} google events to mozdef'.format(len(events)))
for e in events:
e.send()
# record the time we started as
# the start time for next time.
state.data['lastrun'] = lastrun
state.write_state_file()
except Exception as e:
logger.error("Unhandled exception, terminating: %r" % e)
parser.add_option(
'--num_times',
help='Number of times event is sent to loginput (default: 20)',
default=20)
options, arguments = parser.parse_args()
# Fill in with events you want to write
events = [{
"category": "testcategory",
"details": {
"program": "sshd",
"type": "Success Login",
"username": "******",
"sourceipaddress": '1.2.3.4',
},
"processname": "auth0_cron",
"severity": "INFO",
"source": "auth0",
"summary": "login invalid ldap_count_entries failed",
"tags": ["auth0"],
}]
for num in range(0, options.num_times):
for event in events:
mozmsg = mozdef.MozDefEvent(options.loginput_host + "/events/")
for key, value in event.items():
setattr(mozmsg, key, value)
mozmsg.send()
print("Wrote event to loginput")
time.sleep(0.2)
