def main():
if options.output=='syslog':
logger.addHandler(SysLogHandler(address=(options.sysloghostname,options.syslogport)))
else:
sh=logging.StreamHandler(sys.stderr)
sh.setFormatter(formatter)
logger.addHandler(sh)
logger.debug('started')
#logger.debug(options)
try:
es=pyes.ES((list('{0}'.format(s) for s in options.esservers)))
boto.connect_cloudtrail(aws_access_key_id=options.aws_access_key_id,aws_secret_access_key=options.aws_secret_access_key)
#capture the time we start running so next time we catch any files created while we run.
lastrun=toUTC(datetime.now()).isoformat()
#in case we don't archive files..only look at today and yesterday's files.
yesterday=date.strftime(datetime.utcnow()-timedelta(days=1),'%Y/%m/%d')
today = date.strftime(datetime.utcnow(),'%Y/%m/%d')
for region in boto.cloudtrail.regions():
logger.debug('connecting to AWS region {0}'.format(region.name))
ct=boto.cloudtrail.connect_to_region(region.name,aws_access_key_id=options.aws_access_key_id,aws_secret_access_key=options.aws_secret_access_key)
trails=ct.describe_trails()['trailList']
for trail in trails:
s3 = boto.connect_s3(aws_access_key_id=options.aws_access_key_id,aws_secret_access_key=options.aws_secret_access_key)
ctbucket=s3.get_bucket(trail['S3BucketName'])
#ctbucket.get_all_keys()
filelist=list()
for bfile in ctbucket.list():
if 'CloudTrail' in bfile.key and 'json' in bfile.key:
if today in bfile.key or yesterday in bfile.key:
filelist.append(bfile.key)
else:
if options.purge: #delete old files so we don't try to keep reading them.
s3file=ctbucket.get_key(afile)
s3file.delete()
for afile in filelist:
s3file=ctbucket.get_key(afile)
logger.debug('{0} {1}'.format(afile,s3file.last_modified))
if toUTC(s3file.last_modified)>options.lastrun:
compressedData=s3file.read()
databuf=StringIO(compressedData)
f=gzip.GzipFile(fileobj=databuf)
jlog=json.loads(f.read())
try:
for r in jlog['Records']:
r['utctimestamp']=toUTC(r['eventTime']).isoformat()
jbody=json.dumps(r)
res=es.index(index='events',doc_type='cloudtrail',doc=jbody)
#logger.debug(res)
except Exception as e:
logger.error('Error handling log record {0} {1}'.format(r, e))
continue
setConfig('lastrun',lastrun,options.configfile)
except boto.exception.NoAuthHandlerFound:
logger.error("No auth handler found, check your credentials")
except Exception as e:
logger.error("Unhandled exception, terminating: %r"%e)
def main():
if options.output=='syslog':
logger.addHandler(SysLogHandler(address=(options.sysloghostname,options.syslogport)))
else:
sh=logging.StreamHandler(sys.stderr)
sh.setFormatter(formatter)
logger.addHandler(sh)
logger.debug('started')
logger.debug(options)
try:
es=pyes.ES((list('{0}'.format(s) for s in options.esservers)))
#capture the time we start running so next time we catch any files created while we run.
lastrun=str(options.lastrun.isoformat())
today=datetime.utcnow().isoformat()+'+00:00'
# set the max num of items to 50k. At 600kB per item, that's already ~30MB of json body.
url = options.mighost+'/api/v1/search?type=command&status=success&threatfamily=compliance&report=complianceitems&limit=50000&before='+today+'&after='+lastrun
url = url.replace('+00:00', 'Z')
# Prepare the request, make an authorization token using the local keyring
token = makeToken(options.gpghome, options.keyid)
r = requests.get(url,
headers={'X-PGPAUTHORIZATION': token},
timeout=240, # timeout at 4 minutes. those are big requests.
verify=True)
if r.status_code == 200:
migjson=r.json()
logger.debug(url)
cicnt=0
for items in migjson['collection']['items']:
for dataitem in items['data']:
if dataitem['name'] == 'compliance item':
cicnt += 1
complianceitem = dataitem['value']
# historical data - index the new logs
res=es.index(index='complianceitems',doc_type='history',doc=complianceitem)
# last_known_state data - update the logs
# _id = md5('complianceitems'+check.ref+check.test.value+target)
docid=hashlib.md5('complianceitems'+complianceitem['check']['ref']+complianceitem['check']['test']['value']+complianceitem['target']).hexdigest()
res=es.index(index='complianceitems',id=docid,doc_type='last_known_state',doc=complianceitem)
if cicnt == 0:
logger.debug("No compliance item available, terminating")
setConfig('lastrun',today,options.configfile)
elif r.status_code == 500:
# api returns a 500 with an error body on failures
migjson=r.json()
raise Exception("API returned HTTP code %s and error '%s:%s'" %
(r.status_code,
migjson['collection']['error']['code'],
migjson['collection']['error']['message'])
)
else:
# another type of failure that's unlikely to have an error body
raise Exception("Failed with HTTP code %s" % r.status_code)
except Exception as e:
logger.error("Unhandled exception, terminating: %r"%e)
def main():
if options.output == "syslog":
logger.addHandler(SysLogHandler(address=(options.sysloghostname, options.syslogport)))
else:
sh = logging.StreamHandler(sys.stderr)
sh.setFormatter(formatter)
logger.addHandler(sh)
logger.debug("started")
# logger.debug(options)
try:
es = pyes.ES((list("{0}".format(s) for s in options.esservers)))
s = requests.Session()
s.headers.update({"Accept": "application/json"})
s.headers.update({"Content-type": "application/json"})
s.headers.update({"Authorization": "SSWS {0}".format(options.apikey)})
# capture the time we start running so next time we catch any events created while we run.
lastrun = toUTC(datetime.now()).isoformat()
# in case we don't archive files..only look at today and yesterday's files.
yesterday = date.strftime(datetime.utcnow() - timedelta(days=1), "%Y/%m/%d")
today = date.strftime(datetime.utcnow(), "%Y/%m/%d")
r = s.get(
"https://{0}/api/v1/events?startDate={1}&limit={2}".format(
options.oktadomain, toUTC(options.lastrun).strftime("%Y-%m-%dT%H:%M:%S.000Z"), options.recordlimit
)
)
if r.status_code == 200:
oktaevents = json.loads(r.text)
for event in oktaevents:
if "published" in event.keys():
if toUTC(event["published"]) > options.lastrun:
try:
mozdefEvent = dict()
mozdefEvent["utctimestamp"] = toUTC(event["published"]).isoformat()
mozdefEvent["category"] = "okta"
mozdefEvent["tags"] = ["okta"]
if "action" in event.keys() and "message" in event["action"].keys():
mozdefEvent["summary"] = event["action"]["message"]
mozdefEvent["details"] = event
jbody = json.dumps(mozdefEvent)
res = es.index(index="events", doc_type="okta", doc=jbody)
logger.debug(res)
except Exception as e:
logger.error("Error handling log record {0} {1}".format(r, e))
continue
else:
logger.error("Okta event does not contain published date: {0}".format(event))
setConfig("lastrun", lastrun, options.configfile)
except Exception as e:
logger.error("Unhandled exception, terminating: %r" % e)
def discover():
es = pyes.ES(server=(list('{0}'.format(s) for s in options.esservers)))
indicesManager = pyes.managers.Indices(es)
indices = indicesManager.get_indices()
config_indices = []
config_dobackup = []
config_rotation = []
config_pruning = []
for index in indices.keys():
index_template = index
freq = 'none'
pruning = '0'
if re.search(r'-[0-9]{8}', index):
freq = 'daily'
pruning = '20'
index_template = index[:-9]
elif re.search(r'-[0-9]{6}', index):
freq = 'monthly'
index_template = index[:-7]
if index_template not in config_indices:
config_indices.append(index_template)
config_dobackup.append('1')
config_rotation.append(freq)
config_pruning.append(pruning)
setConfig('backup_indices', ','.join(config_indices), options.configfile)
setConfig('backup_dobackup', ','.join(config_dobackup), options.configfile)
setConfig('backup_rotation', ','.join(config_rotation), options.configfile)
setConfig('backup_pruning', ','.join(config_pruning), options.configfile)
def main():
if options.output=='syslog':
logger.addHandler(SysLogHandler(address=(options.sysloghostname,options.syslogport)))
else:
sh=logging.StreamHandler(sys.stderr)
sh.setFormatter(formatter)
logger.addHandler(sh)
logger.debug('started')
logger.debug(options)
try:
es=pyes.ES((list('{0}'.format(s) for s in options.esservers)))
#capture the time we start running so next time we catch any files created while we run.
lastrun=str(options.lastrun.isoformat())
today=datetime.utcnow().isoformat()+'+00:00'
url = options.mighost+'/api/v1/search?type=command&status=done&threatfamily=compliance&report=complianceitems&limit=1000000&before='+today+'&after='+lastrun
url = url.replace('+00:00', 'Z')
r = requests.get(url,
cert=(options.sslclientcert, options.sslclientkey),
verify=options.sslcacert)
if r.status_code == 200:
migjson=r.json()
logger.debug(url)
cicnt=0
for items in migjson['collection']['items']:
for dataitem in items['data']:
if dataitem['name'] == 'compliance item':
cicnt += 1
complianceitem = dataitem['value']
# historical data - index the new logs
res=es.index(index='complianceitems',doc_type='history',doc=complianceitem)
# last_known_state data - update the logs
# _id = md5('complianceitems'+check.ref+check.test.value+target)
docid=hashlib.md5('complianceitems'+complianceitem['check']['ref']+complianceitem['check']['test']['value']+complianceitem['target']).hexdigest()
res=es.index(index='complianceitems',id=docid,doc_type='last_known_state',doc=complianceitem)
if cicnt == 0:
logger.debug("No compliance item available, terminating")
setConfig('lastrun',today,options.configfile)
except Exception as e:
logger.error("Unhandled exception, terminating: %r"%e)
def makeAttackers():
'''
send events that will be correlated into attackers using pre-defined IPs
'''
try:
#time for us to run?
timetoRun = toUTC(options.lastattacker) + timedelta(
minutes=options.attackersminutesinterval)
if timetoRun > toUTC(datetime.now()):
#print(timetoRun)
return
#print(timetoRun, options.lastalert)
eventfiles = glob.glob(options.alertsglob)
#pick a random number of events to send
for i in range(0, options.alertscount):
#pick a random type of event to send
eventfile = random.choice(eventfiles)
events = json.load(open(eventfile))
target = random.randint(0, len(events))
# if there's only one event in the file..use it.
if len(events) == 1 and target == 1:
target = 0
for event in events[target:target + 1]:
event['timestamp'] = pytz.timezone('UTC').localize(
datetime.utcnow()).isoformat()
#remove stored times
if 'utctimestamp' in event.keys():
del event['utctimestamp']
if 'receivedtimestamp' in event.keys():
del event['receivedtimestamp']
#add demo to the tags so it's clear it's not real data.
if 'tags' not in event.keys():
event['tags'] = list()
event['tags'].append('demodata')
event['tags'].append('demoalert')
#replace potential <randomipaddress> with a random ip address
if 'summary' in event.keys(
) and '<randomipaddress>' in event['summary']:
randomIP = genAttackerIPv4()
event['summary'] = event['summary'].replace(
"<randomipaddress>", randomIP)
if 'details' not in event.keys():
event['details'] = dict()
event['details']['sourceipaddress'] = randomIP
event['details']['sourceipv4address'] = randomIP
if 'duplicate' in event.keys():
# send this event multiple times to trigger an alert
for x in range(0, int(event['duplicate'])):
logcache.put(json.dumps(event))
else:
logcache.put(json.dumps(event))
lastattacker = toUTC(datetime.now()).isoformat()
setConfig('lastattacker', lastattacker, options.configfile)
if not logcache.empty():
time.sleep(.01)
try:
postingProcess = Process(target=postLogs,
args=(logcache, ),
name="json2MozdefDemoData")
postingProcess.start()
except OSError as e:
if e.errno == 35: # resource temporarily unavailable.
print(e)
pass
else:
logger.error('%r' % e)
except KeyboardInterrupt as e:
sys.exit(1)
def main():
if options.output == 'syslog':
logger.addHandler(
SysLogHandler(address=(options.sysloghostname,
options.syslogport)))
else:
sh = logging.StreamHandler(sys.stderr)
sh.setFormatter(formatter)
logger.addHandler(sh)
logger.debug('started')
#logger.debug(options)
try:
es = pyes.ES((list('{0}'.format(s) for s in options.esservers)))
s = requests.Session()
s.headers.update({'Accept': 'application/json'})
s.headers.update({'Content-type': 'application/json'})
s.headers.update({'Authorization': 'SSWS {0}'.format(options.apikey)})
#capture the time we start running so next time we catch any events created while we run.
lastrun = toUTC(datetime.now()).isoformat()
#in case we don't archive files..only look at today and yesterday's files.
yesterday = date.strftime(datetime.utcnow() - timedelta(days=1),
'%Y/%m/%d')
today = date.strftime(datetime.utcnow(), '%Y/%m/%d')
r = s.get('https://{0}/api/v1/events?startDate={1}&limit={2}'.format(
options.oktadomain,
toUTC(options.lastrun).strftime('%Y-%m-%dT%H:%M:%S.000Z'),
options.recordlimit))
if r.status_code == 200:
oktaevents = json.loads(r.text)
for event in oktaevents:
if 'published' in event.keys():
if toUTC(event['published']) > options.lastrun:
try:
mozdefEvent = dict()
mozdefEvent['utctimestamp'] = toUTC(
event['published']).isoformat()
mozdefEvent['category'] = 'okta'
mozdefEvent['tags'] = ['okta']
if 'action' in event.keys(
) and 'message' in event['action'].keys():
mozdefEvent['summary'] = event['action'][
'message']
mozdefEvent['details'] = event
jbody = json.dumps(mozdefEvent)
res = es.index(index='events',
doc_type='okta',
doc=jbody)
logger.debug(res)
except Exception as e:
logger.error(
'Error handling log record {0} {1}'.format(
r, e))
continue
else:
logger.error(
'Okta event does not contain published date: {0}'.
format(event))
setConfig('lastrun', lastrun, options.configfile)
except Exception as e:
logger.error("Unhandled exception, terminating: %r" % e)
def main():
if options.output == 'syslog':
logger.addHandler(
SysLogHandler(address=(options.sysloghostname,
options.syslogport)))
else:
sh = logging.StreamHandler(sys.stderr)
sh.setFormatter(formatter)
logger.addHandler(sh)
logger.debug('started')
try:
# capture the time we start running so next time we catch any events
# created while we run.
lastrun = toUTC(datetime.now()).isoformat()
# get our credentials
mozdefClient = json.loads(open(options.jsoncredentialfile).read())
client_email = mozdefClient['client_email']
private_key = mozdefClient['private_key']
# set the oauth scope we will request
scope = [
'https://www.googleapis.com/auth/admin.reports.audit.readonly',
'https://www.googleapis.com/auth/admin.reports.usage.readonly'
]
# authorize our http object
# we do this as a 'service account' so it's important
# to specify the correct 'sub' option
# or you will get access denied even with correct delegations/scope
credentials = SignedJwtAssertionCredentials(client_email,
private_key,
scope=scope,
sub=options.impersonate)
http = Http()
credentials.authorize(http)
# build a request to the admin sdk
api = build('admin', 'reports_v1', http=http)
response = api.activities().list(
userKey='all',
applicationName='login',
startTime=toUTC(
options.lastrun).strftime('%Y-%m-%dT%H:%M:%S.000Z'),
maxResults=options.recordlimit).execute()
# fix up the event craziness to a flatter format
events = []
if 'items' in response.keys():
for i in response['items']:
# flatten the sub dict/lists to pull out the good parts
event = dict(category='google')
event['tags'] = ['google', 'authentication']
event['severity'] = 'INFO'
event['summary'] = 'google authentication: '
details = dict()
for keyValue in flattenDict(i):
# change key/values like:
# [email protected]
# to actor_email=value
key, value = keyValue.split('=')
key = key.replace('.', '_').lower()
details[key] = value
# find important keys
# and adjust their location/name
if 'ipaddress' in details.keys():
# it's the source ip
details['sourceipaddress'] = details['ipaddress']
del details['ipaddress']
if 'id_time' in details.keys():
event['timestamp'] = details['id_time']
event['utctimestamp'] = details['id_time']
if 'events_name' in details.keys():
event['summary'] += details['events_name'] + ' '
if 'actor_email' in details.keys():
event['summary'] += details['actor_email'] + ' '
event['details'] = details
events.append(event)
# post events to mozdef
logger.debug('posting {0} google events to mozdef'.format(len(events)))
for e in events:
requests.post(options.url, data=json.dumps(e))
# record the time we started as
# the start time for next time.
setConfig('lastrun', lastrun, options.configfile)
except Exception as e:
logger.error("Unhandled exception, terminating: %r" % e)
def main():
if options.output == 'syslog':
logger.addHandler(
SysLogHandler(address=(options.sysloghostname,
options.syslogport)))
else:
sh = logging.StreamHandler(sys.stderr)
sh.setFormatter(formatter)
logger.addHandler(sh)
logger.debug('started')
#logger.debug(options)
try:
es = pyes.ES((list('{0}'.format(s) for s in options.esservers)))
boto.connect_cloudtrail(
aws_access_key_id=options.aws_access_key_id,
aws_secret_access_key=options.aws_secret_access_key)
#capture the time we start running so next time we catch any files created while we run.
lastrun = toUTC(datetime.now()).isoformat()
#in case we don't archive files..only look at today and yesterday's files.
yesterday = date.strftime(datetime.utcnow() - timedelta(days=1),
'%Y/%m/%d')
today = date.strftime(datetime.utcnow(), '%Y/%m/%d')
for region in boto.cloudtrail.regions():
logger.debug('connecting to AWS region {0}'.format(region.name))
ct = boto.cloudtrail.connect_to_region(
region.name,
aws_access_key_id=options.aws_access_key_id,
aws_secret_access_key=options.aws_secret_access_key)
trails = ct.describe_trails()['trailList']
for trail in trails:
s3 = boto.connect_s3(
aws_access_key_id=options.aws_access_key_id,
aws_secret_access_key=options.aws_secret_access_key)
ctbucket = s3.get_bucket(trail['S3BucketName'])
#ctbucket.get_all_keys()
filelist = list()
for bfile in ctbucket.list():
if 'CloudTrail' in bfile.key and 'json' in bfile.key:
if today in bfile.key or yesterday in bfile.key:
filelist.append(bfile.key)
else:
if options.purge: #delete old files so we don't try to keep reading them.
s3file = ctbucket.get_key(afile)
s3file.delete()
for afile in filelist:
s3file = ctbucket.get_key(afile)
logger.debug('{0} {1}'.format(afile, s3file.last_modified))
if toUTC(s3file.last_modified) > options.lastrun:
compressedData = s3file.read()
databuf = StringIO(compressedData)
f = gzip.GzipFile(fileobj=databuf)
jlog = json.loads(f.read())
try:
for r in jlog['Records']:
r['utctimestamp'] = toUTC(
r['eventTime']).isoformat()
jbody = json.dumps(r)
res = es.index(index='events',
doc_type='cloudtrail',
doc=jbody)
#logger.debug(res)
except Exception as e:
logger.error(
'Error handling log record {0} {1}'.format(
r, e))
continue
setConfig('lastrun', lastrun, options.configfile)
except boto.exception.NoAuthHandlerFound:
logger.error("No auth handler found, check your credentials")
except Exception as e:
logger.error("Unhandled exception, terminating: %r" % e)
def makeAttackers():
'''
send events that will be correlated into attackers using pre-defined IPs
'''
try:
#time for us to run?
timetoRun=toUTC(options.lastattacker) + timedelta(minutes=options.attackersminutesinterval)
if timetoRun > toUTC(datetime.now()):
#print(timetoRun)
return
#print(timetoRun, options.lastalert)
eventfiles = glob.glob(options.alertsglob)
#pick a random number of events to send
for i in range(0, options.alertscount):
#pick a random type of event to send
eventfile = random.choice(eventfiles)
events = json.load(open(eventfile))
target = random.randint(0, len(events))
# if there's only one event in the file..use it.
if len(events) == 1 and target == 1:
target = 0
for event in events[target:target+1]:
event['timestamp'] = pytz.timezone('UTC').localize(datetime.utcnow()).isoformat()
#remove stored times
if 'utctimestamp' in event.keys():
del event['utctimestamp']
if 'receivedtimestamp' in event.keys():
del event['receivedtimestamp']
#add demo to the tags so it's clear it's not real data.
if 'tags' not in event.keys():
event['tags'] = list()
event['tags'].append('demodata')
event['tags'].append('demoalert')
#replace potential <randomipaddress> with a random ip address
if 'summary' in event.keys() and '<randomipaddress>' in event['summary']:
randomIP = genAttackerIPv4()
event['summary'] = event['summary'].replace("<randomipaddress>", randomIP)
if 'details' not in event.keys():
event['details'] = dict()
event['details']['sourceipaddress'] = randomIP
event['details']['sourceipv4address'] = randomIP
if 'duplicate' in event.keys():
# send this event multiple times to trigger an alert
for x in range(0, int(event['duplicate'])):
logcache.put(json.dumps(event))
else:
logcache.put(json.dumps(event))
lastattacker=toUTC(datetime.now()).isoformat()
setConfig('lastattacker',lastattacker,options.configfile)
if not logcache.empty():
time.sleep(.01)
try:
postingProcess=Process(target=postLogs,args=(logcache,),name="json2MozdefDemoData")
postingProcess.start()
except OSError as e:
if e.errno==35: #resource temporarily unavailable.
print(e)
pass
else:
logger.error('%r'%e)
except KeyboardInterrupt as e:
sys.exit(1)
def main():
if options.output=='syslog':
logger.addHandler(SysLogHandler(address=(options.sysloghostname,options.syslogport)))
else:
sh=logging.StreamHandler(sys.stderr)
sh.setFormatter(formatter)
logger.addHandler(sh)
logger.debug('started')
try:
# capture the time we start running so next time we catch any events
# created while we run.
lastrun=toUTC(datetime.now()).isoformat()
# get our credentials
mozdefClient=json.loads(open(options.jsoncredentialfile).read())
client_email = mozdefClient['client_email']
private_key=mozdefClient['private_key']
# set the oauth scope we will request
scope=[
'https://www.googleapis.com/auth/admin.reports.audit.readonly',
'https://www.googleapis.com/auth/admin.reports.usage.readonly'
]
# authorize our http object
# we do this as a 'service account' so it's important
# to specify the correct 'sub' option
# or you will get access denied even with correct delegations/scope
credentials = SignedJwtAssertionCredentials(client_email,
private_key,
scope=scope,
sub=options.impersonate)
http = Http()
credentials.authorize(http)
# build a request to the admin sdk
api = build('admin', 'reports_v1', http=http)
response = api.activities().list(userKey='all',
applicationName='login',
startTime=toUTC(options.lastrun).strftime('%Y-%m-%dT%H:%M:%S.000Z'),
maxResults=options.recordlimit).execute()
# fix up the event craziness to a flatter format
events=[]
if 'items' in response.keys():
for i in response['items']:
# flatten the sub dict/lists to pull out the good parts
event=dict(category='google')
event['tags']=['google','authentication']
event['severity']='INFO'
event['summary']='google authentication: '
details=dict()
for keyValue in flattenDict(i):
# change key/values like:
# [email protected]
# to actor_email=value
key,value =keyValue.split('=')
key=key.replace('.','_').lower()
details[key]=value
# find important keys
# and adjust their location/name
if 'ipaddress' in details.keys():
# it's the source ip
details['sourceipaddress']=details['ipaddress']
del details['ipaddress']
if 'id_time' in details.keys():
event['timestamp']=details['id_time']
event['utctimestamp']=details['id_time']
if 'events_name' in details.keys():
event['summary']+= details['events_name'] + ' '
if 'actor_email' in details.keys():
event['summary']+= details['actor_email'] + ' '
event['details']=details
events.append(event)
# post events to mozdef
logger.debug('posting {0} google events to mozdef'.format(len(events)))
for e in events:
requests.post(options.url,data=json.dumps(e))
# record the time we started as
# the start time for next time.
setConfig('lastrun',lastrun,options.configfile)
except Exception as e:
logger.error("Unhandled exception, terminating: %r"%e)