def main(self):
date_timedelta = dict(minutes=30)
self.config_file = './unauth_ssh_pyes.conf'
self.config = None
self.initConfiguration()
must = [
pyes.TermFilter('_type', 'event'),
pyes.TermFilter('category', 'syslog'),
pyes.TermFilter('details.program', 'sshd'),
pyes.QueryFilter(
pyes.QueryStringQuery('details.hostname: /{}/'.format(
self.config.hostfilter))),
pyes.QueryFilter(
pyes.MatchQuery('summary',
'Accepted publickey {}'.format(
self.config.user),
operator='and'))
]
must_not = []
for x in self.config.skiphosts:
must_not.append(pyes.QueryFilter(pyes.MatchQuery('summary', x)))
self.filtersManual(date_timedelta, must=must, must_not=must_not)
self.searchEventsSimple()
self.walkEvents()
def searchESForBROAttackers(es, threshold):
begindateUTC = toUTC(datetime.now() - timedelta(hours=2))
enddateUTC = toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
qBro = pyes.QueryFilter(pyes.MatchQuery('category', 'bro_notice',
'phrase'))
qErr = pyes.QueryFilter(
pyes.MatchQuery('details.note',
'MozillaHTTPErrors::Excessive_HTTP_Errors_Attacker',
'phrase'))
q = pyes.FilteredQuery(q, pyes.BoolFilter(must=[qBro, qErr]))
results = es.search(q, size=1000, indices='events,events-previous')
# grab results as native es results to avoid pyes iteration bug
# and get easier access to es metadata fields
rawresults = results._search_raw()['hits']['hits']
# Hit count is buried in the 'sub' field
# as: 'sub': u'6 in 1.0 hr, eps: 0'
# cull the records for hitcounts over the threshold before returning
attackers = list()
for r in rawresults:
hitcount = int(r['_source']['details']['sub'].split()[0])
if hitcount > threshold:
attackers.append(r)
return attackers
def main(self):
# look for events in last 15 mins
date_timedelta = dict(minutes=15)
# Configure filters using pyes
must = [
pyes.TermFilter('_type', 'event'),
pyes.TermFilter('eventsource', 'systemslogs'),
pyes.ExistsFilter('details.sourceipaddress'),
pyes.QueryFilter(pyes.MatchQuery('summary', 'failed', 'phrase')),
pyes.TermFilter('program', 'sshd'),
pyes.QueryFilter(
pyes.MatchQuery('summary', 'login ldap_count_entries',
'boolean'))
]
must_not = [
pyes.QueryFilter(
pyes.MatchQuery('summary', '10.22.8.128', 'phrase')),
pyes.QueryFilter(pyes.MatchQuery('summary', '10.8.75.35',
'phrase')),
pyes.QueryFilter(
pyes.MatchQuery('summary', '208.118.237.', 'phrase'))
]
self.filtersManual(date_timedelta, must=must, must_not=must_not)
# Search aggregations on field 'sourceipaddress', keep 50 samples of events at most
self.searchEventsAggreg('sourceipaddress', samplesLimit=50)
# alert when >= X matching events in an aggregation
self.walkAggregations(threshold=10)
def main(self):
# look for events in last X mins
date_timedelta = dict(minutes=2)
# Configure filters using pyes
must = [
pyes.TermFilter('_type', 'event'),
pyes.QueryFilter(pyes.MatchQuery('summary','failed','phrase')),
pyes.TermFilter('program','sshd'),
pyes.QueryFilter(pyes.MatchQuery('summary', 'login invalid ldap_count_entries', 'boolean'))
]
must_not = [
pyes.QueryFilter(pyes.MatchQuery('summary','10.22.75.203','phrase')),
pyes.QueryFilter(pyes.MatchQuery('summary','10.8.75.144','phrase'))
]
self.filtersManual(date_timedelta, must=must, must_not=must_not)
# Search aggregations on field 'sourceipaddress', keep X samples of events at most
self.searchEventsAggreg('sourceipaddress', samplesLimit=10)
# alert when >= X matching events in an aggregation
self.walkAggregations(threshold=10)
def main(self):
# look for events in last x
date_timedelta = dict(minutes=15)
# Configure filters using pyes
must = [
pyes.TermFilter('category', 'ldapChange'),
pyes.TermFilter('changetype', 'modify'),
pyes.QueryFilter(pyes.MatchQuery("summary","groups"))
]
self.filtersManual(date_timedelta, must=must)
# Search events
self.searchEventsSimple()
self.walkEvents()
def main(self):
# look for events in last 15 mins
date_timedelta = dict(minutes=15)
# Configure filters using pyes
must = [
pyes.QueryFilter(pyes.MatchQuery('summary','Failsafe Duo login','phrase'))
]
self.filtersManual(date_timedelta, must=must)
# Search aggregations on field 'sourceipaddress', keep X samples of events at most
self.searchEventsAggregated('details.hostname', samplesLimit=10)
# alert when >= X matching events in an aggregation
# in this case, always
self.walkAggregations(threshold=1)
def main(self):
# look for events in last 10 mins
date_timedelta = dict(minutes=10)
# Configure filters using pyes
must = [
pyes.TermFilter('_type', 'event'),
pyes.TermFilter('program', 'fail2ban'),
pyes.QueryFilter(pyes.MatchQuery("summary","has been banned","phrase"))
]
self.filtersManual(date_timedelta, must=must)
# Search events
self.searchEventsSimple()
self.walkEvents()
def main(self):
# look for events in last x
date_timedelta = dict(minutes=15)
# Configure filters
# looking for pwdAccountLockedTime setting by admin
must = [
pyes.TermFilter('category', 'ldapChange'),
pyes.TermFilter("actor", "cn=admin,dc=mozilla"),
pyes.QueryFilter(pyes.MatchQuery('changepairs', 'replace:pwdAccountLockedTime','phrase'))
]
self.filtersManual(date_timedelta, must=must)
# Search events
self.searchEventsSimple()
self.walkEvents()
def main(self):
# look for events in last 15 mins
date_timedelta = dict(minutes=15)
# Configure filters using pyes
must = [
pyes.TermFilter('_type', 'bro'),
pyes.TermFilter('eventsource', 'nsm'),
pyes.TermFilter('category', 'bronotice'),
pyes.ExistsFilter('details.sourceipaddress'),
pyes.QueryFilter(pyes.MatchQuery('details.note','SSH::Password_Guessing','phrase')),
]
self.filtersManual(date_timedelta, must=must)
# Search events
self.searchEventsSimple()
self.walkEvents()
def main(self):
# look for events in last 15 mins
date_timedelta = dict(minutes=15)
# Configure filters using pyes
must = [
pyes.TermFilter('tags', 'nubis_events_non_prod'),
pyes.TermFilter('tags', 'nubis_events_prod'),
pyes.TermFilter('category', 'syslog'),
pyes.TermFilter('details.__tag', 'ec2.forward.squid.access'),
pyes.QueryFilter(pyes.MatchQuery('details.summary','is DENIED, because it matched','phrase')),
]
self.filtersManual(date_timedelta, must=must)
# Search events
self.searchEventsSimple()
self.walkEvents()
def main(self):
# look for events in last X mins
date_timedelta = dict(minutes=2)
# Configure filters using pyes
must = [
pyes.TermFilter('_type', 'bro'),
pyes.TermFilter('eventsource', 'nsm'),
pyes.TermFilter('category', 'brointel'),
pyes.ExistsFilter('seenindicator'),
pyes.QueryFilter(pyes.MatchQuery('hostname', 'sensor1 sensor2 sensor3', 'boolean'))
]
self.filtersManual(date_timedelta, must=must)
# Search aggregations on field 'seenindicator', keep X samples of events at most
self.searchEventsAggregated('details.seenindicator', samplesLimit=10)
# alert when >= X matching events in an aggregation
self.walkAggregations(threshold=10)
def esSearch(es, macassignments=None, begindateUTC=None, enddateUTC=None):
'''
Search ES for an event that ties a username to a mac address
This example searches for junos wifi correlations on authentication success
Expecting an event like: user: [email protected]; mac: 5c:f9:38:b1:de:cf; author reason: roamed session; ssid: ANSSID; AP 46/2\n
'''
usermacre=re.compile(r'''user: (?P<username>.*?); mac: (?P<macaddress>.*?); ''',re.IGNORECASE)
correlations={} # list of dicts to populate hits we find
if begindateUTC is None:
begindateUTC = toUTC(datetime.now() - timedelta(minutes=options.correlationminutes))
if enddateUTC is None:
enddateUTC = toUTC(datetime.now())
try:
# search for events within the date range
qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
q=pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(pyes.BoolFilter(must=[
qDate,
pyes.TermFilter("program","AUTHORIZATION-SUCCESS")
],
must_not=[
pyes.QueryFilter(
pyes.MatchQuery("summary","last-resort","phrase")
)
]))
results = es.search(q, size=10000, indices=['events', 'events-previous'])
rawresults=results._search_raw()
for r in rawresults['hits']['hits']:
fields = re.search(usermacre,r['_source']['summary'])
if fields:
if '{0} {1}'.format(fields.group('username'),fields.group('macaddress')) not in correlations.keys():
if fields.group('macaddress')[0:8].lower() in macassignments.keys():
entity=macassignments[fields.group('macaddress')[0:8].lower()]
else:
entity='unknown'
correlations['{0} {1}'.format(fields.group('username'),fields.group('macaddress'))]=dict(username=fields.group('username'),
macaddress=fields.group('macaddress'),
entity=entity,
utctimestamp=r['_source']['utctimestamp'])
return correlations
except pyes.exceptions.NoServerAvailable:
logger.error('Elastic Search server could not be reached, check network connectivity')
def searchForSSHKeys(es):
begindateUTC = toUTC(datetime.now() - timedelta(minutes=5))
enddateUTC = toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange(
'utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'event')
qEvents = pyes.TermFilter("program", "sshd")
q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(
pyes.BoolFilter(must=[
qType, qDate, qEvents,
pyes.QueryFilter(
pyes.MatchQuery("summary",
"found matching key accepted publickey",
"boolean"))
]))
results = es.search(q, size=10000, indices='events')
# return raw search to avoid pyes iteration bug
return results._search_raw()
def filtersFromKibanaDash(self, fp, date_timedelta):
"""
Import filters from a kibana dashboard
fp is the file path of the json file
date_timedelta is a dict in timedelta format
see https://docs.python.org/2/library/datetime.html#timedelta-objects
"""
f = open(fp)
data = json.load(f)
must = []
should = []
must_not = []
for filtid in data['services']['filter']['list'].keys():
filt = data['services']['filter']['list'][filtid]
if filt['active'] and 'query' in filt.keys():
value = filt['query'].split(':')[-1]
fieldname = filt['query'].split(':')[0].split('.')[-1]
# self.log.info(fieldname)
# self.log.info(value)
# field: fieldname
# query: value
if 'field' in filt.keys():
fieldname = filt['field']
value = filt['query']
if '\"' in value:
value = value.split('\"')[1]
pyesfilt = pyes.QueryFilter(
pyes.MatchQuery(fieldname, value, 'phrase'))
else:
pyesfilt = pyes.TermFilter(fieldname, value)
else:
# _exists_:field
if filt['query'].startswith('_exists_:'):
pyesfilt = pyes.ExistsFilter(value.split('.')[-1])
# self.log.info('exists %s' % value.split('.')[-1])
# _missing_:field
elif filt['query'].startswith('_missing_:'):
pyesfilt = pyes.filters.MissingFilter(
value.split('.')[-1])
# self.log.info('missing %s' % value.split('.')[-1])
# field:"value"
elif '\"' in value:
value = value.split('\"')[1]
pyesfilt = pyes.QueryFilter(
pyes.MatchQuery(fieldname, value, 'phrase'))
# self.log.info("phrase %s %s" % (fieldname, value))
# field:(value1 value2 value3)
elif '(' in value and ')' in value:
value = value.split('(')[1]
value = value.split('(')[0]
pyesfilt = pyes.QueryFilter(
pyes.MatchQuery(fieldname, value, "boolean"))
# field:value
else:
pyesfilt = pyes.TermFilter(fieldname, value)
# self.log.info("terms %s %s" % (fieldname, value))
if filt['mandate'] == 'must':
must.append(pyesfilt)
elif filt['mandate'] == 'either':
should.append(pyesfilt)
elif filt['mandate'] == 'mustNot':
must_not.append(pyesfilt)
# self.log.info(must)
f.close()
self.filtersManual(date_timedelta,
must=must,
should=should,
must_not=must_not)
