def main(self): date_timedelta = dict(minutes=30) self.config_file = './unauth_ssh_pyes.conf' self.config = None self.initConfiguration() must = [ pyes.TermFilter('_type', 'event'), pyes.TermFilter('category', 'syslog'), pyes.TermFilter('details.program', 'sshd'), pyes.QueryFilter( pyes.QueryStringQuery('details.hostname: /{}/'.format( self.config.hostfilter))), pyes.QueryFilter( pyes.MatchQuery('summary', 'Accepted publickey {}'.format( self.config.user), operator='and')) ] must_not = [] for x in self.config.skiphosts: must_not.append(pyes.QueryFilter(pyes.MatchQuery('summary', x))) self.filtersManual(date_timedelta, must=must, must_not=must_not) self.searchEventsSimple() self.walkEvents()
def searchESForBROAttackers(es, threshold): begindateUTC = toUTC(datetime.now() - timedelta(hours=2)) enddateUTC = toUTC(datetime.now()) qDate = pyes.RangeQuery(qrange=pyes.ESRange( 'utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) q = pyes.ConstantScoreQuery(pyes.MatchAllQuery()) qBro = pyes.QueryFilter(pyes.MatchQuery('category', 'bro_notice', 'phrase')) qErr = pyes.QueryFilter( pyes.MatchQuery('details.note', 'MozillaHTTPErrors::Excessive_HTTP_Errors_Attacker', 'phrase')) q = pyes.FilteredQuery(q, pyes.BoolFilter(must=[qBro, qErr])) results = es.search(q, size=1000, indices='events,events-previous') # grab results as native es results to avoid pyes iteration bug # and get easier access to es metadata fields rawresults = results._search_raw()['hits']['hits'] # Hit count is buried in the 'sub' field # as: 'sub': u'6 in 1.0 hr, eps: 0' # cull the records for hitcounts over the threshold before returning attackers = list() for r in rawresults: hitcount = int(r['_source']['details']['sub'].split()[0]) if hitcount > threshold: attackers.append(r) return attackers
def main(self): # look for events in last 15 mins date_timedelta = dict(minutes=15) # Configure filters using pyes must = [ pyes.TermFilter('_type', 'event'), pyes.TermFilter('eventsource', 'systemslogs'), pyes.ExistsFilter('details.sourceipaddress'), pyes.QueryFilter(pyes.MatchQuery('summary', 'failed', 'phrase')), pyes.TermFilter('program', 'sshd'), pyes.QueryFilter( pyes.MatchQuery('summary', 'login ldap_count_entries', 'boolean')) ] must_not = [ pyes.QueryFilter( pyes.MatchQuery('summary', '10.22.8.128', 'phrase')), pyes.QueryFilter(pyes.MatchQuery('summary', '10.8.75.35', 'phrase')), pyes.QueryFilter( pyes.MatchQuery('summary', '208.118.237.', 'phrase')) ] self.filtersManual(date_timedelta, must=must, must_not=must_not) # Search aggregations on field 'sourceipaddress', keep 50 samples of events at most self.searchEventsAggreg('sourceipaddress', samplesLimit=50) # alert when >= X matching events in an aggregation self.walkAggregations(threshold=10)
def main(self): # look for events in last X mins date_timedelta = dict(minutes=2) # Configure filters using pyes must = [ pyes.TermFilter('_type', 'event'), pyes.QueryFilter(pyes.MatchQuery('summary','failed','phrase')), pyes.TermFilter('program','sshd'), pyes.QueryFilter(pyes.MatchQuery('summary', 'login invalid ldap_count_entries', 'boolean')) ] must_not = [ pyes.QueryFilter(pyes.MatchQuery('summary','10.22.75.203','phrase')), pyes.QueryFilter(pyes.MatchQuery('summary','10.8.75.144','phrase')) ] self.filtersManual(date_timedelta, must=must, must_not=must_not) # Search aggregations on field 'sourceipaddress', keep X samples of events at most self.searchEventsAggreg('sourceipaddress', samplesLimit=10) # alert when >= X matching events in an aggregation self.walkAggregations(threshold=10)
def main(self): # look for events in last x date_timedelta = dict(minutes=15) # Configure filters using pyes must = [ pyes.TermFilter('category', 'ldapChange'), pyes.TermFilter('changetype', 'modify'), pyes.QueryFilter(pyes.MatchQuery("summary","groups")) ] self.filtersManual(date_timedelta, must=must) # Search events self.searchEventsSimple() self.walkEvents()
def main(self): # look for events in last 15 mins date_timedelta = dict(minutes=15) # Configure filters using pyes must = [ pyes.QueryFilter(pyes.MatchQuery('summary','Failsafe Duo login','phrase')) ] self.filtersManual(date_timedelta, must=must) # Search aggregations on field 'sourceipaddress', keep X samples of events at most self.searchEventsAggregated('details.hostname', samplesLimit=10) # alert when >= X matching events in an aggregation # in this case, always self.walkAggregations(threshold=1)
def main(self): # look for events in last 10 mins date_timedelta = dict(minutes=10) # Configure filters using pyes must = [ pyes.TermFilter('_type', 'event'), pyes.TermFilter('program', 'fail2ban'), pyes.QueryFilter(pyes.MatchQuery("summary","has been banned","phrase")) ] self.filtersManual(date_timedelta, must=must) # Search events self.searchEventsSimple() self.walkEvents()
def main(self): # look for events in last x date_timedelta = dict(minutes=15) # Configure filters # looking for pwdAccountLockedTime setting by admin must = [ pyes.TermFilter('category', 'ldapChange'), pyes.TermFilter("actor", "cn=admin,dc=mozilla"), pyes.QueryFilter(pyes.MatchQuery('changepairs', 'replace:pwdAccountLockedTime','phrase')) ] self.filtersManual(date_timedelta, must=must) # Search events self.searchEventsSimple() self.walkEvents()
def main(self): # look for events in last 15 mins date_timedelta = dict(minutes=15) # Configure filters using pyes must = [ pyes.TermFilter('_type', 'bro'), pyes.TermFilter('eventsource', 'nsm'), pyes.TermFilter('category', 'bronotice'), pyes.ExistsFilter('details.sourceipaddress'), pyes.QueryFilter(pyes.MatchQuery('details.note','SSH::Password_Guessing','phrase')), ] self.filtersManual(date_timedelta, must=must) # Search events self.searchEventsSimple() self.walkEvents()
def main(self): # look for events in last 15 mins date_timedelta = dict(minutes=15) # Configure filters using pyes must = [ pyes.TermFilter('tags', 'nubis_events_non_prod'), pyes.TermFilter('tags', 'nubis_events_prod'), pyes.TermFilter('category', 'syslog'), pyes.TermFilter('details.__tag', 'ec2.forward.squid.access'), pyes.QueryFilter(pyes.MatchQuery('details.summary','is DENIED, because it matched','phrase')), ] self.filtersManual(date_timedelta, must=must) # Search events self.searchEventsSimple() self.walkEvents()
def main(self): # look for events in last X mins date_timedelta = dict(minutes=2) # Configure filters using pyes must = [ pyes.TermFilter('_type', 'bro'), pyes.TermFilter('eventsource', 'nsm'), pyes.TermFilter('category', 'brointel'), pyes.ExistsFilter('seenindicator'), pyes.QueryFilter(pyes.MatchQuery('hostname', 'sensor1 sensor2 sensor3', 'boolean')) ] self.filtersManual(date_timedelta, must=must) # Search aggregations on field 'seenindicator', keep X samples of events at most self.searchEventsAggregated('details.seenindicator', samplesLimit=10) # alert when >= X matching events in an aggregation self.walkAggregations(threshold=10)
def esSearch(es, macassignments=None, begindateUTC=None, enddateUTC=None): ''' Search ES for an event that ties a username to a mac address This example searches for junos wifi correlations on authentication success Expecting an event like: user: [email protected]; mac: 5c:f9:38:b1:de:cf; author reason: roamed session; ssid: ANSSID; AP 46/2\n ''' usermacre=re.compile(r'''user: (?P<username>.*?); mac: (?P<macaddress>.*?); ''',re.IGNORECASE) correlations={} # list of dicts to populate hits we find if begindateUTC is None: begindateUTC = toUTC(datetime.now() - timedelta(minutes=options.correlationminutes)) if enddateUTC is None: enddateUTC = toUTC(datetime.now()) try: # search for events within the date range qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) q=pyes.ConstantScoreQuery(pyes.MatchAllQuery()) q.filters.append(pyes.BoolFilter(must=[ qDate, pyes.TermFilter("program","AUTHORIZATION-SUCCESS") ], must_not=[ pyes.QueryFilter( pyes.MatchQuery("summary","last-resort","phrase") ) ])) results = es.search(q, size=10000, indices=['events', 'events-previous']) rawresults=results._search_raw() for r in rawresults['hits']['hits']: fields = re.search(usermacre,r['_source']['summary']) if fields: if '{0} {1}'.format(fields.group('username'),fields.group('macaddress')) not in correlations.keys(): if fields.group('macaddress')[0:8].lower() in macassignments.keys(): entity=macassignments[fields.group('macaddress')[0:8].lower()] else: entity='unknown' correlations['{0} {1}'.format(fields.group('username'),fields.group('macaddress'))]=dict(username=fields.group('username'), macaddress=fields.group('macaddress'), entity=entity, utctimestamp=r['_source']['utctimestamp']) return correlations except pyes.exceptions.NoServerAvailable: logger.error('Elastic Search server could not be reached, check network connectivity')
def searchForSSHKeys(es): begindateUTC = toUTC(datetime.now() - timedelta(minutes=5)) enddateUTC = toUTC(datetime.now()) qDate = pyes.RangeQuery(qrange=pyes.ESRange( 'utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) qType = pyes.TermFilter('_type', 'event') qEvents = pyes.TermFilter("program", "sshd") q = pyes.ConstantScoreQuery(pyes.MatchAllQuery()) q.filters.append( pyes.BoolFilter(must=[ qType, qDate, qEvents, pyes.QueryFilter( pyes.MatchQuery("summary", "found matching key accepted publickey", "boolean")) ])) results = es.search(q, size=10000, indices='events') # return raw search to avoid pyes iteration bug return results._search_raw()
def filtersFromKibanaDash(self, fp, date_timedelta): """ Import filters from a kibana dashboard fp is the file path of the json file date_timedelta is a dict in timedelta format see https://docs.python.org/2/library/datetime.html#timedelta-objects """ f = open(fp) data = json.load(f) must = [] should = [] must_not = [] for filtid in data['services']['filter']['list'].keys(): filt = data['services']['filter']['list'][filtid] if filt['active'] and 'query' in filt.keys(): value = filt['query'].split(':')[-1] fieldname = filt['query'].split(':')[0].split('.')[-1] # self.log.info(fieldname) # self.log.info(value) # field: fieldname # query: value if 'field' in filt.keys(): fieldname = filt['field'] value = filt['query'] if '\"' in value: value = value.split('\"')[1] pyesfilt = pyes.QueryFilter( pyes.MatchQuery(fieldname, value, 'phrase')) else: pyesfilt = pyes.TermFilter(fieldname, value) else: # _exists_:field if filt['query'].startswith('_exists_:'): pyesfilt = pyes.ExistsFilter(value.split('.')[-1]) # self.log.info('exists %s' % value.split('.')[-1]) # _missing_:field elif filt['query'].startswith('_missing_:'): pyesfilt = pyes.filters.MissingFilter( value.split('.')[-1]) # self.log.info('missing %s' % value.split('.')[-1]) # field:"value" elif '\"' in value: value = value.split('\"')[1] pyesfilt = pyes.QueryFilter( pyes.MatchQuery(fieldname, value, 'phrase')) # self.log.info("phrase %s %s" % (fieldname, value)) # field:(value1 value2 value3) elif '(' in value and ')' in value: value = value.split('(')[1] value = value.split('(')[0] pyesfilt = pyes.QueryFilter( pyes.MatchQuery(fieldname, value, "boolean")) # field:value else: pyesfilt = pyes.TermFilter(fieldname, value) # self.log.info("terms %s %s" % (fieldname, value)) if filt['mandate'] == 'must': must.append(pyesfilt) elif filt['mandate'] == 'either': should.append(pyesfilt) elif filt['mandate'] == 'mustNot': must_not.append(pyesfilt) # self.log.info(must) f.close() self.filtersManual(date_timedelta, must=must, should=should, must_not=must_not)