def filtersManual(self, date_timedelta, must=[], should=[], must_not=[]): """ Configure filters manually date_timedelta is a dict in timedelta format see https://docs.python.org/2/library/datetime.html#timedelta-objects must, should and must_not are pyes filter objects lists see http://pyes.readthedocs.org/en/latest/references/pyes.filters.html """ self.begindateUTC = toUTC(datetime.now() - timedelta(**date_timedelta)) self.enddateUTC = toUTC(datetime.now()) qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp', from_value=self.begindateUTC, to_value=self.enddateUTC)) q = pyes.ConstantScoreQuery(pyes.MatchAllQuery()) #Don't fire on already alerted events if pyes.ExistsFilter('alerttimestamp') not in must_not: must_not.append(pyes.ExistsFilter('alerttimestamp')) must.append(qDate) q.filters.append(pyes.BoolFilter( must=must, should=should, must_not=must_not)) self.filter = q
def esBroXSSEvents(): begindateUTC = toUTC(datetime.now() - timedelta(minutes=30)) enddateUTC = toUTC(datetime.now()) qDate = pyes.RangeQuery(qrange=pyes.ESRange( 'utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) qType = pyes.TermFilter('_type', 'event') qEvents = pyes.TermFilter("category", "broxsslog") qalerted = pyes.ExistsFilter('alerttimestamp') q = pyes.ConstantScoreQuery(pyes.MatchAllQuery()) q.filters.append( pyes.BoolFilter(must=[qType, qDate, qEvents, pyes.ExistsFilter('uri')], must_not=[qalerted])) return q
def esSearch(es, begindateUTC=None, enddateUTC=None): resultsList = list() if begindateUTC is None: begindateUTC = toUTC(datetime.now() - timedelta(minutes=60)) if enddateUTC is None: enddateUTC = toUTC(datetime.now()) try: #search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp) qDate = pyes.RangeQuery(qrange=pyes.ESRange( 'utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) qType = pyes.TermFilter('_type', 'event') qEvents = pyes.TermsFilter('category', ['brointel']) qalerted = pyes.ExistsFilter('alerttimestamp') qdetails = pyes.ExistsFilter('details') qindicator = pyes.ExistsFilter('seenindicator') pyesresults = es.search(pyes.ConstantScoreQuery( pyes.BoolFilter(must=[qType, qDate, qEvents, qdetails, qindicator], must_not=[qalerted])), size=1000) #uncomment for debugging to recreate alerts for events that already have an alerttimestamp #results=es.search(pyes.ConstantScoreQuery(pyes.BoolFilter(must=[qcloud,qDate,qEvents]))) #logger.debug(results.count()) #correlate any matches by the seenindicator field. #make a simple list of indicator values that can be counted/summarized by Counter resultsIndicators = list() #bug in pyes..capture results as raw list or it mutates after first access: #copy the hits.hits list as our resusts, which is the same as the official elastic search library returns. results = pyesresults._search_raw()['hits']['hits'] for r in results: resultsIndicators.append(r['_source']['details']['seenindicator']) #use the list of tuples ('indicator',count) to create a dictionary with: #indicator,count,es records #and add it to a list to return. indicatorList = list() for i in Counter(resultsIndicators).most_common(): idict = dict(indicator=i[0], count=i[1], events=[]) for r in results: if r['_source']['details']['seenindicator'].encode( 'ascii', 'ignore') == i[0]: idict['events'].append(r) indicatorList.append(idict) return indicatorList except pyes.exceptions.NoServerAvailable: logger.error( 'Elastic Search server could not be reached, check network connectivity' )
def main(self): # look for events in last 15 mins date_timedelta = dict(minutes=15) # Configure filters using pyes must = [ pyes.TermFilter('_type', 'event'), pyes.TermFilter('eventsource', 'systemslogs'), pyes.ExistsFilter('details.sourceipaddress'), pyes.QueryFilter(pyes.MatchQuery('summary', 'failed', 'phrase')), pyes.TermFilter('program', 'sshd'), pyes.QueryFilter( pyes.MatchQuery('summary', 'login ldap_count_entries', 'boolean')) ] must_not = [ pyes.QueryFilter( pyes.MatchQuery('summary', '10.22.8.128', 'phrase')), pyes.QueryFilter(pyes.MatchQuery('summary', '10.8.75.35', 'phrase')), pyes.QueryFilter( pyes.MatchQuery('summary', '208.118.237.', 'phrase')) ] self.filtersManual(date_timedelta, must=must, must_not=must_not) # Search aggregations on field 'sourceipaddress', keep 50 samples of events at most self.searchEventsAggreg('sourceipaddress', samplesLimit=50) # alert when >= X matching events in an aggregation self.walkAggregations(threshold=10)
def esBroIntelEvents(): begindateUTC = toUTC(datetime.now() - timedelta(minutes=30)) enddateUTC = toUTC(datetime.now()) #search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp) qDate = pyes.RangeQuery(qrange=pyes.ESRange( 'utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) qType = pyes.TermFilter('_type', 'event') qEvents = pyes.TermsFilter('category', ['brointel']) qalerted = pyes.ExistsFilter('alerttimestamp') q = pyes.ConstantScoreQuery(pyes.MatchAllQuery()) q.filters.append( pyes.BoolFilter( must=[qType, qDate, qEvents, pyes.ExistsFilter('seenindicator')], must_not=[qalerted])) return q
def esCloudTrailSearch(es, begindateUTC=None, enddateUTC=None): resultsList = list() if begindateUTC is None: begindateUTC = toUTC(datetime.now() - timedelta(hours=160)) if enddateUTC is None: enddateUTC = toUTC(datetime.now()) try: #search for actions within the date range that haven't already been alerted (i.e. given an alerttimestamp) qDate = pyes.RangeQuery(qrange=pyes.ESRange( 'utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) qcloud = pyes.TermFilter('_type', 'cloudtrail') qEvents = pyes.TermsFilter( 'eventName', ['runinstances', 'stopinstances', 'startinstances']) qalerted = pyes.ExistsFilter('alerttimestamp') results = es.search(pyes.ConstantScoreQuery( pyes.BoolFilter(must=[qcloud, qDate, qEvents], must_not=[qalerted])), indices='events') #uncomment for debugging to recreate alerts for events that already have an alerttimestamp #results=es.search(pyes.ConstantScoreQuery(pyes.BoolFilter(must=[qcloud,qDate,qEvents]))) return (results._search_raw()['hits']['hits']) except pyes.exceptions.NoServerAvailable: logger.error( 'Elastic Search server could not be reached, check network connectivity' )
def main(self): # look for events in last 15 mins date_timedelta = dict(minutes=15) # Configure filters using pyes must = [ pyes.ExistsFilter('details.hostname'), pyes.QueryFilter(pyes.MatchQuery('summary','DuoAPI contact failed','phrase')) ] self.filtersManual(date_timedelta, must=must) # Search aggregations on field 'sourceipaddress', keep X samples of events at most self.searchEventsAggreg('hostname', samplesLimit=10) # alert when >= X matching events in an aggregation # in this case, always self.walkAggregations(threshold=1)
def main(self): # look for events in last 30 mins date_timedelta = dict(minutes=30) # Configure filters using pyes must = [ pyes.TermFilter('_type', 'event'), pyes.TermFilter('category', 'brointel'), pyes.ExistsFilter('seenindicator') ] self.filtersManual(date_timedelta, must=must) # Search aggregations on field 'seenindicator', keep 50 samples of events at most self.searchEventsAggregated('details.seenindicator', samplesLimit=50) # alert when >= 5 matching events in an aggregation self.walkAggregations(threshold=5)
def main(self): # look for events in last 15 mins date_timedelta = dict(minutes=15) # Configure filters using pyes must = [ pyes.TermFilter('_type', 'bro'), pyes.TermFilter('eventsource', 'nsm'), pyes.TermFilter('category', 'bronotice'), pyes.ExistsFilter('details.sourceipaddress'), pyes.QueryFilter(pyes.MatchQuery('details.note','SSH::Password_Guessing','phrase')), ] self.filtersManual(date_timedelta, must=must) # Search events self.searchEventsSimple() self.walkEvents()
def main(self): # look for events in last 15 mins date_timedelta = dict(minutes=15) # Configure filters using pyes must = [ pyes.TermFilter('_type', 'bro'), pyes.TermFilter('eventsource', 'nsm'), pyes.TermFilter('category', 'brointel'), pyes.TermFilter('details.sources', 'abuse.ch SSLBL'), pyes.ExistsFilter('details.sourceipaddress') ] self.filtersManual(date_timedelta, must=must) # Search events self.searchEventsSimple() self.walkEvents()
def main(self): # look for events in last X mins date_timedelta = dict(minutes=2) # Configure filters using pyes must = [ pyes.TermFilter('_type', 'bro'), pyes.TermFilter('eventsource', 'nsm'), pyes.TermFilter('category', 'brointel'), pyes.ExistsFilter('seenindicator'), pyes.QueryFilter(pyes.MatchQuery('hostname', 'sensor1 sensor2 sensor3', 'boolean')) ] self.filtersManual(date_timedelta, must=must) # Search aggregations on field 'seenindicator', keep X samples of events at most self.searchEventsAggregated('details.seenindicator', samplesLimit=10) # alert when >= X matching events in an aggregation self.walkAggregations(threshold=10)
def filtersFromKibanaDash(self, fp, date_timedelta): """ Import filters from a kibana dashboard fp is the file path of the json file date_timedelta is a dict in timedelta format see https://docs.python.org/2/library/datetime.html#timedelta-objects """ f = open(fp) data = json.load(f) must = [] should = [] must_not = [] for filtid in data['services']['filter']['list'].keys(): filt = data['services']['filter']['list'][filtid] if filt['active'] and 'query' in filt.keys(): value = filt['query'].split(':')[-1] fieldname = filt['query'].split(':')[0].split('.')[-1] # self.log.info(fieldname) # self.log.info(value) # field: fieldname # query: value if 'field' in filt.keys(): fieldname = filt['field'] value = filt['query'] if '\"' in value: value = value.split('\"')[1] pyesfilt = pyes.QueryFilter( pyes.MatchQuery(fieldname, value, 'phrase')) else: pyesfilt = pyes.TermFilter(fieldname, value) else: # _exists_:field if filt['query'].startswith('_exists_:'): pyesfilt = pyes.ExistsFilter(value.split('.')[-1]) # self.log.info('exists %s' % value.split('.')[-1]) # _missing_:field elif filt['query'].startswith('_missing_:'): pyesfilt = pyes.filters.MissingFilter( value.split('.')[-1]) # self.log.info('missing %s' % value.split('.')[-1]) # field:"value" elif '\"' in value: value = value.split('\"')[1] pyesfilt = pyes.QueryFilter( pyes.MatchQuery(fieldname, value, 'phrase')) # self.log.info("phrase %s %s" % (fieldname, value)) # field:(value1 value2 value3) elif '(' in value and ')' in value: value = value.split('(')[1] value = value.split('(')[0] pyesfilt = pyes.QueryFilter( pyes.MatchQuery(fieldname, value, "boolean")) # field:value else: pyesfilt = pyes.TermFilter(fieldname, value) # self.log.info("terms %s %s" % (fieldname, value)) if filt['mandate'] == 'must': must.append(pyesfilt) elif filt['mandate'] == 'either': should.append(pyesfilt) elif filt['mandate'] == 'mustNot': must_not.append(pyesfilt) # self.log.info(must) f.close() self.filtersManual(date_timedelta, must=must, should=should, must_not=must_not)