def getFrontendStats(es): begindateUTC = toUTC(datetime.now() - timedelta(minutes=15), options.defaulttimezone) enddateUTC = toUTC(datetime.now(), options.defaulttimezone) qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) qType = pyes.TermFilter('_type', 'mozdefhealth') qMozdef = pyes.TermsFilter('category', ['mozdef']) qLatest = pyes.TermsFilter('tags', ['latest']) pyesresults = es.search(pyes.ConstantScoreQuery(pyes.BoolFilter( must=[qDate, qType, qLatest, qMozdef])), indices='events') return pyesresults._search_raw()['hits']['hits']
def esCloudTrailSearch(es, begindateUTC=None, enddateUTC=None): resultsList = list() if begindateUTC is None: begindateUTC = toUTC(datetime.now() - timedelta(hours=160)) if enddateUTC is None: enddateUTC = toUTC(datetime.now()) try: #search for actions within the date range that haven't already been alerted (i.e. given an alerttimestamp) qDate = pyes.RangeQuery(qrange=pyes.ESRange( 'utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) qcloud = pyes.TermFilter('_type', 'cloudtrail') qEvents = pyes.TermsFilter( 'eventName', ['runinstances', 'stopinstances', 'startinstances']) qalerted = pyes.ExistsFilter('alerttimestamp') results = es.search(pyes.ConstantScoreQuery( pyes.BoolFilter(must=[qcloud, qDate, qEvents], must_not=[qalerted])), indices='events') #uncomment for debugging to recreate alerts for events that already have an alerttimestamp #results=es.search(pyes.ConstantScoreQuery(pyes.BoolFilter(must=[qcloud,qDate,qEvents]))) return (results._search_raw()['hits']['hits']) except pyes.exceptions.NoServerAvailable: logger.error( 'Elastic Search server could not be reached, check network connectivity' )
def main(self): # look for events in last x hours date_timedelta = dict(hours=1) # Configure filters using pyes must = [ pyes.TermFilter('_type', 'cloudtrail'), pyes.TermsFilter('eventName',['runinstances','stopinstances','startinstances']) ] self.filtersManual(date_timedelta, must=must) # Search events self.searchEventsSimple() self.walkEvents()
def esSearch(es, begindateUTC=None, enddateUTC=None): resultsList = list() if begindateUTC is None: begindateUTC = toUTC(datetime.now() - timedelta(minutes=60)) if enddateUTC is None: enddateUTC = toUTC(datetime.now()) try: #search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp) qDate = pyes.RangeQuery(qrange=pyes.ESRange( 'utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) qType = pyes.TermFilter('_type', 'event') qEvents = pyes.TermsFilter('category', ['brointel']) qalerted = pyes.ExistsFilter('alerttimestamp') qdetails = pyes.ExistsFilter('details') qindicator = pyes.ExistsFilter('seenindicator') pyesresults = es.search(pyes.ConstantScoreQuery( pyes.BoolFilter(must=[qType, qDate, qEvents, qdetails, qindicator], must_not=[qalerted])), size=1000) #uncomment for debugging to recreate alerts for events that already have an alerttimestamp #results=es.search(pyes.ConstantScoreQuery(pyes.BoolFilter(must=[qcloud,qDate,qEvents]))) #logger.debug(results.count()) #correlate any matches by the seenindicator field. #make a simple list of indicator values that can be counted/summarized by Counter resultsIndicators = list() #bug in pyes..capture results as raw list or it mutates after first access: #copy the hits.hits list as our resusts, which is the same as the official elastic search library returns. results = pyesresults._search_raw()['hits']['hits'] for r in results: resultsIndicators.append(r['_source']['details']['seenindicator']) #use the list of tuples ('indicator',count) to create a dictionary with: #indicator,count,es records #and add it to a list to return. indicatorList = list() for i in Counter(resultsIndicators).most_common(): idict = dict(indicator=i[0], count=i[1], events=[]) for r in results: if r['_source']['details']['seenindicator'].encode( 'ascii', 'ignore') == i[0]: idict['events'].append(r) indicatorList.append(idict) return indicatorList except pyes.exceptions.NoServerAvailable: logger.error( 'Elastic Search server could not be reached, check network connectivity' )
def esBroIntelEvents(): begindateUTC = toUTC(datetime.now() - timedelta(minutes=30)) enddateUTC = toUTC(datetime.now()) #search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp) qDate = pyes.RangeQuery(qrange=pyes.ESRange( 'utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) qType = pyes.TermFilter('_type', 'event') qEvents = pyes.TermsFilter('category', ['brointel']) qalerted = pyes.ExistsFilter('alerttimestamp') q = pyes.ConstantScoreQuery(pyes.MatchAllQuery()) q.filters.append( pyes.BoolFilter( must=[qType, qDate, qEvents, pyes.ExistsFilter('seenindicator')], must_not=[qalerted])) return q