def esSearch(es, begindateUTC=None, enddateUTC=None): resultsList = list() if begindateUTC is None: begindateUTC = toUTC(datetime.now() - timedelta(minutes=options.aggregationminutes)) if enddateUTC is None: enddateUTC = toUTC(datetime.now()) try: # search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp) qDate = pyes.RangeQuery(qrange=pyes.ESRange( 'utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) q = pyes.ConstantScoreQuery(pyes.MatchAllQuery()) q = pyes.FilteredQuery(q, pyes.BoolFilter(must=[qDate])) q = q.search() qagg = pyes.aggs.TermsAgg(name='category', field='category') q.agg.add(qagg) results = es.search(query=q, indices=['events']) mozdefstats = dict(utctimestamp=toUTC(datetime.now()).isoformat()) mozdefstats['summary'] = 'Aggregated category counts' mozdefstats['processid'] = os.getpid() mozdefstats['processname'] = sys.argv[0] mozdefstats['details'] = dict(counts=list()) for bucket in results.aggs['category']['buckets']: entry = dict() entry[bucket['key']] = bucket['doc_count'] mozdefstats['details']['counts'].append(entry) return mozdefstats except pyes.exceptions.NoServerAvailable: logger.error( 'Elastic Search server could not be reached, check network connectivity' )
def esCloudTrailSearch(es, begindateUTC=None, enddateUTC=None): resultsList = list() if begindateUTC is None: begindateUTC = toUTC(datetime.now() - timedelta(hours=160)) if enddateUTC is None: enddateUTC = toUTC(datetime.now()) try: #search for actions within the date range that haven't already been alerted (i.e. given an alerttimestamp) qDate = pyes.RangeQuery(qrange=pyes.ESRange( 'utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) qcloud = pyes.TermFilter('_type', 'cloudtrail') qEvents = pyes.TermsFilter( 'eventName', ['runinstances', 'stopinstances', 'startinstances']) qalerted = pyes.ExistsFilter('alerttimestamp') results = es.search(pyes.ConstantScoreQuery( pyes.BoolFilter(must=[qcloud, qDate, qEvents], must_not=[qalerted])), indices='events') #uncomment for debugging to recreate alerts for events that already have an alerttimestamp #results=es.search(pyes.ConstantScoreQuery(pyes.BoolFilter(must=[qcloud,qDate,qEvents]))) return (results._search_raw()['hits']['hits']) except pyes.exceptions.NoServerAvailable: logger.error( 'Elastic Search server could not be reached, check network connectivity' )
def searchESForBROAttackers(es, threshold): begindateUTC = toUTC(datetime.now() - timedelta(hours=2)) enddateUTC = toUTC(datetime.now()) qDate = pyes.RangeQuery(qrange=pyes.ESRange( 'utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) q = pyes.ConstantScoreQuery(pyes.MatchAllQuery()) qBro = pyes.QueryFilter(pyes.MatchQuery('category', 'bro_notice', 'phrase')) qErr = pyes.QueryFilter( pyes.MatchQuery('details.note', 'MozillaHTTPErrors::Excessive_HTTP_Errors_Attacker', 'phrase')) q = pyes.FilteredQuery(q, pyes.BoolFilter(must=[qBro, qErr])) results = es.search(q, size=1000, indices='events,events-previous') # grab results as native es results to avoid pyes iteration bug # and get easier access to es metadata fields rawresults = results._search_raw()['hits']['hits'] # Hit count is buried in the 'sub' field # as: 'sub': u'6 in 1.0 hr, eps: 0' # cull the records for hitcounts over the threshold before returning attackers = list() for r in rawresults: hitcount = int(r['_source']['details']['sub'].split()[0]) if hitcount > threshold: attackers.append(r) return attackers
def filtersManual(self, date_timedelta, must=[], should=[], must_not=[]): """ Configure filters manually date_timedelta is a dict in timedelta format see https://docs.python.org/2/library/datetime.html#timedelta-objects must, should and must_not are pyes filter objects lists see http://pyes.readthedocs.org/en/latest/references/pyes.filters.html """ self.begindateUTC = toUTC(datetime.now() - timedelta(**date_timedelta)) self.enddateUTC = toUTC(datetime.now()) qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp', from_value=self.begindateUTC, to_value=self.enddateUTC)) q = pyes.ConstantScoreQuery(pyes.MatchAllQuery()) #Don't fire on already alerted events if pyes.ExistsFilter('alerttimestamp') not in must_not: must_not.append(pyes.ExistsFilter('alerttimestamp')) must.append(qDate) q.filters.append(pyes.BoolFilter( must=must, should=should, must_not=must_not)) self.filter = q
def esSearch(es, begindateUTC=None, enddateUTC=None): resultsList = list() if begindateUTC is None: begindateUTC = toUTC(datetime.now() - timedelta(minutes=options.aggregationminutes)) if enddateUTC is None: enddateUTC = toUTC(datetime.now()) try: # search for aggregated event stats summaries within the date range qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) q = pyes.ConstantScoreQuery(pyes.MatchAllQuery()) q = pyes.FilteredQuery(q,pyes.BoolFilter(must=[qDate, pyes.TermFilter('_type', 'mozdefstats')])) results=es.search(query=q,size=100,indices=['events','events-previous']) #avoid pyes iteration bug rawresults = results._search_raw() #examine the results #for each details.counts, append the count #as a list to the stats dict stats=dict() for r in rawresults['hits']['hits']: for i in r['_source']['details']['counts']: #print(i.values()[0]) if i.keys()[0] not in stats.keys(): stats[i.keys()[0]]=list() stats[i.keys()[0]].append(i.values()[0]) # make a dictionairy of user-defined # aggregation threshold percentages aggregationthresholds = dict(zip(options.aggregations, options.aggregationthresholds)) #for our running history of counts per category #do some simple stats math to see if we #should alert on anything for s in stats: alert = False smean=round(numpy.mean(stats[s])) sstd=round(numpy.std(stats[s])) stat = round((sstd/smean)*100, 2) if s in aggregationthresholds.keys(): if stat > aggregationthresholds[s]: alert = True elif stat > options.defaultthreshold: alert = True if alert: print('{0} {1}%: \n\t\t{2} \n\t\t{3} \n\t\t{4}'.format( s, stat, stats[s], smean, sstd ) ) except pyes.exceptions.NoServerAvailable: logger.error('Elastic Search server could not be reached, check network connectivity')
def esLdapResults(begindateUTC=None, enddateUTC=None): '''an ES query/facet to count success/failed logins''' resultsList = list() if begindateUTC is None: begindateUTC = datetime.now() - timedelta(hours=1) begindateUTC = toUTC(begindateUTC) if enddateUTC is None: enddateUTC = datetime.now() enddateUTC = toUTC(enddateUTC) try: es = pyes.ES((list('{0}'.format(s) for s in options.esservers))) qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) q = pyes.MatchAllQuery() q = pyes.FilteredQuery(q, qDate) q = pyes.FilteredQuery(q, pyes.TermFilter('tags', 'ldap')) q = pyes.FilteredQuery(q, pyes.TermFilter('details.result', 'ldap_invalid_credentials')) q2 = q.search() q2.facet.add_term_facet('details.result') q2.facet.add_term_facet('details.dn', size=20) results = es.search(q2, indices='events') stoplist = ('o', 'mozilla', 'dc', 'com', 'mozilla.com', 'mozillafoundation.org', 'org') for t in results.facets['details.dn'].terms: if t['term'] in stoplist: continue #print(t['term']) failures = 0 success = 0 dn = t['term'] #re-query with the terms of the details.dn qt = pyes.MatchAllQuery() qt = pyes.FilteredQuery(qt, qDate) qt = pyes.FilteredQuery(qt, pyes.TermFilter('tags', 'ldap')) qt = pyes.FilteredQuery(qt, pyes.TermFilter('details.dn', t['term'])) qt2 = qt.search() qt2.facet.add_term_facet('details.result') results = es.search(qt2) #sys.stdout.write('{0}\n'.format(results.facets['details.result'].terms)) for t in results.facets['details.result'].terms: #print(t['term'],t['count']) if t['term'] == 'ldap_success': success = t['count'] if t['term'] == 'ldap_invalid_credentials': failures = t['count'] resultsList.append(dict(dn=dn, failures=failures, success=success, begin=begindateUTC.isoformat(), end=enddateUTC.isoformat())) return(json.dumps(resultsList)) except pyes.exceptions.NoServerAvailable: sys.stderr.write('Elastic Search server could not be reached, check network connectivity\n')
def make_query(): qrange = pyes.RangeQuery(pyes.ESRange('utctimestamp', start_date, end_date)) qterm = pyes.TermQuery('_type', 'alert') q = pyes.BoolQuery(must=[qrange, qterm]) results = esconn.search(q, indices=indexname) ret = [] for x in results: na = Alert() if na.parse_result(x): ret.append(na) return ret
def getESAlerts(es): begindateUTC = toUTC(datetime.now() - timedelta(minutes=50)) enddateUTC = toUTC(datetime.now()) qDate = pyes.RangeQuery(qrange=pyes.ESRange( 'utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) qType = pyes.TermFilter('_type', 'alert') q = pyes.ConstantScoreQuery(pyes.MatchAllQuery()) q.filters.append(pyes.BoolFilter(must=[qDate, qType])) results = es.search(q, size=10000, indices='alerts') # return raw search to avoid pyes iteration bug return results._search_raw()
def getFrontendStats(es): begindateUTC = toUTC(datetime.now() - timedelta(minutes=1)) enddateUTC = toUTC(datetime.now()) qDate = pyes.RangeQuery(qrange=pyes.ESRange( 'utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) qType = pyes.TermFilter('_type', 'mozdefhealth') qMozdef = pyes.TermsFilter('category', ['mozdef']) pyesresults = es.search(pyes.ConstantScoreQuery( pyes.BoolFilter(must=[qType, qDate, qMozdef])), indices='events') return pyesresults._search_raw()['hits']['hits']
def esBroXSSEvents(): begindateUTC = toUTC(datetime.now() - timedelta(minutes=30)) enddateUTC = toUTC(datetime.now()) qDate = pyes.RangeQuery(qrange=pyes.ESRange( 'utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) qType = pyes.TermFilter('_type', 'event') qEvents = pyes.TermFilter("category", "broxsslog") qalerted = pyes.ExistsFilter('alerttimestamp') q = pyes.ConstantScoreQuery(pyes.MatchAllQuery()) q.filters.append( pyes.BoolFilter(must=[qType, qDate, qEvents, pyes.ExistsFilter('uri')], must_not=[qalerted])) return q
def esSearch(es, begindateUTC=None, enddateUTC=None): resultsList = list() if begindateUTC is None: begindateUTC = toUTC(datetime.now() - timedelta(minutes=60)) if enddateUTC is None: enddateUTC = toUTC(datetime.now()) try: #search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp) qDate = pyes.RangeQuery(qrange=pyes.ESRange( 'utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) qType = pyes.TermFilter('_type', 'event') qEvents = pyes.TermsFilter('category', ['brointel']) qalerted = pyes.ExistsFilter('alerttimestamp') qdetails = pyes.ExistsFilter('details') qindicator = pyes.ExistsFilter('seenindicator') pyesresults = es.search(pyes.ConstantScoreQuery( pyes.BoolFilter(must=[qType, qDate, qEvents, qdetails, qindicator], must_not=[qalerted])), size=1000) #uncomment for debugging to recreate alerts for events that already have an alerttimestamp #results=es.search(pyes.ConstantScoreQuery(pyes.BoolFilter(must=[qcloud,qDate,qEvents]))) #logger.debug(results.count()) #correlate any matches by the seenindicator field. #make a simple list of indicator values that can be counted/summarized by Counter resultsIndicators = list() #bug in pyes..capture results as raw list or it mutates after first access: #copy the hits.hits list as our resusts, which is the same as the official elastic search library returns. results = pyesresults._search_raw()['hits']['hits'] for r in results: resultsIndicators.append(r['_source']['details']['seenindicator']) #use the list of tuples ('indicator',count) to create a dictionary with: #indicator,count,es records #and add it to a list to return. indicatorList = list() for i in Counter(resultsIndicators).most_common(): idict = dict(indicator=i[0], count=i[1], events=[]) for r in results: if r['_source']['details']['seenindicator'].encode( 'ascii', 'ignore') == i[0]: idict['events'].append(r) indicatorList.append(idict) return indicatorList except pyes.exceptions.NoServerAvailable: logger.error( 'Elastic Search server could not be reached, check network connectivity' )
def esBroIntelEvents(): begindateUTC = toUTC(datetime.now() - timedelta(minutes=30)) enddateUTC = toUTC(datetime.now()) #search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp) qDate = pyes.RangeQuery(qrange=pyes.ESRange( 'utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) qType = pyes.TermFilter('_type', 'event') qEvents = pyes.TermsFilter('category', ['brointel']) qalerted = pyes.ExistsFilter('alerttimestamp') q = pyes.ConstantScoreQuery(pyes.MatchAllQuery()) q.filters.append( pyes.BoolFilter( must=[qType, qDate, qEvents, pyes.ExistsFilter('seenindicator')], must_not=[qalerted])) return q
def search_range_time(field, start_date, date_range, index_name, index_type): if type(date_range) == type(-1) and date_range != -1: #start_da = datetime.datetime.strptime(start_date, "%Y-%m-%dT%H:%M:%SZ").date() start_da = datetime.datetime.strptime(start_date, "%Y-%m-%d").date() end_date = (start_da + datetime.timedelta(days=date_range)).strftime('%Y-%m-%d') must = pyes.RangeQuery( pyes.ESRange(field, from_value=start_date, to_value=end_date)) query = pyes.BoolQuery(must=must) dd = [ i for i in CONN_ES.search( query=query, indices=index_name, doc_types=index_type) ] return dd else: raise
def esSearch(es, macassignments=None, begindateUTC=None, enddateUTC=None): ''' Search ES for an event that ties a username to a mac address This example searches for junos wifi correlations on authentication success Expecting an event like: user: [email protected]; mac: 5c:f9:38:b1:de:cf; author reason: roamed session; ssid: ANSSID; AP 46/2\n ''' usermacre=re.compile(r'''user: (?P<username>.*?); mac: (?P<macaddress>.*?); ''',re.IGNORECASE) correlations={} # list of dicts to populate hits we find if begindateUTC is None: begindateUTC = toUTC(datetime.now() - timedelta(minutes=options.correlationminutes)) if enddateUTC is None: enddateUTC = toUTC(datetime.now()) try: # search for events within the date range qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) q=pyes.ConstantScoreQuery(pyes.MatchAllQuery()) q.filters.append(pyes.BoolFilter(must=[ qDate, pyes.TermFilter("program","AUTHORIZATION-SUCCESS") ], must_not=[ pyes.QueryFilter( pyes.MatchQuery("summary","last-resort","phrase") ) ])) results = es.search(q, size=10000, indices=['events', 'events-previous']) rawresults=results._search_raw() for r in rawresults['hits']['hits']: fields = re.search(usermacre,r['_source']['summary']) if fields: if '{0} {1}'.format(fields.group('username'),fields.group('macaddress')) not in correlations.keys(): if fields.group('macaddress')[0:8].lower() in macassignments.keys(): entity=macassignments[fields.group('macaddress')[0:8].lower()] else: entity='unknown' correlations['{0} {1}'.format(fields.group('username'),fields.group('macaddress'))]=dict(username=fields.group('username'), macaddress=fields.group('macaddress'), entity=entity, utctimestamp=r['_source']['utctimestamp']) return correlations except pyes.exceptions.NoServerAvailable: logger.error('Elastic Search server could not be reached, check network connectivity')
def searchForSSHKeys(es): begindateUTC = toUTC(datetime.now() - timedelta(minutes=5)) enddateUTC = toUTC(datetime.now()) qDate = pyes.RangeQuery(qrange=pyes.ESRange( 'utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) qType = pyes.TermFilter('_type', 'event') qEvents = pyes.TermFilter("program", "sshd") q = pyes.ConstantScoreQuery(pyes.MatchAllQuery()) q.filters.append( pyes.BoolFilter(must=[ qType, qDate, qEvents, pyes.QueryFilter( pyes.MatchQuery("summary", "found matching key accepted publickey", "boolean")) ])) results = es.search(q, size=10000, indices='events') # return raw search to avoid pyes iteration bug return results._search_raw()
def search_err(self, program, receiver): start = "%s CST" % (datetime.astimezonenow() - timedelta( hours=self.config['RANGE'])).daysstrftime('%Y/%m/%d %H:%M:%S') end = "%s CST" % datetime.astimezonenow().strftime('%Y/%m/%d %H:%M:%S') index = "%s_%s" % (program, datetime.astimezonenow().strftime('%Y-%m')) q = pyes.RangeQuery( pyes.ESRange('datetime', from_value=start, to_value=end)) q = pyes.FilteredQuery(q, pyes.TermQuery("level", "EROR")) conn = pyes.ES(self.config['ES']) results = conn.search(indices=index, query=q) num = len(results) if num == 0: return _content = "" for r in results: _content += "%s, %s<br/>" % (r['message'], r['datetime']) title = self.config['MAIL']['MAIL_TITLE'] % program content = "<b>Program:%s</b></b><br/>From:%s<br/>To:%s<br/><b>Total Num: %d</b><br/><br/><b>Content are:</b><br/>%s" % ( program, start, end, num, _content) self.sendmail(receiver, title, content)
def query_submissions(q='', fq=None, sort='score desc', start=0, docs=10, date=None, facets=None): (sort_field, sort_order) = sort.split(' ') if sort_field == 'score': sort_field = '_score' sort = {sort_field: {'order': sort_order}} query = pyes.query.BoolQuery() query.add_must(pyes.StringQuery(q, default_operator="AND")) rest = True x = 0 result = [] while rest: y = fq.find(":", x) if y == -1: break temp = fq[x:y] x = y + 1 if fq[x:x + 5] == """: y = fq.find(""", x + 5) if y == -1: break result.append((temp, fq[x + 5:y])) x = y + 6 if x > len(fq): break else: y = fq.find(";", x) if y == -1: result.append((temp, fq[x:len(fq)])) break else: result.append((temp, fq[x:y])) x = y + 1 for sfq in result: if sfq[0] == 'date': (year, month) = sfq[1].split('-') date_start = datetime.datetime(int(year), int(month), 1) date_end = date_start + dateutil.relativedelta.relativedelta( months=+1, seconds=-1) query.add_must( pyes.RangeQuery( qrange=pyes.ESRange('date', date_start, date_end))) else: query.add_must(pyes.TermQuery(field=sfq[0], value=sfq[1])) search = pyes.query.Search(query=query, fields=[''], start=start, size=docs, sort=sort) search.facet.add_term_facet('type') search.facet.add_term_facet('rs') search.facet.add_term_facet('committee') search.facet.add_date_facet(field='date', name='date', interval='month') es = pyes.ES(app.config['ES_HOST'] + ':' + str(app.config['ES_PORT'])) es.default_indices = [app.config['ES_INDEX_NAME_PREFIX'] + '-latest'] es.refresh() result = es.search(search, model=lambda x, y: y) ret = { 'numhits': result.total, 'maxscore': result.max_score, 'result': [], 'facets': {} } if result.max_score is not None: ret['maxscore'] = result.max_score for key in result.facets: ret['facets'][key] = {} if result.facets[key]['_type'] == 'date_histogram': for subval in result.facets[key]['entries']: ret['facets'][key][datetime.datetime.fromtimestamp( int(subval['time']) / 1000).strftime('%Y-%m')] = subval['count'] if result.facets[key]['_type'] == 'terms': for subval in result.facets[key]['terms']: ret['facets'][key][subval['term']] = subval['count'] for r in result: ret['result'].append({'_id': str(r['_id']), 'score': r['_score']}) return ret