def main(): ''' Get aggregated statistics on incoming events to use in alerting/notices/queries about event patterns over time ''' logger.debug('starting') logger.debug(options) es = ElasticsearchClient((list('{0}'.format(s) for s in options.esservers))) index = options.index stats = esSearch(es) logger.debug(json.dumps(stats)) sleepcycles = 0 try: while not es.index_exists(index): sleep(3) if sleepcycles == 3: logger.debug("The index is not created. Terminating eventStats.py cron job.") exit(1) sleepcycles += 1 if es.index_exists(index): # post to elastic search servers directly without going through # message queues in case there is an availability issue es.save_event(index=index, body=json.dumps(stats)) except Exception as e: logger.error("Exception %r when gathering statistics " % e) logger.debug('finished')
def clearESCache(): es = esConnect(None) indexes = es.get_indices() # assums index names like events-YYYYMMDD etc. # used to avoid operating on current indexes dtNow = datetime.utcnow() indexSuffix = date.strftime(dtNow, '%Y%m%d') previousSuffix = date.strftime(dtNow - timedelta(days=1), '%Y%m%d') for targetindex in sorted(indexes): if indexSuffix not in targetindex and previousSuffix not in targetindex: url = '{0}/{1}/_stats'.format(random.choice(options.esservers), targetindex) r = requests.get(url) if r.status_code == 200: indexstats = json.loads(r.text) if indexstats['_all']['total']['search']['query_current'] == 0: fielddata = indexstats['_all']['total']['fielddata']['memory_size_in_bytes'] if fielddata > 0: logger.info('target: {0}: field data {1}'.format(targetindex, indexstats['_all']['total']['fielddata']['memory_size_in_bytes'])) clearurl = '{0}/{1}/_cache/clear'.format(random.choice(options.esservers), targetindex) clearRequest = requests.post(clearurl) logger.info(clearRequest.text) # stop at one? if options.conservative: return else: logger.debug('{0}: <ignoring due to current search > field data {1}'.format(targetindex, indexstats['_all']['total']['fielddata']['memory_size_in_bytes'])) else: logger.error('{0} returned {1}'.format(url, r.status_code))
def process_file(self, s3file): logger.debug("Fetching %s" % s3file.name) compressedData = s3file.read() databuf = StringIO(compressedData) gzip_file = gzip.GzipFile(fileobj=databuf) json_logs = json.loads(gzip_file.read()) return json_logs['Records']
def esPruneIndexes(): logger.debug('started') try: es = ElasticsearchClient((list('{0}'.format(s) for s in options.esservers))) indices = es.get_indices() # do the pruning for (index, dobackup, rotation, pruning) in zip(options.indices, options.dobackup, options.rotation, options.pruning): try: if pruning != '0': index_to_prune = index if rotation == 'daily': idate = date.strftime(toUTC(datetime.now()) - timedelta(days=int(pruning)), '%Y%m%d') index_to_prune += '-%s' % idate elif rotation == 'monthly': idate = date.strftime(datetime.utcnow() - timedelta(days=31 * int(pruning)), '%Y%m') index_to_prune += '-%s' % idate if index_to_prune in indices: logger.debug('Deleting index: %s' % index_to_prune) es.delete_index(index_to_prune, True) else: logger.error('Error deleting index %s, index missing' % index_to_prune) except Exception as e: logger.error("Unhandled exception while deleting %s, terminating: %r" % (index_to_prune, e)) except Exception as e: logger.error("Unhandled exception, terminating: %r" % e)
def fetch_ip_list(aws_key_id, aws_secret_key, s3_bucket, ip_list_filename): logger.debug("Fetching ip list from s3") s3 = boto.connect_s3( aws_access_key_id=aws_key_id, aws_secret_access_key=aws_secret_key ) bucket = s3.get_bucket(s3_bucket) ip_list_key = bucket.lookup(ip_list_filename) contents = ip_list_key.get_contents_as_string().rstrip() return contents.split("\n")
def download_generic_alerts(repo_url, save_location, deploy_key): git_obj = cmd.Git(save_location) git_ssh_cmd = 'ssh -i %s' % deploy_key git_obj.update_environment(GIT_SSH_COMMAND=git_ssh_cmd) if not os.path.isdir(save_location): logger.debug("Cloning " + str(repo_url) + " into " + str(save_location)) Repo.clone_from(repo_url, save_location, env={'GIT_SSH_COMMAND': git_ssh_cmd}) else: logger.debug("Updating " + str(save_location)) git_obj.pull()
def main(): logger.debug('starting') logger.debug(options) es = ElasticsearchClient((list('{0}'.format(s) for s in options.esservers))) client = MongoClient(options.mongohost, options.mongoport) # use meteor db mongo = client.meteor writeFrontendStats(getFrontendStats(es), mongo) writeSqsStats(getSqsStats(es), mongo) writeEsClusterStats(es.get_cluster_health(), mongo) writeEsNodesStats(getEsNodesStats(), mongo) writeEsHotThreads(getEsHotThreads(), mongo)
def main(): logger.debug('starting') logger.debug(options) try: es = ElasticsearchClient((list('{0}'.format(s) for s in options.esservers))) client = MongoClient(options.mongohost, options.mongoport) mozdefdb = client.meteor ensureIndexes(mozdefdb) esResults = getESAlerts(es) updateMongo(mozdefdb, esResults) except Exception as e: logger.error("Exception %r sending health to mongo" % e)
def main(): logger.debug('starting') logger.debug(options) try: es = ElasticsearchClient((list('{0}'.format(s) for s in options.esservers))) client = MongoClient(options.mongohost, options.mongoport) # use meteor db mozdefdb = client.meteor esResults = searchESForBROAttackers(es, 100) updateMongoWithESEvents(mozdefdb, esResults) searchMongoAlerts(mozdefdb) except ValueError as e: logger.error("Exception %r collecting attackers to mongo" % e)
def esCloseIndices(): logger.debug('started') try: es = ElasticsearchClient((list('{0}'.format(s) for s in options.esservers))) indices = es.get_indices() except Exception as e: logger.error("Unhandled exception while connecting to ES, terminating: %r" % (e)) # examine each index pulled from get_indice # to determine if it meets aging criteria month_ago_date = toUTC(datetime.now()) - timedelta(days=int(options.index_age)) month_ago_date = month_ago_date.replace(tzinfo=None) for index in indices: if 'events' in index: index_date = index.rsplit('-', 1)[1] logger.debug("Checking to see if Index: %s can be closed." % (index)) if len(index_date) == 8: index_date_obj = datetime.strptime(index_date, '%Y%m%d') try: if month_ago_date > index_date_obj: logger.debug("Index: %s will be closed." % (index)) es.close_index(index) else: logger.debug("Index: %s does not meet aging criteria and will not be closed." % (index)) except Exception as e: logger.error("Unhandled exception while closing indices, terminating: %r" % (e))
def download_generic_alerts(repo_url, save_location, deploy_key): git_obj = cmd.Git(save_location) git_ssh_cmd = 'ssh -i %s' % deploy_key git_obj.update_environment(GIT_SSH_COMMAND=git_ssh_cmd) if not os.path.isdir(save_location): logger.debug("Cloning " + str(repo_url) + " into " + str(save_location)) Repo.clone_from(repo_url, save_location, env={'GIT_SSH_COMMAND': git_ssh_cmd}) else: logger.debug("Updating " + str(save_location)) git_obj.pull()
def main(): logger.debug('starting') logger.debug(options) try: es = ElasticsearchClient( (list('{0}'.format(s) for s in options.esservers))) client = MongoClient(options.mongohost, options.mongoport) # use meteor db mozdefdb = client.meteor esResults = searchESForBROAttackers(es, 100) updateMongoWithESEvents(mozdefdb, esResults) searchMongoAlerts(mozdefdb) except ValueError as e: logger.error("Exception %r collecting attackers to mongo" % e)
def run(self): while True: try: records = self.sqs_queue.receive_messages( MaxNumberOfMessages=options.prefetch) for msg in records: body_message = msg.body event = json.loads(body_message) if not event['Message']: logger.error( 'Invalid message format for cloudtrail SQS messages' ) logger.error('Malformed Message: %r' % body_message) continue if event['Message'] == 'CloudTrail validation message.': # We don't care about these messages continue message_json = json.loads(event['Message']) if 's3ObjectKey' not in message_json: logger.error( 'Invalid message format, expecting an s3ObjectKey in Message' ) logger.error('Malformed Message: %r' % body_message) continue s3_log_files = message_json['s3ObjectKey'] for log_file in s3_log_files: logger.debug('Downloading and parsing ' + log_file) s3_obj = self.s3_client.get_object( Bucket=message_json['s3Bucket'], Key=log_file) events = self.parse_s3_file(s3_obj) for event in events: self.on_message(event) msg.delete() except (SSLEOFError, SSLError, socket.error): logger.info('Received network related error...reconnecting') time.sleep(5) self.sqs_queue = connect_sqs( region_name=options.region, aws_access_key_id=options.accesskey, aws_secret_access_key=options.secretkey, task_exchange=options.taskexchange) time.sleep(options.sleep_time)
def updateAttackerGeoIP(mozdefdb, attackerID, eventDictionary): '''given an attacker ID and a dictionary of an elastic search event look for a valid geoIP in the dict and update the attacker's geo coordinates ''' # geo ip should be in eventDictionary['details']['sourceipgeolocation'] # "sourceipgeolocation": { # "city": "Polska", # "region_code": "73", # "area_code": 0, # "time_zone": "Europe/Warsaw", # "dma_code": 0, # "metro_code": null, # "country_code3": "POL", # "latitude": 52.59309999999999, # "postal_code": null, # "longitude": 19.089400000000012, # "country_code": "PL", # "country_name": "Poland", # "continent": "EU" # } # logger.debug(eventDictionary) if 'details' in eventDictionary: if 'sourceipgeolocation' in eventDictionary['details']: attackers = mozdefdb['attackers'] attacker = attackers.find_one({'_id': attackerID}) if attacker is not None: attacker['geocoordinates'] = dict(countrycode='', longitude=0, latitude=0) if 'country_code' in eventDictionary['details'][ 'sourceipgeolocation']: attacker['geocoordinates'][ 'countrycode'] = eventDictionary['details'][ 'sourceipgeolocation']['country_code'] if 'longitude' in eventDictionary['details'][ 'sourceipgeolocation']: attacker['geocoordinates']['longitude'] = eventDictionary[ 'details']['sourceipgeolocation']['longitude'] if 'latitude' in eventDictionary['details'][ 'sourceipgeolocation']: attacker['geocoordinates']['latitude'] = eventDictionary[ 'details']['sourceipgeolocation']['latitude'] attackers.save(attacker) else: logger.debug('no details in the dictionary') logger.debug(eventDictionary)
def run(self): self.taskQueue.set_message_class(RawMessage) while True: try: records = self.taskQueue.get_messages(options.prefetch) for msg in records: body_message = msg.get_body() event = json.loads(body_message) if not event['Message']: logger.error('Invalid message format for cloudtrail SQS messages') logger.error('Malformed Message: %r' % body_message) continue if event['Message'] == 'CloudTrail validation message.': # We don't care about these messages continue message_json = json.loads(event['Message']) if 's3ObjectKey' not in message_json: logger.error('Invalid message format, expecting an s3ObjectKey in Message') logger.error('Malformed Message: %r' % body_message) continue s3_log_files = message_json['s3ObjectKey'] for log_file in s3_log_files: logger.debug('Downloading and parsing ' + log_file) bucket = self.s3_connection.get_bucket(message_json['s3Bucket']) log_file_lookup = bucket.lookup(log_file) events = self.process_file(log_file_lookup) for event in events: self.on_message(event) self.taskQueue.delete_message(msg) except (SSLEOFError, SSLError, socket.error): logger.info('Received network related error...reconnecting') time.sleep(5) self.connection, self.taskQueue = connect_sqs( task_exchange=options.taskexchange, **get_aws_credentials( options.region, options.accesskey, options.secretkey)) self.taskQueue.set_message_class(RawMessage)
def broadcastAttacker(attacker): ''' send this attacker info to our message queue ''' try: connString = 'amqp://{0}:{1}@{2}:{3}/{4}'.format(options.mquser, options.mqpassword, options.mqserver, options.mqport, options.mqvhost) if options.mqprotocol == 'amqps': mqSSL = True else: mqSSL = False mqConn = Connection(connString, ssl=mqSSL) alertExchange = Exchange( name=options.alertexchange, type='topic', durable=True) alertExchange(mqConn).declare() mqproducer = mqConn.Producer(serializer='json') logger.debug('Kombu configured') except Exception as e: logger.error('Exception while configuring kombu for alerts: {0}'.format(e)) try: # generate an 'alert' structure for this attacker: mqAlert = dict(severity='NOTICE', category='attacker') if 'datecreated' in attacker: mqAlert['utctimestamp'] = attacker['datecreated'].isoformat() mqAlert['summary'] = 'New Attacker: {0} events: {1}, alerts: {2}'.format(attacker['indicators'], attacker['eventscount'], attacker['alertscount']) logger.debug(mqAlert) ensurePublish = mqConn.ensure( mqproducer, mqproducer.publish, max_retries=10) ensurePublish( mqAlert, exchange=alertExchange, routing_key=options.routingkey ) except Exception as e: logger.error('Exception while publishing attacker: {0}'.format(e))
def updateMongoWithESEvents(mozdefdb, results): logger.debug('Looping through events identified as malicious from bro') attackers = mozdefdb['attackers'] for r in results: if 'sourceipaddress' in r['_source']['details']: if netaddr.valid_ipv4(r['_source']['details']['sourceipaddress']): sourceIP = netaddr.IPNetwork( r['_source']['details']['sourceipaddress']) # expand it to a /24 CIDR # todo: lookup ipwhois for asn_cidr value # potentially with a max mask value (i.e. asn is /8, limit attackers to /24) sourceIP.prefixlen = 24 if not sourceIP.ip.is_loopback( ) and not sourceIP.ip.is_private( ) and not sourceIP.ip.is_reserved(): esrecord = dict(documentid=r['_id'], documenttype=r['_type'], documentindex=r['_index'], documentsource=r['_source']) logger.debug('Trying to find existing attacker at ' + str(sourceIP)) attacker = attackers.find_one( {'indicators.ipv4address': str(sourceIP)}) if attacker is None: # new attacker # generate a meteor-compatible ID # save the ES document type, index, id # and add a sub list for future events logger.debug('Creating new attacker from ' + str(sourceIP)) newAttacker = genNewAttacker() # expand the source ip to a /24 for the indicator match. sourceIP.prefixlen = 24 # str sourceIP to get the ip/cidr rather than netblock cidr. newAttacker['indicators'].append( dict(ipv4address=str(sourceIP))) newAttacker['eventscount'] = 1 newAttacker['lastseentimestamp'] = esrecord[ 'documentsource']['utctimestamp'] attackers.insert(newAttacker) updateAttackerGeoIP(mozdefdb, newAttacker['_id'], esrecord['documentsource']) else: logger.debug( 'Attacker found, increasing eventscount and modding geoip' ) attacker['eventscount'] += 1 attacker['lastseentimestamp'] = esrecord[ 'documentsource']['utctimestamp'] attackers.save(attacker) # geo ip could have changed, update it updateAttackerGeoIP(mozdefdb, attacker['_id'], esrecord['documentsource'])
def broadcastAttacker(attacker): ''' send this attacker info to our message queue ''' try: connString = 'amqp://{0}:{1}@{2}:{3}/{4}'.format( options.mquser, options.mqpassword, options.mqserver, options.mqport, options.mqvhost) if options.mqprotocol == 'amqps': mqSSL = True else: mqSSL = False mqConn = Connection(connString, ssl=mqSSL) alertExchange = Exchange(name=options.alertexchange, type='topic', durable=True) alertExchange(mqConn).declare() mqproducer = mqConn.Producer(serializer='json') logger.debug('Kombu configured') except Exception as e: logger.error( 'Exception while configuring kombu for alerts: {0}'.format(e)) try: # generate an 'alert' structure for this attacker: mqAlert = dict(severity='NOTICE', category='attacker') if 'datecreated' in attacker: mqAlert['utctimestamp'] = attacker['datecreated'].isoformat() mqAlert[ 'summary'] = 'New Attacker: {0} events: {1}, alerts: {2}'.format( attacker['indicators'], attacker['eventscount'], attacker['alertscount']) logger.debug(mqAlert) ensurePublish = mqConn.ensure(mqproducer, mqproducer.publish, max_retries=10) ensurePublish(mqAlert, exchange=alertExchange, routing_key=options.routingkey) except Exception as e: logger.error('Exception while publishing attacker: {0}'.format(e))
def isJVMMemoryHigh(): url = "{0}/_nodes/stats?pretty=true".format(random.choice(options.esservers)) r = requests.get(url) logger.debug(r) if r.status_code == 200: nodestats = r.json() for node in nodestats['nodes']: loadaverage = nodestats['nodes'][node]['os']['cpu']['load_average'] cpuusage = nodestats['nodes'][node]['os']['cpu']['percent'] nodename = nodestats['nodes'][node]['name'] jvmused = nodestats['nodes'][node]['jvm']['mem']['heap_used_percent'] logger.debug('{0}: cpu {1}% jvm {2}% load average: {3}'.format(nodename, cpuusage, jvmused, loadaverage)) if jvmused > options.jvmlimit: logger.info('{0}: cpu {1}% jvm {2}% load average: {3} recommending cache clear'.format(nodename, cpuusage, jvmused, loadaverage)) return True return False else: logger.error(r) return False
def isJVMMemoryHigh(): url = "{0}/_nodes/stats?pretty=true".format(random.choice(options.esservers)) r = requests.get(url) logger.debug(r) if r.status_code == 200: nodestats = r.json() for node in nodestats['nodes']: loadaverage = nodestats['nodes'][node]['os']['cpu']['load_average'] cpuusage = nodestats['nodes'][node]['os']['cpu']['percent'] nodename = nodestats['nodes'][node]['name'] jvmused = nodestats['nodes'][node]['jvm']['mem']['heap_used_percent'] logger.debug('{0}: cpu {1}% jvm {2}% load average: {3}'.format(nodename, cpuusage, jvmused, loadaverage)) if jvmused > options.jvmlimit: logger.info('{0}: cpu {1}% jvm {2}% load average: {3} recommending cache clear'.format(nodename, cpuusage, jvmused, loadaverage)) return True return False else: logger.error(r) return False
def verify_events(options): es_client = ElasticsearchClient(options.esservers) for required_field in options.required_fields: logger.debug('Looking for events without ' + required_field) search_query = SearchQuery(hours=12) search_query.add_must_not(ExistsMatch(required_field)) # Exclude all events that are mozdef related health and stats search_query.add_must_not(TermMatch('_type', 'mozdefstats')) search_query.add_must_not(TermMatch('_type', 'mozdefhealth')) search_query.add_aggregation(Aggregation('_type')) # We don't care about the actual events, we only want the numbers results = search_query.execute(es_client, size=1) for aggreg_term in results['aggregations']['_type']['terms']: count = aggreg_term['count'] category = aggreg_term['key'] logger.error( "Found {0} bad events of _type '{1}' missing '{2}' field". format(count, category, required_field))
def update_alert_schedules(): '''an endpoint to return alerts schedules''' if not request.body: response.status = 503 return response alert_schedules = json.loads(request.body.read()) request.body.close() response.content_type = "application/json" mongoclient = MongoClient(options.mongohost, options.mongoport) schedulers_db = mongoclient.meteor['alertschedules'] schedulers_db.remove() for alert_name, alert_schedule in alert_schedules.items(): logger.debug( "Inserting schedule for {0} into mongodb".format(alert_name)) schedulers_db.insert(alert_schedule) response.status = 200 return response
def aggregateAttackerIPs(attackers): iplist = [] # Set the attacker age timestamp attackerage = datetime.now() - timedelta(days=options.attackerage) ips = attackers.aggregate([ {"$sort": {"lastseentimestamp": -1}}, {"$match": {"category": options.category}}, {"$match": {"lastseentimestamp": {"$gte": attackerage}}}, {"$match": {"indicators.ipv4address": {"$exists": True}}}, {"$group": {"_id": {"ipv4address": "$indicators.ipv4address"}}}, {"$unwind": "$_id.ipv4address"}, {"$limit": options.iplimit} ]) for i in ips: whitelisted = False logger.debug('working {0}'.format(i)) ip = i['_id']['ipv4address'] ipcidr = netaddr.IPNetwork(ip) if not ipcidr.ip.is_loopback() and not ipcidr.ip.is_private() and not ipcidr.ip.is_reserved(): for whitelist_range in options.ipwhitelist: whitelist_network = netaddr.IPNetwork(whitelist_range) if ipcidr in whitelist_network: logger.debug(str(ipcidr) + " is whitelisted as part of " + str(whitelist_network)) whitelisted = True # strip any host bits 192.168.10/24 -> 192.168.0/24 ipcidrnet = str(ipcidr.cidr) if ipcidrnet not in iplist and not whitelisted: iplist.append(ipcidrnet) else: logger.debug('invalid:' + ip) return iplist
def makerequest(self, query, stime, etime, maxid): payload = { 'min_time': calendar.timegm(stime.utctimetuple()), 'max_time': calendar.timegm(etime.utctimetuple()), 'q': query } if maxid is not None: payload['max_id'] = maxid hdrs = {'X-Papertrail-Token': self._apikey} max_retries = 3 total_retries = 0 while True: logger.debug("Sending request to papertrail API") resp = requests.get(self._papertrail_api, headers=hdrs, params=payload) if resp.status_code == 200: break else: logger.debug("Received invalid status code: {0}: {1}".format(resp.status_code, resp.text)) total_retries += 1 if total_retries < max_retries: logger.debug("Sleeping a bit then retrying") time.sleep(2) else: logger.error("Received too many error messages...exiting") logger.error("Last malformed response: {0}: {1}".format(resp.status_code, resp.text)) sys.exit(1) return self.parse_events(resp.json())
def getQueueSizes(): logger.debug('starting') logger.debug(options) es = ElasticsearchClient(options.esservers) sqs_client = boto3.client("sqs", region_name=options.region, aws_access_key_id=options.accesskey, aws_secret_access_key=options.secretkey) queues_stats = { 'queues': [], 'total_feeds': len(options.taskexchange), 'total_messages_ready': 0, 'username': '******' } for queue_name in options.taskexchange: logger.debug('Looking for sqs queue stats in queue' + queue_name) queue_url = sqs_client.get_queue_url(QueueName=queue_name)['QueueUrl'] queue_attributes = sqs_client.get_queue_attributes( QueueUrl=queue_url, AttributeNames=['All'])['Attributes'] queue_stats = { 'queue': queue_name, } if 'ApproximateNumberOfMessages' in queue_attributes: queue_stats['messages_ready'] = int( queue_attributes['ApproximateNumberOfMessages']) queues_stats['total_messages_ready'] += queue_stats[ 'messages_ready'] if 'ApproximateNumberOfMessagesNotVisible' in queue_attributes: queue_stats['messages_inflight'] = int( queue_attributes['ApproximateNumberOfMessagesNotVisible']) if 'ApproximateNumberOfMessagesDelayed' in queue_attributes: queue_stats['messages_delayed'] = int( queue_attributes['ApproximateNumberOfMessagesDelayed']) queues_stats['queues'].append(queue_stats) # setup a log entry for health/status. sqsid = '{0}-{1}'.format(options.account, options.region) healthlog = dict(utctimestamp=toUTC(datetime.now()).isoformat(), hostname=sqsid, processid=os.getpid(), processname=sys.argv[0], severity='INFO', summary='mozdef health/status', category='mozdef', source='aws-sqs', tags=[], details=queues_stats) healthlog['tags'] = ['mozdef', 'status', 'sqs'] healthlog['type'] = 'mozdefhealth' # post to elasticsearch servers directly without going through # message queues in case there is an availability issue es.save_event(index=options.index, body=json.dumps(healthlog)) # post another doc with a static docid and tag # for use when querying for the latest sqs status healthlog['tags'] = ['mozdef', 'status', 'sqs-latest'] es.save_event(index=options.index, doc_id=getDocID(sqsid), body=json.dumps(healthlog))
def makerequest(self, query, stime, etime, maxid): payload = { 'min_time': calendar.timegm(stime.utctimetuple()), 'max_time': calendar.timegm(etime.utctimetuple()), 'q': query } if maxid is not None: payload['max_id'] = maxid hdrs = {'X-Papertrail-Token': self._apikey} max_retries = 3 total_retries = 0 while True: logger.debug("Sending request to papertrail API") resp = requests.get(self._papertrail_api, headers=hdrs, params=payload) if resp.status_code == 200: break else: logger.debug("Received invalid status code: {0}: {1}".format( resp.status_code, resp.text)) total_retries += 1 if total_retries < max_retries: logger.debug("Sleeping a bit then retrying") time.sleep(2) else: logger.error("Received too many error messages...exiting") logger.error("Last malformed response: {0}: {1}".format( resp.status_code, resp.text)) sys.exit(1) return self.parse_events(resp.json())
def updateAttackerGeoIP(mozdefdb, attackerID, eventDictionary): '''given an attacker ID and a dictionary of an elastic search event look for a valid geoIP in the dict and update the attacker's geo coordinates ''' # geo ip should be in eventDictionary['details']['sourceipgeolocation'] # "sourceipgeolocation": { # "city": "Polska", # "region_code": "73", # "area_code": 0, # "time_zone": "Europe/Warsaw", # "dma_code": 0, # "metro_code": null, # "country_code3": "POL", # "latitude": 52.59309999999999, # "postal_code": null, # "longitude": 19.089400000000012, # "country_code": "PL", # "country_name": "Poland", # "continent": "EU" # } # logger.debug(eventDictionary) if 'details' in eventDictionary: if 'sourceipgeolocation' in eventDictionary['details']: attackers=mozdefdb['attackers'] attacker = attackers.find_one({'_id': attackerID}) if attacker is not None: attacker['geocoordinates'] = dict(countrycode='', longitude=0, latitude=0) if 'country_code' in eventDictionary['details']['sourceipgeolocation']: attacker['geocoordinates']['countrycode'] = eventDictionary['details']['sourceipgeolocation']['country_code'] if 'longitude' in eventDictionary['details']['sourceipgeolocation']: attacker['geocoordinates']['longitude'] = eventDictionary['details']['sourceipgeolocation']['longitude'] if 'latitude' in eventDictionary['details']['sourceipgeolocation']: attacker['geocoordinates']['latitude'] = eventDictionary['details']['sourceipgeolocation']['latitude'] attackers.save(attacker) else: logger.debug('no details in the dictionary') logger.debug(eventDictionary)
def aggregateAttackerIPs(attackers): iplist = [] # Set the attacker age timestamp attackerage = datetime.now() - timedelta(days=options.attackerage) ips = attackers.aggregate([{ "$sort": { "lastseentimestamp": -1 } }, { "$match": { "category": options.category } }, { "$match": { "lastseentimestamp": { "$gte": attackerage } } }, { "$match": { "indicators.ipv4address": { "$exists": True } } }, { "$group": { "_id": { "ipv4address": "$indicators.ipv4address" } } }, { "$unwind": "$_id.ipv4address" }, { "$limit": options.iplimit }]) for i in ips: whitelisted = False logger.debug('working {0}'.format(i)) ip = i['_id']['ipv4address'] ipcidr = netaddr.IPNetwork(ip) if not ipcidr.ip.is_loopback() and not ipcidr.ip.is_private( ) and not ipcidr.ip.is_reserved(): for whitelist_range in options.ipwhitelist: whitelist_network = netaddr.IPNetwork(whitelist_range) if ipcidr in whitelist_network: logger.debug( str(ipcidr) + " is whitelisted as part of " + str(whitelist_network)) whitelisted = True # strip any host bits 192.168.10/24 -> 192.168.0/24 ipcidrnet = str(ipcidr.cidr) if ipcidrnet not in iplist and not whitelisted: iplist.append(ipcidrnet) else: logger.debug('invalid:' + ip) return iplist
def verify_events(options): es_client = ElasticsearchClient(options.esservers) for required_field in options.required_fields: logger.debug('Looking for events without ' + required_field) search_query = SearchQuery(hours=12) search_query.add_must_not(ExistsMatch(required_field)) # Exclude all events that are mozdef related health and stats search_query.add_must_not(TermMatch('_type', 'mozdefstats')) search_query.add_must_not(TermMatch('_type', 'mozdefhealth')) search_query.add_aggregation(Aggregation('_type')) # We don't care about the actual events, we only want the numbers results = search_query.execute(es_client, size=1) for aggreg_term in results['aggregations']['_type']['terms']: count = aggreg_term['count'] category = aggreg_term['key'] logger.error("Found {0} bad events of _type '{1}' missing '{2}' field".format( count, category, required_field ))
def esPruneIndexes(): logger.debug('started') try: es = ElasticsearchClient( (list('{0}'.format(s) for s in options.esservers))) indices = es.get_indices() # do the pruning for (index, dobackup, rotation, pruning) in zip(options.indices, options.dobackup, options.rotation, options.pruning): try: if pruning != '0': index_to_prune = index if rotation == 'daily': idate = date.strftime( toUTC(datetime.now()) - timedelta(days=int(pruning)), '%Y%m%d') index_to_prune += '-%s' % idate elif rotation == 'monthly': idate = date.strftime( datetime.utcnow() - timedelta(days=31 * int(pruning)), '%Y%m') index_to_prune += '-%s' % idate if index_to_prune in indices: logger.debug('Deleting index: %s' % index_to_prune) es.delete_index(index_to_prune, True) else: logger.error('Error deleting index %s, index missing' % index_to_prune) except Exception as e: logger.error( "Unhandled exception while deleting %s, terminating: %r" % (index_to_prune, e)) except Exception as e: logger.error("Unhandled exception, terminating: %r" % e)
def save_ip_list(save_path, ips): ip_list_contents = '\n'.join(ips) logger.debug("Saving ip list") if os.path.isfile(save_path): logger.debug("Overwriting ip list file in " + str(save_path)) else: logger.debug("Creating new ip list file at " + str(save_path)) with open(save_path, "w+") as text_file: text_file.write(ip_list_contents)
def save_ip_list(save_path, ips): ip_list_contents = '\n'.join(ips) logger.debug("Saving ip list") if os.path.isfile(save_path): logger.debug("Overwriting ip list file in " + str(save_path)) else: logger.debug("Creating new ip list file at " + str(save_path)) with open(save_path, "w+") as text_file: text_file.write(ip_list_contents)
def clearESCache(): es = esConnect(None) indexes = es.get_indices() # assums index names like events-YYYYMMDD etc. # used to avoid operating on current indexes dtNow = datetime.utcnow() indexSuffix = date.strftime(dtNow, '%Y%m%d') previousSuffix = date.strftime(dtNow - timedelta(days=1), '%Y%m%d') for targetindex in sorted(indexes): if indexSuffix not in targetindex and previousSuffix not in targetindex: url = '{0}/{1}/_stats'.format(random.choice(options.esservers), targetindex) r = requests.get(url) if r.status_code == 200: indexstats = json.loads(r.text) if indexstats['_all']['total']['search']['query_current'] == 0: fielddata = indexstats['_all']['total']['fielddata'][ 'memory_size_in_bytes'] if fielddata > 0: logger.info('target: {0}: field data {1}'.format( targetindex, indexstats['_all']['total'] ['fielddata']['memory_size_in_bytes'])) clearurl = '{0}/{1}/_cache/clear'.format( random.choice(options.esservers), targetindex) clearRequest = requests.post(clearurl) logger.info(clearRequest.text) # stop at one? if options.conservative: return else: logger.debug( '{0}: <ignoring due to current search > field data {1}' .format( targetindex, indexstats['_all']['total'] ['fielddata']['memory_size_in_bytes'])) else: logger.error('{0} returned {1}'.format(url, r.status_code))
def assume_role(self, role_arn, role_session_name='unknown', policy=None): '''Return a boto.sts.credential.Credential object given a role_arn. First check if a Credential oject exists in the local self.credentials cache that is not expired. If there isn't one, assume the role of role_arn store the Credential in the credentials cache and return it''' logger.debug("Connecting to sts") if role_arn in self.credentials: if not self.credentials[role_arn] or not self.credentials[ role_arn].is_expired(): # Return the cached value if it's False (indicating a permissions issue) or if # it hasn't expired. return self.credentials[role_arn] try: self.credentials[role_arn] = self.conn_sts.assume_role( role_arn=role_arn, role_session_name=role_session_name, policy=policy).credentials logger.debug("Assumed new role with credential %s" % self.credentials[role_arn].to_dict()) except Exception, e: print e logger.error("Unable to assume role %s due to exception %s" % (role_arn, e.message)) self.credentials[role_arn] = False
def assume_role(self, role_arn, role_session_name='unknown', policy=None): '''Return a boto.sts.credential.Credential object given a role_arn. First check if a Credential oject exists in the local self.credentials cache that is not expired. If there isn't one, assume the role of role_arn store the Credential in the credentials cache and return it''' logger.debug("Connecting to sts") if role_arn in self.credentials: if not self.credentials[role_arn] or not self.credentials[role_arn].is_expired(): # Return the cached value if it's False (indicating a permissions issue) or if # it hasn't expired. return self.credentials[role_arn] try: self.credentials[role_arn] = self.conn_sts.assume_role( role_arn=role_arn, role_session_name=role_session_name, policy=policy).credentials logger.debug("Assumed new role with credential %s" % self.credentials[role_arn].to_dict()) except Exception, e: print e logger.error("Unable to assume role %s due to exception %s" % (role_arn, e.message)) self.credentials[role_arn] = False
def save_db_data(save_path, db_data): temp_save_path = save_path + ".tmp" logger.debug("Saving db data to " + temp_save_path) with open(temp_save_path, "wb+") as text_file: text_file.write(db_data) logger.debug("Testing temp geolite db file") geo_ip = GeoIP(temp_save_path) # Do a generic lookup to verify we don't get any errors (malformed data) geo_ip.lookup_ip('8.8.8.8') logger.debug("Moving temp file to " + save_path) os.rename(temp_save_path, save_path)
def save_db_data(save_path, db_data): temp_save_path = save_path + ".tmp" logger.debug("Saving db data to " + temp_save_path) with open(temp_save_path, "wb+") as text_file: text_file.write(db_data) logger.debug("Testing temp geolite db file") geo_ip = GeoIP(temp_save_path) # Do a generic lookup to verify we don't get any errors (malformed data) geo_ip.lookup_ip('8.8.8.8') logger.debug("Moving temp file to " + save_path) os.rename(temp_save_path, save_path)
def updateMongoWithESEvents(mozdefdb, results): logger.debug('Looping through events identified as malicious from bro') attackers = mozdefdb['attackers'] for r in results: if 'sourceipaddress' in r['_source']['details']: if netaddr.valid_ipv4(r['_source']['details']['sourceipaddress']): sourceIP = netaddr.IPNetwork(r['_source']['details']['sourceipaddress']) # expand it to a /24 CIDR # todo: lookup ipwhois for asn_cidr value # potentially with a max mask value (i.e. asn is /8, limit attackers to /24) sourceIP.prefixlen = 24 if not sourceIP.ip.is_loopback() and not sourceIP.ip.is_private() and not sourceIP.ip.is_reserved(): esrecord = dict( documentid=r['_id'], documentindex=r['_index'], documentsource=r['_source'] ) logger.debug('Trying to find existing attacker at ' + str(sourceIP)) attacker = attackers.find_one({'indicators.ipv4address': str(sourceIP)}) if attacker is None: # new attacker # generate a meteor-compatible ID # save the ES document type, index, id # and add a sub list for future events logger.debug('Creating new attacker from ' + str(sourceIP)) newAttacker = genNewAttacker() # expand the source ip to a /24 for the indicator match. sourceIP.prefixlen = 24 # str sourceIP to get the ip/cidr rather than netblock cidr. newAttacker['indicators'].append(dict(ipv4address=str(sourceIP))) newAttacker['eventscount'] = 1 newAttacker['lastseentimestamp'] = esrecord['documentsource']['utctimestamp'] attackers.insert(newAttacker) updateAttackerGeoIP(mozdefdb, newAttacker['_id'], esrecord['documentsource']) else: logger.debug('Attacker found, increasing eventscount and modding geoip') attacker['eventscount'] += 1 attacker['lastseentimestamp'] = esrecord['documentsource']['utctimestamp'] attackers.save(attacker) # geo ip could have changed, update it updateAttackerGeoIP(mozdefdb, attacker['_id'], esrecord['documentsource'])
def fetch_db_data(db_download_location): logger.debug('Fetching db data from ' + db_download_location) auth_creds = None if options.account_id != '' and options.license_key != '': logger.debug('Using credentials for maxmind') auth_creds = (options.account_id, options.license_key) response = requests.get(db_download_location, auth=auth_creds) if not response.ok: raise Exception( "Received bad response from maxmind server: {0}".format( response.text)) db_raw_data = response.content with tempfile.NamedTemporaryFile(mode='wb') as temp: logger.debug('Writing compressed gzip to temp file: ' + temp.name) temp.write(db_raw_data) temp.flush() logger.debug('Extracting gzip data from ' + temp.name) gfile = gzip.GzipFile(temp.name, "rb") data = gfile.read() return data
def main(): logger.debug('Starting') logger.debug(options) ips = fetch_ip_list(options.aws_access_key_id, options.aws_secret_access_key, options.aws_bucket_name, options.aws_document_key_name) for manual_addition in options.manual_additions: if manual_addition == '': continue logger.debug("Adding manual addition: " + manual_addition) ips.append(manual_addition) if len(ips) < options.ips_list_threshold: raise LookupError('IP List contains less than ' + str(options.ips_list_threshold) + ' entries...something is probably up here.') save_ip_list(options.local_ip_list_path, ips)
def fetch_db_data(db_download_location): logger.debug('Fetching db data from ' + db_download_location) response = urllib2.urlopen(db_download_location) db_raw_data = response.read() with tempfile.NamedTemporaryFile(mode='wb') as temp: logger.debug('Writing compressed gzip to temp file: ' + temp.name) temp.write(db_raw_data) temp.flush() logger.debug('Extracting gzip data from ' + temp.name) tar = tarfile.open(temp.name) for tarinfo in tar: if tarinfo.name.endswith('GeoLite2-City.mmdb'): extracted_file = tar.extractfile(tarinfo.name) return extracted_file.read()
def fetch_db_data(db_download_location): logger.debug('Fetching db data from ' + db_download_location) response = urllib.request.urlopen(db_download_location) db_raw_data = response.read() with tempfile.NamedTemporaryFile(mode='wb') as temp: logger.debug('Writing compressed gzip to temp file: ' + temp.name) temp.write(db_raw_data) temp.flush() logger.debug('Extracting gzip data from ' + temp.name) tar = tarfile.open(temp.name) for tarinfo in tar: if tarinfo.name.endswith('GeoLite2-City.mmdb'): extracted_file = tar.extractfile(tarinfo.name) return extracted_file.read()
def save_db_data(db_file, db_data): save_path = path.join(options.db_store_location, db_file) fd, temp_path = mkstemp(suffix='.tmp', prefix=db_file, dir=options.db_store_location) with open(temp_path, 'wb') as temp: logger.debug("Saving db data to " + temp_path) temp.write(db_data) fsync(temp.fileno()) temp.flush() logger.debug("Testing temp geolite db file") geo_ip = GeoIP(temp_path) # Do a generic lookup to verify we don't get any errors (malformed data) geo_ip.lookup_ip('8.8.8.8') logger.debug("Moving temp file to " + save_path) close(fd) rename(temp_path, save_path)
def main(): ''' Look for events that contain username and a mac address Add the correlation to the intelligence index. ''' logger.debug('starting') logger.debug(options) es = ElasticsearchClient((list('{0}'.format(s) for s in options.esservers))) # create intelligence index if it's not already there es.create_index('intelligence') # read in the OUI file for mac prefix to vendor dictionary macassignments = readOUIFile(options.ouifilename) # search ES for events containing username and mac address correlations = esSearch(es, macassignments=macassignments) # store the correlation in the intelligence index esStoreCorrelations(es, correlations) logger.debug('finished')
def fetch_db_data(db_file): db_download_location = 'https://updates.maxmind.com/geoip/databases/' + db_file[: -5] + '/update' logger.debug('Fetching db data from ' + db_download_location) auth_creds = (options.account_id, options.license_key) response = requests.get(db_download_location, auth=auth_creds) if not response.ok: raise Exception( "Received bad response from maxmind server: {0}".format( response.text)) db_raw_data = response.content with tempfile.NamedTemporaryFile(mode='wb', prefix=db_file + '.zip.', suffix='.tmp', dir=options.db_store_location) as temp: logger.debug('Writing compressed gzip to temp file: ' + temp.name) temp.write(db_raw_data) temp.flush() logger.debug('Extracting gzip data from ' + temp.name) gfile = gzip.GzipFile(temp.name, "rb") data = gfile.read() return data
def main(): args_parser = argparse.ArgumentParser( description='Task to update IP Intel JSON') args_parser.add_argument('-c', '--configfile', help='Path to JSON configuration file to use.') args = args_parser.parse_args() cfg = Config.load(args.configfile) initLogger() logger.debug('Downloading IP intel JSON') ip_intel_json = download_intel_file(cfg.source_url) logger.debug('Writing intel JSON to file') with open(cfg.download_location, 'w') as download_location: json.dump(ip_intel_json, download_location) logger.debug('Terminating successfully')
def reauth_timer(self): while True: time.sleep(self.flush_wait_time) logger.debug('Recycling credentials and reassuming role') self.authenticate()
def searchMongoAlerts(mozdefdb): attackers = mozdefdb['attackers'] alerts = mozdefdb['alerts'] # search the last X alerts for IP addresses # aggregated by CIDR mask/24 # aggregate IPv4 addresses in the most recent alerts # to find common attackers. ipv4TopHits = alerts.aggregate([ # reverse sort the current alerts {"$sort": {"utcepoch": -1}}, # most recent 100 {"$limit": 100}, # must have an ip address {"$match": {"events.documentsource.details.sourceipaddress": {"$exists": True}}}, # must not be already related to an attacker {"$match": {"attackerid": {"$exists": False}}}, # make each event into it's own doc {"$unwind": "$events"}, {"$project": { "_id": 0, # emit the source ip only "sourceip": "$events.documentsource.details.sourceipaddress" }}, # count by ip {"$group": {"_id": "$sourceip", "hitcount": {"$sum": 1}}}, # limit to those with X observances {"$match": {"hitcount": {"$gt": options.ipv4attackerhitcount}}}, # sort {"$sort": SON([("hitcount", -1), ("_id", -1)])}, # top 10 {"$limit": 10} ]) for ip in ipv4TopHits: # sanity check ip['_id'] which should be the ipv4 address if isIPv4(ip['_id']) and ip['_id'] not in netaddr.IPSet(['0.0.0.0']): ipcidr = netaddr.IPNetwork(ip['_id']) # set CIDR # todo: lookup ipwhois for asn_cidr value # potentially with a max mask value (i.e. asn is /8, limit attackers to /24) ipcidr.prefixlen = options.ipv4attackerprefixlength # append to or create attacker. # does this match an existing attacker's indicators if not ipcidr.ip.is_loopback() and not ipcidr.ip.is_private() and not ipcidr.ip.is_reserved(): logger.debug('Searching for existing attacker with ip ' + str(ipcidr)) attacker = attackers.find_one({'indicators.ipv4address': str(ipcidr)}) if attacker is None: logger.debug('Attacker not found, creating new one') # new attacker # generate a meteor-compatible ID # save the ES document type, index, id newAttacker = genNewAttacker() # str to get the ip/cidr rather than netblock cidr. # i.e. '1.2.3.4/24' not '1.2.3.0/24' newAttacker['indicators'].append(dict(ipv4address=str(ipcidr))) matchingalerts = alerts.find( {"events.documentsource.details.sourceipaddress": str(ipcidr.ip), }) total_events = 0 if matchingalerts is not None: # update list of alerts this attacker matched. for alert in matchingalerts: newAttacker['alerts'].append( dict(alertid=alert['_id']) ) # update alert with attackerID alert['attackerid'] = newAttacker['_id'] alerts.save(alert) total_events += len(alert['events']) if len(alert['events']) > 0: newAttacker['lastseentimestamp'] = toUTC(alert['events'][-1]['documentsource']['utctimestamp']) newAttacker['alertscount'] = len(newAttacker['alerts']) newAttacker['eventscount'] = total_events attackers.insert(newAttacker) # update geoIP info latestGeoIP = [a['events'] for a in alerts.find( {"events.documentsource.details.sourceipaddress": str(ipcidr.ip), })][-1][0]['documentsource'] updateAttackerGeoIP(mozdefdb, newAttacker['_id'], latestGeoIP) if options.broadcastattackers: broadcastAttacker(newAttacker) else: logger.debug('Found existing attacker') # if alert not present in this attackers list # append this to the list # todo: trim the list at X (i.e. last 100) # search alerts without attackerid matchingalerts = alerts.find( {"events.documentsource.details.sourceipaddress": str(ipcidr.ip), "attackerid":{"$exists": False} }) if matchingalerts is not None: logger.debug('Matched alert with attacker') # update list of alerts this attacker matched. for alert in matchingalerts: attacker['alerts'].append( dict(alertid=alert['_id']) ) # update alert with attackerID alert['attackerid'] = attacker['_id'] alerts.save(alert) attacker['eventscount'] += len(alert['events']) attacker['lastseentimestamp'] = toUTC(alert['events'][-1]['documentsource']['utctimestamp']) # geo ip could have changed, update it to the latest updateAttackerGeoIP(mozdefdb, attacker['_id'], alert['events'][-1]['documentsource']) # update counts attacker['alertscount'] = len(attacker['alerts']) attackers.save(attacker) # should we autocategorize the attacker # based on their alerts? if attacker['category'] == 'unknown' and options.autocategorize: # take a look at recent alerts for this attacker # and if they are all the same category # auto-categorize the attacker matchingalerts = alerts.find( {"attackerid": attacker['_id']} ).sort('utcepoch', -1).limit(50) # summarize the alert categories # returns list of tuples: [(u'bruteforce', 8)] categoryCounts= mostCommon(matchingalerts,'category') # are the alerts all the same category? if len(categoryCounts) == 1: # is the alert category mapped to an attacker category? for category in options.categorymapping: if category.keys()[0] == categoryCounts[0][0]: attacker['category'] = category[category.keys()[0]] attackers.save(attacker)
def main(): logger.debug('Starting') logger.debug(options) getQueueSizes()
def reauth_timer(self): while True: time.sleep(self.flush_wait_time) logger.debug('Recycling credentials and reassuming role') self.authenticate()
def getQueueSizes(): logger.debug('starting') logger.debug(options) es = ElasticsearchClient(options.esservers) sqslist = {} sqslist['queue_stats'] = {} qcount = len(options.taskexchange) qcounter = qcount - 1 mqConn = boto.sqs.connect_to_region( options.region, aws_access_key_id=options.accesskey, aws_secret_access_key=options.secretkey ) while qcounter >= 0: for exchange in options.taskexchange: logger.debug('Looking for sqs queue stats in queue' + exchange) eventTaskQueue = mqConn.get_queue(exchange) # get queue stats taskQueueStats = eventTaskQueue.get_attributes('All') sqslist['queue_stats'][qcounter] = taskQueueStats sqslist['queue_stats'][qcounter]['name'] = exchange qcounter -= 1 # setup a log entry for health/status. sqsid = '{0}-{1}'.format(options.account, options.region) healthlog = dict( utctimestamp=toUTC(datetime.now()).isoformat(), hostname=sqsid, processid=os.getpid(), processname=sys.argv[0], severity='INFO', summary='mozdef health/status', category='mozdef', source='aws-sqs', tags=[], details=[]) healthlog['details'] = dict(username='******') healthlog['details']['queues']= list() healthlog['details']['total_messages_ready'] = 0 healthlog['details']['total_feeds'] = qcount healthlog['tags'] = ['mozdef', 'status', 'sqs'] ready = 0 qcounter = qcount - 1 for q in sqslist['queue_stats'].keys(): queuelist = sqslist['queue_stats'][qcounter] if 'ApproximateNumberOfMessages' in queuelist: ready1 = int(queuelist['ApproximateNumberOfMessages']) ready = ready1 + ready healthlog['details']['total_messages_ready'] = ready if 'ApproximateNumberOfMessages' in queuelist: messages = int(queuelist['ApproximateNumberOfMessages']) if 'ApproximateNumberOfMessagesNotVisible' in queuelist: inflight = int(queuelist['ApproximateNumberOfMessagesNotVisible']) if 'ApproximateNumberOfMessagesDelayed' in queuelist: delayed = int(queuelist['ApproximateNumberOfMessagesDelayed']) if 'name' in queuelist: name = queuelist['name'] queueinfo=dict( queue=name, messages_delayed=delayed, messages_ready=messages, messages_inflight=inflight) healthlog['details']['queues'].append(queueinfo) qcounter -= 1 # post to elasticsearch servers directly without going through # message queues in case there is an availability issue es.save_event(index=options.index, doc_type='mozdefhealth', body=json.dumps(healthlog)) # post another doc with a static docid and tag # for use when querying for the latest sqs status healthlog['tags'] = ['mozdef', 'status', 'sqs-latest'] es.save_event(index=options.index, doc_type='mozdefhealth', doc_id=getDocID(sqsid), body=json.dumps(healthlog))
def run(self): while True: try: records = self.sqs_queue.receive_messages( MaxNumberOfMessages=options.prefetch) for msg in records: # msg.id is the id, # get_body() should be json # pre process the message a bit tmp = msg.body try: msgbody = json.loads(tmp) except ValueError: # If Boto wrote to the queue, it might be base64 encoded, so let's decode that try: tmp = base64.b64decode(tmp) msgbody = json.loads(tmp) except Exception as e: logger.error( 'Invalid message, not JSON <dropping message and continuing>: %r' % msg.get_body()) msg.delete() continue # If this is still not a dict, # let's just drop the message and move on if type(msgbody) is not dict: logger.debug( "Message is not a dictionary, dropping message.") msg.delete() continue event = dict() event = msgbody # Was this message sent by fluentd-sqs fluentd_sqs_specific_fields = { 'az', 'instance_id', '__tag' } if fluentd_sqs_specific_fields.issubset(set( msgbody.keys())): # Until we can influence fluentd-sqs to set the # 'customendpoint' key before submitting to SQS, we'll # need to do it here # TODO : Change nubis fluentd output to include # 'customendpoint' event['customendpoint'] = True if 'tags' in event: event['tags'].extend([options.taskexchange]) else: event['tags'] = [options.taskexchange] # process message self.on_message(event, msg) # delete message from queue msg.delete() time.sleep(.1) except ValueError as e: logger.exception('Exception while handling message: %r' % e) msg.delete() except (SSLEOFError, SSLError, socket.error): logger.info('Received network related error...reconnecting') time.sleep(5) self.sqs_queue = connect_sqs(options.region, options.accesskey, options.secretkey, options.taskexchange)
def main(): logger.debug('Starting') logger.debug(options) getQueueSizes()
def main(): logger.debug('Starting') logger.debug(options) verify_events(options)
def main(): logger.debug('Starting') logger.debug(options) download_generic_alerts(options.alert_repo_url, options.alert_data_location, options.deploy_key_location)
def main(): logger.debug('started') state = State(options.state_file_name) try: # capture the time we start running so next time we catch any events # created while we run. lastrun = toUTC(datetime.now()).isoformat() scope = [ 'https://www.googleapis.com/auth/admin.reports.audit.readonly', 'https://www.googleapis.com/auth/admin.reports.usage.readonly' ] # get our credentials credentials = service_account.Credentials.from_service_account_file( options.jsoncredentialfile, scopes=scope, subject=options.impersonate) # build a request to the admin sdk api = googleapiclient.discovery.build('admin', 'reports_v1', credentials=credentials) response = api.activities().list( userKey='all', applicationName='login', startTime=toUTC( state.data['lastrun']).strftime('%Y-%m-%dT%H:%M:%S.000Z'), maxResults=options.recordlimit).execute() # fix up the event craziness to a flatter format events = [] if 'items' in response: for i in response['items']: # flatten the sub dict/lists to pull out the good parts mozmsg = mozdef.MozDefEvent(options.url) mozmsg.category = 'google' mozmsg.tags = ['google', 'authentication'] mozmsg.severity = 'INFO' mozmsg.summary = 'google authentication: ' details = dict() for keyValue in flattenDict(i): # change key/values like: # [email protected] # to actor_email=value try: key, value = keyValue.split('=') except ValueError as e: continue key = key.replace('.', '_').lower() details[key] = value # find important keys # and adjust their location/name if 'ipaddress' in details: # it's the source ip details['sourceipaddress'] = details['ipaddress'] del details['ipaddress'] if 'id_time' in details: mozmsg.timestamp = details['id_time'] mozmsg.utctimestamp = details['id_time'] if 'events_name' in details: mozmsg.summary += details['events_name'] + ' ' if 'actor_email' in details: mozmsg.summary += details['actor_email'] + ' ' mozmsg.details = details events.append(mozmsg) # post events to mozdef logger.debug('posting {0} google events to mozdef'.format(len(events))) for e in events: e.send() # record the time we started as # the start time for next time. state.data['lastrun'] = lastrun state.write_state_file() except Exception as e: logger.error("Unhandled exception, terminating: %r" % e)
def main(): logger.debug('starting') logger.debug(options) try: # connect to mongo client = MongoClient(options.mongohost, options.mongoport) mozdefdb = client.meteor ipblocklist = mozdefdb['ipblocklist'] attackers = mozdefdb['attackers'] # ensure indexes ipblocklist.create_index([('dateExpiring', -1)]) attackers.create_index([('lastseentimestamp', -1)]) attackers.create_index([('category', 1)]) # First, gather IP addresses from recent attackers and add to the block list attackerIPList = aggregateAttackerIPs(attackers) # add attacker IPs to the blocklist # first delete ones we've created from an attacker ipblocklist.delete_many({'creator': 'mozdef', 'reference': 'attacker'}) # delete any that expired ipblocklist.delete_many({'dateExpiring': {"$lte": datetime.utcnow() - timedelta(days=options.expireage)}}) # add the aggregations we've found recently for ip in attackerIPList: ipblocklist.insert_one( {'_id': genMeteorID(), 'address': ip, 'reference': 'attacker', 'creator': 'mozdef', 'dateAdded': datetime.utcnow()}) # Lastly, export the combined blocklist ipCursor = mozdefdb['ipblocklist'].aggregate([ {"$sort": {"dateAdded": -1}}, {"$match": {"address": {"$exists": True}}}, {"$match": { "$or": [ {"dateExpiring": {"$gte": datetime.utcnow()}}, {"dateExpiring": {"$exists": False}}, ]}, }, {"$project": {"address": 1}}, {"$limit": options.iplimit} ]) ips = [] for ip in ipCursor: ips.append(ip['address']) uniq_ranges = netaddr.cidr_merge(ips) # to text with open(options.outputfile, 'w') as outputfile: for ip in uniq_ranges: outputfile.write("{0}\n".format(ip)) outputfile.close() # to s3? if len(options.aws_bucket_name) > 0: s3_upload_file(options.outputfile, options.aws_bucket_name, options.aws_document_key_name) except ValueError as e: logger.error("Exception %r generating IP block list" % e)