def get_initial_beacon_doc(self, implant_id): """ Get the initial beacon document from cobaltstrike or return False if none found """ query = f'implant.id:{implant_id} AND c2.program: cobaltstrike AND c2.log.type:implant_newimplant' initial_beacon_doc = get_query(query, size=1, index='rtops-*') initial_beacon_doc = initial_beacon_doc[0] if len(initial_beacon_doc) > 0 else False self.logger.debug('Initial beacon line [%s]: %s', implant_id, initial_beacon_doc) return initial_beacon_doc
def enrich_beacon_data(self): """ Get all lines in rtops that have not been enriched yet (for CS) """ es_query = f'implant.id:* AND c2.program: cobaltstrike AND NOT c2.log.type:implant_newimplant AND NOT tags:{info["submodule"]}' not_enriched_results = get_query(es_query, size=10000, index='rtops-*') # Created a dict grouped by implant ID implant_ids = {} for not_enriched in not_enriched_results: implant_id = get_value('_source.implant.id', not_enriched) if implant_id in implant_ids: implant_ids[implant_id].append(not_enriched) else: implant_ids[implant_id] = [not_enriched] hits = [] # For each implant ID, get the initial beacon line for implant_id, implant_val in implant_ids.items(): initial_beacon_doc = self.get_initial_beacon_doc(implant_id) # If not initial beacon line found, skip the beacon ID if not initial_beacon_doc: continue for doc in implant_val: # Fields to copy: host.*, implant.*, process.*, user.* res = self.copy_data_fields(initial_beacon_doc, doc, ['host', 'implant', 'user', 'process']) if res: hits.append(res) return hits
def alarm_dummy(self): """ This check returns the last IOC in rtops-* that have not been alarmed yet """ es_query = 'c2.log.type:ioc AND NOT tags:alarm_*' es_results = get_query(es_query, 1, index='rtops-*') self.logger.debug(es_results) return es_results
def alarm_check(self): # pylint: disable=no-self-use """ This check queries for UA's that are listed in any blacklist_useragents.conf and do talk to c2* paths on redirectors We will dig trough ALL data finding specific IP related lines and tag them reading the useragents we trigger on. """ file_name = '/etc/redelk/rogue_useragents.conf' with open(file_name, encoding='utf-8') as file: content = file.readlines() ua_list = [] for line in content: if not line.startswith('#'): ua_list.append(line.strip()) keywords = ua_list es_subquery = '' # add keywords (UA's) to query for keyword in keywords: if es_subquery == '': es_subquery = f'(http.headers.useragent:{keyword}' else: es_subquery = es_subquery + f' OR http.headers.useragent:{keyword}' es_subquery = es_subquery + ') ' # q = "%s AND redir.backendname:c2* AND tags:enrich_* AND NOT tags:alarm_* "%qSub es_query = f'{es_subquery} AND redir.backend.name:c2* AND NOT tags:alarm_useragent' es_results = get_query(es_query, 10000) report = {} report['hits'] = es_results return report
def alarm_check(self): """ This check queries for calls to backends that have *alarm* in their name """ es_query = f'redir.backend.name:*alarm* AND NOT tags:{info["submodule"]}' es_results = get_query(es_query, 10000) report = { 'hits': es_results } return report
def alarm_check(self): """ This check queries for IP's that aren't listed in any iplist* but do talk to c2* paths on redirectors """ es_query = '*' i = get_hits_count(es_query) i = min(i, 10000) es_result = get_query(es_query, i) report = {} report['hits'] = [] report['hits'].append(es_result[0]) report['hits'].append(es_result[1]) return report
def sync_iplist(self, iplist='redteam'): """ Sync data between ES iplist and config files """ # Get data from config file iplist cfg_iplist = self.get_cfg_ips(iplist) # If the config file doesn't exist, skip the sync if cfg_iplist is None: return [] # Get data from ES iplist query = f'iplist.name:{iplist}' es_iplist_docs = get_query(query, size=10000, index='redelk-*') # Check if config IP is in ES and source = config_file es_iplist = [] for doc in es_iplist_docs: ip = get_value('_source.iplist.ip', doc) # pylint: disable=invalid-name if ip: es_iplist.append((ip, doc)) for ipc, comment in cfg_iplist: found = [item for item in es_iplist if ipc in item] if not found: self.logger.debug('IP not found in ES: %s', ipc) # if not, add it self.add_es_ip(ipc, iplist, comment) toadd = [] for ipe, doc in es_iplist: # Check if ES IP is in config file found = [item for item in cfg_iplist if ipe in item] if not found: # if not, check if source = config_file if get_value('_source.iplist.source', doc) == 'config_file': # if yes, remove IP from ES self.remove_es_ip(doc, iplist) else: # if not, add it comment = get_value('_source.iplist.comment', doc) if comment: ipa = f'{ipe} # From ES -- {comment}' else: ipa = f'{ipe} # From ES' toadd.append(ipa) self.add_cfg_ips(toadd, iplist) return toadd
def alarm_check(self): """ This check queries public sources given a list of md5 hashes. If a hash was seen we set an alarm """ es_query = 'c2.log.type:ioc AND NOT tags:alarm_filehash AND ioc.type:file' alarmed_md5_q = { 'aggs': { 'interval_filter': { 'filter': { 'range': { 'alarm.last_checked': { 'gte': f'now-{self.interval}s', 'lt': 'now' } } }, 'aggs': { 'md5_interval': { 'terms': { 'field': 'file.hash.md5' } } } }, 'alarmed_filter': { 'filter': { 'terms': { 'tags': ['alarm_filehash'] } }, 'aggs': { 'md5_alarmed': { 'terms': { 'field': 'file.hash.md5' } } } } } } report = {} iocs = [] self.logger.debug('Running query %s', es_query) # First, get all IOCs of type 'file' that have not been alarmed yet iocs = get_query(es_query, 10000, index='rtops-*') self.logger.debug('found ioc: %s', iocs) # Then we get an aggregation of all md5 alarmed within the last 'interval' time self.logger.debug('Running query %s', alarmed_md5_q) already_alarmed_result = raw_search(alarmed_md5_q, index='rtops-*') already_checked = [] already_alarmed = [] if already_alarmed_result: self.logger.debug(already_alarmed_result['aggregations']) # add md5 hashes that have been checked within the 'interval' in 'already_checked' for hit in already_alarmed_result['aggregations'][ 'interval_filter']['md5_interval']['buckets']: already_checked.append(hit['key']) # add md5 hashes that have been alarmed previously in 'already_alarmed' for hit in already_alarmed_result['aggregations'][ 'alarmed_filter']['md5_alarmed']['buckets']: already_alarmed.append(hit['key']) # Group all hits per md5 hash md5_dict = self.group_hits(iocs, already_alarmed, already_checked) # Create an array with all md5 hashes to send to the different providers # we now have an array with unique md5's to go test md5_list = [] for md5 in md5_dict: md5_list.append(md5) self.logger.debug('md5 hashes to check: %s', md5_list) # Run the checks check_results = self.check_hashes(md5_list) # Get the alarmed hashes with their corresponding mutations alarmed_hashes = self.get_mutations(check_results) # Get the report report = self.build_report(md5_dict, alarmed_hashes) return report