def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # XXX get rid of verify=False self.sip_client = pysip.Client(saq.CONFIG['sip']['remote_address'], saq.CONFIG['sip']['api_key'], verify=False)
def set_sip_indicator_status(indicator_id, status): """Sets the given indicator to the given status. Returns True if the operation succeeded.""" assert isinstance(indicator_id, int) assert isinstance(status, str) import pysip sip_client = pysip.Client(saq.CONFIG['sip']['remote_address'], saq.CONFIG['sip']['api_key'], verify=False) return sip_client.put(f'indicators/{indicator_id}', data={"status" : status})
def query_sip_indicator(indicator_id): """Queries SIP for indicator details. Returns the dictionary containing the information (see the SIP documenation for dictionary schema.)""" assert isinstance(indicator_id, int) import pysip sip_client = pysip.Client(saq.CONFIG['sip']['remote_address'], saq.CONFIG['sip']['api_key'], verify=False) return sip_client.get(f'indicators/{indicator_id}')
def setUp(self): ACEModuleTestCase.setUp(self) if not saq.CONFIG['sip'].getboolean('enabled'): return # XXX get rid of verify=False self.sip_client = pysip.Client(saq.CONFIG['sip']['remote_address'], saq.CONFIG['sip']['api_key'], verify=False) # insert the indicator we'll test against self.test_indicator = self.sip_client.post('indicators', { 'type': 'Email - Address', 'value': '*****@*****.**' }) self.test_indicator_id = self.test_indicator['id'] self.assertTrue(isinstance(self.test_indicator_id, int))
def setUp(self): ACEBasicTestCase.setUp(self) # XXX get rid of verify=False if saq.CONFIG['sip'].getboolean('enabled'): self.sip_client = pysip.Client(saq.CONFIG['sip']['remote_address'], saq.CONFIG['sip']['api_key'], verify=False) self.test_indicators = [] # insert the indicator(s) we'll test against for indicator in [ { 'type': 'URI - URL', 'value': 'http://whackadoodle.net/dunno.html', 'status': 'Analyzed' }, #{ 'type': 'Address - ipv4-addr', 'value': '165.45.66.45', 'status': 'Analyzed' }, { 'type': 'URI - Path', 'value': '/follow/the/white/rabbit.html', 'status': 'Analyzed' }, { 'type': 'Windows - FileName', 'value': 'ultimate.txt', 'status': 'Analyzed' }, ]: self.test_indicators.append(self.sip_client.post('indicators', indicator)) self.target_urls = [ 'http://whackadoodle.net/dunno.html', #'http://165.45.66.45/whatever.asp', 'http://www.g00gle.com/follow/the/white/rabbit.html', 'http://www.c00kie.com/ultimate.txt' ]
def main(): parser = argparse.ArgumentParser( description="SIP Indicator CbR Search and ACE Alert.") parser.add_argument('-d', '--debug', action="store_true", help="set logging to DEBUG", default=False) args = parser.parse_args() # load config config = ConfigParser() config.read('etc/config.ini') # load SIP indicator specs so we know how to get the indicators we want indicator_specs = {} with open(config['SIP']['indicator_specifications'], 'r') as stream: try: indicator_specs = yaml.safe_load(stream) logging.info( "Successfully loaded indicator specifications: {}".format( indicator_specs)) except yaml.YAMLError as e: logging.error("Couldn't load indicator specs : {}".format(e)) return # Load ACE API ace_api.set_default_remote_host(config['ACE']['ace_address']) ace_api.set_default_ssl_ca_path(config['ACE']['ca_chain_path']) # Create SIP Client and load indicators sip_ssl = config['SIP'].getboolean('ssl_verify') sc = pysip.Client(config['SIP']['sip_address'], config['SIP']['sip_api_key'], verify=sip_ssl) status = indicator_specs[ 'status'] if 'status' in indicator_specs else 'Analyzed' indicators = {} for i_type in indicator_specs['type']: handle_proxy(config['SIP']) indicators[i_type] = sc.get('/indicators?type={}&status={}'.format( i_type, status)) # load field mappings field_map = ConfigParser() field_map.read(config['GLOBAL']['field_mappings']) sip_cbr_map = field_map['SIP-TO-CBR'] sip_ace_map = field_map['SIP-TO-ACE'] cbr_ace_map = field_map['CBR-TO-ACE'] submitted_alerts = [] # Query Carbon Black Response for our indicators #cbq = CBquery(profile=config['CbR']['profile']) handle_proxy(config['CbR']) cb = CbResponseAPI(profile=config['CbR']['profile']) for i_type in indicator_specs['type']: for i in indicators[i_type]: query = '{}:"{}"'.format(sip_cbr_map[i_type], i['value']) logging.debug('Querying CbR for indicator:{} query:{}'.format( i['id'], query)) procs = cb.select(Process).where(query).group_by('id') if procs: # alert ACE Alert = ace_api.Analysis(description='CbR - SIP:{}'.format( i['value']), analysis_mode='correlation', tool='SipCbrAce') print(Alert.description) Alert.add_indicator(i['id']) # get sip tags and tag Alert handle_proxy(config['SIP']) i_details = sc.get('/indicators/{}'.format(i['id'])) handle_proxy(config['CbR']) for tag in i_details['tags']: Alert.add_tag(tag) alert_details = {} alert_details['total_results'] = len(procs) max_results = config['GLOBAL'].getint('alert_max_results') alert_details['included_results'] = 0 alert_details['process_details'] = [] for proc in procs: if alert_details['included_results'] > max_results: break alert_details['process_details'].append(str(proc)) alert_details['included_results'] += 1 Alert.add_hostname(proc.hostname) Alert.add_md5(proc.process_md5) Alert.add_ipv4(proc.comms_ip) Alert.add_ipv4(proc.interface_ip) Alert.add_process_guid(proc.id) Alert.add_user(proc.username) Alert.add_file_name(proc.process_name) Alert.add_file_path(proc.path) #Alert.add_file_location('{}@{}'.format(proc.hostname, proc.path)) #Alert.submit_kwargs['details'] = alert_details handle_proxy(config['ACE']) print(Alert.description) submitted_alerts.append(Alert.submit()) logger.info( "Submitted alert to ACE: {UUID} - URL=https://{HOST}/ace/analysis?direct={UUID}" .format(UUID=Alert.uuid, HOST=Alert.remote_host)) print(submitted_alerts)
requests.packages.urllib3.disable_warnings(InsecureRequestWarning) if __name__ == "__main__": # remove proxy if it's set if 'http_proxy' in os.environ: del os.environ['http_proxy'] if 'https_proxy' in os.environ: del os.environ['https_proxy'] # load configuration config = ConfigParser() config.read(os.path.expanduser("~")+'/.sipit.ini') sip_client = pysip.Client(config['sip']['end_point'],config['sip']['api_key'],verify=False) parser = argparse.ArgumentParser(description="Add Indicators and query SIP") subparsers = parser.add_subparsers(dest='command') commands = [ 'create', 'query','update' ] query_parser = subparsers.add_parser('query',help="query aspects of SIP. query -h for more") query_parser.add_argument('-t','--types',default=False,action='store_true',help='list indicator types') query_parser.add_argument('-s','--sources',default=False,action='store_true',help='list sources') query_parser.add_argument('-c','--campaigns',default=False,action='store_true',help='list campaigns') query_parser.add_argument('--tags',default=False,action='store_true',help='list tags') query_parser.add_argument('-v','--value',default=False,dest='value',help='search for an indicator value') query_parser.add_argument('-d','--details',default=False,action='store_true',help='all information about an indicator value') query_parser.add_argument('--status',default=False,action='store_true',help='list possible status values for indicators') query_parser.add_argument('-id','--indicator-id',dest='id',help='query the specific indicator information for a sip id')
def update_local_cache(): import pysip # XXX remove verify=False sip_client = pysip.Client(saq.CONFIG['sip']['remote_address'], saq.CONFIG['sip']['api_key'], verify=False) cache_path = os.path.join(saq.DATA_DIR, saq.CONFIG['sip']['cache_db_path']) # the actual file should be a symlink if os.path.exists(cache_path) and not os.path.islink(cache_path): logging.error( "{} should be a symlink but it's not!".format(cache_path)) return False # get the file the symlink points to current_cache_path = None if os.path.exists(cache_path): current_cache_path = os.path.realpath(cache_path) else: current_cache_path = '{}.b'.format(cache_path) # there are two files that end with .a and .b if not current_cache_path.endswith( '.a') and not current_cache_path.endswith('.b'): logging.error( "expecting {} to end with .a or .b!".format(current_cache_path)) return False # we edit the other one base_cache_path = current_cache_path[:-2] if current_cache_path.endswith('.a'): target_cache_path = '{}.b'.format(base_cache_path) else: target_cache_path = '{}.a'.format(base_cache_path) logging.info("updating {}".format(target_cache_path)) if os.path.exists(target_cache_path): try: logging.info( "deleting existing crits cache {}".format(target_cache_path)) os.remove(target_cache_path) except Exception as e: logging.error("unable to delete {}: {}".format( target_cache_path, e)) return False cache_db = sqlite3.connect(target_cache_path) db_cursor = cache_db.cursor() db_cursor.execute("""CREATE TABLE indicators ( id TEXT PRIMARY KEY, type TEXT NOT NULL, value TEXT NOT NULL )""") db_cursor.execute( "CREATE INDEX i_type_value_index ON indicators ( type, value )") logging.info("caching indicators...") c = 0 for indicator in sip_client.get( '/api/indicators?status=Analyzed&bulk=True'): db_cursor.execute( "INSERT INTO indicators ( id, type, value ) VALUES ( ?, ?, LOWER(?) )", (str(indicator['id']), indicator['type'], indicator['value'])) c += 1 logging.info("comitting changes to database...") cache_db.commit() logging.info("updating symlink...") # now point current link to our new database # leaving the old one in place for current processes to keep using try: try: os.remove(cache_path) except: pass os.symlink(os.path.basename(target_cache_path), cache_path) except Exception as e: logging.error("failed to update symlink: {}".format(e)) logging.info("done") logging.debug("loaded {} indicators".format(c)) return True
def main(): parser = argparse.ArgumentParser( description= "The tool maps Cofense IOCs from their JSON integration into SIP indicators." ) parser.add_argument('-d', '--debug', action='store_true', help="Turn on debug logging.") parser.add_argument( '--logging-config', required=False, default='etc/logging.ini', dest='logging_config', help="Path to logging configuration file. Defaults to etc/logging.ini" ) parser.add_argument( '-c', '--config', required=False, default='etc/config.ini', dest='config_path', help="Path to configuration file. Defaults to etc/config.ini") parser.add_argument('-p', '--print-indicator-summary', action='store_true', help="Print a summary of all indicators incoming.") parser.add_argument('-rd', '--target-report-dir', action='store', default=None, help="only evaluate this report directory") parser.add_argument('-grh', '--get-report-html', action='store_true', help="get report html") args = parser.parse_args() # work out of home dir os.chdir(HOME_PATH) # initialize logging try: logging.config.fileConfig(args.logging_config) except Exception as e: sys.stderr.write( "ERROR: unable to load logging config from {0}: {1}".format( args.logging_config, str(e))) sys.exit(1) coloredlogs.install(level='INFO', logger=logging.getLogger()) if args.debug: coloredlogs.install(level='DEBUG', logger=logging.getLogger()) config = configparser.ConfigParser() config.read(args.config_path) sip_map = config['sip_mappings'] # for turning indicator creation on/off by type indicator_filter = config['indicator_filter'] # variables # - keep a throttle on indicators created per day # - track threats indicators_created_today = 0 max_indicators_per_day = config['collect'].getint('max_indicators_per_day') indicator_creation_count_file = os.path.join( HOME_PATH, 'var', f"indicator_count_for_{datetime.now().strftime('%Y-%m-%d')}") if not os.path.exists(indicator_creation_count_file): logging.info(f"reseting indicator count for a new day..") for old_file in glob.glob( f"{os.path.join(HOME_PATH, 'var')}/indicator_count_for_*"): logging.info(f"deleting old variable file: {old_file}") os.remove(old_file) with open(indicator_creation_count_file, 'w') as f: f.write(str(0)) else: with open(indicator_creation_count_file, 'r') as f: indicators_created_today = f.read() indicators_created_today = int(indicators_created_today) unique_threat_tracker = {} unique_threat_tracker_file = os.path.join(HOME_PATH, 'var', "unique_threat_tracker.json") if os.path.exists(unique_threat_tracker_file): with open(unique_threat_tracker_file, 'r') as fp: unique_threat_tracker = json.load(fp) # connect to sip verify_ssl = config['sip'].get('verify_ssl') if not os.path.exists(verify_ssl): verify_ssl = config['sip'].getboolean('verify_ssl') sip = pysip.Client( f"{config['sip'].get('server')}:{config['sip'].get('port')}", config['sip']['api_key'], verify=verify_ssl) def _sip_indicator(type: str, value: str, reference: dict, tags: list, username=config['sip'].get('user'), case_sensitive=False) -> dict: # A sip indicator with some defaults defined. return { 'type': type, 'status': 'New', 'confidence': 'low', 'impact': 'unknown', 'value': value, 'references': [{ 'source': "Cofense", 'reference': json.dumps(reference) }], 'username': username, 'case_sensitive': case_sensitive, 'tags': list(set(tags)) } resume_report_id = None processed_reports = [] total_incoming_indicators = [] for report in report_iterator(target_report_dir=args.target_report_dir): if args.get_report_html: get_html_report(report['reportURL']) return True global_tags = [] for threat in report['malwareFamilySet']: if threat['familyName'] == 'Credential Phishing': global_tags.append('creds_harvesting') continue if threat['familyName'] not in unique_threat_tracker: logging.info( f"found previously un-seen threat name: {threat['familyName']} - {threat['description']}" ) unique_threat_tracker[ threat['familyName']] = threat['description'] if threat['familyName'] not in global_tags: global_tags.append(threat['familyName']) reference = { 'id': report['id'], 'reportURL': report['reportURL'], # TODO 'executiveSummary': report['executiveSummary'] } ref_length = len(json.dumps(reference)) if ref_length > 512: # Max length of SIP reference field is 512. cut_length = len( reference['executiveSummary']) - (ref_length - 512) reference['executiveSummary'] = reference[ 'executiveSummary'][:cut_length] # report indicators to post to SIP potential_indicators = [] # process blocksSet for block in report['blockSet']: _tags = global_tags # deduplication happens later if block['role'] == "InfURL": _tags.append('phishing_url') else: _tags.append(block['role']) itype = sip_map[block['blockType']] idata = _sip_indicator(type=itype, value=block['data'], reference=reference, tags=_tags) if indicator_filter.getboolean(itype): potential_indicators.append(idata) # create more indicators if block['blockType'] == "URL": if block['role'] == "InfURL": # this was a phishing url itype = 'Email - Content - Domain Name' value = block['data_1']['host'] idata = _sip_indicator(type=itype, value=value, reference=reference, tags=_tags) idata['tags'].append('domain_in_url') if indicator_filter.getboolean(itype): potential_indicators.append(idata) # uri path? value = block['data_1']['path'] if value and len(value) > 9: idata = _sip_indicator(type='URI - Path', value=value, reference=reference, tags=_tags) if indicator_filter.getboolean('URI - Path'): potential_indicators.append(idata) # default assume domain name itype = 'URI - Domain Name' value = block['data_1']['host'] if is_ipv4(block['data_1']['host']): itype = "Address - ipv4-addr" idata = _sip_indicator(type=itype, value=value, reference=reference, tags=_tags) if indicator_filter.getboolean(itype): potential_indicators.append(idata) # process executableSet for malfile in report['executableSet']: fileName = malfile.get('fileName') if fileName and indicator_filter.getboolean(sip_map['fileName']): potential_indicators.append( _sip_indicator(type=sip_map['fileName'], value=fileName, reference=reference, tags=global_tags)) sha256Hex = malfile.get('sha256Hex') if sha256Hex and indicator_filter.getboolean(sip_map['fileName']): potential_indicators.append( _sip_indicator(type=sip_map['sha256Hex'], value=sha256Hex, reference=reference, tags=global_tags)) md5Hex = malfile.get('md5Hex') if md5Hex and indicator_filter.getboolean(sip_map['fileName']): potential_indicators.append( _sip_indicator(type=sip_map['md5Hex'], value=md5Hex, reference=reference, tags=global_tags)) # process subjectSet for subjectSet in report['subjectSet']: subject = subjectSet.get('subject') if subject and indicator_filter.getboolean(sip_map['fileName']): potential_indicators.append( _sip_indicator(type='Email - Subject', value=subject, reference=reference, tags=global_tags)) if args.print_indicator_summary: total_incoming_indicators.extend(potential_indicators) continue for indicator in potential_indicators: if indicators_created_today >= max_indicators_per_day: resume_report_id = report['id'] logging.warning( f"maximum indicators created for the day. Will resume report {resume_report_id} tomorrow." ) break if create_sip_indicator(sip, indicator): indicators_created_today += 1 if resume_report_id is not None: break if args.print_indicator_summary: summary_data = [] for i in total_incoming_indicators: summary_data.append({ 'Report ID': json.loads(i['references'][0]['reference'])['id'], 'type': i['type'], 'value': i['value'] }) #print(f" + type:{i['type']}\t\t\tvalue:{i['value']}") print(tabulate(summary_data, headers="keys")) #, tablefmt="github")) print(f"\nTotal Incoming Indicators: {len(total_incoming_indicators)}") return # update records try: with open(indicator_creation_count_file, 'w') as fp: fp.write(str(indicators_created_today)) except Exception as e: logging.error(f"Problem writing indicator count file: {e}") try: with open(unique_threat_tracker_file, 'w') as fp: json.dump(unique_threat_tracker, fp) except Exception as e: logging.error(f"Problem writing unique threat tracker: {e}") # archive processed reports for report_id, report_path in PROCESSED_REPORTS.items(): if resume_report_id == report_id: continue archive_dir = os.path.dirname(report_path).replace( INCOMING_DIR_NAME, ARCHIVE_DIR_NAME) if not os.path.exists(archive_dir): os.mkdir(archive_dir) archive_path = os.path.join(archive_dir, f"{report_id}.json") try: os.rename(report_path, archive_path) except Exception as e: logging.error(f"couldn't archive report: {e}") logging.info(f"archived {report_path} to {archive_path}") # delete empty dirs for report_dir in glob.glob(f"{os.path.join(INCOMING_DIR)}/*"): if os.path.isdir(report_dir): files = os.listdir(report_dir) if len(files) == 0: os.rmdir(report_dir) logging.info(f"deleted empty report dir: {report_dir}")
# get default config items server = config['default']['server'] port = config['default']['port'] api_key = config['default']['api_key'] verify_ssl = config['default']['verify_ssl'] whitelist_tags = config['default']['whitelist_tags'] ignore_proxy = config['default']['ignore_proxy'] if ignore_proxy: if 'https_proxy' in os.environ: del os.environ['https_proxy'] if 'http_proxy' in os.environ: del os.environ['http_proxy'] # create sip client and sipwhitelist instance sip_client = pysip.Client(server + ':' + port, api_key, verify=verify_ssl) swl = SIPWhitelist(whitelist_tags, sip_client) indicator_types = [ i['value'] for i in sip_client.get('/api/indicators/type') ] istatus_types = [ s['value'] for s in sip_client.get('/api/indicators/status') ] isources = [s['value'] for s in sip_client.get('/api/intel/source')] whitelisted_itypes = swl.whitelist.keys() parser = argparse.ArgumentParser( description="Add Indicators and query SIP") parser.add_argument('-d', '--debug',