def __init__(self, api_route, api_key, sec_key, temp_file_location='/tmp'): self.isight_api = ISightAPI(api_route, api_key, sec_key) self.cache_db_name = os.path.join(temp_file_location, 'isight_cache.db') self.cache_db = sqlite3.connect(self.cache_db_name) self.set_up_cache()
class Bridge(object): RISK_RATING_MAP = { 'LOW': 50, 'MEDIUM': 60, 'HIGH': 70, 'CRITICAL': 100 } DEFAULT_RISK = 'MEDIUM' UNKNOWN_RISK = 100 INTERESTED_CHARACTERIZATIONS = ['Attacker', 'Compromised'] def __init__(self, api_route, api_key, sec_key, temp_file_location='/tmp'): self.isight_api = ISightAPI(api_route, api_key, sec_key) self.cache_db_name = os.path.join(temp_file_location, 'isight_cache.db') self.cache_db = sqlite3.connect(self.cache_db_name) self.set_up_cache() def set_up_cache(self): self.cache_db.execute('''create table if not exists isight_cache (report_id text, report_publish_date int, report_json_content text)''') def perform(self, history_in_days=30): _logger.info("Contacting iSIGHT for IOCs for last {0:d} days".format(history_in_days)) reports = json.loads(self.isight_api.get_iocs(history_in_days, format='json')) report_set = set() for report in reports['message']: report_set.add((report['reportLink'].split('/')[-1], long(report['publishDate']))) reports = self.process_reports(report_set) return reports def perform_iocs(self, history_in_days=30, iocs_only=True): _logger.info("Contacting iSIGHT for IOCs for last {0:d} days".format(history_in_days)) reports = {} if iocs_only: iocs = StringIO(self.isight_api.get_iocs(history_in_days, format='csv')) else: iocs = StringIO(self.isight_api.get_i_and_w(history_in_days, format='csv')) for row in csv.DictReader(iocs): report_id = row['reportId'] try: timestamp = int(row['publishDate']) except ValueError as e: _logger.error("Invalid publishDate for reportId %s: %s. Setting to today's date." % (report_id, row['publishDate'])) timestamp = int(time.time()) if report_id not in reports.keys(): # add new report metadata reports[report_id] = { 'id': report_id, 'title': row['title'], 'link': row['webLink'], 'timestamp': timestamp, 'iocs': defaultdict(set) } report = reports[report_id] if row['cidr']: cidr_block = unicode(row['cidr']) cidr_block = ipaddr.IPv4Network(cidr_block) if cidr_block.numhosts <= 256: report['iocs']['ipv4'].update([unicode(x) for x in cidr_block.iterhosts()]) else: _logger.info("Ignoring larger than /24 netblock ({0:s})".format(cidr_block)) if row['ip']: report['iocs']['ipv4'].add(unicode(row['ip'])) if row['domain']: report['iocs']['dns'].add(row['domain']) if row['md5']: report['iocs']['md5'].add(row['md5']) reports_with_iocs = [v for k, v in reports.iteritems() if len(v['iocs'])] for report in reports_with_iocs: for ioc_type in report['iocs']: report['iocs'][ioc_type] = list(report['iocs'][ioc_type]) return reports_with_iocs def process_reports(self, report_set): reports = [] # filter for reports that actually have content for report_id in report_set: raw_report = self.get_report(report_id) if not raw_report: continue if 'message' not in raw_report: continue raw_report = raw_report['message'] if 'report' not in raw_report: continue raw_report = raw_report['report'] report = dict(raw_report=raw_report, iocs={}, id=report_id[0], timestamp=report_id[1]) reports.append(report) # get indicators and score for each report for report_entry in reports: report = report_entry['raw_report'] report_entry['title'] = report.get('title', '') report_entry['link'] = "https://mysight.isightpartners.com/report/full/{0:s}".format(report_entry['id']) risk_rating = report.get('riskRating', Bridge.DEFAULT_RISK) report_entry['score'] = Bridge.RISK_RATING_MAP.get(risk_rating, Bridge.UNKNOWN_RISK) if 'tagSection' in report: if 'networks' in report['tagSection']: _logger.debug("adding network section to {0:s}".format(report_entry['id'])) report_entry['iocs'].update( self.parse_network_iocs( [ioc for ioc in report['tagSection']['networks']['network'] if ioc.get('identifier', None) in Bridge.INTERESTED_CHARACTERIZATIONS])) if 'files' in report['tagSection']: _logger.debug("adding file section to {0:s}".format(report_entry['id'])) report_entry['iocs'].update( self.parse_file_iocs( [ioc for ioc in report['tagSection']['files']['file'] if ioc.get('identifier', None) in Bridge.INTERESTED_CHARACTERIZATIONS])) return [r for r in reports if len(r['iocs'])] def parse_network_iocs(self, report_iocs): ipaddrs = set() domains = set() for ioc in report_iocs: if 'cidr' in ioc: # expand out ip addresses if we are <= /24 cidr_block = unicode(ioc['cidr']) cidr_block = ipaddr.IPv4Network(cidr_block) if cidr_block.numhosts <= 256: ipaddrs.update([unicode(x) for x in cidr_block.iterhosts()]) else: _logger.info("Ignoring larger than /24 netblock ({0:s})".format(cidr_block)) if 'ip' in ioc: ipaddrs.add(unicode(ioc['ip'])) if 'domain' in ioc: domains.add(ioc['domain']) ret = {} if len(ipaddrs): ret['ipv4'] = list(ipaddrs) if len(domains): ret['dns'] = list(domains) return ret def parse_file_iocs(self, report_iocs): md5s = set() for ioc in report_iocs: if 'md5' in ioc: md5s.add(ioc['md5']) ret = {} if len(md5s): ret['md5'] = list(md5s) return ret # TODO: add error checking def get_report(self, report_key): cur = self.cache_db.cursor() req = cur.execute("select report_json_content from isight_cache where report_id=? and report_publish_date=?", report_key) content = req.fetchone() if content: _logger.debug("Found cached report for {0:s} published on {1:d}".format(report_key[0], report_key[1])) content = json.loads(content[0]) else: try: content = self.isight_api.get_report(report_key[0], format='json') except: _logger.debug("Exception retrieving report {0:s}: ".format(report_key[0]) + traceback.format_exc()) return None _logger.debug( "Inserting report {0:s} published on {1:d} into local cache".format(report_key[0], report_key[1])) cur.execute('''insert into isight_cache(report_id, report_publish_date, report_json_content) values (?, ?, ?)''', (report_key[0], report_key[1], content)) self.cache_db.commit() content = json.loads(content) return content