def __init__(self, api_route, api_key, sec_key, temp_file_location='/tmp'):
self.isight_api = ISightAPI(api_route, api_key, sec_key)
self.cache_db_name = os.path.join(temp_file_location, 'isight_cache.db')
self.cache_db = sqlite3.connect(self.cache_db_name)
self.set_up_cache()
class Bridge(object):
RISK_RATING_MAP = {
'LOW': 50,
'MEDIUM': 60,
'HIGH': 70,
'CRITICAL': 100
}
DEFAULT_RISK = 'MEDIUM'
UNKNOWN_RISK = 100
INTERESTED_CHARACTERIZATIONS = ['Attacker', 'Compromised']
def __init__(self, api_route, api_key, sec_key, temp_file_location='/tmp'):
self.isight_api = ISightAPI(api_route, api_key, sec_key)
self.cache_db_name = os.path.join(temp_file_location, 'isight_cache.db')
self.cache_db = sqlite3.connect(self.cache_db_name)
self.set_up_cache()
def set_up_cache(self):
self.cache_db.execute('''create table if not exists isight_cache
(report_id text, report_publish_date int, report_json_content text)''')
def perform(self, history_in_days=30):
_logger.info("Contacting iSIGHT for IOCs for last {0:d} days".format(history_in_days))
reports = json.loads(self.isight_api.get_iocs(history_in_days, format='json'))
report_set = set()
for report in reports['message']:
report_set.add((report['reportLink'].split('/')[-1], long(report['publishDate'])))
reports = self.process_reports(report_set)
return reports
def perform_iocs(self, history_in_days=30, iocs_only=True):
_logger.info("Contacting iSIGHT for IOCs for last {0:d} days".format(history_in_days))
reports = {}
if iocs_only:
iocs = StringIO(self.isight_api.get_iocs(history_in_days, format='csv'))
else:
iocs = StringIO(self.isight_api.get_i_and_w(history_in_days, format='csv'))
for row in csv.DictReader(iocs):
report_id = row['reportId']
try:
timestamp = int(row['publishDate'])
except ValueError as e:
_logger.error("Invalid publishDate for reportId %s: %s. Setting to today's date." % (report_id,
row['publishDate']))
timestamp = int(time.time())
if report_id not in reports.keys():
# add new report metadata
reports[report_id] = {
'id': report_id,
'title': row['title'],
'link': row['webLink'],
'timestamp': timestamp,
'iocs': defaultdict(set)
}
report = reports[report_id]
if row['cidr']:
cidr_block = unicode(row['cidr'])
cidr_block = ipaddr.IPv4Network(cidr_block)
if cidr_block.numhosts <= 256:
report['iocs']['ipv4'].update([unicode(x) for x in cidr_block.iterhosts()])
else:
_logger.info("Ignoring larger than /24 netblock ({0:s})".format(cidr_block))
if row['ip']:
report['iocs']['ipv4'].add(unicode(row['ip']))
if row['domain']:
report['iocs']['dns'].add(row['domain'])
if row['md5']:
report['iocs']['md5'].add(row['md5'])
reports_with_iocs = [v for k, v in reports.iteritems() if len(v['iocs'])]
for report in reports_with_iocs:
for ioc_type in report['iocs']:
report['iocs'][ioc_type] = list(report['iocs'][ioc_type])
return reports_with_iocs
def process_reports(self, report_set):
reports = []
# filter for reports that actually have content
for report_id in report_set:
raw_report = self.get_report(report_id)
if not raw_report:
continue
if 'message' not in raw_report:
continue
raw_report = raw_report['message']
if 'report' not in raw_report:
continue
raw_report = raw_report['report']
report = dict(raw_report=raw_report, iocs={}, id=report_id[0], timestamp=report_id[1])
reports.append(report)
# get indicators and score for each report
for report_entry in reports:
report = report_entry['raw_report']
report_entry['title'] = report.get('title', '')
report_entry['link'] = "https://mysight.isightpartners.com/report/full/{0:s}".format(report_entry['id'])
risk_rating = report.get('riskRating', Bridge.DEFAULT_RISK)
report_entry['score'] = Bridge.RISK_RATING_MAP.get(risk_rating, Bridge.UNKNOWN_RISK)
if 'tagSection' in report:
if 'networks' in report['tagSection']:
_logger.debug("adding network section to {0:s}".format(report_entry['id']))
report_entry['iocs'].update(
self.parse_network_iocs(
[ioc for ioc in report['tagSection']['networks']['network']
if ioc.get('identifier', None) in Bridge.INTERESTED_CHARACTERIZATIONS]))
if 'files' in report['tagSection']:
_logger.debug("adding file section to {0:s}".format(report_entry['id']))
report_entry['iocs'].update(
self.parse_file_iocs(
[ioc for ioc in report['tagSection']['files']['file']
if ioc.get('identifier', None) in Bridge.INTERESTED_CHARACTERIZATIONS]))
return [r for r in reports if len(r['iocs'])]
def parse_network_iocs(self, report_iocs):
ipaddrs = set()
domains = set()
for ioc in report_iocs:
if 'cidr' in ioc:
# expand out ip addresses if we are <= /24
cidr_block = unicode(ioc['cidr'])
cidr_block = ipaddr.IPv4Network(cidr_block)
if cidr_block.numhosts <= 256:
ipaddrs.update([unicode(x) for x in cidr_block.iterhosts()])
else:
_logger.info("Ignoring larger than /24 netblock ({0:s})".format(cidr_block))
if 'ip' in ioc:
ipaddrs.add(unicode(ioc['ip']))
if 'domain' in ioc:
domains.add(ioc['domain'])
ret = {}
if len(ipaddrs):
ret['ipv4'] = list(ipaddrs)
if len(domains):
ret['dns'] = list(domains)
return ret
def parse_file_iocs(self, report_iocs):
md5s = set()
for ioc in report_iocs:
if 'md5' in ioc:
md5s.add(ioc['md5'])
ret = {}
if len(md5s):
ret['md5'] = list(md5s)
return ret
# TODO: add error checking
def get_report(self, report_key):
cur = self.cache_db.cursor()
req = cur.execute("select report_json_content from isight_cache where report_id=? and report_publish_date=?",
report_key)
content = req.fetchone()
if content:
_logger.debug("Found cached report for {0:s} published on {1:d}".format(report_key[0], report_key[1]))
content = json.loads(content[0])
else:
try:
content = self.isight_api.get_report(report_key[0], format='json')
except:
_logger.debug("Exception retrieving report {0:s}: ".format(report_key[0]) + traceback.format_exc())
return None
_logger.debug(
"Inserting report {0:s} published on {1:d} into local cache".format(report_key[0], report_key[1]))
cur.execute('''insert into isight_cache(report_id, report_publish_date, report_json_content)
values (?, ?, ?)''', (report_key[0], report_key[1], content))
self.cache_db.commit()
content = json.loads(content)
return content
