def create_feed(options):
# generate the required feed information fields
# based on command-line arguments
#
feedinfo = {'name': options.name,
'display_name': options.display_name,
'provider_url': options.url,
'summary': options.summary,
'tech_data': options.techdata}
# if an icon was provided, encode as base64 and
# include in the feed information
#
if options.icon:
bytes = base64.b64encode(open(options.icon).read())
feedinfo['icon'] = bytes
# build a CbFeedInfo instance
# this does field validation
#
feedinfo = CbFeedInfo(**feedinfo)
# build a list of reports (always one report in this
# case). the single report will include all the IOCs
#
reports = build_reports(options)
# build a CbFeed instance
# this does field validation (including on the report data)
#
feed = CbFeed(feedinfo, reports)
return feed.dump()
def run(self):
try:
with file_lock(os.path.join(CbIntegrationBridge.WORKING_FILE_ROOT, '%s.pid' % self.integration_name)):
reports = self.perform()
if not reports:
return True
if self.debug:
self.print_reports(reports)
# Create the feed
feed = CbFeed(self.feed_metadata, reports)
if not feed.validate():
self.logger.error("Feed {0:s} did not validate, not updating".format(self.integration_name))
return False
raw_feed_data = feed.dump()
with NamedTemporaryFile(dir=self.working_file_root, delete=False) as fp:
fp.write(raw_feed_data)
self.logger.info("Creating {0:s} feed at {1:s}".format(self.integration_name, self.feed_file_name))
os.rename(fp.name, self.feed_file_name)
c = connect_local_cbapi()
feed_id = c.feed_get_id_by_name(self.integration_name)
if not feed_id:
self.logger.info("Creating {0:s} feed for the first time".format(self.integration_name))
c.feed_add_from_url("file://" + self.feed_file_name, True, False, False)
# force a synchronization
c.feed_synchronize(self.integration_name)
except AlreadyRunningError:
self.logger.error("{0:s} is already running".format(self.integration_name))
def create():
nodes = get_tor_nodes()
reports = build_reports(nodes)
feedinfo = {'name': 'tor',
'display_name': "Tor Exit Nodes",
'provider_url': 'https://www.torproject.org/',
'summary': "This feed is a list of Tor Node IP addresses, updated every 30 minutes.",
'tech_data': "There are no requirements to share any data to receive this feed.",
'icon': 'tor.png',
'icon_small': 'tor.small.jpg',
'category': 'Open Source',
}
# lazy way out to get right icon path. sorry.
old_cwd = os.getcwd()
os.chdir(os.path.dirname(os.path.realpath(__file__)))
feedinfo = CbFeedInfo(**feedinfo)
feed = CbFeed(feedinfo, reports)
created_feed = feed.dump()
os.chdir(old_cwd)
return created_feed
def create(localcsv=None):
if localcsv:
lines = open(localcsv, "r").readlines()
else:
r = requests.get("http://www.malwaredomainlist.com/mdlcsv.php", stream=True)
lines = r.text.split("\r\n")
reports = reports_from_csv(lines)
feedinfo = {'name': 'mdl',
'display_name': "Malware Domain List",
'provider_url': "http://www.malwaredomainlist.com/mdl.php",
'summary': "Malware Domain List is a non-commercial community project to track domains used by malware." +
" This feed contains the most recent 180 days of entries.",
'tech_data': "There are no requirements to share any data to receive this feed.",
"icon": "mdl.png"
}
# lazy way out
old_cwd = os.getcwd()
os.chdir(os.path.dirname(os.path.realpath(__file__)))
feedinfo = CbFeedInfo(**feedinfo)
feed = CbFeed(feedinfo, reports)
bytes = feed.dump()
os.chdir(old_cwd)
return bytes
def create():
nodes = get_et_fwips()
reports = build_reports(nodes)
feedinfo = {'name': 'Emerging_Threats',
'display_name': "Emerging Threats IPs",
'provider_url': 'https://www.emergingthreats.net/',
'summary': "This feed is a list of IP addresses, from Emerging Threats public feed.",
'tech_data': "There are no requirements to share any data to receive this feed.",
'icon': 'et_image.jpg',
'icon_small': 'et_image_small.jpg',
'category': 'Open Source',
}
# lazy way out to get right icon path. sorry.
old_cwd = os.getcwd()
os.chdir(os.path.dirname(os.path.realpath(__file__)))
feedinfo = CbFeedInfo(**feedinfo)
feed = CbFeed(feedinfo, reports)
created_feed = feed.dump()
os.chdir(old_cwd)
return created_feed
def create():
reports = []
reports.extend(get_zeus())
reports.extend(get_palevo())
reports.extend(get_spyeye())
feedinfo = {'name': 'abusech',
'display_name': "abuse.ch Malware Domains",
'provider_url': "http://www.abuse.ch",
'summary': "abuse.ch tracks C&C servers for Zeus, SpyEye and Palevo malware. " +
"This feed combines the three domain names blocklists.",
'tech_data': "There are no requirements to share any data to receive this feed.",
"icon": "abuse.ch.jpg"
}
# the lazy way to the icon
old_cwd = os.getcwd()
os.chdir(os.path.dirname(os.path.realpath(__file__)))
feedinfo = CbFeedInfo(**feedinfo)
feed = CbFeed(feedinfo, reports)
bytes = feed.dump()
os.chdir(old_cwd)
return bytes
def build_feed_data(feed_name: str, display_name: str, feed_summary: str, site: str, icon_link: str,
reports: List[Dict[str, Any]]) -> str:
"""
Return a feed definition as a JSON string definition.
:param feed_name: the short name of the feed
:param display_name: the display name of the feed
:param feed_summary: the feed summary
:param site: the site name
:param icon_link: path to the icon source
:param reports: List of gathered reports
:return: feed as JSON string
"""
feedinfo = {'name': feed_name,
'display_name': display_name,
'provider_url': 'http://' + site,
'summary': feed_summary,
'tech_data': "There are no requirements to share any data to receive this feed.",
}
# handle optionals
if icon_link:
feedinfo['icon'] = icon_link
feedinfo = CbFeedInfo(**feedinfo)
reports = remove_duplicate_reports(reports)
feed = CbFeed(feedinfo, reports)
return feed.dump()
def create_feed(options):
# generate the required feed information fields
# based on command-line arguments
#
feedinfo = {
'name': options.name,
'display_name': options.display_name,
'provider_url': options.url,
'summary': options.summary,
'tech_data': options.techdata
}
# if an icon was provided, encode as base64 and
# include in the feed information
#
if options.icon:
b64bytes = base64.b64encode(open(options.icon).read()).decode("utf-8")
feedinfo['icon'] = b64bytes
# if a small icon was provided, encode as base64 and
# include in the feed information
#
if options.small_icon:
b64bytes = base64.b64encode(open(
options.small_icon).read()).decode("utf-8")
feedinfo['icon_small'] = b64bytes
# if a feed category was provided, include it in the feed information
#
if options.category:
feedinfo['category'] = options.category
# build a CbFeedInfo instance
# this does field validation
#
feedinfo = CbFeedInfo(**feedinfo)
# build a list of reports (always one report in this
# case). the single report will include all the IOCs
#
reports = build_reports(options)
# build a CbFeed instance
# this does field validation (including on the report data)
#
feed = CbFeed(feedinfo, reports)
return feed.dump()
def create():
nodes = get_tor_nodes()
reports = build_reports(nodes)
feedinfo = {'name': 'tor',
'display_name': "Tor Exit Nodes",
'provider_url': 'https://www.torproject.org/',
'summary': "This feed is a list of Tor Node IP addresses, updated every 30 minutes.",
'tech_data': "There are no requirements to share any data to receive this feed.",
'icon': 'images/tor.png'}
feedinfo = CbFeedInfo(**feedinfo)
feed = CbFeed(feedinfo, reports)
return feed.dump()
def create():
nodes = get_tor_nodes()
reports = build_reports(nodes)
feedinfo = {'name': 'tor',
'display_name': "Tor Exit Nodes",
'provider_url': 'https://www.torproject.org/',
'summary': "This feed is a list of Tor Node IP addresses, updated every 30 minutes.",
'tech_data': "There are no requirements to share any data to receive this feed.",
'icon': 'images/tor.png'}
feedinfo = CbFeedInfo(**feedinfo)
feed = CbFeed(feedinfo, reports)
return feed.dump()
def create(input_source):
reports = build_reports(input_source)
# ****************************
# TODO - you probably want to change these values to reflect your
# local input source
feedinfo = {'name': 'stiximport',
'display_name': "STIX Package Import",
'provider_url': 'http://stix.mitre.org',
'summary': "This feed was imported from stix package(s) at %s" % input_source,
'tech_data': "There are no requirements to share any data to receive this feed.",
'icon': 'images/stix.gif'
}
feedinfo = CbFeedInfo(**feedinfo)
feed = CbFeed(feedinfo, reports)
return feed.dump()
def build_feed_data(feed_name, feed_description, site, icon_link, reports):
"""
:return:feed as bytes to be written out
"""
feedinfo = {'name': feed_name,
'display_name': feed_description,
'provider_url': 'http://' + site,
'summary': "TAXII Feed %s" % feed_description,
'tech_data': "There are no requirements to share any data to receive this feed.",
'icon': icon_link
}
feedinfo = CbFeedInfo(**feedinfo)
reports = remove_duplicate_reports(reports)
feed = CbFeed(feedinfo, reports)
return feed.dump()
def run(self):
try:
with file_lock(
os.path.join(CbIntegrationBridge.WORKING_FILE_ROOT,
'%s.pid' % self.integration_name)):
reports = self.perform()
if not reports:
return True
if self.debug:
self.print_reports(reports)
# Create the feed
feed = CbFeed(self.feed_metadata, reports)
if not feed.validate():
self.logger.error(
"Feed {0:s} did not validate, not updating".format(
self.integration_name))
return False
raw_feed_data = feed.dump()
with NamedTemporaryFile(dir=self.working_file_root,
delete=False) as fp:
fp.write(raw_feed_data)
self.logger.info("Creating {0:s} feed at {1:s}".format(
self.integration_name, self.feed_file_name))
os.rename(fp.name, self.feed_file_name)
c = connect_local_cbapi()
feed_id = c.feed_get_id_by_name(self.integration_name)
if not feed_id:
self.logger.info(
"Creating {0:s} feed for the first time".format(
self.integration_name))
c.feed_add_from_url("file://" + self.feed_file_name, True,
False, False)
# force a synchronization
c.feed_synchronize(self.integration_name)
except AlreadyRunningError:
self.logger.error("{0:s} is already running".format(
self.integration_name))
def create(input_source):
reports = build_reports(input_source)
# ****************************
# TODO - you probably want to change these values to reflect your
# local input source
feedinfo = {
'name': 'stiximport',
'display_name': "STIX Package Import",
'provider_url': 'http://stix.mitre.org',
'summary':
"This feed was imported from stix package(s) at %s" % input_source,
'tech_data':
"There are no requirements to share any data to receive this feed.",
'icon': 'images/stix.gif'
}
feedinfo = CbFeedInfo(**feedinfo)
feed = CbFeed(feedinfo, reports)
return feed.dump()
def build_feed_data(feed_name, display_name, feed_summary, site, icon_link,
reports):
"""
:return:feed as bytes to be written out
"""
feedinfo = {
'name': feed_name,
'display_name': display_name,
'provider_url': 'http://' + site,
'summary': feed_summary,
'tech_data':
"There are no requirements to share any data to receive this feed.",
'icon': icon_link
}
feedinfo = CbFeedInfo(**feedinfo)
reports = remove_duplicate_reports(reports)
feed = CbFeed(feedinfo, reports)
return feed.dump()
def create():
reports = []
tmp = get_zeus()
if tmp:
reports.extend(tmp)
tmp = get_feodo()
if tmp:
reports.extend(tmp)
tmp = get_ransomware()
if tmp:
reports.extend(tmp)
feedinfo = {
'name': 'abusech',
'display_name': "abuse.ch Malware Domains",
'provider_url': "http://www.abuse.ch",
'summary':
"abuse.ch tracks C&C servers for Zeus and Palevo malware. " +
"This feed combines the two domain names blocklists.",
'tech_data':
"There are no requirements to share any data to receive this feed.",
'icon': "abuse.ch.jpg",
'icon_small': "abuse.ch.small.jpg",
'category': "Open Source"
}
# the lazy way to the icon
old_cwd = os.getcwd()
os.chdir(os.path.dirname(os.path.realpath(__file__)))
feedinfo = CbFeedInfo(**feedinfo)
feed = CbFeed(feedinfo, reports)
feed_bytes = feed.dump()
os.chdir(old_cwd)
return feed_bytes
def create(config_file, existing_csv=None, reports_to_skip=[]):
# parse the configuration file
# this configuration file includes the keys needed to talk to the
# iSight report server, etc.
#
#print "-> Parsing iSight configuration..."
cfg = isight_config.ISightConfig(config_file)
# instantiate a local iSight API object
#
#print "-> Instantiating an iSight API object..."
api = isight_api.ISightAPI(cfg.iSightRemoteImportUrl,
cfg.iSightRemoteImportUsername,
cfg.iSightRemoteImportPassword,
cfg.iSightRemoteImportPublicKey,
cfg.iSightRemoteImportPrivateKey)
if not existing_csv:
# query the iSight report server for raw CSV report data
# query 'back' the specified number of days
#
#print "-> Querying iSight server for last %d days of reports..." % (cfg.iSightRemoteImportDaysBack)
#
# @todo iSIGHT has a new-and-improved REST API which could be used instead of this legacy API
#
raw_report_data = api.get_i_and_w(cfg.iSightRemoteImportDaysBack)
# save off the raw report data for future reference
#
#print "-> Saving iSight report data to iSight.csv..."
f = open('iSight.csv', 'w')
f.write(raw_report_data)
f.close()
else:
raw_report_data = open(existing_csv, "r").read()
# convert the raw report data into something more managable
# in particular, a list of dictionaries, with each dictionary describing a report
# this helper routine accounts for the fact that report data is spread across
# multiple lines of the raw CSV blob
#
results = isight_helpers.isight_csv_to_iocs_dict([raw_report_data])
# set up a dictionary for basic stat tracking
#
stats = {'md5' : {'total' : 0, 'max' : 0},
'ipaddr' : {'total' : 0, 'max' : 0},
'domain' : {'total' : 0, 'max' : 0}}
for report_id in results.keys():
stats['md5']['total'] += len(results[report_id]['md5'])
if len(results[report_id]['md5']) > stats['md5']['max']:
stats['md5']['max'] = len(results[report_id]['md5'])
stats['ipaddr']['total'] += len(results[report_id]['ipaddr'])
if len(results[report_id]['ipaddr']) > stats['ipaddr']['max']:
stats['ipaddr']['max'] = len(results[report_id]['ipaddr'])
stats['domain']['total'] += len(results[report_id]['domain'])
if len(results[report_id]['domain']) > stats['domain']['max']:
stats['domain']['max'] = len(results[report_id]['domain'])
#print " -> Total Reports: %d" % (len(results.keys()))
#print " -> ----------------------------------------------- ---"
#print " -> Maximum number of MD5s in one report: %d" % (stats['md5']['max'])
#print " -> Total MD5s across all reports: %d" % (stats['md5']['total'])
#print " -> Maximum number of IPv4 addresses in one report: %d" % (stats['ipaddr']['max'])
#print " -> Total IPv4 addresses in all reports: %d" % (stats['ipaddr']['total'])
#print " -> Maximum number of DNS names in one report: %d" % (stats['domain']['max'])
#print " -> Total DNS names in all reports: %d" % (stats['domain']['total'])
# generate the feed data from the raw iSight report data
#
#print "-> Generating feed data..."
reports = generate_reports(results, api)
# shim to skip entire reports
reports = [report for report in reports if report.data['id'] not in reports_to_skip]
# generate the feed metadata (feed information)
# this is a static description of the feed itself
#
# lazy way out
cwd_old = os.getcwd()
os.chdir(os.path.dirname(os.path.realpath(__file__)))
#print "-> Generating feed metadata..."
feedinfo = generate_feed_information()
# write out feed document
#
feed = CbFeed(feedinfo, reports)
#print "-> Writing out completed feed document..."
return feed.dump()
os.chdir(cwd_old)
def perform(configpath, export_mode):
if not os.path.exists(configpath):
raise FatalError("Config File %s does not exist!" % configpath)
config = ConfigParser.RawConfigParser(defaults=
{
'iSightRemoteImportUrl': 'https://api.isightpartners.com',
'iSightRemoteImportDaysBack': 80,
'iSightDefaultScore': 50,
'iSightFeedName': 'isightconnector',
'iSightGetReports': 'false'
}
)
config.read(configpath)
if not config.has_section('cb-isight'):
raise FatalError("Config File must have cb-isight section")
for option in ['iSightRemoteImportPublicKey', 'iSightRemoteImportPrivateKey', 'carbonblack_server_token']:
if not config.has_option('cb-isight', option):
raise FatalError("Config file not complete: missing option {0:s}".format(option))
api_key = config.get("cb-isight", "iSightRemoteImportPublicKey")
sec_key = config.get("cb-isight", "iSightRemoteImportPrivateKey")
days_back = config.getint("cb-isight", "iSightRemoteImportDaysBack")
api_route = config.get("cb-isight", "iSightRemoteImportUrl")
default_score = config.get("cb-isight", "iSightDefaultScore")
feed_name = config.get("cb-isight", "iSightFeedName")
read_reports = config.getboolean("cb-isight", "iSightGetReports")
if config.has_option('cb-isight', 'https_proxy'):
os.environ['HTTPS_PROXY'] = config.get('cb-isight', 'https_proxy')
os.environ['no_proxy'] = '127.0.0.1,localhost'
isight_bridge = Bridge(api_route, api_key, sec_key)
if read_reports:
reports = isight_bridge.perform(days_back)
else:
reports = isight_bridge.perform_iocs(days_back)
cb_reports = []
for report in reports:
cb_id = 'isight-%s' % report['id']
feed_entry = dict((k, report[k]) for k in ('title', 'link', 'iocs', 'timestamp'))
feed_entry['id'] = cb_id
feed_entry['score'] = default_score
cb_reports.append(feed_entry)
feed = CbFeed(generate_feed_metadata(feed_name), cb_reports)
raw_feed_data = feed.dump()
if export_mode:
print raw_feed_data
else:
with NamedTemporaryFile(dir=CB_ISIGHT_ROOT, delete=False) as fp:
fp.write(raw_feed_data)
destination_filename = os.path.join(CB_ISIGHT_ROOT, 'isight_feed.json')
_logger.info("Creating iSIGHT feed at {0:s}".format(destination_filename))
os.rename(fp.name, destination_filename)
os.chmod(destination_filename, 0755)
if config.has_option('cb-isight', 'carbonblack_server_url'):
local_cb_server = config.get('cb-isight', 'carbonblack_server_url')
else:
local_cb_server = 'https://127.0.0.1'
c = connect_local_cbapi(local_cb_server, config.get('cb-isight', 'carbonblack_server_token'))
feed_id = c.feed_get_id_by_name(feed_name)
if not feed_id:
_logger.info("Creating iSIGHT feed for the first time")
c.feed_add_from_url("file://" + CB_ISIGHT_ROOT + '/isight_feed.json', True, False, False)
# force a synchronization
c.feed_synchronize(feed_name)
def create(config_file, existing_csv=None, reports_to_skip=[]):
# parse the configuration file
# this configuration file includes the keys needed to talk to the
# iSight report server, etc.
#
#print "-> Parsing iSight configuration..."
cfg = isight_config.ISightConfig(config_file)
# instantiate a local iSight API object
#
#print "-> Instantiating an iSight API object..."
api = isight_api.ISightAPI(cfg.iSightRemoteImportUrl,
cfg.iSightRemoteImportUsername,
cfg.iSightRemoteImportPassword,
cfg.iSightRemoteImportPublicKey,
cfg.iSightRemoteImportPrivateKey)
if not existing_csv:
# query the iSight report server for raw CSV report data
# query 'back' the specified number of days
#
#print "-> Querying iSight server for last %d days of reports..." % (cfg.iSightRemoteImportDaysBack)
#
# @todo iSIGHT has a new-and-improved REST API which could be used instead of this legacy API
#
raw_report_data = api.get_i_and_w(cfg.iSightRemoteImportDaysBack)
# save off the raw report data for future reference
#
#print "-> Saving iSight report data to iSight.csv..."
f = open('iSight.csv', 'w')
f.write(raw_report_data)
f.close()
else:
raw_report_data = open(existing_csv, "r").read()
# convert the raw report data into something more managable
# in particular, a list of dictionaries, with each dictionary describing a report
# this helper routine accounts for the fact that report data is spread across
# multiple lines of the raw CSV blob
#
results = isight_helpers.isight_csv_to_iocs_dict([raw_report_data])
# set up a dictionary for basic stat tracking
#
stats = {
'md5': {
'total': 0,
'max': 0
},
'ipaddr': {
'total': 0,
'max': 0
},
'domain': {
'total': 0,
'max': 0
}
}
for report_id in results.keys():
stats['md5']['total'] += len(results[report_id]['md5'])
if len(results[report_id]['md5']) > stats['md5']['max']:
stats['md5']['max'] = len(results[report_id]['md5'])
stats['ipaddr']['total'] += len(results[report_id]['ipaddr'])
if len(results[report_id]['ipaddr']) > stats['ipaddr']['max']:
stats['ipaddr']['max'] = len(results[report_id]['ipaddr'])
stats['domain']['total'] += len(results[report_id]['domain'])
if len(results[report_id]['domain']) > stats['domain']['max']:
stats['domain']['max'] = len(results[report_id]['domain'])
#print " -> Total Reports: %d" % (len(results.keys()))
#print " -> ----------------------------------------------- ---"
#print " -> Maximum number of MD5s in one report: %d" % (stats['md5']['max'])
#print " -> Total MD5s across all reports: %d" % (stats['md5']['total'])
#print " -> Maximum number of IPv4 addresses in one report: %d" % (stats['ipaddr']['max'])
#print " -> Total IPv4 addresses in all reports: %d" % (stats['ipaddr']['total'])
#print " -> Maximum number of DNS names in one report: %d" % (stats['domain']['max'])
#print " -> Total DNS names in all reports: %d" % (stats['domain']['total'])
# generate the feed data from the raw iSight report data
#
#print "-> Generating feed data..."
reports = generate_reports(results, api)
# shim to skip entire reports
reports = [
report for report in reports
if report.data['id'] not in reports_to_skip
]
# generate the feed metadata (feed information)
# this is a static description of the feed itself
#
# lazy way out
cwd_old = os.getcwd()
os.chdir(os.path.dirname(os.path.realpath(__file__)))
#print "-> Generating feed metadata..."
feedinfo = generate_feed_information()
# write out feed document
#
feed = CbFeed(feedinfo, reports)
#print "-> Writing out completed feed document..."
return feed.dump()
os.chdir(cwd_old)