def __init__(self, cb, bridge_options):
try:
Action.__init__(self, cb) # TODO -- maybe a ThreadedAction class?
threading.Thread.__init__(self)
self.flask_feed = FlaskFeed(__name__)
self.bridge_options = bridge_options
self.data_dir = "/usr/share/cb/integrations/infoblox"
self.sync_needed = False
self.feed_name = "infoblox"
self.display_name = "Infoblox"
self.feed = {}
self.feed_domains = defaultdict(dict)
self.feed_lock = threading.Lock()
self.directory = self.data_dir
self.cb_image_path = "/carbonblack.png"
self.integration_image_path = "/infoblox.png"
self.json_feed_path = "/infoblox/json"
self.flask_feed.app.add_url_rule(
self.cb_image_path, view_func=self.handle_cb_image_request)
self.flask_feed.app.add_url_rule(
self.integration_image_path,
view_func=self.handle_integration_image_request)
self.flask_feed.app.add_url_rule(
self.json_feed_path,
view_func=self.handle_json_feed_request,
methods=['GET'])
self.flask_feed.app.add_url_rule(
"/", view_func=self.handle_index_request, methods=['GET'])
self.flask_feed.app.add_url_rule(
"/feed.html",
view_func=self.handle_html_feed_request,
methods=['GET'])
except:
logger.error('%s' % traceback.format_exc())
def __init__(self, name, configfile, logfile=None, pidfile=None, debug=False):
CbIntegrationDaemon.__init__(self, name, configfile=configfile, logfile=logfile,
pidfile=pidfile, debug=debug)
template_folder = "/usr/share/cb/integrations/threatexchange/content"
self.flask_feed = FlaskFeed(__name__, False, template_folder)
self.bridge_options = {}
self.bridge_auth = {}
if 'bridge' in self.options:
self.debug = self.options['bridge'].get("debug", 0)
if self.debug:
self.logger.setLevel(logging.DEBUG)
self.validated_config = self.validate_config()
self.cb = None
self.feed_name = "threatexchangeconnector"
self.display_name = "ThreatExchange"
self.directory = template_folder
self.cb_image_path = "/carbonblack.png"
self.integration_image_path = "/threatexchange.png"
self.integration_small_image_path = "/threatexchange-small.png"
self.json_feed_path = "/threatexchange/json"
self.feed_lock = threading.RLock()
self.flask_feed.app.add_url_rule(self.cb_image_path, view_func=self.handle_cb_image_request)
self.flask_feed.app.add_url_rule(self.integration_image_path, view_func=self.handle_integration_image_request)
self.flask_feed.app.add_url_rule(self.json_feed_path, view_func=self.handle_json_feed_request, methods=['GET'])
self.flask_feed.app.add_url_rule("/", view_func=self.handle_index_request, methods=['GET'])
self.flask_feed.app.add_url_rule("/feed.html", view_func=self.handle_html_feed_request, methods=['GET'])
self.logger.debug("generating feed metadata")
with self.feed_lock:
self.feed = self.create_feed()
self.last_sync = "No sync performed"
self.last_successful_sync = "No sync performed"
def __init__(self, name, configfile, logfile=None, pidfile=None, debug=False, dbfile=None):
CbIntegrationDaemon.__init__(self, name, configfile=configfile, logfile=logfile,
pidfile=pidfile, debug=debug)
template_folder = "/usr/share/cb/integrations/threatexchange/content"
self.db_file = dbfile or "/usr/share/cb/integrations/threatexchange/db/threatexchange.db"
self.flask_feed = FlaskFeed(__name__, False, template_folder)
self.bridge_options = {}
self.bridge_auth = {}
if 'bridge' in self.options:
self.debug = self.options['bridge'].get("debug", 0)
if self.debug:
self.logger.setLevel(logging.DEBUG)
self.validated_config = self.validate_config()
self.cb = None
self.feed_name = "threatexchangeconnector"
self.display_name = "ThreatExchange"
self.directory = template_folder
self.cb_image_path = "/carbonblack.png"
self.integration_image_path = "/threatexchange.png"
self.integration_small_image_path = "/threatexchange-small.png"
self.json_feed_path = "/threatexchange/json"
self.feed_lock = threading.RLock()
self.flask_feed.app.add_url_rule(self.cb_image_path, view_func=self.handle_cb_image_request)
self.flask_feed.app.add_url_rule(self.integration_image_path, view_func=self.handle_integration_image_request)
self.flask_feed.app.add_url_rule(self.json_feed_path, view_func=self.handle_json_feed_request, methods=['GET'])
self.flask_feed.app.add_url_rule("/", view_func=self.handle_index_request, methods=['GET'])
self.flask_feed.app.add_url_rule("/feed.html", view_func=self.handle_html_feed_request, methods=['GET'])
self.logger.debug("generating feed metadata")
with self.feed_lock:
self.feed = self.create_feed()
self.last_sync = "No sync performed"
self.last_successful_sync = "No sync performed"
class ThreatExchangeConnector(CbIntegrationDaemon):
def __init__(self, name, configfile, logfile=None, pidfile=None, debug=False):
CbIntegrationDaemon.__init__(self, name, configfile=configfile, logfile=logfile,
pidfile=pidfile, debug=debug)
template_folder = "/usr/share/cb/integrations/threatexchange/content"
self.flask_feed = FlaskFeed(__name__, False, template_folder)
self.bridge_options = {}
self.bridge_auth = {}
if 'bridge' in self.options:
self.debug = self.options['bridge'].get("debug", 0)
if self.debug:
self.logger.setLevel(logging.DEBUG)
self.validated_config = self.validate_config()
self.cb = None
self.feed_name = "threatexchangeconnector"
self.display_name = "ThreatExchange"
self.directory = template_folder
self.cb_image_path = "/carbonblack.png"
self.integration_image_path = "/threatexchange.png"
self.integration_small_image_path = "/threatexchange-small.png"
self.json_feed_path = "/threatexchange/json"
self.feed_lock = threading.RLock()
self.flask_feed.app.add_url_rule(self.cb_image_path, view_func=self.handle_cb_image_request)
self.flask_feed.app.add_url_rule(self.integration_image_path, view_func=self.handle_integration_image_request)
self.flask_feed.app.add_url_rule(self.json_feed_path, view_func=self.handle_json_feed_request, methods=['GET'])
self.flask_feed.app.add_url_rule("/", view_func=self.handle_index_request, methods=['GET'])
self.flask_feed.app.add_url_rule("/feed.html", view_func=self.handle_html_feed_request, methods=['GET'])
self.logger.debug("generating feed metadata")
with self.feed_lock:
self.feed = self.create_feed()
self.last_sync = "No sync performed"
self.last_successful_sync = "No sync performed"
def create_feed(self):
return FeedHandler(generate_feed(
self.feed_name,
summary="Connector for Threat intelligence data from Facebook ThreatExchange",
tech_data="""This connector enables members of the Facebook ThreatExchange to import threat indicators
from the ThreatExchange, including domain names, IPs, hashes, and behavioral indicators, into Carbon
Black. The Facebook ThreatExchange and its members provide and maintain this data. This connector
requires an Access Token to the Facebook ThreatExchange API. For more information, visit:
https://developers.facebook.com/products/threat-exchange/""",
provider_url="https://developers.facebook.com/products/threat-exchange",
icon_path="%s/%s" % (self.directory, self.integration_image_path),
small_icon_path="%s/%s" % (self.directory, self.integration_small_image_path),
display_name=self.display_name,
category="Partner"))
def serve(self):
address = self.bridge_options.get('listener_address', '127.0.0.1')
port = self.bridge_options.get('listener_port', 6120)
self.logger.info("starting flask server: %s:%s" % (address, port))
self.flask_feed.app.run(port=port, debug=self.debug,
host=address, use_reloader=False)
def handle_json_feed_request(self):
with self.feed_lock:
json = self.flask_feed.generate_json_feed(self.feed.retrieve_feed())
return json
def handle_html_feed_request(self):
with self.feed_lock:
html = self.flask_feed.generate_html_feed(self.feed.retrieve_feed(), self.display_name)
return html
def handle_index_request(self):
with self.feed_lock:
index = self.flask_feed.generate_html_index(self.feed.retrieve_feed(), self.bridge_options, self.display_name,
self.cb_image_path, self.integration_image_path,
self.json_feed_path, self.last_sync)
return index
def handle_cb_image_request(self):
return self.flask_feed.generate_image_response(image_path="%s%s" % (self.directory, self.cb_image_path))
def handle_integration_image_request(self):
return self.flask_feed.generate_image_response(image_path="%s%s" %
(self.directory, self.integration_image_path))
def on_start(self):
if self.debug:
self.logger.setLevel(logging.DEBUG)
def on_stopping(self):
if self.debug:
self.logger.setLevel(logging.DEBUG)
def get_or_create_feed(self):
feed_id = self.cb.feed_get_id_by_name(self.feed_name)
self.logger.info("Feed id for %s: %s" % (self.feed_name, feed_id))
if not feed_id:
feed_url = "http://%s:%d%s" % (self.bridge_options["feed_host"], int(self.bridge_options["listener_port"]),
self.json_feed_path)
self.logger.info("Creating %s feed @ %s for the first time" % (self.feed_name, feed_url))
# TODO: clarification of feed_host vs listener_address
result = self.cb.feed_add_from_url(feed_url, True, False, False)
# TODO: defensive coding around these self.cb calls
feed_id = result.get('id', 0)
return feed_id
def run(self):
self.logger.info("starting Carbon Black <-> ThreatExchange Connector | version %s" % __version__)
self.debug = self.get_config_boolean('debug', False)
if self.debug:
self.logger.setLevel(logging.DEBUG)
self.cb = cbapi.CbApi(self.get_config_string('carbonblack_server_url', 'https://127.0.0.1'),
token=self.get_config_string('carbonblack_server_token'),
ssl_verify=self.get_config_boolean('carbonblack_server_sslverify', False))
self.logger.debug("starting continuous feed retrieval thread")
work_thread = threading.Thread(target=self.perform_continuous_feed_retrieval)
work_thread.setDaemon(True)
work_thread.start()
self.logger.debug("starting flask")
self.serve()
def check_required_options(self, opts):
CbIntegrationDaemon.check_required_options(self, opts)
for opt in opts:
val = self.cfg.get("bridge", opt)
if not val or len(val) == 0:
raise ConfigurationError("Configuration file has option %s in [bridge] section but not set to value " %
opt)
def validate_config(self):
super(ThreatExchangeConnector, self).validate_config()
self.check_required_options(["tx_app_id", "tx_secret_key", "carbonblack_server_token"])
self.bridge_options["listener_port"] = self.get_config_integer("listener_port", 6120)
self.bridge_options["feed_host"] = self.get_config_string("feed_host", "127.0.0.1")
self.bridge_options["listener_address"] = self.get_config_string("listener_address", "0.0.0.0")
self.bridge_auth["app_id"] = self.get_config_string("tx_app_id")
self.bridge_auth["secret_key"] = self.get_config_string("tx_secret_key")
access_token.init(self.bridge_auth["app_id"], self.bridge_auth["secret_key"])
ioc_types = self.get_config_string("tx_ioc_types", None)
if not ioc_types or len(ioc_types.strip()) == 0:
ioc_types = processing_engines.ALL_INDICATOR_TYPES
ioc_types = ioc_types.split(',')
self.bridge_options["ioc_types"] = []
for ioc_type in ioc_types:
if ioc_type not in processing_engines.INDICATOR_PROCESSORS:
self.logger.warning("%s is not a valid IOC type, ignoring" % ioc_type)
else:
self.bridge_options["ioc_types"].append(ioc_type)
self.bridge_options["historical_days"] = self.get_config_integer("tx_historical_days", 7)
# retrieve once a day by default
self.bridge_options["feed_retrieval_minutes"] = self.get_config_integer("feed_retrieval_minutes", 120)
self.bridge_options["minimum_severity"] = self.get_config_string("tx_minimum_severity", "WARNING")
self.bridge_options["minimum_confidence"] = int(self.get_config_string("tx_minimum_confidence", 50))
status_filter = self.get_config_string("tx_status_filter", None)
if type(status_filter) == str:
self.bridge_options["status_filter"] = status_filter.split(',')
else:
self.bridge_options["status_filter"] = None
return True
def perform_continuous_feed_retrieval(self):
if not self.validated_config:
self.validate_config()
sleep_secs = int(self.bridge_options["feed_retrieval_minutes"]) * 60
while True:
self.logger.info("Beginning Feed Retrieval")
try:
with Timer() as t:
self.perform_feed_retrieval()
self.logger.info("Facebook ThreatExchange feed retrieval succeeded after %0.2f seconds" % t.interval)
self.get_or_create_feed()
self.cb.feed_synchronize(self.feed_name)
self.logger.info("Sleeping for %d seconds" % sleep_secs)
time.sleep(sleep_secs)
except Exception as e:
self.logger.exception("Exception during feed retrieval. Will retry in 60 seconds")
time.sleep(60)
def perform_feed_retrieval(self):
new_feed = self.create_feed()
tx_limit = self.bridge_options.get('tx_request_limit', 500) or 500
tx_retries = self.bridge_options.get('tx_request_retries', 5) or 5
now = datetime.utcnow()
since_date = now - timedelta(days=self.bridge_options["historical_days"])
since_date_str = since_date.strftime("%Y-%m-%d")
until_date = since_date
while until_date < now + timedelta(1):
until_date += timedelta(days=1)
until_date_str = until_date.strftime("%Y-%m-%d")
for ioc_type in self.bridge_options["ioc_types"]:
self.logger.info("Pulling %s IOCs (%s to %s)" % (ioc_type, since_date_str, until_date_str))
try:
count = 0
for result in ThreatDescriptor.objects(since=since_date_str,
until=until_date_str,
type_=ioc_type,
dict_generator=True,
limit=tx_limit,
retries=tx_retries,
fields="raw_indicator,owner,indicator{id,indicator},type,last_updated,share_level,severity,description,report_urls,status,confidence"):
new_reports = processing_engines.process_ioc(ioc_type,
result,
minimum_severity=self.bridge_options["minimum_severity"],
status_filter=self.bridge_options["status_filter"],
minimum_confidence=self.bridge_options["minimum_confidence"])
for report in new_reports:
new_feed.add_report(report)
count += 1
if count % 1000 == 0:
self.logger.info("%s added %d reports for this iteration" % (ioc_type, count))
self.logger.info("%s added %d reports TOTAL" % (ioc_type, count))
except pytxFetchError:
self.logger.warning("Could not retrieve some IOCs of type %s. Continuing." % ioc_type)
except Exception:
self.logger.exception("Unknown exception retrieving IOCs of type %s." % ioc_type)
# update the start date
since_date_str = until_date_str
with self.feed_lock:
self.feed = new_feed
class ThreatExchangeConnector(CbIntegrationDaemon):
def __init__(self, name, configfile, logfile=None, pidfile=None, debug=False, dbfile=None):
CbIntegrationDaemon.__init__(self, name, configfile=configfile, logfile=logfile,
pidfile=pidfile, debug=debug)
template_folder = "/usr/share/cb/integrations/threatexchange/content"
self.db_file = dbfile or "/usr/share/cb/integrations/threatexchange/db/threatexchange.db"
self.flask_feed = FlaskFeed(__name__, False, template_folder)
self.bridge_options = {}
self.bridge_auth = {}
if 'bridge' in self.options:
self.debug = self.options['bridge'].get("debug", 0)
if self.debug:
self.logger.setLevel(logging.DEBUG)
self.validated_config = self.validate_config()
self.cb = None
self.feed_name = "threatexchangeconnector"
self.display_name = "ThreatExchange"
self.directory = template_folder
self.cb_image_path = "/carbonblack.png"
self.integration_image_path = "/threatexchange.png"
self.integration_small_image_path = "/threatexchange-small.png"
self.json_feed_path = "/threatexchange/json"
self.feed_lock = threading.RLock()
self.flask_feed.app.add_url_rule(self.cb_image_path, view_func=self.handle_cb_image_request)
self.flask_feed.app.add_url_rule(self.integration_image_path, view_func=self.handle_integration_image_request)
self.flask_feed.app.add_url_rule(self.json_feed_path, view_func=self.handle_json_feed_request, methods=['GET'])
self.flask_feed.app.add_url_rule("/", view_func=self.handle_index_request, methods=['GET'])
self.flask_feed.app.add_url_rule("/feed.html", view_func=self.handle_html_feed_request, methods=['GET'])
self.logger.debug("generating feed metadata")
with self.feed_lock:
self.feed = self.create_feed()
self.last_sync = "No sync performed"
self.last_successful_sync = "No sync performed"
def create_feed(self):
return FeedHandler(generate_feed(
self.feed_name,
summary="Connector for Threat intelligence data from Facebook ThreatExchange",
tech_data="""This connector enables members of the Facebook ThreatExchange to import threat indicators
from the ThreatExchange, including domain names, IPs, hashes, and behavioral indicators, into Carbon
Black. The Facebook ThreatExchange and its members provide and maintain this data. This connector
requires an Access Token to the Facebook ThreatExchange API. For more information, visit:
https://developers.facebook.com/products/threat-exchange/""",
provider_url="https://developers.facebook.com/products/threat-exchange",
icon_path="%s/%s" % (self.directory, self.integration_image_path),
small_icon_path="%s/%s" % (self.directory, self.integration_small_image_path),
display_name=self.display_name,
category="Partner"))
def serve(self):
address = self.bridge_options.get('listener_address', '127.0.0.1')
port = self.bridge_options.get('listener_port', 6120)
self.logger.info("starting flask server: %s:%s" % (address, port))
self.flask_feed.app.run(port=port, debug=self.debug,
host=address, use_reloader=False)
def handle_json_feed_request(self):
with self.feed_lock:
json = self.flask_feed.generate_json_feed(self.feed.retrieve_feed())
return json
def handle_html_feed_request(self):
with self.feed_lock:
html = self.flask_feed.generate_html_feed(self.feed.retrieve_feed(), self.display_name)
return html
def handle_index_request(self):
with self.feed_lock:
index = self.flask_feed.generate_html_index(self.feed.retrieve_feed(), self.bridge_options, self.display_name,
self.cb_image_path, self.integration_image_path,
self.json_feed_path, self.last_sync)
return index
def handle_cb_image_request(self):
return self.flask_feed.generate_image_response(image_path="%s%s" % (self.directory, self.cb_image_path))
def handle_integration_image_request(self):
return self.flask_feed.generate_image_response(image_path="%s%s" %
(self.directory, self.integration_image_path))
def on_start(self):
if self.debug:
self.logger.setLevel(logging.DEBUG)
def on_stopping(self):
if self.debug:
self.logger.setLevel(logging.DEBUG)
def get_or_create_feed(self):
feed_id = self.cb.feed_get_id_by_name(self.feed_name)
self.logger.info("Feed id for %s: %s" % (self.feed_name, feed_id))
if not feed_id:
feed_url = "http://%s:%d%s" % (self.bridge_options["feed_host"], int(self.bridge_options["listener_port"]),
self.json_feed_path)
self.logger.info("Creating %s feed @ %s for the first time" % (self.feed_name, feed_url))
# TODO: clarification of feed_host vs listener_address
result = self.cb.feed_add_from_url(feed_url, True, False, False)
# TODO: defensive coding around these self.cb calls
feed_id = result.get('id', 0)
return feed_id
def run(self):
self.logger.info("starting Carbon Black <-> ThreatExchange Connector | version %s" % __version__)
self.debug = self.get_config_boolean('debug', False)
if self.debug:
self.logger.setLevel(logging.DEBUG)
self.cb = cbapi.CbApi(self.get_config_string('carbonblack_server_url', 'https://127.0.0.1'),
token=self.get_config_string('carbonblack_server_token'),
ssl_verify=self.get_config_boolean('carbonblack_server_sslverify', False),
ignore_system_proxy=True)
self.logger.debug("starting continuous feed retrieval thread")
work_thread = threading.Thread(target=self.perform_continuous_feed_retrieval)
work_thread.setDaemon(True)
work_thread.start()
self.logger.debug("starting flask")
self.serve()
def check_required_options(self, opts):
CbIntegrationDaemon.check_required_options(self, opts)
for opt in opts:
val = self.cfg.get("bridge", opt)
if not val or len(val) == 0:
raise ConfigurationError("Configuration file has option %s in [bridge] section but not set to value " %
opt)
def validate_config(self):
super(ThreatExchangeConnector, self).validate_config()
self.check_required_options(["tx_app_id", "tx_secret_key", "carbonblack_server_token"])
self.bridge_options["listener_port"] = self.get_config_integer("listener_port", 6120)
self.bridge_options["feed_host"] = self.get_config_string("feed_host", "127.0.0.1")
self.bridge_options["listener_address"] = self.get_config_string("listener_address", "0.0.0.0")
self.bridge_auth["app_id"] = self.get_config_string("tx_app_id")
self.bridge_auth["secret_key"] = self.get_config_string("tx_secret_key")
access_token(self.bridge_auth["app_id"], self.bridge_auth["secret_key"])
ioc_types = self.get_config_string("tx_ioc_types", None)
if not ioc_types or len(ioc_types.strip()) == 0:
ioc_types = processing_engines.ALL_INDICATOR_TYPES
ioc_types = ioc_types.split(',')
self.bridge_options["ioc_types"] = []
for ioc_type in ioc_types:
if ioc_type not in processing_engines.INDICATOR_PROCESSORS:
self.logger.warning("%s is not a valid IOC type, ignoring" % ioc_type)
else:
self.bridge_options["ioc_types"].append(ioc_type)
self.bridge_options["historical_days"] = self.get_config_integer("tx_historical_days", 7)
# retrieve once a day by default
self.bridge_options["feed_retrieval_minutes"] = self.get_config_integer("feed_retrieval_minutes", 120)
self.bridge_options["minimum_severity"] = self.get_config_string("tx_minimum_severity", "WARNING")
self.bridge_options["minimum_confidence"] = int(self.get_config_string("tx_minimum_confidence", 50))
status_filter = self.get_config_string("tx_status_filter", None)
if type(status_filter) == str:
self.bridge_options["status_filter"] = status_filter.split(',')
else:
self.bridge_options["status_filter"] = None
return True
def perform_continuous_feed_retrieval(self):
if not self.validated_config:
self.validate_config()
proxy_host = self.get_config_string("https_proxy", None)
if proxy_host:
os.environ['HTTPS_PROXY'] = proxy_host
os.environ['no_proxy'] = '127.0.0.1,localhost'
sleep_secs = int(self.bridge_options["feed_retrieval_minutes"]) * 60
while True:
self.logger.info("Beginning Feed Retrieval")
try:
with Timer() as t:
self.perform_feed_retrieval()
self.logger.info("Facebook ThreatExchange feed retrieval succeeded after %0.2f seconds" % t.interval)
self.get_or_create_feed()
self.cb.feed_synchronize(self.feed_name)
self.logger.info("Sleeping for %d seconds" % sleep_secs)
time.sleep(sleep_secs)
except Exception as e:
self.logger.exception("Exception during feed retrieval. Will retry in 60 seconds")
time.sleep(60)
def perform_feed_retrieval(self):
new_feed = self.create_feed()
with ThreatExchangeDb(self.db_file) as db:
now = datetime.utcnow()
since_date = now - timedelta(days=self.bridge_options["historical_days"])
db.cull_before(since_date)
tx_limit = self.bridge_options.get('tx_request_limit', 500) or 500
tx_retries = self.bridge_options.get('tx_request_retries', 5) or 5
db.update(self.bridge_options["ioc_types"], tx_limit, tx_retries)
minimum_severity = self.bridge_options["minimum_severity"]
status_filter = self.bridge_options["status_filter"]
minimum_confidence = self.bridge_options["minimum_confidence"]
for report in db.generate_reports(minimum_severity, status_filter, minimum_confidence):
new_feed.add_report(report)
with self.feed_lock:
self.feed = new_feed
class FeedAction(threading.Thread, Action):
def __init__(self, cb, bridge_options):
try:
Action.__init__(self, cb) # TODO -- maybe a ThreadedAction class?
threading.Thread.__init__(self)
self.flask_feed = FlaskFeed(__name__)
self.bridge_options = bridge_options
self.data_dir = "/usr/share/cb/integrations/infoblox"
self.sync_needed = False
self.feed_name = "infoblox"
self.display_name = "Infoblox"
self.feed = {}
self.feed_domains = defaultdict(dict)
self.feed_lock = threading.Lock()
self.directory = self.data_dir
self.cb_image_path = "/carbonblack.png"
self.integration_image_path = "/infoblox.png"
self.json_feed_path = "/infoblox/json"
self.flask_feed.app.add_url_rule(
self.cb_image_path, view_func=self.handle_cb_image_request)
self.flask_feed.app.add_url_rule(
self.integration_image_path,
view_func=self.handle_integration_image_request)
self.flask_feed.app.add_url_rule(
self.json_feed_path,
view_func=self.handle_json_feed_request,
methods=['GET'])
self.flask_feed.app.add_url_rule(
"/", view_func=self.handle_index_request, methods=['GET'])
self.flask_feed.app.add_url_rule(
"/feed.html",
view_func=self.handle_html_feed_request,
methods=['GET'])
except:
logger.error('%s' % traceback.format_exc())
def name(self):
return 'Add results to feed'
def run(self):
try:
# make data directories as required
#
ensure_directory_exists(self.data_dir)
# restore alerts from disk if so configured
#
num_restored = 0
if int(self.bridge_options.get('restore_feed_on_restart', 1)):
logger.info("Restoring saved feed...")
num_restored = self.restore_feed_files()
self.feed = self.generate_feed()
logger.info("Restored %d alerts" % num_restored)
logger.info("starting feed server")
self.serve()
except:
logger.error('%s' % traceback.format_exc())
def get_or_create_feed(self):
feed = None
try:
feeds = get_object_by_name_or_id(self.cb,
Feed,
name=self.feed_name)
except Exception as e:
logger.error(e.message)
feeds = None
if not feeds:
logger.info(
"Feed {} was not found, so we are going to create it".format(
self.feed_name))
f = self.cb.create(Feed)
f.feed_url = "http://%s:%d%s" % (
self.bridge_options.get('feed_host', '127.0.0.1'),
int(self.bridge_options['listener_port']), self.json_feed_path)
f.enabled = True
f.use_proxy = False
f.validate_server_cert = False
try:
f.save()
except ServerError as se:
if se.error_code == 500:
logger.info("Could not add feed:")
logger.info(
" Received error code 500 from server. This is usually because the server cannot retrieve the feed."
)
logger.info(
" Check to ensure the Cb server has network connectivity and the credentials are correct."
)
else:
logger.info("Could not add feed: {0:s}".format(str(se)))
except Exception as e:
logger.info("Could not add feed: {0:s}".format(str(e)))
else:
logger.info("Feed data: {0:s}".format(str(f)))
logger.info("Added feed. New feed ID is {0:d}".format(f.id))
f.synchronize(False)
self.feed_object = f
elif len(feeds) > 1:
logger.warning("Multiple feeds found, selecting Feed id {}".format(
feeds[0].id))
elif feeds:
feed_id = feeds[0].id
logger.info("Feed {} was found as Feed ID {}".format(
self.feed_name, feed_id))
feeds[0].synchronize(False)
feed = feeds[0]
self.feed_object = feed
def serve(self):
address = self.bridge_options.get('listener_address', '0.0.0.0')
port = int(self.bridge_options['listener_port'])
logger.info("starting flask server: %s:%d" % (address, port))
self.flask_feed.app.run(port=port,
debug=True,
host=address,
use_reloader=False)
def generate_feed(self):
logger.info("Generating feed")
icon_path = "%s/%s" % (self.directory, self.integration_image_path)
logger.info("icon_path: %s" % icon_path)
ret = cbint.utils.feed.generate_feed(
self.feed_name,
summary="Infoblox secure DNS domain connector",
tech_data=
"There are no requirements to share any data with Carbon Black to use this feed.",
provider_url="http://www.infoblox.com/",
icon_path=icon_path,
display_name=self.display_name,
category="Connectors")
ret['reports'] = []
with self.feed_lock:
for domain in self.feed_domains.keys():
report = {
'id': "Domain-%s" % domain,
'link': 'http://www.infoblox.com',
'score': 100,
'timestamp': self.feed_domains[domain]['timestamp'],
'iocs': {
'dns': [domain]
},
'title': "Domain-%s" % domain
}
logger.info("Adding domain %s to feed" % domain)
ret["reports"].append(report)
return ret
def handle_json_feed_request(self):
return self.flask_feed.generate_json_feed(self.feed)
def handle_html_feed_request(self):
return self.flask_feed.generate_html_feed(self.feed, self.display_name)
def handle_index_request(self):
return self.flask_feed.generate_html_index(self.feed,
self.bridge_options,
self.display_name,
self.cb_image_path,
self.integration_image_path,
self.json_feed_path)
def handle_cb_image_request(self):
return self.flask_feed.generate_image_response(
image_path="%s%s" % (self.directory, self.cb_image_path))
def handle_integration_image_request(self):
return self.flask_feed.generate_image_response(
image_path="%s%s" % (self.directory, self.integration_image_path))
def restore_feed_files(self):
fn = '%s/%s' % (self.data_dir, 'infoblox_domains.json')
new_domains = {}
if os.path.isfile(fn):
try:
new_domains = json.load(
open('%s/%s' % (self.data_dir, 'infoblox_domains.json')))
self.feed_domains.update(new_domains)
except:
pass
return len(new_domains)
def action(self, sensors, domain):
"""
add a infoblox domain determination to a feed
"""
# TODO: we need a timeout feature so domains will age out of the feed over time
try:
logger.info("Adding domain: %s" % domain)
with self.feed_lock:
if domain not in self.feed_domains:
self.sync_needed = True
self.feed_domains[domain]['timestamp'] = time.time()
json.dump(
self.feed_domains,
open('%s/%s' % (self.data_dir, 'infoblox_domains.json'),
'w'))
self.feed = self.generate_feed()
if self.sync_needed and self.feed:
self.feed_object.synchronize(False)
self.sync_needed = False
except:
logger.error('%s' % traceback.format_exc())