class CbTaxiiFeedConverter(object): """ Class to convert TAXII feeds into EDR feeds. """ def __init__(self, config_file_path: str, debug_mode: bool = False, import_dir: str = '', export_dir: Optional[str] = None, strict_mode: bool = False): """ Parse config file and save off the information we need. NOTE: At present, import path is unused :param config_file_path: configuration file location :param debug_mode: If True, operate in debug mode :param import_dir: feed import directory :param export_dir: export directory (optional) :param strict_mode: It True, be harsher wit config """ try: config_dict = parse_config(config_file_path, strict_mode=strict_mode) except TaxiiConfigurationException as err: _logger.error(f"{err}", exc_info=False) sys.exit(-1) if debug_mode: _logger.debug(f"Config: {config_dict}") self.server_url = config_dict.get('server_url', 'https://127.0.0.1') self.api_token = config_dict.get('api_token', '') self.sites = config_dict.get('sites', []) self.debug = config_dict.get('debug', False) self.export_dir = export_dir self.import_dir = import_dir self.integration_name = 'Cb Taxii Connector 1.6.5' self.http_proxy_url = config_dict.get('http_proxy_url', None) self.https_proxy_url = config_dict.get('https_proxy_url', None) # if exporting, make sure the directory exists if self.export_dir and not os.path.exists(self.export_dir): os.mkdir(self.export_dir) # Test Cb Response connectivity try: self.cb = CbResponseAPI(url=self.server_url, token=self.api_token, ssl_verify=False, integration_name=self.integration_name) self.cb.info() except Exception as err: _logger.error(f"Failed to make connection: {err}", exc_info=True) sys.exit(-1) @staticmethod def write_to_temp_file(message: AnyStr) -> Tuple[tempfile.NamedTemporaryFile, str]: """ Write text to a temp file for later use. :param message: text to be saved :return: Tuple of (NamedTemporaryFile, tempfile name) """ temp_file = tempfile.NamedTemporaryFile() temp_file.write(message) temp_file.flush() return temp_file, temp_file.name # NOTE: currently unused; retained for future need # noinspection PyUnusedFunction def read_from_xml(self) -> List[str]: """ Walk the import dir and return all filenames. We are assuming all xml files. :return: List of filenames """ the_list = [] if self.import_dir is None: # possible if input dir is not specified _logger.warning("Input directory was not specified -- skipping xml read") return the_list for (dirpath, dirnames, filenames) in os.walk(self.import_dir): the_list.extend(filenames) break return the_list def export_xml(self, feed_name: str, start_time: str, end_time: str, block_num: int, message: AnyStr) -> None: """ :param feed_name: name of the feed, for the holding directory name :param start_time: start time :param end_time: end time :param block_num: write block number (for uniqueness) :param message: feed text """ # create a directory to store all content blocks dir_name = f"{feed_name}".replace(' ', '_') full_dir_name = os.path.join(self.export_dir, dir_name) # Make sure the directory exists if not os.path.exists(os.path.join(self.export_dir, dir_name)): os.mkdir(full_dir_name) # Actually write the file file_name = f"{start_time}-{end_time}-{block_num}".replace(' ', "_") full_file_name = os.path.join(full_dir_name, file_name) with open(full_file_name, 'wb') as file_handle: file_handle.write(message) def _import_collection(self, client: Union[Client10, Client11], site: dict, collection: CabbyCollection, data_set: bool = False) -> int: """ Import a taxii client collectio into a feed. :param client: Taxii spec client v1.0 or v1.1 :param site: site definition :param collection: cabby collection :param data_set: True if DATA_SET, False otherwise :return: the EDR feed id, or -1 if not available """ global BINDING_CHOICES collection_name = collection.name display_name = f"{site.get('site')} {collection_name}" sanitized_feed_name = cleanup_string(site.get('site') + collection_name) feed_summary = display_name available = collection.available collection_type = collection.type default_score = site.get('default_score') _logger.info(f"Working on SITE {site.get('site')}, NAME {collection_name}, FEED {sanitized_feed_name}, " f"AVAIL {available}, TYPE {collection_type}") _logger.info('-' * 80) # if not available, nothing else to do if not available: return -1 # Sanity check on start date; provide a bare minimum start_date_str = site.get('start_date') if not start_date_str or len(start_date_str) == 0: start_date_str = "2019-01-01 00:00:00" # Create a feed helper object feed_helper = FeedHelper(site.get('output_path'), sanitized_feed_name, site.get('minutes_to_advance'), start_date_str, reset_start_date=site.get('reset_start_date', False)) if not data_set: _logger.info("Feed start time %s" % feed_helper.start_date) # # Build up the URI for polling # if not site.get('poll_path', ''): uri: Optional[str] = None else: uri: str = '' if site.get('use_https'): uri += 'https://' else: uri += 'http://' uri += site.get('site') uri += site.get('poll_path') _logger.info(f'Poll path: {uri}') # build up all the reports for the feed reports: List[Dict[str, Any]] = [] while True: num_times_empty_content_blocks = 0 try: try: _logger.info(f"Polling Collection: {collection.name} ...") content_blocks = client.poll(collection_name=collection.name, begin_date=feed_helper.start_date, end_date=feed_helper.end_date, content_bindings=BINDING_CHOICES, uri=uri) except Exception as e: _logger.info(f"{e}") content_blocks = [] # # Iterate through all content_blocks # num_blocks = 0 if not data_set: _logger.info(f" ... start_date: {feed_helper.start_date}, end_date: {feed_helper.end_date}") for block in content_blocks: _logger.debug(block.content) # # if in export mode then save off this content block # if self.export_dir: self.export_xml(collection_name, feed_helper.start_date, feed_helper.end_date, num_blocks, block.content) # # This code accounts for a case found with ThreatCentral.io where the content is url encoded. # etree.fromstring can parse this data. # try: root = etree.fromstring(block.content) content = root.find('.//{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content') if content is not None and len(content) == 0 and len(list(content)) == 0: # # Content has no children. So lets make sure we parse the xml text for content and re-add # it as valid XML so we can parse # new_stix_package = etree.fromstring(root.find( "{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}" "Content_Block/{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content").text) content.append(new_stix_package) # # Since we modified the xml, we need create a new xml message string to parse # message = etree.tostring(root) # # Write the content block to disk so we can parse with python stix # file_handle, file_path = self.write_to_temp_file(message) # # Parse STIX data # stix_package = STIXPackage.from_xml(file_path) # # if it is a DATA_SET make feed_summary from the stix_header description # NOTE: this is for RecordedFuture, also note that we only do this for data_sets. # to date I have only seen RecordedFuture use data_sets # if data_set and stix_package.stix_header and stix_package.stix_header.descriptions: for desc in stix_package.stix_header.descriptions: feed_summary = f"{desc.value}: {collection_name}" break # # Get the timestamp of the STIX Package so we can use this in our feed # timestamp = dt_to_seconds(stix_package.timestamp) # check for empty content in this block; we break out after 10 empty blocks if not stix_package.indicators and not stix_package.observables: num_times_empty_content_blocks += 1 if num_times_empty_content_blocks > 10: break # Go through all STIX indicators if stix_package.indicators: for indicator in stix_package.indicators: if not indicator or not indicator.observable: continue if indicator.confidence: if str(indicator.confidence.value).isdigit(): # # Get the confidence score and use it for our score # score = int(indicator.confidence.to_dict().get("value", default_score)) else: if str(indicator.confidence.value).lower() == "high": score = 75 elif str(indicator.confidence.value).lower() == "medium": score = 50 elif str(indicator.confidence.value).lower() == "low": score = 25 else: score = default_score else: score = default_score if not indicator.timestamp: timestamp = 0 else: timestamp = int((indicator.timestamp - EPOCH).total_seconds()) # Cybox observable returns a list reports.extend(cybox_parse_observable(indicator.observable, indicator, timestamp, score)) # # Now lets find some data. Iterate through all observables and parse # if stix_package.observables: for observable in stix_package.observables: if not observable: continue # Cybox observable returns a list reports.extend(cybox_parse_observable(observable, None, timestamp, default_score)) # # Delete our temporary file # file_handle.close() # increase block count num_blocks += 1 except Exception as e: _logger.info(f"{e}") continue _logger.info(f"content blocks read: {num_blocks}") _logger.info(f"current number of reports: {len(reports)}") if len(reports) > site.get('reports_limit'): _logger.info(f"We have reached the reports limit of {site.get('reports_limit')}") break except Exception as e: _logger.info(f"{e}") # If it is just a data_set, the data is unordered, so we can just break out of the while loop if data_set: break if feed_helper.advance(): continue else: break _logger.info(f"Found {len(reports)} new reports.") if not data_set: # We only want to concatenate if we are NOT a data set, otherwise we want to refresh all the reports _logger.info("Adding existing reports...") reports = feed_helper.load_existing_feed_data() + reports _logger.info(f"Total number of reports: {len(reports)}") if site.get('reports_limit') < len(reports): _logger.info("Truncating reports to length {0}".format(site.get('reports_limit'))) reports = reports[:site.get('reports_limit')] try: use_icon = site.get('icon_link') if not os.path.exists(use_icon): _logger.warning(f"Unable to find feed icon at path {use_icon}") use_icon = None data = build_feed_data(sanitized_feed_name, display_name, feed_summary, site.get('site'), use_icon, reports) except Exception as err: _logger.warning(f"Failed to create feed data for {sanitized_feed_name}: {err}") return -1 if feed_helper.write_feed(data): feed_helper.save_details() # # Create Cb Response Feed if necessary # feed_id = None try: feeds = get_object_by_name_or_id(self.cb, Feed, name=sanitized_feed_name) if not feeds: _logger.info(f"Feed {sanitized_feed_name} was not found, so we are going to create it...") elif len(feeds) > 1: _logger.warning(f"Multiple feeds found, selecting Feed id {feeds[0].id}") feed_id = feeds[0].id elif feeds: feed_id = feeds[0].id _logger.info(f"Feed `{sanitized_feed_name}` was found as Feed ID {feed_id}") except Exception as e: _logger.info(f"{e}") if not feed_id: _logger.info(f" ... creating {sanitized_feed_name} feed for the first time") f = self.cb.create(Feed) f.feed_url = "file://" + feed_helper.path f.enabled = site.get('feeds_enable') f.use_proxy = False f.validate_server_cert = False try: f.save() except ServerError as se: if se.error_code == 500: _logger.warning(" ... could not add feed:") _logger.warning(" Received error code 500 from server. This is usually because " "the server cannot retrieve the feed.") _logger.warning(" Check to ensure the Cb server has network connectivity " "and the credentials are correct.") _logger.warning("!" * 80 + "\n") else: info = feed_helper.dump_feedinfo() _logger.info(f" ... Could not add feed: {se}\n >> {info}") except Exception as e: info = feed_helper.dump_feedinfo() _logger.warning(f" ... Could not add feed: {e}\n >> {info}") _logger.warning("!" * 80 + "\n") else: _logger.info(f"Feed data: {f}") _logger.info(f"Added feed. New feed ID is {f.id}") feed_id = f.id return feed_id def perform(self) -> None: """ Perform the taxii hailing service. """ for site in self.sites: client: Union[Client10, Client11] = create_client(site.get('site'), use_https=site.get('use_https'), discovery_path=site.get('discovery_path')) # # Set verify_ssl and ca_cert inside the client # client.set_auth(verify_ssl=site.get('ssl_verify'), ca_cert=site.get('ca_cert')) # # Proxy Settings # proxy_dict = dict() if self.http_proxy_url: _logger.info(f"Found HTTP Proxy: {self.http_proxy_url}") proxy_dict['http'] = self.http_proxy_url if self.https_proxy_url: _logger.info(f"Found HTTPS Proxy: {self.https_proxy_url}") proxy_dict['https'] = self.https_proxy_url if proxy_dict: client.set_proxies(proxy_dict) # If a username is supplied use basic authentication if site.get('username') or site.get('cert_file'): _logger.info("Found Username in config, using basic auth...") client.set_auth(username=site.get('username'), password=site.get('password'), verify_ssl=site.get('ssl_verify'), ca_cert=site.get('ca_cert'), cert_file=site.get('cert_file'), key_file=site.get('key_file')) if not site.get('collection_management_path', ''): collections = client.get_collections() else: uri = '' if site.get('use_https'): uri += 'https://' else: uri += 'http://' uri += site.get('site') uri += site.get('collection_management_path') _logger.info('Collection Management Path: {}'.format(uri)) collections: List[CabbyCollection] = client.get_collections(uri=uri) if len(collections) == 0: _logger.info('Unable to find any collections. Exiting...') sys.exit(0) _logger.info("=" * 80) for collection in collections: _logger.info(f'Collection Name: {collection.name}, Collection Type: {collection.type}') _logger.info("=" * 80 + "\n") desired_collections = [x.strip() for x in site.get('collections').lower().split(',')] want_all = False if '*' in desired_collections: want_all = True for collection in collections: if collection.type != 'DATA_FEED' and collection.type != 'DATA_SET': continue if collection.type == 'DATA_SET': data_set = True else: data_set = False if want_all or collection.name.lower() in desired_collections: self._import_collection(client, site, collection, data_set)
class CarbonBlackThreatConnectBridge(CbIntegrationDaemon): def __init__(self, name, configfile, logfile=None, pidfile=None, debug=False): CbIntegrationDaemon.__init__(self, name, configfile=configfile, logfile=logfile, pidfile=pidfile, debug=debug) template_folder = "/usr/share/cb/integrations/cb-threatconnect-connector/content" self.flask_feed = cbint.utils.flaskfeed.FlaskFeed( __name__, False, template_folder) self.bridge_options = {} self.bridge_auth = {} self.api_urns = {} self.validated_config = False self.cb = None self.sync_needed = False self.feed_name = "threatconnectintegration" self.display_name = "ThreatConnect" self.feed = {} self.directory = template_folder self.cb_image_path = "/carbonblack.png" self.integration_image_path = "/threatconnect.png" self.integration_image_small_path = "/threatconnect-small.png" self.json_feed_path = "/threatconnect/json" self.feed_lock = threading.RLock() self.logfile = logfile self.flask_feed.app.add_url_rule( self.cb_image_path, view_func=self.handle_cb_image_request) self.flask_feed.app.add_url_rule( self.integration_image_path, view_func=self.handle_integration_image_request) self.flask_feed.app.add_url_rule( self.json_feed_path, view_func=self.handle_json_feed_request, methods=['GET']) self.flask_feed.app.add_url_rule("/", view_func=self.handle_index_request, methods=['GET']) self.flask_feed.app.add_url_rule( "/feed.html", view_func=self.handle_html_feed_request, methods=['GET']) self.initialize_logging() logger.debug("generating feed metadata") with self.feed_lock: self.feed = cbint.utils.feed.generate_feed( self.feed_name, summary= "Threat intelligence data provided by ThreatConnect to the Carbon Black Community", tech_data= "There are no requirements to share any data to receive this feed.", provider_url="http://www.threatconnect.com/", icon_path="%s/%s" % (self.directory, self.integration_image_path), small_icon_path="%s/%s" % (self.directory, self.integration_image_small_path), display_name=self.display_name, category="Partner") self.last_sync = "No sync performed" self.last_successful_sync = "No sync performed" def initialize_logging(self): if not self.logfile: log_path = "/var/log/cb/integrations/%s/" % self.name cbint.utils.filesystem.ensure_directory_exists(log_path) self.logfile = "%s%s.log" % (log_path, self.name) root_logger = logging.getLogger() root_logger.setLevel(logging.INFO) root_logger.handlers = [] rlh = RotatingFileHandler(self.logfile, maxBytes=524288, backupCount=10) rlh.setFormatter( logging.Formatter( fmt="%(asctime)s: %(module)s: %(levelname)s: %(message)s")) root_logger.addHandler(rlh) @property def integration_name(self): return 'Cb ThreatConnect Connector 1.2.9' def serve(self): if "https_proxy" in self.bridge_options: os.environ['HTTPS_PROXY'] = self.bridge_options.get( "https_proxy", "") os.environ['no_proxy'] = '127.0.0.1,localhost' address = self.bridge_options.get('listener_address', '127.0.0.1') port = self.bridge_options['listener_port'] logger.info("starting flask server: %s:%s" % (address, port)) self.flask_feed.app.run(port=port, debug=self.debug, host=address, use_reloader=False) def handle_json_feed_request(self): with self.feed_lock: json = self.flask_feed.generate_json_feed(self.feed) return json def handle_html_feed_request(self): with self.feed_lock: html = self.flask_feed.generate_html_feed(self.feed, self.display_name) return html def handle_index_request(self): with self.feed_lock: index = self.flask_feed.generate_html_index( self.feed, self.bridge_options, self.display_name, self.cb_image_path, self.integration_image_path, self.json_feed_path, self.last_sync) return index def handle_cb_image_request(self): return self.flask_feed.generate_image_response( image_path="%s%s" % (self.directory, self.cb_image_path)) def handle_integration_image_request(self): return self.flask_feed.generate_image_response( image_path="%s%s" % (self.directory, self.integration_image_path)) def run(self): logger.info( "starting Carbon Black <-> ThreatConnect Connector | version %s" % version.__version__) logger.debug("starting continuous feed retrieval thread") work_thread = threading.Thread( target=self.perform_continuous_feed_retrieval) work_thread.setDaemon(True) work_thread.start() logger.debug("starting flask") self.serve() def validate_config(self): self.validated_config = True logger.info("Validating configuration file ...") if 'bridge' in self.options: self.bridge_options = self.options['bridge'] else: logger.error("Configuration does not contain a [bridge] section") return False if 'auth' in self.options: self.bridge_auth = self.options['auth'] else: logger.error("configuration does not contain a [auth] section") return False if 'sources' in self.options: self.api_urns = self.options["sources"] else: logger.error("configuration does not contain a [sources] section") return False opts = self.bridge_options auth = self.bridge_auth config_valid = True msgs = [] if len(self.api_urns) <= 0: msgs.append('No data sources are configured under [sources]') config_valid = False item = 'listener_port' if not (item in opts and opts[item].isdigit() and 0 < int(opts[item]) <= 65535): msgs.append( 'the config option listener_port is required and must be a valid port number' ) config_valid = False else: opts[item] = int(opts[item]) item = 'listener_address' if not (item in opts and opts[item] is not ""): msgs.append( 'the config option listener_address is required and cannot be empty' ) config_valid = False item = 'feed_retrieval_minutes' if not (item in opts and opts[item].isdigit() and 0 < int(opts[item])): msgs.append( 'the config option feed_retrieval_minutes is required and must be greater than 1' ) config_valid = False else: opts[item] = int(opts[item]) item = 'ioc_min_score' if item in opts: if not (opts[item].isdigit() and 0 <= int(opts[item]) <= 100): msgs.append( 'The config option ioc_min_score must be a number in the range 0 - 100' ) config_valid = False else: opts[item] = int(opts[item]) else: logger.warning("No value provided for ioc_min_score. Using 1") opts[item] = 1 item = 'api_key' if not (item in auth and auth[item].isdigit()): msgs.append( 'The config option api_key is required under section [auth] and must be a numeric value' ) config_valid = False item = 'url' if not (item in auth and auth[item] is not ""): msgs.append( 'The config option url is required under section [auth] and cannot be blank' ) config_valid = False if 'secret_key_encrypted' in auth and 'secret_key' not in auth: msgs.append( "Encrypted API secret key no longer supported. Use unencrypted 'secret_key' form." ) config_valid = False elif 'secret_key' in auth and auth['secret_key'] != "": auth['api_secret_key'] = self.bridge_auth.get("secret_key") else: msgs.append( 'The config option secret_key under section [auth] must be provided' ) config_valid = False # Convert all 1 or 0 values to true/false opts["ignore_ioc_md5"] = opts.get("disable_ioc_md5", "0") == "1" opts["ignore_ioc_ip"] = opts.get("disable_ioc_ip", "0") == "1" opts["ignore_ioc_host"] = opts.get("disable_ioc_host", "0") == "1" # create a cbapi instance ssl_verify = self.get_config_boolean("carbonblack_server_sslverify", False) server_url = self.get_config_string("carbonblack_server_url", "https://127.0.0.1") server_token = self.get_config_string("carbonblack_server_token", "") try: self.cb = CbResponseAPI(url=server_url, token=server_token, ssl_verify=False, integration_name=self.integration_name) self.cb.info() except: logger.error(traceback.format_exc()) return False if not config_valid: for msg in msgs: sys.stderr.write("%s\n" % msg) logger.error(msg) return False else: return True def _filter_results(self, results): logger.debug("Number of IOCs before filtering applied: %d", len(results)) opts = self.bridge_options filter_min_score = opts["ioc_min_score"] # Filter out those scores lower than the minimum score if filter_min_score > 0: results = filter(lambda x: x["score"] >= filter_min_score, results) logger.debug( "Number of IOCs after scores less than %d discarded: %d", filter_min_score, len(results)) # For end user simplicity we call "dns" entries "host" and ipv4 entries "ip" # format: {"flag_name" : ("official_name", "friendly_name")} ignore_ioc_mapping = { "ignore_ioc_md5": ("md5", "md5"), "ignore_ioc_ip": ("ipv4", "ip"), "ignore_ioc_host": ("dns", "host") } # On a per flag basis discard md5s, ips, or host if the user has requested we do so # If we don't discard then check if an exclusions file has been specified and discard entries # that match those in the exclusions file for ignore_flag in ignore_ioc_mapping: exclude_type = ignore_ioc_mapping[ignore_flag][0] exclude_type_friendly_name = ignore_ioc_mapping[ignore_flag][1] if opts[ignore_flag]: results = filter(lambda x: exclude_type not in x["iocs"], results) logger.debug("Number of IOCs after %s entries discarded: %d", exclude_type, len(results)) elif 'exclusions' in self.options and exclude_type_friendly_name in self.options[ 'exclusions']: file_path = self.options['exclusions'][ exclude_type_friendly_name] if not os.path.exists(file_path): logger.debug("Exclusions file %s not found", file_path) continue with open(file_path, 'r') as exclude_file: data = frozenset([line.strip() for line in exclude_file]) results = filter( lambda x: exclude_type not in x["iocs"] or x["iocs"][ exclude_type][0] not in data, results) logger.debug( "Number of IOCs after %s exclusions file applied: %d", exclude_type_friendly_name, len(results)) return results def perform_continuous_feed_retrieval(self, loop_forever=True): try: # config validation is critical to this connector working correctly if not self.validated_config: self.validate_config() opts = self.bridge_options auth = self.bridge_auth while True: logger.debug("Starting retrieval iteration") try: tc = ThreatConnectFeedGenerator(auth["api_key"], auth['api_secret_key'], auth["url"], self.api_urns.items()) tmp = tc.get_threatconnect_iocs() tmp = self._filter_results(tmp) with self.feed_lock: self.feed["reports"] = tmp self.last_sync = strftime( "%a, %d %b %Y %H:%M:%S +0000", gmtime()) self.last_successful_sync = strftime( "%a, %d %b %Y %H:%M:%S +0000", gmtime()) logger.info("Successfully retrieved data at %s" % self.last_successful_sync) except ConnectionException as e: logger.error("Error connecting to Threat Connect: %s" % e.value) self.last_sync = self.last_successful_sync + " (" + str( e.value) + ")" if not loop_forever: sys.stderr.write( "Error connecting to Threat Connect: %s\n" % e.value) sys.exit(2) except Exception as e: logger.error(traceback.format_exc()) time.sleep(opts.get('feed_retrieval_minutes') * 60) # synchronize feed with Carbon Black server if not "skip_cb_sync" in opts: try: feeds = get_object_by_name_or_id(self.cb, Feed, name=self.feed_name) except Exception as e: logger.error(e.message) feeds = None if not feeds: logger.info( "Feed {} was not found, so we are going to create it" .format(self.feed_name)) f = self.cb.create(Feed) f.feed_url = "http://{0}:{1}/threatconnect/json".format( self.bridge_options.get('feed_host', '127.0.0.1'), self.bridge_options.get('listener_port', '6100')) f.enabled = True f.use_proxy = False f.validate_server_cert = False try: f.save() except ServerError as se: if se.error_code == 500: logger.info("Could not add feed:") logger.info( " Received error code 500 from server. This is usually because the server cannot retrieve the feed." ) logger.info( " Check to ensure the Cb server has network connectivity and the credentials are correct." ) else: logger.info("Could not add feed: {0:s}".format( str(se))) except Exception as e: logger.info("Could not add feed: {0:s}".format( str(e))) else: logger.info("Feed data: {0:s}".format(str(f))) logger.info( "Added feed. New feed ID is {0:d}".format( f.id)) f.synchronize(False) elif len(feeds) > 1: logger.warning( "Multiple feeds found, selecting Feed id {}". format(feeds[0].id)) elif feeds: feed_id = feeds[0].id logger.info("Feed {} was found as Feed ID {}".format( self.feed_name, feed_id)) feeds[0].synchronize(False) logger.debug("ending feed retrieval loop") # Function should only ever return when loop_forever is set to false if not loop_forever: return self.flask_feed.generate_json_feed(self.feed).data time.sleep(opts.get('feed_retrieval_minutes') * 60) except Exception: # If an exception makes us exit then log what we can for our own sake logger.fatal( "FEED RETRIEVAL LOOP IS EXITING! Daemon should be restarted to restore functionality! " ) logger.fatal("Fatal Error Encountered:\n %s" % traceback.format_exc()) sys.stderr.write( "FEED RETRIEVAL LOOP IS EXITING! Daemon should be restarted to restore functionality!\n" ) sys.stderr.write("Fatal Error Encountered:\n %s\n" % traceback.format_exc()) sys.exit(3) # If we somehow get here the function is going to exit. # This is not normal so we LOUDLY log the fact logger.fatal( "FEED RETRIEVAL LOOP IS EXITING! Daemon should be restarted to restore functionality!" )
class CarbonBlackThreatConnectBridge(CbIntegrationDaemon): def __init__(self, name, configfile, logfile=None, pidfile=None, debug=False): CbIntegrationDaemon.__init__(self, name, configfile=configfile, logfile=logfile, pidfile=pidfile, debug=debug) template_folder = "/usr/share/cb/integrations/cb-threatconnect-connector/content" self.flask_feed = cbint.utils.flaskfeed.FlaskFeed( __name__, False, template_folder) self.bridge_options = {} self.tc_config = {} self.api_urns = {} self.validated_config = False self.cb = None self.sync_needed = False self.feed_name = "threatconnectintegration" self.display_name = "ThreatConnect" self.feed = {} self.directory = template_folder self.cb_image_path = "/carbonblack.png" self.integration_image_path = "/threatconnect.png" self.integration_image_small_path = "/threatconnect-small.png" self.json_feed_path = "/threatconnect/json" self.feed_lock = threading.RLock() self.logfile = logfile self.debug = debug self.pretty_print_json = False self.flask_feed.app.add_url_rule( self.cb_image_path, view_func=self.handle_cb_image_request) self.flask_feed.app.add_url_rule( self.integration_image_path, view_func=self.handle_integration_image_request) self.flask_feed.app.add_url_rule( self.json_feed_path, view_func=self.handle_json_feed_request, methods=['GET']) self.flask_feed.app.add_url_rule("/", view_func=self.handle_index_request, methods=['GET']) self.flask_feed.app.add_url_rule( "/feed.html", view_func=self.handle_html_feed_request, methods=['GET']) self.initialize_logging() logger.debug("generating feed metadata") with self.feed_lock: self.feed = cbint.utils.feed.generate_feed( self.feed_name, summary= "Threat intelligence data provided by ThreatConnect to the Carbon Black Community", tech_data= "There are no requirements to share any data to receive this feed.", provider_url="http://www.threatconnect.com/", icon_path="%s/%s" % (self.directory, self.integration_image_path), small_icon_path="%s/%s" % (self.directory, self.integration_image_small_path), display_name=self.display_name, category="Partner") self.last_sync = TimeStamp() self.last_successful_sync = TimeStamp() self.feed_ready = False def _read_cached(self): with self.feed_lock: if self.feed_ready: return folder = self.bridge_options.get("cache_folder", "./") cbint.utils.filesystem.ensure_directory_exists(folder) try: with open(os.path.join(folder, "reports.cache"), "r") as f: reports = json.loads(f.read()) with self.feed_lock: if not self.feed_ready: self.feed["reports"] = reports self.feed_ready = True logger.info("Reports loaded from cache.") except IOError as e: logger.warning("Cache file missing or invalid: {0}".format(e)) def initialize_logging(self): if not self.logfile: log_path = "/var/log/cb/integrations/%s/" % self.name cbint.utils.filesystem.ensure_directory_exists(log_path) self.logfile = "%s%s.log" % (log_path, self.name) root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG if self.debug else logging.INFO) root_logger.handlers = [] rlh = RotatingFileHandler(self.logfile, maxBytes=524288, backupCount=10) rlh.setFormatter( logging.Formatter( fmt="%(asctime)s - %(levelname)-7s - %(module)s - %(message)s") ) root_logger.addHandler(rlh) self.logger = root_logger @property def integration_name(self): return 'Cb ThreatConnect Connector {0}'.format(version.__version__) def serve(self): if "https_proxy" in self.bridge_options: os.environ['HTTPS_PROXY'] = self.bridge_options.get( "https_proxy", "") os.environ['no_proxy'] = '127.0.0.1,localhost' address = self.bridge_options.get('listener_address', '127.0.0.1') port = self.bridge_options['listener_port'] logger.info("starting flask server: %s:%s" % (address, port)) self.flask_feed.app.run(port=port, debug=self.debug, host=address, use_reloader=False) def handle_json_feed_request(self): with self.feed_lock: json = self.flask_feed.generate_json_feed(self.feed) return json def handle_html_feed_request(self): with self.feed_lock: html = self.flask_feed.generate_html_feed(self.feed, self.display_name) return html def handle_index_request(self): with self.feed_lock: index = self.flask_feed.generate_html_index( self.feed, self.bridge_options, self.display_name, self.cb_image_path, self.integration_image_path, self.json_feed_path, str(self.last_sync)) return index def handle_cb_image_request(self): return self.flask_feed.generate_image_response( image_path="%s%s" % (self.directory, self.cb_image_path)) def handle_integration_image_request(self): return self.flask_feed.generate_image_response( image_path="%s%s" % (self.directory, self.integration_image_path)) def on_starting(self): self._read_cached() ThreatConnectDriver.initialize(self.tc_config) def run(self): logger.info( "starting Carbon Black <-> ThreatConnect Connector | version %s" % version.__version__) logger.debug("starting continuous feed retrieval thread") work_thread = threading.Thread( target=self.perform_continuous_feed_retrieval) work_thread.setDaemon(True) work_thread.start() logger.debug("starting flask") self.serve() def validate_config(self): config_valid = True if self.validated_config: return True self.validated_config = True logger.debug("Validating configuration file ...") if 'bridge' in self.options: self.bridge_options = self.options['bridge'] else: sys.stderr.write( "Configuration does not contain a [bridge] section\n") logger.error("Configuration does not contain a [bridge] section") return False self.debug = self.bridge_options.get( 'debug', 'F') in ['1', 't', 'T', 'True', 'true'] log_level = self.bridge_options.get('log_level', 'INFO').upper() log_level = log_level if log_level in [ "INFO", "WARNING", "DEBUG", "ERROR" ] else "INFO" self.logger.setLevel( logging.DEBUG if self.debug else logging.getLevelName(log_level)) tc_options = self.options.get('threatconnect', {}) if not tc_options: sys.stderr.write( "Configuration does not contain a [threatconnect] section or section is empty.\n" ) logger.error( "configuration does not contain a [threatconnect] section or section is empty." ) return False try: self.tc_config = ThreatConnectConfig(**tc_options) except Exception as e: sys.stderr.write("Error: {0}\n".format(e)) logger.error(e) return False self.pretty_print_json = self.bridge_options.get( 'pretty_print_json', 'F') in ['1', 't', 'T', 'True', 'true'] ca_file = os.environ.get("REQUESTS_CA_BUNDLE", None) if ca_file: logger.info("Using CA Cert file: {0}".format(ca_file)) else: logger.info("No CA Cert file found.") opts = self.bridge_options msgs = [] item = 'listener_port' if not (item in opts and opts[item].isdigit() and 0 < int(opts[item]) <= 65535): msgs.append( 'the config option listener_port is required and must be a valid port number' ) config_valid = False else: opts[item] = int(opts[item]) item = 'listener_address' if not (item in opts and opts[item]): msgs.append( 'the config option listener_address is required and cannot be empty' ) config_valid = False item = 'feed_retrieval_minutes' if not (item in opts and opts[item].isdigit() and 0 < int(opts[item])): msgs.append( 'the config option feed_retrieval_minutes is required and must be greater than 1' ) config_valid = False else: opts[item] = int(opts[item]) # Create a cbapi instance server_url = self.get_config_string("carbonblack_server_url", "https://127.0.0.1") server_token = self.get_config_string("carbonblack_server_token", "") try: self.cb = CbResponseAPI(url=server_url, token=server_token, ssl_verify=False, integration_name=self.integration_name) self.cb.info() except Exception: logger.error(traceback.format_exc()) return False if not config_valid: for msg in msgs: sys.stderr.write("%s\n" % msg) logger.error(msg) return False else: return True def perform_continuous_feed_retrieval(self, loop_forever=True): try: self.validate_config() opts = self.bridge_options folder = self.bridge_options.get("cache_folder", "./") cbint.utils.filesystem.ensure_directory_exists(folder) while True: logger.info("Starting feed retrieval.") errored = True try: start = timer() tc = ThreatConnectDriver(self.tc_config) reports = tc.generate_reports() logger.debug( "Retrieved reports ({0:.3f} seconds).".format(timer() - start)) if reports: write_start = timer() # Instead of rewriting the cache file directly, we're writing to a temporary file # and then moving it onto the cache file so that we don't have a situation where # the cache file is only partially written and corrupt or empty. with open(os.path.join(folder, "reports.cache_new"), "w") as f: if self.pretty_print_json: f.write(json.dumps(reports, indent=2)) else: f.write(json.dumps(reports)) # This is a quick operation that will not leave the file in an invalid state. shutil.move(os.path.join(folder, "reports.cache_new"), os.path.join(folder, "reports.cache")) logger.debug( "Finished writing reports to cache ({0:.3f} seconds)." .format(timer() - write_start)) with self.feed_lock: if reports: self.feed["reports"] = reports self.last_successful_sync.stamp() logger.info( "Successfully retrieved data at {0} ({1:.3f} seconds total)" .format(self.last_successful_sync, timer() - start)) errored = False self._sync_cb_feed() except Exception as e: logger.exception( "Error occurred while attempting to retrieve feed: {0}" .format(e)) self.last_sync.stamp() logger.debug("Feed report retrieval completed{0}.".format( " (Errored)" if errored else "")) if not loop_forever: return self.flask_feed.generate_json_feed(self.feed).data # Full sleep interval is taken between feed retrieval work. time.sleep(opts.get('feed_retrieval_minutes') * 60) except Exception: # If an exception makes us exit then log what we can for our own sake logger.fatal( "FEED RETRIEVAL LOOP IS EXITING! Daemon should be restarted to restore functionality! " ) logger.fatal("Fatal Error Encountered:\n %s" % traceback.format_exc()) sys.stderr.write( "FEED RETRIEVAL LOOP IS EXITING! Daemon should be restarted to restore functionality!\n" ) sys.stderr.write("Fatal Error Encountered:\n %s\n" % traceback.format_exc()) sys.exit(3) # If we somehow get here the function is going to exit. # This is not normal so we LOUDLY log the fact logger.fatal( "FEED RETRIEVAL LOOP IS EXITING! Daemon should be restarted to restore functionality!" ) def _sync_cb_feed(self): opts = self.bridge_options if "skip_cb_sync" in opts: return try: feeds = get_object_by_name_or_id(self.cb, Feed, name=self.feed_name) except Exception as e: logger.error(e.message) feeds = None if not feeds: logger.info( "Feed {} was not found, so we are going to create it".format( self.feed_name)) f = self.cb.create(Feed) f.feed_url = "http://{0}:{1}/threatconnect/json".format( self.bridge_options.get('feed_host', '127.0.0.1'), self.bridge_options.get('listener_port', '6100')) f.enabled = True f.use_proxy = False f.validate_server_cert = False try: f.save() except ServerError as se: if se.error_code == 500: logger.info("Could not add feed:") logger.info( " Received error code 500 from server. " "This is usually because the server cannot retrieve the feed." ) logger.info( " Check to ensure the Cb server has network connectivity and the credentials are correct." ) else: logger.info("Could not add feed: {0:s}".format(str(se))) except Exception as e: logger.info("Could not add feed: {0:s}".format(str(e))) else: logger.info("Feed data: {0:s}".format(str(f))) logger.info("Added feed. New feed ID is {0:d}".format(f.id)) f.synchronize(False) elif len(feeds) > 1: logger.warning("Multiple feeds found, selecting Feed id {}".format( feeds[0].id)) elif feeds: feed_id = feeds[0].id logger.info("Feed {} was found as Feed ID {}".format( self.feed_name, feed_id)) feeds[0].synchronize(False)
class CbTaxiiFeedConverter(object): def __init__(self, config_file_path, debug_mode=False, import_dir='', export_dir=''): # # parse config file and save off the information we need # config_dict = parse_config(config_file_path) self.server_url = config_dict.get('server_url', 'https://127.0.0.1') self.api_token = config_dict.get('api_token', '') self.sites = config_dict.get('sites', []) self.debug = config_dict.get('debug', False) self.export_dir = export_dir self.import_dir = import_dir self.integration_name = 'Cb Taxii Connector 1.6.5' self.http_proxy_url = config_dict.get('http_proxy_url', None) self.https_proxy_url = config_dict.get('https_proxy_url', None) if self.export_dir and not os.path.exists(self.export_dir): os.mkdir(self.export_dir) # # Test Cb Response connectivity # try: self.cb = CbResponseAPI(url=self.server_url, token=self.api_token, ssl_verify=False, integration_name=self.integration_name) self.cb.info() except: logger.error(traceback.format_exc()) sys.exit(-1) def write_to_temp_file(self, message): temp_file = tempfile.NamedTemporaryFile() temp_file.write(message) temp_file.flush() return temp_file, temp_file.name def read_from_xml(self): """ Walk the import dir and return all filenames. We are assuming all xml files :return: """ f = [] for (dirpath, dirnames, filenames) in os.walk(self.import_dir): f.extend(filenames) break return f def export_xml(self, feed_name, start_time, end_time, block_num, message): """ :param feed_name: :param start_time: :param end_time: :param block_num: :param message: :return: """ # # create a directory to store all content blocks # dir_name = "{}".format(feed_name).replace(' ', '_') full_dir_name = os.path.join(self.export_dir, dir_name) # # Make sure the directory exists # if not os.path.exists(os.path.join(self.export_dir, dir_name)): os.mkdir(full_dir_name) # # Actually write the file # file_name = "{}-{}-{}".format(start_time, end_time, block_num).replace(' ', "_") full_file_name = os.path.join(full_dir_name, file_name) with open(full_file_name, 'wb') as file_handle: file_handle.write(message) def _import_collection(self, client, site, collection, data_set=False): collection_name = collection.name sanitized_feed_name = cleanup_string( "%s%s" % (site.get('site'), collection_name)) feed_summary = "%s %s" % (site.get('site'), collection_name) available = collection.available collection_type = collection.type default_score = site.get('default_score') logger.info("%s,%s,%s,%s,%s" % (site.get('site'), collection_name, sanitized_feed_name, available, collection_type)) if not available: return False # # Sanity check on start date # start_date_str = site.get('start_date') if not start_date_str or len(start_date_str) == 0: start_date_str = "2019-01-01 00:00:00" # # Create a feed helper object # feed_helper = FeedHelper(site.get('output_path'), sanitized_feed_name, site.get('minutes_to_advance'), start_date_str) if not data_set: logger.info("Feed start time %s" % feed_helper.start_date) logger.info("polling Collection: {}...".format(collection.name)) # # Build up the URI for polling # if not site.get('poll_path', ''): uri = None else: uri = '' if site.get('use_https'): uri += 'https://' else: uri += 'http://' uri += site.get('site') uri += site.get('poll_path') logger.info('Poll path: {}'.format(uri)) reports = [] while True: num_times_empty_content_blocks = 0 try: try: logger.info("Polling Collection: {0}".format( collection.name)) content_blocks = client.poll( uri=uri, collection_name=collection.name, begin_date=feed_helper.start_date, end_date=feed_helper.end_date, content_bindings=BINDING_CHOICES) except Exception as e: logger.info(e.message) content_blocks = [] # # Iterate through all content_blocks # num_blocks = 0 if not data_set: logger.info("polling start_date: {}, end_date: {}".format( feed_helper.start_date, feed_helper.end_date)) for block in content_blocks: logger.debug(block.content) # # if in export mode then save off this content block # if self.export_dir: self.export_xml(collection_name, feed_helper.start_date, feed_helper.end_date, num_blocks, block.content) # # This code accounts for a case found with ThreatCentral.io where the content is url encoded. # etree.fromstring can parse this data. # try: root = etree.fromstring(block.content) content = root.find( './/{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content' ) if content is not None and len(content) == 0 and len( list(content)) == 0: # # Content has no children. So lets make sure we parse the xml text for content and re-add # it as valid XML so we can parse # new_stix_package = etree.fromstring( root.find( "{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content_Block/{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content" ).text) content.append(new_stix_package) # # Since we modified the xml, we need create a new xml message string to parse # message = etree.tostring(root) # # Write the content block to disk so we can parse with python stix # file_handle, file_path = self.write_to_temp_file( message) # # Parse STIX data # stix_package = STIXPackage.from_xml(file_path) # # if it is a DATA_SET make feed_summary from the stix_header description # NOTE: this is for RecordedFuture, also note that we only do this for data_sets. # to date I have only seen RecordedFuture use data_sets # if data_set and stix_package.stix_header and stix_package.stix_header.descriptions: for desc in stix_package.stix_header.descriptions: feed_summary = "{}: {}".format( desc.value, collection_name) break # # Get the timestamp of the STIX Package so we can use this in our feed # timestamp = total_seconds(stix_package.timestamp) if not stix_package.indicators and not stix_package.observables: num_times_empty_content_blocks += 1 if num_times_empty_content_blocks > 10: break if stix_package.indicators: for indicator in stix_package.indicators: if not indicator or not indicator.observable: continue if indicator.confidence: if str(indicator.confidence.value).isdigit( ): # # Get the confidence score and use it for our score # score = int( indicator.confidence.to_dict().get( "value", default_score)) else: if str(indicator.confidence.value ).lower() == "high": score = 75 elif str(indicator.confidence.value ).lower() == "medium": score = 50 elif str(indicator.confidence.value ).lower() == "low": score = 25 else: score = default_score else: score = default_score if not indicator.timestamp: timestamp = 0 else: timestamp = int( (indicator.timestamp - datetime.datetime(1970, 1, 1).replace( tzinfo=dateutil.tz.tzutc()) ).total_seconds()) reports.extend( cybox_parse_observable( indicator.observable, indicator, timestamp, score)) # # Now lets find some data. Iterate through all observables and parse # if stix_package.observables: for observable in stix_package.observables: if not observable: continue # # Cybox observable returns a list # reports.extend( cybox_parse_observable( observable, None, timestamp, default_score)) # # Delete our temporary file # file_handle.close() num_blocks += 1 # # end for loop through content blocks # except Exception as e: # logger.info(traceback.format_exc()) logger.info(e.message) continue logger.info("content blocks read: {}".format(num_blocks)) logger.info("current number of reports: {}".format( len(reports))) if len(reports) > site.get('reports_limit'): logger.info( "We have reached the reports limit of {0}".format( site.get('reports_limit'))) break # # DEBUG CODE # # if len(reports) > 10: # break # # Attempt to advance the start time and end time # except Exception as e: logger.info(traceback.format_exc()) # # If it is just a data_set, the data is unordered, so we can just break out of the while loop # if data_set: break if feed_helper.advance(): continue else: break # # end While True # logger.info("Found {} new reports.".format(len(reports))) if not data_set: # # We only want to concatenate if we are NOT a data set, otherwise we want to refresh all the reports # logger.info("Adding existing reports...") reports = feed_helper.load_existing_feed_data() + reports logger.info("Total number of reports: {}".format(len(reports))) if site.get('reports_limit') < len(reports): logger.info("Truncating reports to length {0}".format( site.get('reports_limit'))) reports = reports[:site.get('reports_limit')] data = build_feed_data(sanitized_feed_name, "%s %s" % (site.get('site'), collection_name), feed_summary, site.get('site'), site.get('icon_link'), reports) if feed_helper.write_feed(data): feed_helper.save_details() # # Create Cb Response Feed if necessary # feed_id = None try: feeds = get_object_by_name_or_id(self.cb, Feed, name=sanitized_feed_name) if not feeds: logger.info( "Feed {} was not found, so we are going to create it". format(sanitized_feed_name)) elif len(feeds) > 1: logger.warning( "Multiple feeds found, selecting Feed id {}".format( feeds[0].id)) feed_id = feeds[0].id elif feeds: feed_id = feeds[0].id logger.info("Feed {} was found as Feed ID {}".format( sanitized_feed_name, feed_id)) except Exception as e: logger.info(e.message) if not feed_id: logger.info("Creating {} feed for the first time".format( sanitized_feed_name)) f = self.cb.create(Feed) f.feed_url = "file://" + feed_helper.path f.enabled = site.get('feeds_enable') f.use_proxy = False f.validate_server_cert = False try: f.save() except ServerError as se: if se.error_code == 500: logger.info("Could not add feed:") logger.info( " Received error code 500 from server. This is usually because the server cannot retrieve the feed." ) logger.info( " Check to ensure the Cb server has network connectivity and the credentials are correct." ) else: logger.info("Could not add feed: {0:s}".format(str(se))) except Exception as e: logger.info("Could not add feed: {0:s}".format(str(e))) else: logger.info("Feed data: {0:s}".format(str(f))) logger.info("Added feed. New feed ID is {0:d}".format(f.id)) feed_id = f.id return feed_id def perform(self): """ :param self: :param enumerate_collections_only: :return: """ for site in self.sites: client = create_client(site.get('site'), use_https=site.get('use_https'), discovery_path=site.get('discovery_path')) # # Set verify_ssl and ca_cert inside the client # client.set_auth(verify_ssl=site.get('ssl_verify'), ca_cert=site.get('ca_cert')) # # Proxy Settings # proxy_dict = dict() if self.http_proxy_url: logger.info("Found HTTP Proxy: {}".format(self.http_proxy_url)) proxy_dict['http'] = self.http_proxy_url if self.https_proxy_url: logger.info("Found HTTPS Proxy: {}".format( self.https_proxy_url)) proxy_dict['https'] = self.https_proxy_url if proxy_dict: client.set_proxies(proxy_dict) if site.get('username') or site.get('cert_file'): # # If a username is supplied use basic authentication # logger.info("Found Username in config, using basic auth...") client.set_auth(username=site.get('username'), password=site.get('password'), verify_ssl=site.get('ssl_verify'), ca_cert=site.get('ca_cert'), cert_file=site.get('cert_file'), key_file=site.get('key_file')) if not site.get('collection_management_path', ''): collections = client.get_collections() else: uri = '' if site.get('use_https'): uri += 'https://' else: uri += 'http://' uri += site.get('site') uri += site.get('collection_management_path') logger.info('Collection Management Path: {}'.format(uri)) collections = client.get_collections(uri=uri) for collection in collections: logger.info('Collection Name: {}, Collection Type: {}'.format( collection.name, collection.type)) if len(collections) == 0: logger.info('Unable to find any collections. Exiting...') sys.exit(0) desired_collections = [ x.strip() for x in site.get('collections').lower().split(',') ] want_all = False if '*' in desired_collections: want_all = True for collection in collections: if collection.type != 'DATA_FEED' and collection.type != 'DATA_SET': continue if collection.type == 'DATA_SET': data_set = True else: data_set = False if want_all or collection.name.lower() in desired_collections: self._import_collection(client, site, collection, data_set)
class InfobloxBridge(CbIntegrationDaemon): def __init__(self, name, configfile, debug=False): self.config_ready = False CbIntegrationDaemon.__init__(self, name, configfile=configfile, debug=debug) self.cb = None self.worker_queue = Queue.Queue(maxsize=10) self.initialize_logging() self.logfile = None def initialize_logging(self): if not self.logfile: log_path = "/var/log/cb/integrations/%s/" % self.name cbint.utils.filesystem.ensure_directory_exists(log_path) self.logfile = "%s%s.log" % (log_path, self.name) root_logger = logging.getLogger() root_logger.setLevel(logging.INFO) root_logger.handlers = [] rlh = RotatingFileHandler(self.logfile, maxBytes=524288, backupCount=10) rlh.setFormatter( logging.Formatter( fmt="%(asctime)s: %(module)s: %(levelname)s: %(message)s")) root_logger.addHandler(rlh) @property def integration_name(self): return "Cb InfoBlox Connector " + version.__version__ def _set_alert_action(self, feed_id): actions = self.cb.feed_action_enum(feed_id) for action in actions: if action['id'] == 3: # XXX: "3" is the action id associated with creating an alert return self.cb.feed_action_add(feed_id, 3, []) def run(self): self.validate_config() try: logger.warn("CB Infoblox Bridge Starting") sslverify = False if self.bridge_options.get( 'carbonblack_server_sslverify', "0") == "0" else True self.cb = CbResponseAPI( url=self.bridge_options['carbonblack_server_url'], token=self.bridge_options['carbonblack_server_token'], ssl_verify=sslverify, integration_name=self.integration_name) self.cb.info() self.streaming_host = self.bridge_options.get( 'carbonblack_streaming_host') self.streaming_username = self.bridge_options.get( 'carbonblack_streaming_username') self.streaming_password = self.bridge_options.get( 'carbonblack_streaming_password') self.use_cloud_api = True if int( self.bridge_options.get('use_cloud_api', '0')) != 0 else False #start the syslog server normally , otherwise start the rest poller if not (self.use_cloud_api): syslog_server = SyslogServer(10240, self.worker_queue) else: self.api_token = self.bridge_options.get( 'api_token', "PASSWORD") self.poll_interval = self.bridge_options.get( 'rest_poll_interval', "5M") self.api_route = self.bridge_options.get('api_route', "") logger.info("starting rest poller") rest_poller = RestPoller(self.api_route, self.api_token, worker_queue=self.worker_queue, time_increment=self.poll_interval) message_broker = FanOutMessage(self.cb, self.worker_queue) # Set up the built-in feed feed_thread = FeedAction(self.cb, self.bridge_options) feed_thread.start() ctx = feed_thread.flask_feed.app.test_request_context() ctx.push() feed_thread.flask_feed.app.preprocess_request() ctx.pop() logger.info("flask ready") feed_id = feed_thread.get_or_create_feed() #TODO revisit #if self.bridge_options.get('do_alert', False): # self._set_alert_action(feed_id) # Note: it is important to keep the relative order stable here. # we want to make sure that the Cb sensor flush occurs first, before the feed entry is created # and before any other actions are taken (isolation or process termination) # We will always flush the sensor that triggered the action, so that we get the most up-to-date # information into the Cb console. flusher = FlushAction(self.cb) message_broker.add_response_action(flusher) message_broker.add_response_action(feed_thread) # Conditionally create a kill-process action based on the configuration file. kill_option = self.bridge_options.get('do_kill', None) if kill_option == 'api': kill_process_thread = ApiKillProcessAction(self.cb) kill_process_thread.start() message_broker.add_response_action(kill_process_thread) elif kill_option == 'streaming': # # For some reason this must be imported here otherwise the event registry thread does not start # from streaming_kill_process import StreamingKillProcessAction kill_streaming_action = StreamingKillProcessAction( self.cb, self.streaming_host, self.streaming_username, self.streaming_password) message_broker.add_response_action(kill_streaming_action) if self.bridge_options.get('do_isolate', False): isolator = IsolateAction(self.cb) message_broker.add_response_action(isolator) # once everything is up & running, start the message broker then the syslog server message_broker.start() if (self.use_cloud_api): rest_poller.start() else: syslog_server.start() logger.info("Starting event loop") try: while True: time.sleep(5) except KeyboardInterrupt: logger.warn("Stopping Cb Infoblox Connector due to Control-C") logger.warn("Cb Infoblox Connector Stopping") except: logger.error(traceback.format_exc()) sys.exit(1) def validate_config(self): if self.config_ready: return if 'bridge' in self.options: self.bridge_options = self.options['bridge'] else: logger.error("configuration does not contain a [bridge] section") return False config_valid = True msgs = [] if not 'listener_port' in self.bridge_options or not self.bridge_options[ 'listener_port'].isdigit(): msgs.append( 'the config option listener_port is required and must be a valid port number' ) config_valid = False if not 'carbonblack_server_url' in self.bridge_options: msgs.append('the config option carbonblack_server_url is required') config_valid = False if not 'carbonblack_server_token' in self.bridge_options: msgs.append( 'the config option carbonblack_server_token is required') config_valid = False if not 'use_cloud_api' in self.bridge_options: #default to False self.bridge_options['use_cloud_api'] = False if not config_valid: for msg in msgs: sys.stderr.write("%s\n" % msg) logger.error(msg) return False else: self.config_ready = True return True