def _dispatch(self, alert, descriptor):
"""Send ban hash command to CarbonBlack
Publishing:
There is currently no method to control carbonblack's behavior with publishers.
Args:
alert (Alert): Alert instance which triggered a rule
descriptor (str): Output descriptor
Returns:
bool: True if alert was sent successfully, False otherwise
"""
if not alert.context:
LOGGER.error('[%s] Alert must contain context to run actions',
self.__service__)
return False
creds = self._load_creds(descriptor)
if not creds:
return False
client = CbResponseAPI(**creds)
carbonblack_context = alert.context.get('carbonblack', {})
# Get md5 hash 'value' passed from the rules engine function
action = carbonblack_context.get('action')
if action == 'ban':
binary_hash = carbonblack_context.get('value')
# The binary should already exist in CarbonBlack
binary = client.select(Binary, binary_hash)
# Determine if the binary is currently listed as banned
if binary.banned:
# Determine if the banned action is enabled, if true exit
if binary.banned.enabled:
return True
# If the binary is banned and disabled, begin the banning hash operation
banned_hash = client.select(BannedHash, binary_hash)
banned_hash.enabled = True
banned_hash.save()
else:
# Create a new BannedHash object to be saved
banned_hash = client.create(BannedHash)
# Begin the banning hash operation
banned_hash.md5hash = binary.md5
banned_hash.text = "Banned from StreamAlert"
banned_hash.enabled = True
banned_hash.save()
return banned_hash.enabled is True
else:
LOGGER.error('[%s] Action not supported: %s', self.__service__,
action)
return False
class CbTaxiiFeedConverter(object):
"""
Class to convert TAXII feeds into EDR feeds.
"""
def __init__(self, config_file_path: str, debug_mode: bool = False, import_dir: str = '',
export_dir: Optional[str] = None, strict_mode: bool = False):
"""
Parse config file and save off the information we need.
NOTE: At present, import path is unused
:param config_file_path: configuration file location
:param debug_mode: If True, operate in debug mode
:param import_dir: feed import directory
:param export_dir: export directory (optional)
:param strict_mode: It True, be harsher wit config
"""
try:
config_dict = parse_config(config_file_path, strict_mode=strict_mode)
except TaxiiConfigurationException as err:
_logger.error(f"{err}", exc_info=False)
sys.exit(-1)
if debug_mode:
_logger.debug(f"Config: {config_dict}")
self.server_url = config_dict.get('server_url', 'https://127.0.0.1')
self.api_token = config_dict.get('api_token', '')
self.sites = config_dict.get('sites', [])
self.debug = config_dict.get('debug', False)
self.export_dir = export_dir
self.import_dir = import_dir
self.integration_name = 'Cb Taxii Connector 1.6.5'
self.http_proxy_url = config_dict.get('http_proxy_url', None)
self.https_proxy_url = config_dict.get('https_proxy_url', None)
# if exporting, make sure the directory exists
if self.export_dir and not os.path.exists(self.export_dir):
os.mkdir(self.export_dir)
# Test Cb Response connectivity
try:
self.cb = CbResponseAPI(url=self.server_url, token=self.api_token,
ssl_verify=False, integration_name=self.integration_name)
self.cb.info()
except Exception as err:
_logger.error(f"Failed to make connection: {err}", exc_info=True)
sys.exit(-1)
@staticmethod
def write_to_temp_file(message: AnyStr) -> Tuple[tempfile.NamedTemporaryFile, str]:
"""
Write text to a temp file for later use.
:param message: text to be saved
:return: Tuple of (NamedTemporaryFile, tempfile name)
"""
temp_file = tempfile.NamedTemporaryFile()
temp_file.write(message)
temp_file.flush()
return temp_file, temp_file.name
# NOTE: currently unused; retained for future need
# noinspection PyUnusedFunction
def read_from_xml(self) -> List[str]:
"""
Walk the import dir and return all filenames. We are assuming all xml files.
:return: List of filenames
"""
the_list = []
if self.import_dir is None: # possible if input dir is not specified
_logger.warning("Input directory was not specified -- skipping xml read")
return the_list
for (dirpath, dirnames, filenames) in os.walk(self.import_dir):
the_list.extend(filenames)
break
return the_list
def export_xml(self, feed_name: str, start_time: str, end_time: str, block_num: int, message: AnyStr) -> None:
"""
:param feed_name: name of the feed, for the holding directory name
:param start_time: start time
:param end_time: end time
:param block_num: write block number (for uniqueness)
:param message: feed text
"""
# create a directory to store all content blocks
dir_name = f"{feed_name}".replace(' ', '_')
full_dir_name = os.path.join(self.export_dir, dir_name)
# Make sure the directory exists
if not os.path.exists(os.path.join(self.export_dir, dir_name)):
os.mkdir(full_dir_name)
# Actually write the file
file_name = f"{start_time}-{end_time}-{block_num}".replace(' ', "_")
full_file_name = os.path.join(full_dir_name, file_name)
with open(full_file_name, 'wb') as file_handle:
file_handle.write(message)
def _import_collection(self, client: Union[Client10, Client11], site: dict, collection: CabbyCollection,
data_set: bool = False) -> int:
"""
Import a taxii client collectio into a feed.
:param client: Taxii spec client v1.0 or v1.1
:param site: site definition
:param collection: cabby collection
:param data_set: True if DATA_SET, False otherwise
:return: the EDR feed id, or -1 if not available
"""
global BINDING_CHOICES
collection_name = collection.name
display_name = f"{site.get('site')} {collection_name}"
sanitized_feed_name = cleanup_string(site.get('site') + collection_name)
feed_summary = display_name
available = collection.available
collection_type = collection.type
default_score = site.get('default_score')
_logger.info(f"Working on SITE {site.get('site')}, NAME {collection_name}, FEED {sanitized_feed_name}, "
f"AVAIL {available}, TYPE {collection_type}")
_logger.info('-' * 80)
# if not available, nothing else to do
if not available:
return -1
# Sanity check on start date; provide a bare minimum
start_date_str = site.get('start_date')
if not start_date_str or len(start_date_str) == 0:
start_date_str = "2019-01-01 00:00:00"
# Create a feed helper object
feed_helper = FeedHelper(site.get('output_path'), sanitized_feed_name, site.get('minutes_to_advance'),
start_date_str, reset_start_date=site.get('reset_start_date', False))
if not data_set:
_logger.info("Feed start time %s" % feed_helper.start_date)
#
# Build up the URI for polling
#
if not site.get('poll_path', ''):
uri: Optional[str] = None
else:
uri: str = ''
if site.get('use_https'):
uri += 'https://'
else:
uri += 'http://'
uri += site.get('site')
uri += site.get('poll_path')
_logger.info(f'Poll path: {uri}')
# build up all the reports for the feed
reports: List[Dict[str, Any]] = []
while True:
num_times_empty_content_blocks = 0
try:
try:
_logger.info(f"Polling Collection: {collection.name} ...")
content_blocks = client.poll(collection_name=collection.name, begin_date=feed_helper.start_date,
end_date=feed_helper.end_date, content_bindings=BINDING_CHOICES,
uri=uri)
except Exception as e:
_logger.info(f"{e}")
content_blocks = []
#
# Iterate through all content_blocks
#
num_blocks = 0
if not data_set:
_logger.info(f" ... start_date: {feed_helper.start_date}, end_date: {feed_helper.end_date}")
for block in content_blocks:
_logger.debug(block.content)
#
# if in export mode then save off this content block
#
if self.export_dir:
self.export_xml(collection_name,
feed_helper.start_date,
feed_helper.end_date,
num_blocks,
block.content)
#
# This code accounts for a case found with ThreatCentral.io where the content is url encoded.
# etree.fromstring can parse this data.
#
try:
root = etree.fromstring(block.content)
content = root.find('.//{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content')
if content is not None and len(content) == 0 and len(list(content)) == 0:
#
# Content has no children. So lets make sure we parse the xml text for content and re-add
# it as valid XML so we can parse
#
new_stix_package = etree.fromstring(root.find(
"{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}"
"Content_Block/{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content").text)
content.append(new_stix_package)
#
# Since we modified the xml, we need create a new xml message string to parse
#
message = etree.tostring(root)
#
# Write the content block to disk so we can parse with python stix
#
file_handle, file_path = self.write_to_temp_file(message)
#
# Parse STIX data
#
stix_package = STIXPackage.from_xml(file_path)
#
# if it is a DATA_SET make feed_summary from the stix_header description
# NOTE: this is for RecordedFuture, also note that we only do this for data_sets.
# to date I have only seen RecordedFuture use data_sets
#
if data_set and stix_package.stix_header and stix_package.stix_header.descriptions:
for desc in stix_package.stix_header.descriptions:
feed_summary = f"{desc.value}: {collection_name}"
break
#
# Get the timestamp of the STIX Package so we can use this in our feed
#
timestamp = dt_to_seconds(stix_package.timestamp)
# check for empty content in this block; we break out after 10 empty blocks
if not stix_package.indicators and not stix_package.observables:
num_times_empty_content_blocks += 1
if num_times_empty_content_blocks > 10:
break
# Go through all STIX indicators
if stix_package.indicators:
for indicator in stix_package.indicators:
if not indicator or not indicator.observable:
continue
if indicator.confidence:
if str(indicator.confidence.value).isdigit():
#
# Get the confidence score and use it for our score
#
score = int(indicator.confidence.to_dict().get("value", default_score))
else:
if str(indicator.confidence.value).lower() == "high":
score = 75
elif str(indicator.confidence.value).lower() == "medium":
score = 50
elif str(indicator.confidence.value).lower() == "low":
score = 25
else:
score = default_score
else:
score = default_score
if not indicator.timestamp:
timestamp = 0
else:
timestamp = int((indicator.timestamp - EPOCH).total_seconds())
# Cybox observable returns a list
reports.extend(cybox_parse_observable(indicator.observable, indicator, timestamp,
score))
#
# Now lets find some data. Iterate through all observables and parse
#
if stix_package.observables:
for observable in stix_package.observables:
if not observable:
continue
# Cybox observable returns a list
reports.extend(cybox_parse_observable(observable, None, timestamp, default_score))
#
# Delete our temporary file
#
file_handle.close()
# increase block count
num_blocks += 1
except Exception as e:
_logger.info(f"{e}")
continue
_logger.info(f"content blocks read: {num_blocks}")
_logger.info(f"current number of reports: {len(reports)}")
if len(reports) > site.get('reports_limit'):
_logger.info(f"We have reached the reports limit of {site.get('reports_limit')}")
break
except Exception as e:
_logger.info(f"{e}")
# If it is just a data_set, the data is unordered, so we can just break out of the while loop
if data_set:
break
if feed_helper.advance():
continue
else:
break
_logger.info(f"Found {len(reports)} new reports.")
if not data_set:
# We only want to concatenate if we are NOT a data set, otherwise we want to refresh all the reports
_logger.info("Adding existing reports...")
reports = feed_helper.load_existing_feed_data() + reports
_logger.info(f"Total number of reports: {len(reports)}")
if site.get('reports_limit') < len(reports):
_logger.info("Truncating reports to length {0}".format(site.get('reports_limit')))
reports = reports[:site.get('reports_limit')]
try:
use_icon = site.get('icon_link')
if not os.path.exists(use_icon):
_logger.warning(f"Unable to find feed icon at path {use_icon}")
use_icon = None
data = build_feed_data(sanitized_feed_name,
display_name,
feed_summary,
site.get('site'),
use_icon,
reports)
except Exception as err:
_logger.warning(f"Failed to create feed data for {sanitized_feed_name}: {err}")
return -1
if feed_helper.write_feed(data):
feed_helper.save_details()
#
# Create Cb Response Feed if necessary
#
feed_id = None
try:
feeds = get_object_by_name_or_id(self.cb, Feed, name=sanitized_feed_name)
if not feeds:
_logger.info(f"Feed {sanitized_feed_name} was not found, so we are going to create it...")
elif len(feeds) > 1:
_logger.warning(f"Multiple feeds found, selecting Feed id {feeds[0].id}")
feed_id = feeds[0].id
elif feeds:
feed_id = feeds[0].id
_logger.info(f"Feed `{sanitized_feed_name}` was found as Feed ID {feed_id}")
except Exception as e:
_logger.info(f"{e}")
if not feed_id:
_logger.info(f" ... creating {sanitized_feed_name} feed for the first time")
f = self.cb.create(Feed)
f.feed_url = "file://" + feed_helper.path
f.enabled = site.get('feeds_enable')
f.use_proxy = False
f.validate_server_cert = False
try:
f.save()
except ServerError as se:
if se.error_code == 500:
_logger.warning(" ... could not add feed:")
_logger.warning(" Received error code 500 from server. This is usually because "
"the server cannot retrieve the feed.")
_logger.warning(" Check to ensure the Cb server has network connectivity "
"and the credentials are correct.")
_logger.warning("!" * 80 + "\n")
else:
info = feed_helper.dump_feedinfo()
_logger.info(f" ... Could not add feed: {se}\n >> {info}")
except Exception as e:
info = feed_helper.dump_feedinfo()
_logger.warning(f" ... Could not add feed: {e}\n >> {info}")
_logger.warning("!" * 80 + "\n")
else:
_logger.info(f"Feed data: {f}")
_logger.info(f"Added feed. New feed ID is {f.id}")
feed_id = f.id
return feed_id
def perform(self) -> None:
"""
Perform the taxii hailing service.
"""
for site in self.sites:
client: Union[Client10, Client11] = create_client(site.get('site'),
use_https=site.get('use_https'),
discovery_path=site.get('discovery_path'))
#
# Set verify_ssl and ca_cert inside the client
#
client.set_auth(verify_ssl=site.get('ssl_verify'), ca_cert=site.get('ca_cert'))
#
# Proxy Settings
#
proxy_dict = dict()
if self.http_proxy_url:
_logger.info(f"Found HTTP Proxy: {self.http_proxy_url}")
proxy_dict['http'] = self.http_proxy_url
if self.https_proxy_url:
_logger.info(f"Found HTTPS Proxy: {self.https_proxy_url}")
proxy_dict['https'] = self.https_proxy_url
if proxy_dict:
client.set_proxies(proxy_dict)
# If a username is supplied use basic authentication
if site.get('username') or site.get('cert_file'):
_logger.info("Found Username in config, using basic auth...")
client.set_auth(username=site.get('username'),
password=site.get('password'),
verify_ssl=site.get('ssl_verify'),
ca_cert=site.get('ca_cert'),
cert_file=site.get('cert_file'),
key_file=site.get('key_file'))
if not site.get('collection_management_path', ''):
collections = client.get_collections()
else:
uri = ''
if site.get('use_https'):
uri += 'https://'
else:
uri += 'http://'
uri += site.get('site')
uri += site.get('collection_management_path')
_logger.info('Collection Management Path: {}'.format(uri))
collections: List[CabbyCollection] = client.get_collections(uri=uri)
if len(collections) == 0:
_logger.info('Unable to find any collections. Exiting...')
sys.exit(0)
_logger.info("=" * 80)
for collection in collections:
_logger.info(f'Collection Name: {collection.name}, Collection Type: {collection.type}')
_logger.info("=" * 80 + "\n")
desired_collections = [x.strip() for x in site.get('collections').lower().split(',')]
want_all = False
if '*' in desired_collections:
want_all = True
for collection in collections:
if collection.type != 'DATA_FEED' and collection.type != 'DATA_SET':
continue
if collection.type == 'DATA_SET':
data_set = True
else:
data_set = False
if want_all or collection.name.lower() in desired_collections:
self._import_collection(client, site, collection, data_set)
class CbTaxiiFeedConverter(object):
def __init__(self,
config_file_path,
debug_mode=False,
import_dir='',
export_dir=''):
#
# parse config file and save off the information we need
#
config_dict = parse_config(config_file_path)
self.server_url = config_dict.get('server_url', 'https://127.0.0.1')
self.api_token = config_dict.get('api_token', '')
self.sites = config_dict.get('sites', [])
self.debug = config_dict.get('debug', False)
self.export_dir = export_dir
self.import_dir = import_dir
self.integration_name = 'Cb Taxii Connector 1.6.5'
self.http_proxy_url = config_dict.get('http_proxy_url', None)
self.https_proxy_url = config_dict.get('https_proxy_url', None)
if self.export_dir and not os.path.exists(self.export_dir):
os.mkdir(self.export_dir)
#
# Test Cb Response connectivity
#
try:
self.cb = CbResponseAPI(url=self.server_url,
token=self.api_token,
ssl_verify=False,
integration_name=self.integration_name)
self.cb.info()
except:
logger.error(traceback.format_exc())
sys.exit(-1)
def write_to_temp_file(self, message):
temp_file = tempfile.NamedTemporaryFile()
temp_file.write(message)
temp_file.flush()
return temp_file, temp_file.name
def read_from_xml(self):
"""
Walk the import dir and return all filenames. We are assuming all xml files
:return:
"""
f = []
for (dirpath, dirnames, filenames) in os.walk(self.import_dir):
f.extend(filenames)
break
return f
def export_xml(self, feed_name, start_time, end_time, block_num, message):
"""
:param feed_name:
:param start_time:
:param end_time:
:param block_num:
:param message:
:return:
"""
#
# create a directory to store all content blocks
#
dir_name = "{}".format(feed_name).replace(' ', '_')
full_dir_name = os.path.join(self.export_dir, dir_name)
#
# Make sure the directory exists
#
if not os.path.exists(os.path.join(self.export_dir, dir_name)):
os.mkdir(full_dir_name)
#
# Actually write the file
#
file_name = "{}-{}-{}".format(start_time, end_time,
block_num).replace(' ', "_")
full_file_name = os.path.join(full_dir_name, file_name)
with open(full_file_name, 'wb') as file_handle:
file_handle.write(message)
def _import_collection(self, client, site, collection, data_set=False):
collection_name = collection.name
sanitized_feed_name = cleanup_string(
"%s%s" % (site.get('site'), collection_name))
feed_summary = "%s %s" % (site.get('site'), collection_name)
available = collection.available
collection_type = collection.type
default_score = site.get('default_score')
logger.info("%s,%s,%s,%s,%s" %
(site.get('site'), collection_name, sanitized_feed_name,
available, collection_type))
if not available:
return False
#
# Sanity check on start date
#
start_date_str = site.get('start_date')
if not start_date_str or len(start_date_str) == 0:
start_date_str = "2019-01-01 00:00:00"
#
# Create a feed helper object
#
feed_helper = FeedHelper(site.get('output_path'), sanitized_feed_name,
site.get('minutes_to_advance'),
start_date_str)
if not data_set:
logger.info("Feed start time %s" % feed_helper.start_date)
logger.info("polling Collection: {}...".format(collection.name))
#
# Build up the URI for polling
#
if not site.get('poll_path', ''):
uri = None
else:
uri = ''
if site.get('use_https'):
uri += 'https://'
else:
uri += 'http://'
uri += site.get('site')
uri += site.get('poll_path')
logger.info('Poll path: {}'.format(uri))
reports = []
while True:
num_times_empty_content_blocks = 0
try:
try:
logger.info("Polling Collection: {0}".format(
collection.name))
content_blocks = client.poll(
uri=uri,
collection_name=collection.name,
begin_date=feed_helper.start_date,
end_date=feed_helper.end_date,
content_bindings=BINDING_CHOICES)
except Exception as e:
logger.info(e.message)
content_blocks = []
#
# Iterate through all content_blocks
#
num_blocks = 0
if not data_set:
logger.info("polling start_date: {}, end_date: {}".format(
feed_helper.start_date, feed_helper.end_date))
for block in content_blocks:
logger.debug(block.content)
#
# if in export mode then save off this content block
#
if self.export_dir:
self.export_xml(collection_name,
feed_helper.start_date,
feed_helper.end_date, num_blocks,
block.content)
#
# This code accounts for a case found with ThreatCentral.io where the content is url encoded.
# etree.fromstring can parse this data.
#
try:
root = etree.fromstring(block.content)
content = root.find(
'.//{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content'
)
if content is not None and len(content) == 0 and len(
list(content)) == 0:
#
# Content has no children. So lets make sure we parse the xml text for content and re-add
# it as valid XML so we can parse
#
new_stix_package = etree.fromstring(
root.find(
"{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content_Block/{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content"
).text)
content.append(new_stix_package)
#
# Since we modified the xml, we need create a new xml message string to parse
#
message = etree.tostring(root)
#
# Write the content block to disk so we can parse with python stix
#
file_handle, file_path = self.write_to_temp_file(
message)
#
# Parse STIX data
#
stix_package = STIXPackage.from_xml(file_path)
#
# if it is a DATA_SET make feed_summary from the stix_header description
# NOTE: this is for RecordedFuture, also note that we only do this for data_sets.
# to date I have only seen RecordedFuture use data_sets
#
if data_set and stix_package.stix_header and stix_package.stix_header.descriptions:
for desc in stix_package.stix_header.descriptions:
feed_summary = "{}: {}".format(
desc.value, collection_name)
break
#
# Get the timestamp of the STIX Package so we can use this in our feed
#
timestamp = total_seconds(stix_package.timestamp)
if not stix_package.indicators and not stix_package.observables:
num_times_empty_content_blocks += 1
if num_times_empty_content_blocks > 10:
break
if stix_package.indicators:
for indicator in stix_package.indicators:
if not indicator or not indicator.observable:
continue
if indicator.confidence:
if str(indicator.confidence.value).isdigit(
):
#
# Get the confidence score and use it for our score
#
score = int(
indicator.confidence.to_dict().get(
"value", default_score))
else:
if str(indicator.confidence.value
).lower() == "high":
score = 75
elif str(indicator.confidence.value
).lower() == "medium":
score = 50
elif str(indicator.confidence.value
).lower() == "low":
score = 25
else:
score = default_score
else:
score = default_score
if not indicator.timestamp:
timestamp = 0
else:
timestamp = int(
(indicator.timestamp -
datetime.datetime(1970, 1, 1).replace(
tzinfo=dateutil.tz.tzutc())
).total_seconds())
reports.extend(
cybox_parse_observable(
indicator.observable, indicator,
timestamp, score))
#
# Now lets find some data. Iterate through all observables and parse
#
if stix_package.observables:
for observable in stix_package.observables:
if not observable:
continue
#
# Cybox observable returns a list
#
reports.extend(
cybox_parse_observable(
observable, None, timestamp,
default_score))
#
# Delete our temporary file
#
file_handle.close()
num_blocks += 1
#
# end for loop through content blocks
#
except Exception as e:
# logger.info(traceback.format_exc())
logger.info(e.message)
continue
logger.info("content blocks read: {}".format(num_blocks))
logger.info("current number of reports: {}".format(
len(reports)))
if len(reports) > site.get('reports_limit'):
logger.info(
"We have reached the reports limit of {0}".format(
site.get('reports_limit')))
break
#
# DEBUG CODE
#
# if len(reports) > 10:
# break
#
# Attempt to advance the start time and end time
#
except Exception as e:
logger.info(traceback.format_exc())
#
# If it is just a data_set, the data is unordered, so we can just break out of the while loop
#
if data_set:
break
if feed_helper.advance():
continue
else:
break
#
# end While True
#
logger.info("Found {} new reports.".format(len(reports)))
if not data_set:
#
# We only want to concatenate if we are NOT a data set, otherwise we want to refresh all the reports
#
logger.info("Adding existing reports...")
reports = feed_helper.load_existing_feed_data() + reports
logger.info("Total number of reports: {}".format(len(reports)))
if site.get('reports_limit') < len(reports):
logger.info("Truncating reports to length {0}".format(
site.get('reports_limit')))
reports = reports[:site.get('reports_limit')]
data = build_feed_data(sanitized_feed_name,
"%s %s" % (site.get('site'), collection_name),
feed_summary, site.get('site'),
site.get('icon_link'), reports)
if feed_helper.write_feed(data):
feed_helper.save_details()
#
# Create Cb Response Feed if necessary
#
feed_id = None
try:
feeds = get_object_by_name_or_id(self.cb,
Feed,
name=sanitized_feed_name)
if not feeds:
logger.info(
"Feed {} was not found, so we are going to create it".
format(sanitized_feed_name))
elif len(feeds) > 1:
logger.warning(
"Multiple feeds found, selecting Feed id {}".format(
feeds[0].id))
feed_id = feeds[0].id
elif feeds:
feed_id = feeds[0].id
logger.info("Feed {} was found as Feed ID {}".format(
sanitized_feed_name, feed_id))
except Exception as e:
logger.info(e.message)
if not feed_id:
logger.info("Creating {} feed for the first time".format(
sanitized_feed_name))
f = self.cb.create(Feed)
f.feed_url = "file://" + feed_helper.path
f.enabled = site.get('feeds_enable')
f.use_proxy = False
f.validate_server_cert = False
try:
f.save()
except ServerError as se:
if se.error_code == 500:
logger.info("Could not add feed:")
logger.info(
" Received error code 500 from server. This is usually because the server cannot retrieve the feed."
)
logger.info(
" Check to ensure the Cb server has network connectivity and the credentials are correct."
)
else:
logger.info("Could not add feed: {0:s}".format(str(se)))
except Exception as e:
logger.info("Could not add feed: {0:s}".format(str(e)))
else:
logger.info("Feed data: {0:s}".format(str(f)))
logger.info("Added feed. New feed ID is {0:d}".format(f.id))
feed_id = f.id
return feed_id
def perform(self):
"""
:param self:
:param enumerate_collections_only:
:return:
"""
for site in self.sites:
client = create_client(site.get('site'),
use_https=site.get('use_https'),
discovery_path=site.get('discovery_path'))
#
# Set verify_ssl and ca_cert inside the client
#
client.set_auth(verify_ssl=site.get('ssl_verify'),
ca_cert=site.get('ca_cert'))
#
# Proxy Settings
#
proxy_dict = dict()
if self.http_proxy_url:
logger.info("Found HTTP Proxy: {}".format(self.http_proxy_url))
proxy_dict['http'] = self.http_proxy_url
if self.https_proxy_url:
logger.info("Found HTTPS Proxy: {}".format(
self.https_proxy_url))
proxy_dict['https'] = self.https_proxy_url
if proxy_dict:
client.set_proxies(proxy_dict)
if site.get('username') or site.get('cert_file'):
#
# If a username is supplied use basic authentication
#
logger.info("Found Username in config, using basic auth...")
client.set_auth(username=site.get('username'),
password=site.get('password'),
verify_ssl=site.get('ssl_verify'),
ca_cert=site.get('ca_cert'),
cert_file=site.get('cert_file'),
key_file=site.get('key_file'))
if not site.get('collection_management_path', ''):
collections = client.get_collections()
else:
uri = ''
if site.get('use_https'):
uri += 'https://'
else:
uri += 'http://'
uri += site.get('site')
uri += site.get('collection_management_path')
logger.info('Collection Management Path: {}'.format(uri))
collections = client.get_collections(uri=uri)
for collection in collections:
logger.info('Collection Name: {}, Collection Type: {}'.format(
collection.name, collection.type))
if len(collections) == 0:
logger.info('Unable to find any collections. Exiting...')
sys.exit(0)
desired_collections = [
x.strip() for x in site.get('collections').lower().split(',')
]
want_all = False
if '*' in desired_collections:
want_all = True
for collection in collections:
if collection.type != 'DATA_FEED' and collection.type != 'DATA_SET':
continue
if collection.type == 'DATA_SET':
data_set = True
else:
data_set = False
if want_all or collection.name.lower() in desired_collections:
self._import_collection(client, site, collection, data_set)
class CarbonBlackThreatConnectBridge(CbIntegrationDaemon):
def __init__(self,
name,
configfile,
logfile=None,
pidfile=None,
debug=False):
CbIntegrationDaemon.__init__(self,
name,
configfile=configfile,
logfile=logfile,
pidfile=pidfile,
debug=debug)
template_folder = "/usr/share/cb/integrations/cb-threatconnect-connector/content"
self.flask_feed = cbint.utils.flaskfeed.FlaskFeed(
__name__, False, template_folder)
self.bridge_options = {}
self.bridge_auth = {}
self.api_urns = {}
self.validated_config = False
self.cb = None
self.sync_needed = False
self.feed_name = "threatconnectintegration"
self.display_name = "ThreatConnect"
self.feed = {}
self.directory = template_folder
self.cb_image_path = "/carbonblack.png"
self.integration_image_path = "/threatconnect.png"
self.integration_image_small_path = "/threatconnect-small.png"
self.json_feed_path = "/threatconnect/json"
self.feed_lock = threading.RLock()
self.logfile = logfile
self.flask_feed.app.add_url_rule(
self.cb_image_path, view_func=self.handle_cb_image_request)
self.flask_feed.app.add_url_rule(
self.integration_image_path,
view_func=self.handle_integration_image_request)
self.flask_feed.app.add_url_rule(
self.json_feed_path,
view_func=self.handle_json_feed_request,
methods=['GET'])
self.flask_feed.app.add_url_rule("/",
view_func=self.handle_index_request,
methods=['GET'])
self.flask_feed.app.add_url_rule(
"/feed.html",
view_func=self.handle_html_feed_request,
methods=['GET'])
self.initialize_logging()
logger.debug("generating feed metadata")
with self.feed_lock:
self.feed = cbint.utils.feed.generate_feed(
self.feed_name,
summary=
"Threat intelligence data provided by ThreatConnect to the Carbon Black Community",
tech_data=
"There are no requirements to share any data to receive this feed.",
provider_url="http://www.threatconnect.com/",
icon_path="%s/%s" %
(self.directory, self.integration_image_path),
small_icon_path="%s/%s" %
(self.directory, self.integration_image_small_path),
display_name=self.display_name,
category="Partner")
self.last_sync = "No sync performed"
self.last_successful_sync = "No sync performed"
def initialize_logging(self):
if not self.logfile:
log_path = "/var/log/cb/integrations/%s/" % self.name
cbint.utils.filesystem.ensure_directory_exists(log_path)
self.logfile = "%s%s.log" % (log_path, self.name)
root_logger = logging.getLogger()
root_logger.setLevel(logging.INFO)
root_logger.handlers = []
rlh = RotatingFileHandler(self.logfile,
maxBytes=524288,
backupCount=10)
rlh.setFormatter(
logging.Formatter(
fmt="%(asctime)s: %(module)s: %(levelname)s: %(message)s"))
root_logger.addHandler(rlh)
@property
def integration_name(self):
return 'Cb ThreatConnect Connector 1.2.9'
def serve(self):
if "https_proxy" in self.bridge_options:
os.environ['HTTPS_PROXY'] = self.bridge_options.get(
"https_proxy", "")
os.environ['no_proxy'] = '127.0.0.1,localhost'
address = self.bridge_options.get('listener_address', '127.0.0.1')
port = self.bridge_options['listener_port']
logger.info("starting flask server: %s:%s" % (address, port))
self.flask_feed.app.run(port=port,
debug=self.debug,
host=address,
use_reloader=False)
def handle_json_feed_request(self):
with self.feed_lock:
json = self.flask_feed.generate_json_feed(self.feed)
return json
def handle_html_feed_request(self):
with self.feed_lock:
html = self.flask_feed.generate_html_feed(self.feed,
self.display_name)
return html
def handle_index_request(self):
with self.feed_lock:
index = self.flask_feed.generate_html_index(
self.feed, self.bridge_options, self.display_name,
self.cb_image_path, self.integration_image_path,
self.json_feed_path, self.last_sync)
return index
def handle_cb_image_request(self):
return self.flask_feed.generate_image_response(
image_path="%s%s" % (self.directory, self.cb_image_path))
def handle_integration_image_request(self):
return self.flask_feed.generate_image_response(
image_path="%s%s" % (self.directory, self.integration_image_path))
def run(self):
logger.info(
"starting Carbon Black <-> ThreatConnect Connector | version %s" %
version.__version__)
logger.debug("starting continuous feed retrieval thread")
work_thread = threading.Thread(
target=self.perform_continuous_feed_retrieval)
work_thread.setDaemon(True)
work_thread.start()
logger.debug("starting flask")
self.serve()
def validate_config(self):
self.validated_config = True
logger.info("Validating configuration file ...")
if 'bridge' in self.options:
self.bridge_options = self.options['bridge']
else:
logger.error("Configuration does not contain a [bridge] section")
return False
if 'auth' in self.options:
self.bridge_auth = self.options['auth']
else:
logger.error("configuration does not contain a [auth] section")
return False
if 'sources' in self.options:
self.api_urns = self.options["sources"]
else:
logger.error("configuration does not contain a [sources] section")
return False
opts = self.bridge_options
auth = self.bridge_auth
config_valid = True
msgs = []
if len(self.api_urns) <= 0:
msgs.append('No data sources are configured under [sources]')
config_valid = False
item = 'listener_port'
if not (item in opts and opts[item].isdigit()
and 0 < int(opts[item]) <= 65535):
msgs.append(
'the config option listener_port is required and must be a valid port number'
)
config_valid = False
else:
opts[item] = int(opts[item])
item = 'listener_address'
if not (item in opts and opts[item] is not ""):
msgs.append(
'the config option listener_address is required and cannot be empty'
)
config_valid = False
item = 'feed_retrieval_minutes'
if not (item in opts and opts[item].isdigit() and 0 < int(opts[item])):
msgs.append(
'the config option feed_retrieval_minutes is required and must be greater than 1'
)
config_valid = False
else:
opts[item] = int(opts[item])
item = 'ioc_min_score'
if item in opts:
if not (opts[item].isdigit() and 0 <= int(opts[item]) <= 100):
msgs.append(
'The config option ioc_min_score must be a number in the range 0 - 100'
)
config_valid = False
else:
opts[item] = int(opts[item])
else:
logger.warning("No value provided for ioc_min_score. Using 1")
opts[item] = 1
item = 'api_key'
if not (item in auth and auth[item].isdigit()):
msgs.append(
'The config option api_key is required under section [auth] and must be a numeric value'
)
config_valid = False
item = 'url'
if not (item in auth and auth[item] is not ""):
msgs.append(
'The config option url is required under section [auth] and cannot be blank'
)
config_valid = False
if 'secret_key_encrypted' in auth and 'secret_key' not in auth:
msgs.append(
"Encrypted API secret key no longer supported. Use unencrypted 'secret_key' form."
)
config_valid = False
elif 'secret_key' in auth and auth['secret_key'] != "":
auth['api_secret_key'] = self.bridge_auth.get("secret_key")
else:
msgs.append(
'The config option secret_key under section [auth] must be provided'
)
config_valid = False
# Convert all 1 or 0 values to true/false
opts["ignore_ioc_md5"] = opts.get("disable_ioc_md5", "0") == "1"
opts["ignore_ioc_ip"] = opts.get("disable_ioc_ip", "0") == "1"
opts["ignore_ioc_host"] = opts.get("disable_ioc_host", "0") == "1"
# create a cbapi instance
ssl_verify = self.get_config_boolean("carbonblack_server_sslverify",
False)
server_url = self.get_config_string("carbonblack_server_url",
"https://127.0.0.1")
server_token = self.get_config_string("carbonblack_server_token", "")
try:
self.cb = CbResponseAPI(url=server_url,
token=server_token,
ssl_verify=False,
integration_name=self.integration_name)
self.cb.info()
except:
logger.error(traceback.format_exc())
return False
if not config_valid:
for msg in msgs:
sys.stderr.write("%s\n" % msg)
logger.error(msg)
return False
else:
return True
def _filter_results(self, results):
logger.debug("Number of IOCs before filtering applied: %d",
len(results))
opts = self.bridge_options
filter_min_score = opts["ioc_min_score"]
# Filter out those scores lower than the minimum score
if filter_min_score > 0:
results = filter(lambda x: x["score"] >= filter_min_score, results)
logger.debug(
"Number of IOCs after scores less than %d discarded: %d",
filter_min_score, len(results))
# For end user simplicity we call "dns" entries "host" and ipv4 entries "ip"
# format: {"flag_name" : ("official_name", "friendly_name")}
ignore_ioc_mapping = {
"ignore_ioc_md5": ("md5", "md5"),
"ignore_ioc_ip": ("ipv4", "ip"),
"ignore_ioc_host": ("dns", "host")
}
# On a per flag basis discard md5s, ips, or host if the user has requested we do so
# If we don't discard then check if an exclusions file has been specified and discard entries
# that match those in the exclusions file
for ignore_flag in ignore_ioc_mapping:
exclude_type = ignore_ioc_mapping[ignore_flag][0]
exclude_type_friendly_name = ignore_ioc_mapping[ignore_flag][1]
if opts[ignore_flag]:
results = filter(lambda x: exclude_type not in x["iocs"],
results)
logger.debug("Number of IOCs after %s entries discarded: %d",
exclude_type, len(results))
elif 'exclusions' in self.options and exclude_type_friendly_name in self.options[
'exclusions']:
file_path = self.options['exclusions'][
exclude_type_friendly_name]
if not os.path.exists(file_path):
logger.debug("Exclusions file %s not found", file_path)
continue
with open(file_path, 'r') as exclude_file:
data = frozenset([line.strip() for line in exclude_file])
results = filter(
lambda x: exclude_type not in x["iocs"] or x["iocs"][
exclude_type][0] not in data, results)
logger.debug(
"Number of IOCs after %s exclusions file applied: %d",
exclude_type_friendly_name, len(results))
return results
def perform_continuous_feed_retrieval(self, loop_forever=True):
try:
# config validation is critical to this connector working correctly
if not self.validated_config:
self.validate_config()
opts = self.bridge_options
auth = self.bridge_auth
while True:
logger.debug("Starting retrieval iteration")
try:
tc = ThreatConnectFeedGenerator(auth["api_key"],
auth['api_secret_key'],
auth["url"],
self.api_urns.items())
tmp = tc.get_threatconnect_iocs()
tmp = self._filter_results(tmp)
with self.feed_lock:
self.feed["reports"] = tmp
self.last_sync = strftime(
"%a, %d %b %Y %H:%M:%S +0000", gmtime())
self.last_successful_sync = strftime(
"%a, %d %b %Y %H:%M:%S +0000", gmtime())
logger.info("Successfully retrieved data at %s" %
self.last_successful_sync)
except ConnectionException as e:
logger.error("Error connecting to Threat Connect: %s" %
e.value)
self.last_sync = self.last_successful_sync + " (" + str(
e.value) + ")"
if not loop_forever:
sys.stderr.write(
"Error connecting to Threat Connect: %s\n" %
e.value)
sys.exit(2)
except Exception as e:
logger.error(traceback.format_exc())
time.sleep(opts.get('feed_retrieval_minutes') * 60)
# synchronize feed with Carbon Black server
if not "skip_cb_sync" in opts:
try:
feeds = get_object_by_name_or_id(self.cb,
Feed,
name=self.feed_name)
except Exception as e:
logger.error(e.message)
feeds = None
if not feeds:
logger.info(
"Feed {} was not found, so we are going to create it"
.format(self.feed_name))
f = self.cb.create(Feed)
f.feed_url = "http://{0}:{1}/threatconnect/json".format(
self.bridge_options.get('feed_host', '127.0.0.1'),
self.bridge_options.get('listener_port', '6100'))
f.enabled = True
f.use_proxy = False
f.validate_server_cert = False
try:
f.save()
except ServerError as se:
if se.error_code == 500:
logger.info("Could not add feed:")
logger.info(
" Received error code 500 from server. This is usually because the server cannot retrieve the feed."
)
logger.info(
" Check to ensure the Cb server has network connectivity and the credentials are correct."
)
else:
logger.info("Could not add feed: {0:s}".format(
str(se)))
except Exception as e:
logger.info("Could not add feed: {0:s}".format(
str(e)))
else:
logger.info("Feed data: {0:s}".format(str(f)))
logger.info(
"Added feed. New feed ID is {0:d}".format(
f.id))
f.synchronize(False)
elif len(feeds) > 1:
logger.warning(
"Multiple feeds found, selecting Feed id {}".
format(feeds[0].id))
elif feeds:
feed_id = feeds[0].id
logger.info("Feed {} was found as Feed ID {}".format(
self.feed_name, feed_id))
feeds[0].synchronize(False)
logger.debug("ending feed retrieval loop")
# Function should only ever return when loop_forever is set to false
if not loop_forever:
return self.flask_feed.generate_json_feed(self.feed).data
time.sleep(opts.get('feed_retrieval_minutes') * 60)
except Exception:
# If an exception makes us exit then log what we can for our own sake
logger.fatal(
"FEED RETRIEVAL LOOP IS EXITING! Daemon should be restarted to restore functionality! "
)
logger.fatal("Fatal Error Encountered:\n %s" %
traceback.format_exc())
sys.stderr.write(
"FEED RETRIEVAL LOOP IS EXITING! Daemon should be restarted to restore functionality!\n"
)
sys.stderr.write("Fatal Error Encountered:\n %s\n" %
traceback.format_exc())
sys.exit(3)
# If we somehow get here the function is going to exit.
# This is not normal so we LOUDLY log the fact
logger.fatal(
"FEED RETRIEVAL LOOP IS EXITING! Daemon should be restarted to restore functionality!"
)
class CarbonBlackThreatConnectBridge(CbIntegrationDaemon):
def __init__(self,
name,
configfile,
logfile=None,
pidfile=None,
debug=False):
CbIntegrationDaemon.__init__(self,
name,
configfile=configfile,
logfile=logfile,
pidfile=pidfile,
debug=debug)
template_folder = "/usr/share/cb/integrations/cb-threatconnect-connector/content"
self.flask_feed = cbint.utils.flaskfeed.FlaskFeed(
__name__, False, template_folder)
self.bridge_options = {}
self.tc_config = {}
self.api_urns = {}
self.validated_config = False
self.cb = None
self.sync_needed = False
self.feed_name = "threatconnectintegration"
self.display_name = "ThreatConnect"
self.feed = {}
self.directory = template_folder
self.cb_image_path = "/carbonblack.png"
self.integration_image_path = "/threatconnect.png"
self.integration_image_small_path = "/threatconnect-small.png"
self.json_feed_path = "/threatconnect/json"
self.feed_lock = threading.RLock()
self.logfile = logfile
self.debug = debug
self.pretty_print_json = False
self.flask_feed.app.add_url_rule(
self.cb_image_path, view_func=self.handle_cb_image_request)
self.flask_feed.app.add_url_rule(
self.integration_image_path,
view_func=self.handle_integration_image_request)
self.flask_feed.app.add_url_rule(
self.json_feed_path,
view_func=self.handle_json_feed_request,
methods=['GET'])
self.flask_feed.app.add_url_rule("/",
view_func=self.handle_index_request,
methods=['GET'])
self.flask_feed.app.add_url_rule(
"/feed.html",
view_func=self.handle_html_feed_request,
methods=['GET'])
self.initialize_logging()
logger.debug("generating feed metadata")
with self.feed_lock:
self.feed = cbint.utils.feed.generate_feed(
self.feed_name,
summary=
"Threat intelligence data provided by ThreatConnect to the Carbon Black Community",
tech_data=
"There are no requirements to share any data to receive this feed.",
provider_url="http://www.threatconnect.com/",
icon_path="%s/%s" %
(self.directory, self.integration_image_path),
small_icon_path="%s/%s" %
(self.directory, self.integration_image_small_path),
display_name=self.display_name,
category="Partner")
self.last_sync = TimeStamp()
self.last_successful_sync = TimeStamp()
self.feed_ready = False
def _read_cached(self):
with self.feed_lock:
if self.feed_ready:
return
folder = self.bridge_options.get("cache_folder", "./")
cbint.utils.filesystem.ensure_directory_exists(folder)
try:
with open(os.path.join(folder, "reports.cache"), "r") as f:
reports = json.loads(f.read())
with self.feed_lock:
if not self.feed_ready:
self.feed["reports"] = reports
self.feed_ready = True
logger.info("Reports loaded from cache.")
except IOError as e:
logger.warning("Cache file missing or invalid: {0}".format(e))
def initialize_logging(self):
if not self.logfile:
log_path = "/var/log/cb/integrations/%s/" % self.name
cbint.utils.filesystem.ensure_directory_exists(log_path)
self.logfile = "%s%s.log" % (log_path, self.name)
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG if self.debug else logging.INFO)
root_logger.handlers = []
rlh = RotatingFileHandler(self.logfile,
maxBytes=524288,
backupCount=10)
rlh.setFormatter(
logging.Formatter(
fmt="%(asctime)s - %(levelname)-7s - %(module)s - %(message)s")
)
root_logger.addHandler(rlh)
self.logger = root_logger
@property
def integration_name(self):
return 'Cb ThreatConnect Connector {0}'.format(version.__version__)
def serve(self):
if "https_proxy" in self.bridge_options:
os.environ['HTTPS_PROXY'] = self.bridge_options.get(
"https_proxy", "")
os.environ['no_proxy'] = '127.0.0.1,localhost'
address = self.bridge_options.get('listener_address', '127.0.0.1')
port = self.bridge_options['listener_port']
logger.info("starting flask server: %s:%s" % (address, port))
self.flask_feed.app.run(port=port,
debug=self.debug,
host=address,
use_reloader=False)
def handle_json_feed_request(self):
with self.feed_lock:
json = self.flask_feed.generate_json_feed(self.feed)
return json
def handle_html_feed_request(self):
with self.feed_lock:
html = self.flask_feed.generate_html_feed(self.feed,
self.display_name)
return html
def handle_index_request(self):
with self.feed_lock:
index = self.flask_feed.generate_html_index(
self.feed, self.bridge_options, self.display_name,
self.cb_image_path, self.integration_image_path,
self.json_feed_path, str(self.last_sync))
return index
def handle_cb_image_request(self):
return self.flask_feed.generate_image_response(
image_path="%s%s" % (self.directory, self.cb_image_path))
def handle_integration_image_request(self):
return self.flask_feed.generate_image_response(
image_path="%s%s" % (self.directory, self.integration_image_path))
def on_starting(self):
self._read_cached()
ThreatConnectDriver.initialize(self.tc_config)
def run(self):
logger.info(
"starting Carbon Black <-> ThreatConnect Connector | version %s" %
version.__version__)
logger.debug("starting continuous feed retrieval thread")
work_thread = threading.Thread(
target=self.perform_continuous_feed_retrieval)
work_thread.setDaemon(True)
work_thread.start()
logger.debug("starting flask")
self.serve()
def validate_config(self):
config_valid = True
if self.validated_config:
return True
self.validated_config = True
logger.debug("Validating configuration file ...")
if 'bridge' in self.options:
self.bridge_options = self.options['bridge']
else:
sys.stderr.write(
"Configuration does not contain a [bridge] section\n")
logger.error("Configuration does not contain a [bridge] section")
return False
self.debug = self.bridge_options.get(
'debug', 'F') in ['1', 't', 'T', 'True', 'true']
log_level = self.bridge_options.get('log_level', 'INFO').upper()
log_level = log_level if log_level in [
"INFO", "WARNING", "DEBUG", "ERROR"
] else "INFO"
self.logger.setLevel(
logging.DEBUG if self.debug else logging.getLevelName(log_level))
tc_options = self.options.get('threatconnect', {})
if not tc_options:
sys.stderr.write(
"Configuration does not contain a [threatconnect] section or section is empty.\n"
)
logger.error(
"configuration does not contain a [threatconnect] section or section is empty."
)
return False
try:
self.tc_config = ThreatConnectConfig(**tc_options)
except Exception as e:
sys.stderr.write("Error: {0}\n".format(e))
logger.error(e)
return False
self.pretty_print_json = self.bridge_options.get(
'pretty_print_json', 'F') in ['1', 't', 'T', 'True', 'true']
ca_file = os.environ.get("REQUESTS_CA_BUNDLE", None)
if ca_file:
logger.info("Using CA Cert file: {0}".format(ca_file))
else:
logger.info("No CA Cert file found.")
opts = self.bridge_options
msgs = []
item = 'listener_port'
if not (item in opts and opts[item].isdigit()
and 0 < int(opts[item]) <= 65535):
msgs.append(
'the config option listener_port is required and must be a valid port number'
)
config_valid = False
else:
opts[item] = int(opts[item])
item = 'listener_address'
if not (item in opts and opts[item]):
msgs.append(
'the config option listener_address is required and cannot be empty'
)
config_valid = False
item = 'feed_retrieval_minutes'
if not (item in opts and opts[item].isdigit() and 0 < int(opts[item])):
msgs.append(
'the config option feed_retrieval_minutes is required and must be greater than 1'
)
config_valid = False
else:
opts[item] = int(opts[item])
# Create a cbapi instance
server_url = self.get_config_string("carbonblack_server_url",
"https://127.0.0.1")
server_token = self.get_config_string("carbonblack_server_token", "")
try:
self.cb = CbResponseAPI(url=server_url,
token=server_token,
ssl_verify=False,
integration_name=self.integration_name)
self.cb.info()
except Exception:
logger.error(traceback.format_exc())
return False
if not config_valid:
for msg in msgs:
sys.stderr.write("%s\n" % msg)
logger.error(msg)
return False
else:
return True
def perform_continuous_feed_retrieval(self, loop_forever=True):
try:
self.validate_config()
opts = self.bridge_options
folder = self.bridge_options.get("cache_folder", "./")
cbint.utils.filesystem.ensure_directory_exists(folder)
while True:
logger.info("Starting feed retrieval.")
errored = True
try:
start = timer()
tc = ThreatConnectDriver(self.tc_config)
reports = tc.generate_reports()
logger.debug(
"Retrieved reports ({0:.3f} seconds).".format(timer() -
start))
if reports:
write_start = timer()
# Instead of rewriting the cache file directly, we're writing to a temporary file
# and then moving it onto the cache file so that we don't have a situation where
# the cache file is only partially written and corrupt or empty.
with open(os.path.join(folder, "reports.cache_new"),
"w") as f:
if self.pretty_print_json:
f.write(json.dumps(reports, indent=2))
else:
f.write(json.dumps(reports))
# This is a quick operation that will not leave the file in an invalid state.
shutil.move(os.path.join(folder, "reports.cache_new"),
os.path.join(folder, "reports.cache"))
logger.debug(
"Finished writing reports to cache ({0:.3f} seconds)."
.format(timer() - write_start))
with self.feed_lock:
if reports:
self.feed["reports"] = reports
self.last_successful_sync.stamp()
logger.info(
"Successfully retrieved data at {0} ({1:.3f} seconds total)"
.format(self.last_successful_sync,
timer() - start))
errored = False
self._sync_cb_feed()
except Exception as e:
logger.exception(
"Error occurred while attempting to retrieve feed: {0}"
.format(e))
self.last_sync.stamp()
logger.debug("Feed report retrieval completed{0}.".format(
" (Errored)" if errored else ""))
if not loop_forever:
return self.flask_feed.generate_json_feed(self.feed).data
# Full sleep interval is taken between feed retrieval work.
time.sleep(opts.get('feed_retrieval_minutes') * 60)
except Exception:
# If an exception makes us exit then log what we can for our own sake
logger.fatal(
"FEED RETRIEVAL LOOP IS EXITING! Daemon should be restarted to restore functionality! "
)
logger.fatal("Fatal Error Encountered:\n %s" %
traceback.format_exc())
sys.stderr.write(
"FEED RETRIEVAL LOOP IS EXITING! Daemon should be restarted to restore functionality!\n"
)
sys.stderr.write("Fatal Error Encountered:\n %s\n" %
traceback.format_exc())
sys.exit(3)
# If we somehow get here the function is going to exit.
# This is not normal so we LOUDLY log the fact
logger.fatal(
"FEED RETRIEVAL LOOP IS EXITING! Daemon should be restarted to restore functionality!"
)
def _sync_cb_feed(self):
opts = self.bridge_options
if "skip_cb_sync" in opts:
return
try:
feeds = get_object_by_name_or_id(self.cb,
Feed,
name=self.feed_name)
except Exception as e:
logger.error(e.message)
feeds = None
if not feeds:
logger.info(
"Feed {} was not found, so we are going to create it".format(
self.feed_name))
f = self.cb.create(Feed)
f.feed_url = "http://{0}:{1}/threatconnect/json".format(
self.bridge_options.get('feed_host', '127.0.0.1'),
self.bridge_options.get('listener_port', '6100'))
f.enabled = True
f.use_proxy = False
f.validate_server_cert = False
try:
f.save()
except ServerError as se:
if se.error_code == 500:
logger.info("Could not add feed:")
logger.info(
" Received error code 500 from server. "
"This is usually because the server cannot retrieve the feed."
)
logger.info(
" Check to ensure the Cb server has network connectivity and the credentials are correct."
)
else:
logger.info("Could not add feed: {0:s}".format(str(se)))
except Exception as e:
logger.info("Could not add feed: {0:s}".format(str(e)))
else:
logger.info("Feed data: {0:s}".format(str(f)))
logger.info("Added feed. New feed ID is {0:d}".format(f.id))
f.synchronize(False)
elif len(feeds) > 1:
logger.warning("Multiple feeds found, selecting Feed id {}".format(
feeds[0].id))
elif feeds:
feed_id = feeds[0].id
logger.info("Feed {} was found as Feed ID {}".format(
self.feed_name, feed_id))
feeds[0].synchronize(False)
