def __init__(self):
config = load(open(r'conf.yaml', 'rb'))
self.address = config['address']
self.port = config['port']
self.banner = config['banner']
self.protection_url = config['protection']['url']
self.protection_token = config['protection']['token']
self.protection_headers = {'X-Auth-Token': self.protection_token}
self.protection_path = config['protection']['path']
self.protection_interval = config['protection']['interval']
self.protection_counter = config['protection']['counter']
self.response_url = config['response']['url']
self.response_token = config['response']['token']
self.response_path = config['response']['path']
self.archives = config['archives']
self.logs = config['logs']
self.password = config['password']
#~ self.root = getLogger("cbapi")
self.queue = Queue()
self.check_dirs()
setup_logging()
#~ basicConfig(filename=datetime.now().strftime(r'logs/FileCollector_%H_%M_%d_%m_%Y.log'), level=INFO)
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
self.cb = CbApi(self.response_url,
token=self.response_token,
ssl_verify=False)
def __init__(self,
config_file_path,
debug_mode=False,
import_dir='',
export_dir=''):
#
# parse config file and save off the information we need
#
config_dict = parse_config(config_file_path)
self.server_url = config_dict.get('server_url', 'https://127.0.0.1')
self.api_token = config_dict.get('api_token', '')
self.sites = config_dict.get('sites', [])
self.debug = config_dict.get('debug', False)
self.export_dir = export_dir
self.import_dir = import_dir
self.http_proxy_url = config_dict.get('http_proxy_url', None)
self.https_proxy_url = config_dict.get('https_proxy_url', None)
if self.export_dir and not os.path.exists(self.export_dir):
os.mkdir(self.export_dir)
#
# Test Cb Response connectivity
#
try:
self.cb = CbApi(server=self.server_url,
token=self.api_token,
ssl_verify=False)
self.cb.feed_enum()
except:
logger.error(traceback.format_exc())
sys.exit(-1)
def prepare(self):
configuration_dict = splunk.clilib.cli_common.getConfStanza(
'carbonblack', 'cbserver')
cb_server = configuration_dict['cburl']
token = configuration_dict['cbapikey']
self.cb = CbApi(cb_server, token=token, ssl_verify=False)
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
# set up some stat tracking
self.stats = {}
self.stats['total_processes'] = 0
self.stats['total_netconns'] = 0
self.stats['matching_netconns'] = 0
self.stats['output_errors'] = 0
self.stats['process_errors'] = 0
def __init__(self, hosts_mapping):
configuration_dict = splunk.clilib.cli_common.getConfStanza('carbonblack', 'cbserver')
self.cb_server = configuration_dict['cburl']
self.token = configuration_dict['cbapikey']
self.cb = CbApi(self.cb_server, token=self.token, ssl_verify=False)
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.DEBUG)
self.hosts_mapping = hosts_mapping
def __init__(self, config_file_path, debug_mode=False, import_dir='', export_dir=''):
#
# parse config file and save off the information we need
#
config_dict = parse_config(config_file_path)
self.server_url = config_dict.get('server_url', 'https://127.0.0.1')
self.api_token = config_dict.get('api_token', '')
self.sites = config_dict.get('sites', [])
self.debug = config_dict.get('debug', False)
self.export_dir = export_dir
self.import_dir = import_dir
if self.export_dir and not os.path.exists(self.export_dir):
os.mkdir(self.export_dir)
#
# Test Cb Response connectivity
#
try:
self.cb = CbApi(server=self.server_url, token=self.api_token, ssl_verify=False)
self.cb.feed_enum()
except:
logger.error(traceback.format_exc())
sys.exit(-1)
class BinarySearchCommand(GeneratingCommand):
"""Generates a binary search result from Carbon Black from a given MD5 or search query
| binarysearch ${md5}
"""
query = Option(name="query", require=True)
field_names = ['digsig_publisher', 'digsig_result', 'digsig_sign_time', 'host_count', 'is_executable_image',
'last_seen', 'original_filename', 'os_type', 'product_name', 'product_version', 'md5']
def prepare(self):
configuration_dict = splunk.clilib.cli_common.getConfStanza('carbonblack', 'cbserver')
self.cb_server = configuration_dict['cburl']
token = configuration_dict['cbapikey']
self.cb = CbApi(self.cb_server, token=token, ssl_verify=False)
def generate(self):
for bindata in self.cb.binary_search_iter(self.query):
self.logger.info("yielding binary %s" % bindata["md5"])
rawdata = dict((field_name, bindata.get(field_name, "")) for field_name in self.field_names)
try:
rawdata
synthevent = {'sourcetype': 'bit9:carbonblack:json', '_time': time.time(), 'source': self.cb_server,
'_raw': rawdata}
yield synthevent
except Exception:
synthevent = {'sourcetype': 'bit9:carbonblack:json', '_time': time.time(), 'source': self.cb_server,
'_raw': '{"Error":"MD5 not found"}'}
yield synthevent
def prepare(self):
configuration_dict = splunk.clilib.cli_common.getConfStanza('carbonblack', 'cbserver')
self.cb_server = configuration_dict['cburl']
token = configuration_dict['cbapikey']
self.cb = CbApi(self.cb_server, token=token, ssl_verify=False)
def __init__(self, cb_server, token):
self.cb = CbApi(cb_server, token=token, ssl_verify=False)
self.worker_queue = Queue.Queue(maxsize=10)
self.output_queue = Queue.Queue()
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.DEBUG)
logging.basicConfig()
class ProcessSearchCommand(GeneratingCommand):
"""Generates a process search result from Carbon Black from a given IP or search query
| processsearch query=${ip}
"""
query = Option(name="query", require=True)
field_names = [
'cmdline', 'comms_ip', 'hostname', 'id', 'interface_ip', 'last_update',
'os_type', 'parent_md5', 'parent_name', 'parent_pid',
'parent_unique_id', 'path', 'process_md5', 'process_name',
'process_pid', 'regmod_count', 'segment_id', 'sensor_id', 'start',
'unique_id', 'username', 'childproc_count', 'crossproc_count',
'modload_count', 'netconn_count', 'filemod_count', 'group', 'host_type'
]
def prepare(self):
configuration_dict = splunk.clilib.cli_common.getConfStanza(
'carbonblack', 'cbserver')
self.cb_server = configuration_dict['cburl']
self.token = configuration_dict['cbapikey']
self.cb = CbApi(self.cb_server, token=self.token, ssl_verify=False)
def generate(self):
self.logger.info("query %s" % self.query)
i = 0
for bindata in self.cb.process_search_iter(self.query):
i += 1
if i > 1000:
# TODO: let's stop at 1,000 results for now?
self.finish()
return
temp = dict((field_name, bindata[field_name])
for field_name in self.field_names)
temp['sourcetype'] = 'bit9:carbonblack:json'
#
# Sometimes we have seen 'start' be equal to -1
#
try:
temp['_time'] = int(
time.mktime(
dateutil.parser.parse(bindata['start']).timetuple()))
except Exception as e:
self.logger.exception('parsing bindata["start"] %s' %
bindata['start'])
temp['_time'] = 0
temp['link_process'] = self.cb_server + '/#/analyze/' + bindata[
'id'] + "/1"
temp['source'] = 'cbapi'
temp['_raw'] = json.dumps(temp)
yield temp
if i % 10 == 0:
self.flush()
class BinarySearchCommand(GeneratingCommand):
"""Generates a binary search result from Carbon Black from a given MD5 or search query
| binarysearch ${md5}
"""
query = Option(name="query", require=True)
field_names = [
'digsig_publisher', 'digsig_result', 'digsig_sign_time', 'host_count',
'is_executable_image', 'last_seen', 'original_filename', 'os_type',
'product_name', 'product_version', 'md5'
]
def prepare(self):
configuration_dict = splunk.clilib.cli_common.getConfStanza(
'carbonblack', 'cbserver')
cb_server = configuration_dict['cburl']
token = configuration_dict['cbapikey']
self.cb = CbApi(cb_server, token=token, ssl_verify=False)
def generate(self):
for bindata in self.cb.binary_search_iter(self.query):
self.logger.info("yielding binary %s" % bindata["md5"])
yield dict((field_name, bindata.get(field_name, ""))
for field_name in self.field_names)
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
# set up some stat tracking
self.stats = {}
self.stats['total_processes'] = 0
self.stats['total_netconns'] = 0
self.stats['matching_netconns'] = 0
self.stats['output_errors'] = 0
self.stats['process_errors'] = 0
def validate_form(self, form):
if not super(CbModelView, self).validate_form(form):
return False
if not hasattr(form, 'url') or not hasattr(form, 'admin_key'):
return True
cb_server = form.url.data
cb_token = form.admin_key.data
# validate that we can connect to the cb server using the token provided
c = CbApi(cb_server, token=cb_token, ssl_verify=False)
try:
info = c.info()
return True
except:
import traceback
traceback.print_exc()
for field in [form.url, form.admin_key]:
field.errors.append("Could not contact a Cb server at this URL and token")
return False
def on_model_change(self, form, model, is_created):
if is_created:
(already_exist, failed_users, success_users) = bulk_synchronize(
CbApi(model.url, token=model.admin_key, ssl_verify=False)
)
statuses = []
if len(already_exist):
statuses.append("Users {0:s} already existed".format(", ".join(already_exist)))
if len(failed_users):
statuses.append("Failed to add users {0:s}".format(", ".join(failed_users)))
flash(". ".join(statuses))
else:
# all we do here is to update the SAML SP configuration
load_metadata(model.id, model.saml_sp_config)
def main_helper(description, main, custom_required=None, custom_optional=None):
"""
:param description:
:param main:
:return:
"""
default_required = [("-c", "--cburl", "store", None, "server_url",
"CB server's URL. e.g., http://127.0.0.1 "),
("-a", "--apitoken", "store", None, "token",
"API Token for Carbon Black server")]
default_optional = [("-n", "--no-ssl-verify", "store_false", True,
"ssl_verify", "Do not verify server SSL certificate.")
]
if not custom_required:
custom_required = []
if not custom_optional:
custom_optional = []
required = default_required + custom_required
optional = default_optional + custom_optional
parser = build_cli_parser(description, required + optional)
opts, args = parser.parse_args()
for opt in required:
if not getattr(opts, opt[4]):
print
print "** Missing required parameter '%s' **" % opt[4]
print
parser.print_help()
print
sys.exit(-1)
args = {}
for opt in required + optional:
name = opt[4]
args[name] = getattr(opts, name)
cb = CbApi(opts.server_url, ssl_verify=opts.ssl_verify, token=opts.token)
main(cb, args)
class CBQuery(object):
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
def report(self, hostname, user_dictionary):
print ""
print "%s | %s : %s" % ("Hostname", "Process Count", "Username")
print "--------------------------------------------"
for key,value in user_dictionary.items():
print "%s | %s = %s" % (hostname, value, key)
def check(self, hostname):
# print a legend
print ""
print "USER REPORT FOR %s:" % (hostname)
print "--------------------------------------------"
# build the query string
q = "hostname:%s" % (hostname)
#define dictionary
user_dictionary = dict()
# loop over the entire result set
for result in self.cb.process_search_iter(q):
user_name = result.get("username", "<unknown>")
if user_name not in user_dictionary.keys():
print "NEW USER found on %s : %s" % (hostname, user_name)
user_dictionary[user_name] = 1
else:
user_dictionary[user_name] = user_dictionary[user_name] + 1
self.report(hostname, user_dictionary)
class CBQuery(object):
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
def report(self, hostname, user_dictionary):
print ""
print "%s | %s : %s" % ("Hostname", "Process Count", "Username")
print "--------------------------------------------"
for key, value in user_dictionary.items():
print "%s | %s = %s" % (hostname, value, key)
def check(self, hostname):
# print a legend
print ""
print "USER REPORT FOR %s:" % (hostname)
print "--------------------------------------------"
# build the query string
q = "hostname:%s" % (hostname)
#define dictionary
user_dictionary = dict()
# loop over the entire result set
for result in self.cb.process_search_iter(q):
user_name = result.get("username", "<unknown>")
if user_name not in user_dictionary.keys():
print "NEW USER found on %s : %s" % (hostname, user_name)
user_dictionary[user_name] = 1
else:
user_dictionary[user_name] = user_dictionary[user_name] + 1
self.report(hostname, user_dictionary)
class CBQuery(object):
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
def report(self, rundll_query, dll_dictionary, search_match_count):
# CALLED BY: self.report(regex, regex_match_dictionary, search_match_count)
print "--------------------------------------------"
print "%s Command Line Matches:" % (search_match_count)
print "%s : %s" % ("Search Match Count", "Command Line Match")
print "--------------------------------------------"
#ordered_dll_dictionary = collections.OrderedDict(sorted(dll_dictionary.items()))
ordered_dll_dictionary = sorted(dll_dictionary.items(),
key=operator.itemgetter(1))
for value in ordered_dll_dictionary:
print "%s : %s" % (value[1], value[0])
def check(self, regex, ignore_case, group_reference_to_match, count_flag,
matches_only_flag):
# CALLED BY: cb.check(opts.regex, opts.ignore_case, opts.group_reference_to_match, opts.count_flag, opts.matches_only_flag)
# print a legend
print ""
print "Displaying Report for Commandline regular expression matches"
print ""
print "Command Line Strings Matching REGEX: %s" % (regex)
print "============================================================"
print ""
# build the query string
q = "cmdline:*"
#define dictionary
regex_match_dictionary = dict()
search_match_count = 0
#define regexp
# check if we need to ignore case, if so, update regexp
if ignore_case:
regexp = re.compile(regex, re.IGNORECASE)
else:
regexp = re.compile(regex)
for result in self.cb.process_search_iter(q):
cmdline = result.get("cmdline", "<unknown>")
# print "CMD: %s" % (cmdline,)
#SEARCH FOR REGEX IN STRING!!
if matches_only_flag:
# print "-----MATCHES ONLY"
search_match_result = regexp.match(cmdline)
else:
# print "-----EVERYTHING"
search_match_result = regexp.search(cmdline)
if search_match_result is not None:
# print "cmdline: %s" % (cmdline)
# print "result: %s" % (search_match_result)
# print "------------------------------------"
# Iterate TOTAL Search Match Count
search_match_count = search_match_count + 1
# On Match, add to dictionary
# 1st Check group_reference_to_match flag to see if we need to add a specific Group Reference or just the entire Command Line as the regex match
if group_reference_to_match:
# print "cmdline: %s" % (cmdline)
# print"matching GROUP: %s" % (group_reference_to_match)
# print"search_match_result: %s" % (search_match_result)
regex_match_group_reference = search_match_result.group(
int(group_reference_to_match))
if regex_match_group_reference not in regex_match_dictionary.keys(
):
print "%s" % (regex_match_group_reference)
regex_match_dictionary[regex_match_group_reference] = 1
else:
regex_match_dictionary[
regex_match_group_reference] = regex_match_dictionary[
regex_match_group_reference] + 1
else:
if cmdline not in regex_match_dictionary.keys():
print "%s" % (cmdline)
regex_match_dictionary[cmdline] = 1
else:
regex_match_dictionary[
cmdline] = regex_match_dictionary[cmdline] + 1
self.report(regex, regex_match_dictionary, search_match_count)
class CBQuery(object):
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
def report(self, hostname, result):
# return the events associated with this process segment
# this will include netconns, as well as modloads, filemods, etc.
events = self.cb.process_events(result["id"], result["segment_id"])
proc = events["process"]
# for convenience, use locals for some process metadata fields
process_name = result.get("process_name", "<unknown>")
user_name = result.get("username", "<unknown>")
process_md5 = result.get("process_md5", "<unknown>")
# the search criteria (netconn_count:[1 to *]) should ensure that
# all results have at least one netconn
if proc.has_key("netconn_complete"):
# examine each netconn in turn
for netconn in proc["netconn_complete"]:
# split the netconn event into component parts
# note that the port is the remote port in the case of outbound
# netconns, and local port in the case of inbound netconns
ts, ip, port, proto, domain, dir = netconn.split("|")
# get the dotted-quad string representation of the ip
str_ip = socket.inet_ntoa(struct.pack("!i", int(ip)))
# the underlying data model provides the protocol number
# convert this to human-readable strings (tcp or udp)
if "6" == proto:
proto = "tcp"
elif "17" == proto:
proto = "udp"
# the underlying data model provides a boolean indication as to
# if this is an inbound or outbound network connection
if "true" == dir:
dir = "out"
else:
dir = "in"
# pring the record, using pipes as a delimiter
print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s)" % (
hostname, ts, process_name, user_name, process_md5, proto,
str_ip, port, dir, domain)
def check(self, hostname):
# print a legend
print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s)" % (
"hostname", "timestamp", "process name", "username", "process md5",
"protocol", "ip", "port", "direction", "domain")
# build the query string
q = "netconn_count:[1 to *] AND hostname:%s" % (hostname)
# begin with the first result - we'll perform the search in pages
# the default page size is 10 (10 reslts)
start = 0
# loop over the entire result set
while True:
# get the next page of results
procs = self.cb.process_search(q, start=start)
# if there are no results, we are done paging
if len(procs["results"]) == 0:
break
# examine each result individually
# each result represents a single process segment
for result in procs["results"]:
self.report(hostname, result)
# move forward to the next page
start = start + 10
def connect_cb(self):
return CbApi(self.url, token=self.admin_key, ssl_verify=False)
class CBQuery(object):
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
# set up some stat tracking
self.stats = {}
self.stats['total_processes'] = 0
self.stats['total_netconns'] = 0
self.stats['matching_netconns'] = 0
self.stats['output_errors'] = 0
self.stats['process_errors'] = 0
def getStats(self):
"""
return the current statistics
"""
return self.stats
def outputNetConn(self, proc, netconn):
"""
output a single netconn event from a process document
the caller is responsible for ensuring that the document
meets start time and subnet criteria
"""
# for convenience, use locals for some process metadata fields
hostname = proc.get("hostname", "<unknown>")
process_name = proc.get("process_name", "<unknown>")
user_name = proc.get("username", "<unknown>")
process_md5 = proc.get("process_md5", "<unknown>")
cmdline = proc.get("cmdline", "<unknown>")
path = proc.get("path", "<unknown>")
procstarttime = proc.get("start", "<unknown>")
proclastupdate = proc.get("last_update", "<unknown>")
# split the netconn into component parts
ts, ip, port, proto, domain, dir = netconn.split("|")
# get the dotted-quad string representation of the ip
str_ip = socket.inet_ntoa(struct.pack("!i", int(ip)))
# the underlying data model provides the protocol number
# convert this to human-readable strings (tcp or udp)
if "6" == proto:
proto = "tcp"
elif "17" == proto:
proto = "udp"
# the underlying data model provides a boolean indication as to
# if this is an inbound or outbound network connection
if "true" == dir:
dir = "out"
else:
dir = "in"
# print the record, using pipes as a delimiter
print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|" % (procstarttime,proclastupdate,hostname, user_name, proto, str_ip, port, dir, domain, process_name, process_md5, path, cmdline)
def addressInNetwork(self, ip, cidr):
# the ip can be the emtpy string ('') in cases where the connection
# is made via a web proxy. in these cases the sensor cannot report
# the true remote IP as DNS resolution happens on the web proxy (and
# not the endpoint)
if '' == ip:
return False
try:
net = cidr.split('/')[0]
bits = cidr.split('/')[1]
if int(ip) > 0:
ipaddr = struct.unpack('<L', socket.inet_aton(ip))[0]
else:
ipaddr = struct.unpack('<L', socket.inet_aton(".".join(map(lambda n: str(int(ip)>>n & 0xFF), [24,16,8,0]))))[0]
netaddr = struct.unpack('<L', socket.inet_aton(net))[0]
netmask = ((1L << int(bits)) - 1)
return ipaddr & netmask == netaddr & netmask
except:
return False
def report(self, result, subnet):
# return the events associated with this process segment
# this will include netconns, as well as modloads, filemods, etc.
events = self.cb.process_events(result["id"], result["segment_id"])
proc = events["process"]
# the search criteria (netconn_count:[1 to *]) should ensure that
# all results have at least one netconn
if proc.has_key("netconn_complete"):
# examine each netconn in turn
for netconn in proc["netconn_complete"]:
# update the count of total netconns
self.stats['total_netconns'] = self.stats['total_netconns'] + 1
# split the netconn event into component parts
# note that the port is the remote port in the case of outbound
# netconns, and local port in the case of inbound netconns
#
# for the purpose of this example script, eat any errors
ts, ip, port, proto, domain, dir = netconn.split("|")
if self.addressInNetwork(ip, subnet):
try:
self.stats['matching_netconns'] = self.stats['matching_netconns'] + 1
self.outputNetConn(proc, netconn)
except:
self.stats['output_errors'] = self.stats['output_errors'] + 1
pass
def strip_to_int(ip):
"""
convert a dotted-quad string IP to the corresponding int32
"""
return struct.unpack('<L', socket.inet_aton(ip))[0]
def check(self, subnet, datetype, begin, end):
# print a legend
print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|" % ("ProcStartTime", "ProcUpdateTime","hostname", "username", "protocol", "ip", "port", "direction", "domain", "process name", "process md5", "process path", "cmdline")
# build the query string
if not end and not begin:
q = "ipaddr:%s" % (subnet,)
else:
if not end: end = "*"
if not begin: begin = "*"
q = "ipaddr:%s %s:[%s TO %s]" % (subnet, datetype, begin, end)
# begin with the first result - we'll perform the search in pages
# the default page size is 10 (10 results)
start = 0
# loop over the entire result set, paging as required
while True:
# get the next page of results
procs = self.cb.process_search(q, start=start)
# track the total number of matching results
self.stats['total_processes'] = procs['total_results']
# if there are no results, we are done paging
if len(procs["results"]) == 0:
break
# examine each result individually
# each result represents a single segment of a single process
#
# for the purposes of this example script, eat any errors
#
for result in procs["results"]:
try:
self.report(result, subnet)
except Exception, e:
self.stats['process_errors'] = self.stats['process_errors'] + 1
# move forward to the next page
start = start + 10
def get_legacy_cbapi(splunk_service):
cb_server, token = get_creds(splunk_service)
return CbApi(cb_server, ssl_verify=False, token=token)
class CBQuery(object):
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
def report(self, result, search_filename):
# return the events associated with this process segment
# this will include netconns, as well as modloads, filemods, etc.
#
events = self.cb.process_events(result["id"], result["segment_id"])
proc = events["process"]
# for convenience, use locals for some process metadata fields
#
host_name = result.get("hostname", "<unknown>")
process_name = result.get("process_name", "<unknown>")
user_name = result.get("username", "<unknown>")
if proc.has_key("filemod_complete"):
# examine each filemod in turn
#
for filemod in proc["filemod_complete"]:
# filemods are of the form:
# 1|2014-06-19 15:40:05.446|c:\dir\filename||
#
action, ts, filename, filemd5, filetype = filemod.split('|')
# the _document as a whole_ matched the query
# that doesn't mean each _indvidual filemod_ within the document matched
# the user-specified filename to search for
#
# iterate over each filemod and determine if the path matches
# what was specified
#
# make sense? hope so!
#
if search_filename.lower() not in filename.lower():
continue
if "1" == action:
action = "creation"
elif "2" == action or "8" == action:
action = "modification"
elif "4" == action:
action = "deletion"
print "%s|%s|%s|%s|%s|%s" % (host_name, ts, filename,
process_name, user_name, action)
def check(self, filename):
# build the query string
#
q = "filemod:%s" % (filename)
# begin with the first result - we'll perform the search in pages
# the default page size is 10 (10 reslts)
#
start = 0
# loop over the entire result set
#
while True:
# get the next page of results
#
procs = self.cb.process_search(q, start=start)
# if there are no results, we are done paging
#
if len(procs["results"]) == 0:
break
# examine each result individually
# each result represents a single process segment
#
for result in procs["results"]:
self.report(result, filename)
# move forward to the next page
#
start = start + 10
class Device(object):
path = "carbonblack.endpoint"
def __init__(self, hosts_mapping):
configuration_dict = splunk.clilib.cli_common.getConfStanza('carbonblack', 'cbserver')
self.cb_server = configuration_dict['cburl']
self.token = configuration_dict['cbapikey']
self.cb = CbApi(self.cb_server, token=self.token, ssl_verify=False)
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.DEBUG)
self.hosts_mapping = hosts_mapping
def ban_hash(self, md5):
"""
Performs a POST to the Carbon Black Server API for blacklisting an MD5 hash
:param md5:
:return:
"""
print "blacklisting md5:%s" % (md5)
headers = {'X-AUTH-TOKEN': self.token}
data = {"md5hash": md5,
"text": "Blacklist From Splunk",
"last_ban_time": 0,
"ban_count": 0,
"last_ban_host": 0,
"enabled": True}
print "connecting to: %s/api/v1/banning/blacklist..." % (self.cb_server)
r = requests.post("%s/api/v1/banning/blacklist" % (self.cb_server),
headers=headers,
data=json.dumps(data),
verify=False)
if r.status_code == 409:
print "This md5 hash is already blacklisted"
elif r.status_code == 200:
print "Carbon Black Server API Success"
else:
print "CarbonBlack Server API returned an error: %d" % (r.status_code)
print "Be sure to check the Carbon Black API token"
def get_sensor_id_from_ip(self, ip):
filters = {}
sensors = self.cb.sensors(filters)
for sensor in sensors:
src_ip = filter(bool, sensor.get('network_adapters', '').split('|'))
for ip_address in src_ip:
if unicode(ip, "utf-8") == ip_address.split(',')[0]:
return sensor.get('id', None)
return None
def pre_action(self, action_type, data):
if action_type in ['isolate', 'flush']:
src_ip = data.get('src_ip', None) or data.get('local_ip', None)
if not src_ip:
raise PrerequisiteFailedError("No source IP address provided")
sensor_id = self.get_sensor_id_from_ip(src_ip)
if not sensor_id:
raise PrerequisiteFailedError("Cannot find sensor associated with source IP address %s" % src_ip)
return sensor_id
elif action_type in ['killproc']:
proc_id = data.get('process_guid', None)
if not proc_id:
raise PrerequisiteFailedError("No Process GUID provided")
if not isinstance(proc_id, six.string_types):
raise PrerequisiteFailedError("Process GUID not valid: must be a string")
if len(proc_id.split("-")) < 5:
raise PrerequisiteFailedError("Process GUID not valid: must be a GUID")
return proc_id
elif action_type in ['banhash']:
#
# Pull out md5 from 'data'
#
md5 = data.get('md5', None)
if not md5:
#
# Error out if we can't
#
raise PrerequisiteFailedError("Error: Unable to get an MD5 hash from parameters")
return md5
return None
def submit_action(self, settings, data):
"""
This gets called when the user executes a search
:param settings:
:param data:
:return:
"""
action_type = settings.get('action_type', '')
try:
sensor_id = self.pre_action(action_type, data)
except PrerequisiteFailedError as e:
# TODO: how do we signal failure back to ARF ARFARFARF
self.logger.error(e.message)
# TODO: return success
def flush_action(self, sensor_id):
print "Flushing sensor id: %s" % sensor_id
#
# We will always flush the sensor that triggered the action, so that we get the most up-to-date
# information into the Cb console.
#
flusher = FlushAction(self.cb, self.logger)
flusher.action(sensor_id)
def isolate_action(self, sensor_id):
isolator = IsolateAction(self.cb, self.logger)
isolator.action(sensor_id)
def kill_action(self, process_id):
killer = KillProcessAction(self.cb, self.logger)
killer.action(process_id)
def run_action(self, settings, data):
"""
This gets called when the user clicks the validate button
:param settings:
:param data:
:return:
"""
action_type = settings.get('action_type', '')
# get sensor ID if required
try:
action_argument = self.pre_action(action_type, data)
except PrerequisiteFailedError as e:
self.logger.error(e.message)
else:
if action_type == 'banhash':
self.ban_hash(action_argument)
elif action_type == 'isolate':
self.isolate_action(action_argument)
elif action_type == 'flush':
self.flush_action(action_argument)
elif action_type == 'killproc':
self.kill_action(action_argument)
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
class CBQuery(object):
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
# set up some stat tracking
self.stats = {}
self.stats['total_processes'] = 0
self.stats['total_netconns'] = 0
self.stats['matching_netconns'] = 0
self.stats['output_errors'] = 0
self.stats['process_errors'] = 0
def getStats(self):
"""
return the current statistics
"""
return self.stats
def outputNetConn(self, proc, netconn):
"""
output a single netconn event from a process document
the caller is responsible for ensuring that the document
meets start time and subnet criteria
"""
# for convenience, use locals for some process metadata fields
hostname = proc.get("hostname", "<unknown>")
process_name = proc.get("process_name", "<unknown>")
user_name = proc.get("username", "<unknown>")
process_md5 = proc.get("process_md5", "<unknown>")
cmdline = proc.get("cmdline", "<unknown>")
path = proc.get("path", "<unknown>")
procstarttime = proc.get("start", "<unknown>")
proclastupdate = proc.get("last_update", "<unknown>")
# split the netconn into component parts
ts, ip, port, proto, domain, dir = netconn.split("|")
# get the dotted-quad string representation of the ip
str_ip = socket.inet_ntoa(struct.pack("!i", int(ip)))
# the underlying data model provides the protocol number
# convert this to human-readable strings (tcp or udp)
if "6" == proto:
proto = "tcp"
elif "17" == proto:
proto = "udp"
# the underlying data model provides a boolean indication as to
# if this is an inbound or outbound network connection
if "true" == dir:
dir = "out"
else:
dir = "in"
# print the record, using pipes as a delimiter
print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|" % (
procstarttime, proclastupdate, hostname, user_name, proto, str_ip,
port, dir, domain, process_name, process_md5, path, cmdline)
def addressInNetwork(self, ip, cidr):
# the ip can be the emtpy string ('') in cases where the connection
# is made via a web proxy. in these cases the sensor cannot report
# the true remote IP as DNS resolution happens on the web proxy (and
# not the endpoint)
if '' == ip:
return False
try:
net = cidr.split('/')[0]
bits = cidr.split('/')[1]
if int(ip) > 0:
ipaddr = struct.unpack('<L', socket.inet_aton(ip))[0]
else:
ipaddr = struct.unpack(
'<L',
socket.inet_aton(".".join(
map(lambda n: str(int(ip) >> n & 0xFF),
[24, 16, 8, 0]))))[0]
netaddr = struct.unpack('<L', socket.inet_aton(net))[0]
netmask = ((1L << int(bits)) - 1)
return ipaddr & netmask == netaddr & netmask
except:
return False
def report(self, result, subnet):
# return the events associated with this process segment
# this will include netconns, as well as modloads, filemods, etc.
events = self.cb.process_events(result["id"], result["segment_id"])
proc = events["process"]
# the search criteria (netconn_count:[1 to *]) should ensure that
# all results have at least one netconn
if proc.has_key("netconn_complete"):
# examine each netconn in turn
for netconn in proc["netconn_complete"]:
# update the count of total netconns
self.stats['total_netconns'] = self.stats['total_netconns'] + 1
# split the netconn event into component parts
# note that the port is the remote port in the case of outbound
# netconns, and local port in the case of inbound netconns
#
# for the purpose of this example script, eat any errors
ts, ip, port, proto, domain, dir = netconn.split("|")
if self.addressInNetwork(ip, subnet):
try:
self.stats['matching_netconns'] = self.stats[
'matching_netconns'] + 1
self.outputNetConn(proc, netconn)
except:
self.stats[
'output_errors'] = self.stats['output_errors'] + 1
pass
def strip_to_int(ip):
"""
convert a dotted-quad string IP to the corresponding int32
"""
return struct.unpack('<L', socket.inet_aton(ip))[0]
def check(self, subnet, datetype, begin, end):
# print a legend
print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|" % (
"ProcStartTime", "ProcUpdateTime", "hostname", "username",
"protocol", "ip", "port", "direction", "domain", "process name",
"process md5", "process path", "cmdline")
# build the query string
if not end and not begin:
q = "ipaddr:%s" % (subnet, )
else:
if not end: end = "*"
if not begin: begin = "*"
q = "ipaddr:%s %s:[%s TO %s]" % (subnet, datetype, begin, end)
# begin with the first result - we'll perform the search in pages
# the default page size is 10 (10 results)
start = 0
# loop over the entire result set, paging as required
while True:
# get the next page of results
procs = self.cb.process_search(q, start=start)
# track the total number of matching results
self.stats['total_processes'] = procs['total_results']
# if there are no results, we are done paging
if len(procs["results"]) == 0:
break
# examine each result individually
# each result represents a single segment of a single process
#
# for the purposes of this example script, eat any errors
#
for result in procs["results"]:
try:
self.report(result, subnet)
except Exception, e:
self.stats[
'process_errors'] = self.stats['process_errors'] + 1
# move forward to the next page
start = start + 10
class ProcessSearchCommand(GeneratingCommand):
"""Generates a process search result from Carbon Black from a given IP or search query
| processsearch query=${ip}
"""
query = Option(name="query", require=True)
field_names = ['cmdline',
'comms_ip',
'hostname',
'id',
'interface_ip',
'last_update',
'os_type',
'parent_md5',
'parent_name',
'parent_pid',
'parent_unique_id',
'path',
'process_md5',
'process_name',
'process_pid',
'regmod_count',
'segment_id',
'sensor_id',
'start',
'unique_id',
'username',
'childproc_count',
'crossproc_count',
'modload_count',
'netconn_count',
'filemod_count',
'group',
'host_type']
def prepare(self):
configuration_dict = splunk.clilib.cli_common.getConfStanza('carbonblack', 'cbserver')
self.cb_server = configuration_dict['cburl']
self.token = configuration_dict['cbapikey']
self.cb = CbApi(self.cb_server, token=self.token, ssl_verify=False)
def generate(self):
self.logger.info("query %s" % self.query)
i = 0
for bindata in self.cb.process_search_iter(self.query):
i += 1
if i > 1000:
# TODO: let's stop at 1,000 results for now?
self.finish()
return
temp = dict((field_name, bindata[field_name]) for field_name in self.field_names)
temp['sourcetype'] = 'bit9:carbonblack:json'
#
# Sometimes we have seen 'start' be equal to -1
#
try:
temp['_time'] = int(time.mktime(dateutil.parser.parse(bindata['start']).timetuple()))
except Exception as e:
self.logger.exception('parsing bindata["start"] %s' % bindata['start'])
temp['_time'] = 0
temp['link_process'] = self.cb_server + '/#/analyze/' + bindata['id'] + "/1"
temp['source'] = 'cbapi'
temp['_raw'] = json.dumps(temp)
yield temp
if i % 10 == 0:
self.flush()
class CBQuery(object):
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
def report(self, hostname, result):
# return the events associated with this process segment
# this will include netconns, as well as modloads, filemods, etc.
events = self.cb.process_events(result["id"], result["segment_id"])
proc = events["process"]
# for convenience, use locals for some process metadata fields
process_name = result.get("process_name", "<unknown>")
user_name = result.get("username", "<unknown>")
process_md5 = result.get("process_md5", "<unknown>")
# the search criteria (netconn_count:[1 to *]) should ensure that
# all results have at least one netconn
if proc.has_key("netconn_complete"):
# examine each netconn in turn
for netconn in proc["netconn_complete"]:
# split the netconn event into component parts
# note that the port is the remote port in the case of outbound
# netconns, and local port in the case of inbound netconns
ts, ip, port, proto, domain, dir = netconn.split("|")
# get the dotted-quad string representation of the ip
str_ip = socket.inet_ntoa(struct.pack("!i", int(ip)))
# the underlying data model provides the protocol number
# convert this to human-readable strings (tcp or udp)
if "6" == proto:
proto = "tcp"
elif "17" == proto:
proto = "udp"
# the underlying data model provides a boolean indication as to
# if this is an inbound or outbound network connection
if "true" == dir:
dir = "out"
else:
dir = "in"
# pring the record, using pipes as a delimiter
print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s)" % (hostname, ts, process_name, user_name, process_md5, proto, str_ip, port, dir, domain)
def check(self, hostname):
# print a legend
print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s)" % ("hostname", "timestamp", "process name", "username", "process md5", "protocol", "ip", "port", "direction", "domain")
# build the query string
q = "netconn_count:[1 to *] AND hostname:%s" % (hostname)
# begin with the first result - we'll perform the search in pages
# the default page size is 10 (10 reslts)
start = 0
# loop over the entire result set
while True:
# get the next page of results
procs = self.cb.process_search(q, start=start)
# if there are no results, we are done paging
if len(procs["results"]) == 0:
break
# examine each result individually
# each result represents a single process segment
for result in procs["results"]:
self.report(hostname, result)
# move forward to the next page
start = start + 10
class CbTaxiiFeedConverter(object):
def __init__(self, configpath, export_mode=False):
self.export_mode = export_mode
self.sites = []
if self.export_mode:
_logger.warn("CB Taxii %s running (EXPORT MODE)" % __version__)
else:
_logger.warn("CB Taxii %s running" % __version__)
config = ConfigParser.ConfigParser()
if not os.path.exists(configpath):
_logger.error("Config File %s does not exist!" % configpath)
print("Config File %s does not exist!" % configpath)
sys.exit(-1)
config.read(configpath)
# SEE IF THERE's A DIFFERENT SERVER_PORT
server_port = 443
if config.has_section("cbconfig"):
if config.has_option("cbconfig", "server_port"):
server_port = config.getint("cbconfig", "server_port")
for section in config.sections():
# don't do cbconfig
if section.lower() == 'cbconfig':
continue
site = config.get(section, "site")
output_path = config.get(section, "output_path")
icon_link = config.get(section, "icon_link")
username = config.get(section, "username")
password = config.get(section, "password")
feeds_enable = config.getboolean(section, "feeds_enable")
feeds_alerting = config.get(section, "feeds_alerting")
### OPTIONAL ARGUMENTS #######################################################
if config.has_option(section, "start_date"):
start_date = config.get(section, "start_date")
else:
start_date = "2015-01-01 00:00:00"
if config.has_option(section, "use_https"):
use_https=config.getboolean(section, "use_https")
else:
use_https = False
cert_file = None
key_file = None
if config.has_option(section, "cert_file") and config.has_option(section, "key_file"):
cert_file = config.get(section, "cert_file").strip()
if cert_file == "":
cert_file = None
elif not os.path.exists(cert_file):
_logger.error("Cert file supplied but doesn't exist: %s" % (cert_file))
key_file = config.get(section, "key_file").strip()
if key_file == "":
cert_file = None
elif not os.path.exists(key_file):
_logger.error("Key file supplied but doesn't exist: %s" % (key_file))
if config.has_option(section, "minutes_to_advance"):
minutes_to_advance = int(config.get(section, "minutes_to_advance"))
else:
minutes_to_advance = 15
if config.has_option(section, "enable_ip_ranges"):
enable_ip_ranges = config.getboolean(section, "enable_ip_ranges")
else:
enable_ip_ranges = True
_logger.info("Configured Site: %s Path: %s" % (site, output_path))
self.sites.append({"site": site,
"output_path": output_path,
"username": username,
"password": password,
"icon_link": icon_link,
"feeds_enable": feeds_enable,
"feeds_alerting": feeds_alerting,
"enable_ip_ranges": enable_ip_ranges,
"start_date": start_date,
"use_https": use_https,
"key_file": key_file,
"cert_file": cert_file,
"minutes_to_advance": minutes_to_advance})
self.api_token = lookup_admin_api_token()
server_url = "https://127.0.0.1:%d/" % server_port
_logger.info("Using Server URL: %s" % server_url)
self.cb = CbApi(server_url, token=self.api_token, ssl_verify=False)
try:
# TEST CB CONNECTIVITY
self.cb.feed_enum()
except:
e = traceback.format_exc()
_logger.error("Unable to connect to CB using url: %s Error: %s" % (server_url, e))
print("Unable to connect to CB using url: %s Error: %s" % (server_url, e))
sys.exit(-1)
@staticmethod
def _message_to_reports(filepath, site, site_url, collection, enable_ip_ranges):
context = etree.iterparse(filepath, tag='{http://stix.mitre.org/stix-1}STIX_Package')
global _logger
reports = fast_xml_iter(context, stix_element_to_reports, site, site_url, collection, enable_ip_ranges, _logger)
return reports
def _write_message_to_disk(self, message):
fd,path = tempfile.mkstemp()
# os.write(fd, message)
os.close(fd)
f = file(path, 'wb')
f.write(message)
f.close()
return path
def _export_message_to_disk(self, feed_name, start_time, end_time, message):
log_dir = "/var/run/cb/cbtaxii-export"
if not os.path.exists(log_dir):
os.mkdir(log_dir)
path = "%s/%s-%s-%s.xml" % (log_dir, feed_name, start_time, end_time)
path = path.replace(' ', '_')
f = file(path, 'wb')
f.write(message)
f.close()
return path
def _import_collection(self, client, site, collection):
collection_name = collection.get('collection_name', '')
sanitized_feed_name = cleanup_string("%s%s" % (site.get('site'), collection_name))
available = collection.get('available', False)
collection_type = collection.get('collection_type', '').upper()
_logger.info("%s,%s,%s,%s,%s" % (site.get('site'),
collection_name,
sanitized_feed_name,
available,
collection_type))
if not available or collection_type != "DATA_FEED":
return
start_date_str = site.get('start_date')
if not start_date_str or len(start_date_str) == 0:
start_date_str = "2015-04-01 00:00:00"
feed_helper = FeedHelper(site.get('output_path'), sanitized_feed_name, site.get('minutes_to_advance'), start_date_str, self.export_mode)
_logger.info("Feed start time %s" % feed_helper.start_date)
reports = []
# CATCHUP -- TODO, move to a function??
while True:
these_reports = []
tries = 0
while tries < 5:
try:
if feed_helper.start_date > feed_helper.end_date:
break
t1 = time.time()
message = client.retrieve_collection(collection_name, feed_helper.start_date, feed_helper.end_date)
t2 = time.time()
message_len = len(message)
if self.export_mode:
path = self._export_message_to_disk(sanitized_feed_name, feed_helper.start_date, feed_helper.end_date, message)
_logger.info("%s - %s - %s - %d (%f)- %s" % (feed_helper.start_date, feed_helper.end_date, collection_name, message_len, (t2-t1), path))
message = None
else:
filepath = self._write_message_to_disk(message)
message = None
site_url = "%s://%s" % ("https" if site.get('use_https') else "http", site.get('site'))
these_reports = self._message_to_reports(filepath, site.get('site'), site_url, collection_name, site.get('enable_ip_ranges'))
t3 = time.time()
os.remove(filepath)
count = len(these_reports)
_logger.info("%s - %s - %s - %d (%d)(%.2f)(%.2f)" % (feed_helper.start_date, feed_helper.end_date, collection_name, count, message_len, (t2-t1), (t3-t2)))
break
except:
_logger.error("%s" % traceback.format_exc())
time.sleep(5)
tries += 1
if tries == 5:
_logger.error("Giving up for site %s, collection %s" % (site.get('site'), collection))
return
if not self.export_mode:
reports.extend(these_reports)
if not feed_helper.advance():
break
########## end while (for iterating across time)
_logger.info("COMPLETED %s,%s,%s,%s,%s (%d)" % (site.get('site'),
collection_name,
sanitized_feed_name,
available,
collection_type,
len(reports)))
if not self.export_mode:
# TODO -- clean this up
if len(reports) > 0:
# load existing data and convert new data
reports = feed_helper.load_existing_feed_data() + reports
# convert feed info and reports to json
data = build_feed_data(sanitized_feed_name,
"%s %s" % (site.get('site'), collection_name),
site.get('site'),
site.get('icon_link'),
reports)
# SAVE THE DATA: write out the feed file and save the details for when we last queried it
if feed_helper.write_feed(data):
feed_helper.save_details()
# Actually add CB feed if necessary
feed_id = self.cb.feed_get_id_by_name(sanitized_feed_name)
if not feed_id:
data = self.cb.feed_add_from_url("file://" + feed_helper.path,
site.get('feeds_enable'),
False,
False)
# FEED ALERTING!!
feed_id = data.get('id')
url = "https://127.0.0.1/api/v1/feed/%d/action" % feed_id
alert_types = site.get('feeds_alerting', '').split(',')
headers = {'X-Auth-Token' : self.api_token, "Accept" : "application/json"}
for alert in alert_types:
if alert.lower() == "syslog":
action_data = {"action_data": """{"email_recipients":[1]}""", "action_type": 1, "group_id": feed_id, "watchlist_id": ""}
resp = requests.post(url, headers=headers, data=json.dumps(action_data), verify=False)
if resp.status_code != 200:
_logger.warn("Error for syslog action (%d): %s" % (feed_id, resp.content))
elif alert.lower() == "cb":
action_data = {"action_data": """{"email_recipients":[1]}""", "action_type": 3, "group_id": feed_id, "watchlist_id": ""}
resp = requests.post(url, headers=headers, data=json.dumps(action_data), verify=False)
if resp.status_code != 200:
_logger.warn("Error for cb action (%d): %s" % (feed_id, resp.content))
else: # no reports
feed_helper.save_details()
@staticmethod
def perform_from_files(directory):
global _logger
_logger = create_stdout_log("cb-taxii", logging.DEBUG)
files = os.listdir(directory)
for filepath in files:
if not filepath.endswith(".xml"):
continue
pieces = filepath.split('-')
site = pieces[0]
filepath = os.path.join(directory, filepath)
these_reports = CbTaxiiFeedConverter._message_to_reports(filepath, site, site, site, True)
for report in these_reports:
iocs = report.get('iocs')
if "dns" in iocs:
print "%s - %s" % (site, iocs['dns'])
if "ipv4" in iocs:
print "%s - %s" % (site, iocs['ipv4'])
if "query" in iocs:
print "%s - %s" % (site, iocs['query'])
if "hash" in iocs:
print "%s - %s" % (site, iocs['hash'])
def perform(self):
"""
Loops through the sites supplied and adds each one if necessary.
Then downloads new data and appends to existing feed file.
"""
for site in self.sites:
client = TaxiiClient(site.get('site'),
site.get('username'),
site.get('password'),
site.get('use_https'),
site.get('key_file'),
site.get('cert_file'))
try:
collections = client.enumerate_collections(_logger)
if len(collections) == 0:
_logger.warn("No collections returned!")
except UnauthorizedException, e:
_logger.error("Site: %s, Exception: %s" % (site.get('site'), e))
continue
for collection in collections:
self._import_collection(client, site, collection)
cmdline = bprocess["cmdline"]
else:
cmdline = ""
myrow.append(cmdline)
if bprocess.has_key("childproc_complete"):
getchildprocs(bprocess['childproc_complete'])
parser = build_cli_parser()
opts, args = parser.parse_args(sys.argv[1:])
if not opts.server or not opts.token or (
not opts.procnamefile and not opts.procname) or not opts.percentless:
print "Missing required param."
sys.exit(-1)
cb = CbApi(opts.server, ssl_verify=opts.ssl_verify, token=opts.token)
if opts.procnamefile:
searchprocess = processsearchlist(opts.procnamefile)
else:
searchprocess = opts.procname.split(",")
for proc in searchprocess:
start = 0
data = cb.process_search("parent_name:%s" % (proc), rows=1, start=start)
facetterms = data['facets']
for term in reversed(facetterms['process_name']):
termratio = int(float(term['ratio']))
if int(opts.percentless) >= termratio:
start = 0
while True:
q = "parent_name:%s AND process_name:%s" % (proc, term['name'])
data = cb.process_search(q, rows=int(pagesize), start=start)
def __init__(self, configpath, export_mode=False):
self.export_mode = export_mode
self.sites = []
if self.export_mode:
_logger.warn("CB Taxii %s running (EXPORT MODE)" % __version__)
else:
_logger.warn("CB Taxii %s running" % __version__)
config = ConfigParser.ConfigParser()
if not os.path.exists(configpath):
_logger.error("Config File %s does not exist!" % configpath)
print("Config File %s does not exist!" % configpath)
sys.exit(-1)
config.read(configpath)
# SEE IF THERE's A DIFFERENT SERVER_PORT
server_port = 443
if config.has_section("cbconfig"):
if config.has_option("cbconfig", "server_port"):
server_port = config.getint("cbconfig", "server_port")
for section in config.sections():
# don't do cbconfig
if section.lower() == 'cbconfig':
continue
site = config.get(section, "site")
output_path = config.get(section, "output_path")
icon_link = config.get(section, "icon_link")
username = config.get(section, "username")
password = config.get(section, "password")
feeds_enable = config.getboolean(section, "feeds_enable")
feeds_alerting = config.get(section, "feeds_alerting")
### OPTIONAL ARGUMENTS #######################################################
if config.has_option(section, "start_date"):
start_date = config.get(section, "start_date")
else:
start_date = "2015-01-01 00:00:00"
if config.has_option(section, "use_https"):
use_https=config.getboolean(section, "use_https")
else:
use_https = False
cert_file = None
key_file = None
if config.has_option(section, "cert_file") and config.has_option(section, "key_file"):
cert_file = config.get(section, "cert_file").strip()
if cert_file == "":
cert_file = None
elif not os.path.exists(cert_file):
_logger.error("Cert file supplied but doesn't exist: %s" % (cert_file))
key_file = config.get(section, "key_file").strip()
if key_file == "":
cert_file = None
elif not os.path.exists(key_file):
_logger.error("Key file supplied but doesn't exist: %s" % (key_file))
if config.has_option(section, "minutes_to_advance"):
minutes_to_advance = int(config.get(section, "minutes_to_advance"))
else:
minutes_to_advance = 15
if config.has_option(section, "enable_ip_ranges"):
enable_ip_ranges = config.getboolean(section, "enable_ip_ranges")
else:
enable_ip_ranges = True
_logger.info("Configured Site: %s Path: %s" % (site, output_path))
self.sites.append({"site": site,
"output_path": output_path,
"username": username,
"password": password,
"icon_link": icon_link,
"feeds_enable": feeds_enable,
"feeds_alerting": feeds_alerting,
"enable_ip_ranges": enable_ip_ranges,
"start_date": start_date,
"use_https": use_https,
"key_file": key_file,
"cert_file": cert_file,
"minutes_to_advance": minutes_to_advance})
self.api_token = lookup_admin_api_token()
server_url = "https://127.0.0.1:%d/" % server_port
_logger.info("Using Server URL: %s" % server_url)
self.cb = CbApi(server_url, token=self.api_token, ssl_verify=False)
try:
# TEST CB CONNECTIVITY
self.cb.feed_enum()
except:
e = traceback.format_exc()
_logger.error("Unable to connect to CB using url: %s Error: %s" % (server_url, e))
print("Unable to connect to CB using url: %s Error: %s" % (server_url, e))
sys.exit(-1)
class CBQuery(object):
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
def report(self, result, search_filename):
# return the events associated with this process segment
# this will include netconns, as well as modloads, filemods, etc.
#
events = self.cb.process_events(result["id"], result["segment_id"])
proc = events["process"]
# for convenience, use locals for some process metadata fields
#
host_name = result.get("hostname", "<unknown>")
process_name = result.get("process_name", "<unknown>")
user_name = result.get("username", "<unknown>")
if proc.has_key("filemod_complete"):
# examine each filemod in turn
#
for filemod in proc["filemod_complete"]:
# filemods are of the form:
# 1|2014-06-19 15:40:05.446|c:\dir\filename||
#
parts = filemod.split('|')
action, ts, filename, filemd5, filetype = parts[:5]
# the _document as a whole_ matched the query
# that doesn't mean each _indvidual filemod_ within the document matched
# the user-specified filename to search for
#
# iterate over each filemod and determine if the path matches
# what was specified
#
# make sense? hope so!
#
if search_filename.lower() not in filename.lower():
continue
if "1" == action:
action = "creation"
elif "2" == action or "8" == action:
action = "modification"
elif "4" == action:
action = "deletion"
print "%s|%s|%s|%s|%s|%s" % (host_name, ts, filename, process_name, user_name, action)
def check(self, filename):
# build the query string
#
q = "filemod:%s" % (filename)
# begin with the first result - we'll perform the search in pages
# the default page size is 10 (10 reslts)
#
start = 0
# loop over the entire result set
#
while True:
# get the next page of results
#
procs = self.cb.process_search(q, start=start)
# if there are no results, we are done paging
#
if len(procs["results"]) == 0:
break
# examine each result individually
# each result represents a single process segment
#
for result in procs["results"]:
self.report(result, filename)
# move forward to the next page
#
start = start + 10
class CbTaxiiFeedConverter(object):
def __init__(self,
config_file_path,
debug_mode=False,
import_dir='',
export_dir=''):
#
# parse config file and save off the information we need
#
config_dict = parse_config(config_file_path)
self.server_url = config_dict.get('server_url', 'https://127.0.0.1')
self.api_token = config_dict.get('api_token', '')
self.sites = config_dict.get('sites', [])
self.debug = config_dict.get('debug', False)
self.export_dir = export_dir
self.import_dir = import_dir
self.http_proxy_url = config_dict.get('http_proxy_url', None)
self.https_proxy_url = config_dict.get('https_proxy_url', None)
if self.export_dir and not os.path.exists(self.export_dir):
os.mkdir(self.export_dir)
#
# Test Cb Response connectivity
#
try:
self.cb = CbApi(server=self.server_url,
token=self.api_token,
ssl_verify=False)
self.cb.feed_enum()
except:
logger.error(traceback.format_exc())
sys.exit(-1)
def write_to_temp_file(self, message):
temp_file = tempfile.NamedTemporaryFile()
temp_file.write(message)
temp_file.flush()
return temp_file, temp_file.name
def read_from_xml(self):
"""
Walk the import dir and return all filenames. We are assuming all xml files
:return:
"""
f = []
for (dirpath, dirnames, filenames) in os.walk(self.import_dir):
f.extend(filenames)
break
return f
def export_xml(self, feed_name, start_time, end_time, block_num, message):
"""
:param feed_name:
:param start_time:
:param end_time:
:param block_num:
:param message:
:return:
"""
#
# create a directory to store all content blocks
#
dir_name = "{}".format(feed_name).replace(' ', '_')
full_dir_name = os.path.join(self.export_dir, dir_name)
#
# Make sure the directory exists
#
if not os.path.exists(os.path.join(self.export_dir, dir_name)):
os.mkdir(full_dir_name)
#
# Actually write the file
#
file_name = "{}-{}-{}".format(start_time, end_time,
block_num).replace(' ', "_")
full_file_name = os.path.join(full_dir_name, file_name)
with open(full_file_name, 'wb') as file_handle:
file_handle.write(message)
def _import_collection(self, client, site, collection, data_set=False):
collection_name = collection.name
sanitized_feed_name = cleanup_string(
"%s%s" % (site.get('site'), collection_name))
feed_summary = "%s %s" % (site.get('site'), collection_name)
available = collection.available
collection_type = collection.type
default_score = site.get('default_score')
logger.info("%s,%s,%s,%s,%s" %
(site.get('site'), collection_name, sanitized_feed_name,
available, collection_type))
if not available:
return False
#
# Sanity check on start date
#
start_date_str = site.get('start_date')
if not start_date_str or len(start_date_str) == 0:
start_date_str = "2017-01-01 00:00:00"
#
# Create a feed helper object
#
feed_helper = FeedHelper(site.get('output_path'), sanitized_feed_name,
site.get('minutes_to_advance'),
start_date_str)
if not data_set:
logger.info("Feed start time %s" % feed_helper.start_date)
logger.info("polling Collection: {}...".format(collection.name))
#
# Build up the URI for polling
#
if not site.get('poll_path', ''):
uri = None
else:
uri = ''
if site.get('use_https'):
uri += 'https://'
else:
uri += 'http://'
uri += site.get('site')
uri += site.get('poll_path')
logger.info('Poll path: {}'.format(uri))
reports = []
while True:
try:
try:
content_blocks = client.poll(
uri=uri,
collection_name=collection.name,
begin_date=feed_helper.start_date,
end_date=feed_helper.end_date,
content_bindings=BINDING_CHOICES)
except Exception as e:
logger.info(e.message)
content_blocks = []
#
# Iterate through all content_blocks
#
num_blocks = 0
if not data_set:
logger.info("polling start_date: {}, end_date: {}".format(
feed_helper.start_date, feed_helper.end_date))
for block in content_blocks:
#
# if in export mode then save off this content block
#
if self.export_dir:
self.export_xml(collection_name,
feed_helper.start_date,
feed_helper.end_date, num_blocks,
block.content)
#
# This code accounts for a case found with ThreatCentral.io where the content is url encoded.
# etree.fromstring can parse this data.
#
try:
root = etree.fromstring(block.content)
content = root.find(
'.//{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content'
)
if content is not None and len(content) == 0 and len(
list(content)) == 0:
#
# Content has no children. So lets make sure we parse the xml text for content and re-add
# it as valid XML so we can parse
#
new_stix_package = etree.fromstring(
root.find(
"{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content_Block/{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content"
).text)
content.append(new_stix_package)
#
# Since we modified the xml, we need create a new xml message string to parse
#
message = etree.tostring(root)
#
# Write the content block to disk so we can parse with python stix
#
file_handle, file_path = self.write_to_temp_file(
message)
#
# Parse STIX data
#
stix_package = STIXPackage.from_xml(file_path)
#
# if it is a DATA_SET make feed_summary from the stix_header description
# NOTE: this is for RecordedFuture, also note that we only do this for data_sets.
# to date I have only seen RecordedFuture use data_sets
#
if data_set and stix_package.stix_header and stix_package.stix_header.descriptions:
for desc in stix_package.stix_header.descriptions:
feed_summary = desc.value
break
#
# Get the timestamp of the STIX Package so we can use this in our feed
#
timestamp = total_seconds(stix_package.timestamp)
if stix_package.indicators:
for indicator in stix_package.indicators:
if not indicator or not indicator.observable:
continue
if not indicator.timestamp:
timestamp = 0
else:
timestamp = int(
(indicator.timestamp -
datetime.datetime(1970, 1, 1).replace(
tzinfo=dateutil.tz.tzutc())
).total_seconds())
reports.extend(
cybox_parse_observable(
indicator.observable, indicator,
timestamp, default_score))
#
# Now lets find some data. Iterate through all observables and parse
#
if stix_package.observables:
for observable in stix_package.observables:
if not observable:
continue
#
# Cybox observable returns a list
#
reports.extend(
cybox_parse_observable(
observable, None, timestamp,
default_score))
#
# Delete our temporary file
#
file_handle.close()
num_blocks += 1
#
# end for loop through content blocks
#
except Exception as e:
#logger.info(traceback.format_exc())
logger.info(e.message)
continue
logger.info("content blocks read: {}".format(num_blocks))
logger.info("current number of reports: {}".format(
len(reports)))
#
# DEBUG CODE
#
#if len(reports) > 10:
# break
#
# Attempt to advance the start time and end time
#
except Exception as e:
logger.info(traceback.format_exc())
#
# If it is just a data_set, the data is unordered, so we can just break out of the while loop
#
if data_set:
break
if feed_helper.advance():
continue
else:
break
#
# end While True
#
logger.info("Found {} new reports.".format(len(reports)))
reports = feed_helper.load_existing_feed_data() + reports
logger.info("Total number of reports: {}".format(len(reports)))
data = build_feed_data(sanitized_feed_name,
"%s %s" % (site.get('site'), collection_name),
feed_summary, site.get('site'),
site.get('icon_link'), reports)
if feed_helper.write_feed(data):
feed_helper.save_details()
#
# Create Cb Response Feed if necessary
#
feed_id = self.cb.feed_get_id_by_name(sanitized_feed_name)
if not feed_id:
self.cb.feed_add_from_url("file://" + feed_helper.path,
site.get('feeds_enable'), False, False)
def perform(self):
"""
:param self:
:param enumerate_collections_only:
:return:
"""
for site in self.sites:
client = create_client(site.get('site'),
use_https=site.get('use_https'),
discovery_path=site.get('discovery_path'))
#
# Set verify_ssl and ca_cert inside the client
#
client.set_auth(verify_ssl=site.get('ssl_verify'),
ca_cert=site.get('ca_cert'))
#
# Proxy Settings
#
proxy_dict = dict()
if self.http_proxy_url:
logger.info("Found HTTP Proxy: {}".format(self.http_proxy_url))
proxy_dict['http'] = self.http_proxy_url
if self.https_proxy_url:
logger.info("Found HTTPS Proxy: {}".format(
self.https_proxy_url))
proxy_dict['https'] = self.https_proxy_url
if proxy_dict:
client.set_proxies(proxy_dict)
if site.get('username') or site.get('cert_file'):
#
# If a username is supplied use basic authentication
#
logger.info("Found Username in config, using basic auth...")
client.set_auth(username=site.get('username'),
password=site.get('password'),
verify_ssl=site.get('ssl_verify'),
ca_cert=site.get('ca_cert'),
cert_file=site.get('cert_file'),
key_file=site.get('key_file'))
if not site.get('collection_management_path', ''):
collections = client.get_collections()
else:
uri = ''
if site.get('use_https'):
uri += 'https://'
else:
uri += 'http://'
uri += site.get('site')
uri += site.get('collection_management_path')
logger.info('Collection Management Path: {}'.format(uri))
collections = client.get_collections(uri=uri)
for collection in collections:
logger.info('Collection Name: {}, Collection Type: {}'.format(
collection.name, collection.type))
if len(collections) == 0:
logger.info('Unable to find any collections. Exiting...')
sys.exit(0)
desired_collections = [
x.strip() for x in site.get('collections').lower().split(',')
]
want_all = False
if '*' in desired_collections:
want_all = True
for collection in collections:
if collection.type != 'DATA_FEED' and collection.type != 'DATA_SET':
continue
if collection.type == 'DATA_SET':
data_set = True
else:
data_set = False
if want_all or collection.name.lower() in desired_collections:
self._import_collection(client, site, collection, data_set)
class CbTaxiiFeedConverter(object):
def __init__(self, config_file_path, debug_mode=False, import_dir='', export_dir=''):
#
# parse config file and save off the information we need
#
config_dict = parse_config(config_file_path)
self.server_url = config_dict.get('server_url', 'https://127.0.0.1')
self.api_token = config_dict.get('api_token', '')
self.sites = config_dict.get('sites', [])
self.debug = config_dict.get('debug', False)
self.export_dir = export_dir
self.import_dir = import_dir
if self.export_dir and not os.path.exists(self.export_dir):
os.mkdir(self.export_dir)
#
# Test Cb Response connectivity
#
try:
self.cb = CbApi(server=self.server_url, token=self.api_token, ssl_verify=False)
self.cb.feed_enum()
except:
logger.error(traceback.format_exc())
sys.exit(-1)
def write_to_temp_file(self, message):
temp_file = tempfile.NamedTemporaryFile()
temp_file.write(message)
temp_file.flush()
return temp_file, temp_file.name
def read_from_xml(self):
"""
Walk the import dir and return all filenames. We are assuming all xml files
:return:
"""
f = []
for (dirpath, dirnames, filenames) in os.walk(self.import_dir):
f.extend(filenames)
break
return f
def export_xml(self, feed_name, start_time, end_time, block_num, message):
"""
:param feed_name:
:param start_time:
:param end_time:
:param block_num:
:param message:
:return:
"""
#
# create a directory to store all content blocks
#
dir_name = "{}".format(feed_name).replace(' ', '_')
full_dir_name = os.path.join(self.export_dir, dir_name)
#
# Make sure the directory exists
#
if not os.path.exists(os.path.join(self.export_dir, dir_name)):
os.mkdir(full_dir_name)
#
# Actually write the file
#
file_name = "{}-{}-{}".format(start_time, end_time, block_num).replace(' ', "_")
full_file_name = os.path.join(full_dir_name, file_name)
with open(full_file_name, 'wb') as file_handle:
file_handle.write(message)
def _import_collection(self, client, site, collection):
collection_name = collection.name
sanitized_feed_name = cleanup_string("%s%s" % (site.get('site'), collection_name))
available = collection.available
collection_type = collection.type
logger.info("%s,%s,%s,%s,%s" % (site.get('site'),
collection_name,
sanitized_feed_name,
available,
collection_type))
#
# We only care about DATA_FEED type
#
if not available or collection_type != "DATA_FEED":
return False
#
# Sanity check on start date
#
start_date_str = site.get('start_date')
if not start_date_str or len(start_date_str) == 0:
start_date_str = "2016-12-01 00:00:00"
#
# Create a feed helper object
#
feed_helper = FeedHelper(
site.get('output_path'),
sanitized_feed_name,
site.get('minutes_to_advance'),
start_date_str)
logger.info("Feed start time %s" % feed_helper.start_date)
logger.info("polling Collection: {}...".format(collection.name))
#
# Build up the URI for polling
#
if not site.get('poll_path', ''):
uri = None
else:
uri = ''
if site.get('use_https'):
uri += 'https://'
else:
uri += 'http://'
uri += site.get('site')
uri += site.get('poll_path')
logger.info('Poll path: {}'.format(uri))
reports = []
while True:
content_blocks = client.poll(uri=uri,
collection_name=collection.name,
begin_date=feed_helper.start_date,
end_date=feed_helper.end_date,
#content_bindings=BINDING_CHOICES)
content_bindings=[CB_STIX_XML_12])
#
# Iterate through all content_blocks
#
num_blocks = 0
for block in content_blocks:
#
# if in export mode then save off this content block
#
if self.export_dir:
self.export_xml(collection_name,
feed_helper.start_date,
feed_helper.end_date,
num_blocks,
block.content)
#
# This code accounts for a case found with ThreatCentral.io where the content is url encoded.
# etree.fromstring can parse this data.
#
root = etree.fromstring(block.content)
content = root.find('.//{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content')
if content is not None and len(content) == 0 and len(list(content)) == 0:
#
# Content has no children. So lets make sure we parse the xml text for content and re-add
# it as valid XML so we can parse
#
new_stix_package = etree.fromstring(root.find(
"{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content_Block/{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content").text)
content.append(new_stix_package)
#
# Since we modified the xml, we need create a new xml message string to parse
#
message = etree.tostring(root)
#
# Write the content block to disk so we can parse with python stix
#
file_handle, file_path = self.write_to_temp_file(message)
#
# Parse STIX data
#
stix_package = STIXPackage.from_xml(file_path)
#
# Get the timestamp of the STIX Package so we can use this in our feed
#
timestamp = total_seconds(stix_package.timestamp)
#
# Now lets find some data. Iterate through all observables and parse
#
if stix_package.observables:
for observable in stix_package.observables:
#
# Cybox observable returns a list
#
reports.extend(cybox_parse_observable(observable, timestamp))
#
# Delete our temporary file
#
file_handle.close()
num_blocks += 1
#
# end for loop through content blocks
#
logger.info("content blocks read: {}".format(num_blocks))
logger.info("current number of reports: {}".format(len(reports)))
#
# DEBUG CODE
#
#if len(reports) > 10:
# break
#
# Attempt to advance the start time and end time
#
if feed_helper.advance():
continue
else:
break
#
# end While True
#
logger.info("Found {} new reports.".format(len(reports)))
reports = feed_helper.load_existing_feed_data() + reports
logger.info("Total number of reports: {}".format(len(reports)))
data = build_feed_data(sanitized_feed_name,
"%s %s" % (site.get('site'), collection_name),
site.get('site'),
site.get('icon_link'),
reports)
if feed_helper.write_feed(data):
feed_helper.save_details()
#
# Create Cb Response Feed if necessary
#
feed_id = self.cb.feed_get_id_by_name(sanitized_feed_name)
if not feed_id:
data = self.cb.feed_add_from_url("file://" + feed_helper.path,
site.get('feeds_enable'),
False,
False)
def perform(self):
"""
:param self:
:param enumerate_collections_only:
:return:
"""
for site in self.sites:
client = create_client(site.get('site'),
use_https=site.get('use_https'),
discovery_path=site.get('discovery_path'))
if not site.get('collection_management_path', ''):
collections = client.get_collections()
else:
uri = ''
if site.get('use_https'):
uri += 'https://'
else:
uri += 'http://'
uri += site.get('site')
uri += site.get('collection_management_path')
logger.info('Collection Management Path: {}'.format(uri))
collections = client.get_collections(uri=uri)
for collection in collections:
logger.info('Collection Name: {}, Collection Type: {}'.format(collection.name, collection.type))
if len(collections) == 0:
logger.info('Unable to find any collections. Exiting...')
sys.exit(0)
desired_collections = site.get('collections').lower().split(',')
want_all = False
if '*' in desired_collections:
want_all = True
for collection in collections:
if collection.type != 'DATA_FEED':
continue
if want_all or collection.name.lower() in desired_collections:
self._import_collection(client, site, collection)
class CBQuery(object):
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
def report(self, rundll_query, dll_dictionary, search_match_count):
# CALLED BY: self.report(regex, regex_match_dictionary, search_match_count)
print "--------------------------------------------"
print "%s Command Line Matches:" % (search_match_count)
print "%s : %s" % ("Search Match Count", "Command Line Match")
print "--------------------------------------------"
#ordered_dll_dictionary = collections.OrderedDict(sorted(dll_dictionary.items()))
ordered_dll_dictionary = sorted(dll_dictionary.items(), key=operator.itemgetter(1))
for value in ordered_dll_dictionary:
print "%s : %s" % (value[1], value[0])
def check(self, regex, ignore_case, group_reference_to_match, count_flag, matches_only_flag):
# CALLED BY: cb.check(opts.regex, opts.ignore_case, opts.group_reference_to_match, opts.count_flag, opts.matches_only_flag)
# print a legend
print ""
print "Displaying Report for Commandline regular expression matches"
print ""
print "Command Line Strings Matching REGEX: %s" % (regex)
print "============================================================"
print ""
# build the query string
q = "cmdline:*"
#define dictionary
regex_match_dictionary = dict()
search_match_count = 0
#define regexp
# check if we need to ignore case, if so, update regexp
if ignore_case:
regexp = re.compile(regex, re.IGNORECASE)
else:
regexp = re.compile(regex)
for result in self.cb.process_search_iter(q):
cmdline = result.get("cmdline", "<unknown>")
# print "CMD: %s" % (cmdline,)
#SEARCH FOR REGEX IN STRING!!
if matches_only_flag:
# print "-----MATCHES ONLY"
search_match_result = regexp.match(cmdline)
else:
# print "-----EVERYTHING"
search_match_result = regexp.search(cmdline)
if search_match_result is not None:
# print "cmdline: %s" % (cmdline)
# print "result: %s" % (search_match_result)
# print "------------------------------------"
# Iterate TOTAL Search Match Count
search_match_count = search_match_count + 1
# On Match, add to dictionary
# 1st Check group_reference_to_match flag to see if we need to add a specific Group Reference or just the entire Command Line as the regex match
if group_reference_to_match:
# print "cmdline: %s" % (cmdline)
# print"matching GROUP: %s" % (group_reference_to_match)
# print"search_match_result: %s" % (search_match_result)
regex_match_group_reference = search_match_result.group(int(group_reference_to_match))
if regex_match_group_reference not in regex_match_dictionary.keys():
print "%s" % (regex_match_group_reference)
regex_match_dictionary[regex_match_group_reference] = 1
else:
regex_match_dictionary[regex_match_group_reference] = regex_match_dictionary[regex_match_group_reference] + 1
else:
if cmdline not in regex_match_dictionary.keys():
print "%s" % (cmdline)
regex_match_dictionary[cmdline] = 1
else:
regex_match_dictionary[cmdline] = regex_match_dictionary[cmdline] + 1
self.report(regex, regex_match_dictionary, search_match_count)
class CBQuery(object):
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
def report(self, ioc, type, procs, detail=False):
for result in procs["results"]:
# print the results to stdout. you could do anything here -
# log to syslog, send a SMS, fire off a siren and strobe light, etc.
print
print "Found %s IOC for %s in:" % (type, ioc)
print
print "\tPath: %s" % result["path"]
print "\tHostname: %s" % result["hostname"]
print "\tStarted: %s" % result["start"]
print "\tLast Updated: %s" % result["last_update"]
print "\tDetails: %s/#analyze/%s/%s" % (self.cb_url, result["id"], result["segment_id"])
print
if detail:
self.report_detail(ioc, type, result)
def report_detail(self, ioc, type, result):
events = self.cb.process_events(result["id"], result["segment_id"])
proc = events["process"]
if type == "domain" and proc.has_key("netconn_complete"):
for netconn in proc["netconn_complete"]:
ts, ip, port, proto, domain, dir = netconn.split("|")
if ioc in domain:
str_ip = socket.inet_ntoa(struct.pack("!i", int(ip)))
print "%s\t%s (%s:%s)" % (ts, domain, str_ip, port)
elif type == "ipaddr" and proc.has_key("netconn_complete"):
for netconn in proc["netconn_complete"]:
ts, ip, port, proto, domain, direction = netconn.split("|")
packed_ip = struct.unpack("!i", socket.inet_aton(ioc))[0]
#import code; code.interact(local=locals())
if packed_ip == int(ip):
str_ip = socket.inet_ntoa(struct.pack("!i", int(ip)))
print "%s\t%s (%s:%s)" % (ts, domain, str_ip, port)
elif type == "md5" and proc.has_key("modload_complete"):
for modload in proc["modload_complete"]:
ts, md5, path = modload.split("|")
if ioc in md5:
print "%s\t%s %s" % (ts, md5, path)
if result["process_md5"] == ioc:
print "%s\t%s %s" % (result["start"], result["process_md5"], result["path"])
def check(self, iocs, type, detail=False):
# for each ioc, do a search for (type):(ioc)
# e.g,
# domain:bigfish.com
# md5:ce7a81ceccfa03e5e0dfd0d9a7f41466
#
# additionally, if a cron interval is specified, limit searches
# to processes updated in the last CRON_INTERVAL period
#
# note - this is a very inefficient way to do this, since you test only one
# IOC per request - you could build a large OR clause together with a few hundred
# to efficiently and quickly check 1000s of IOCs, at the cost of increased complexity
# when you discover a hit.
#
# note 2 - with a list of flat indicators, what you really want is a CB feed
# see http://github.com/carbonblack/cbfeeds
#
for ioc in iocs:
if CRON_INTERVAL:
q = "%s:%s and last_update:-%s" % (type, ioc, CRON_INTERVAL)
else:
q = "%s:%s" % (type, ioc)
print q
procs = self.cb.process_search(q)
# if there are _any_ hits, give us the details.
# then check the next ioc
if len(procs["results"]) > 0:
self.report(ioc, type, procs, detail)
else:
sys.stdout.write(".")
sys.stdout.flush()
myrow.append(bprocess["id"])
if bprocess.has_key("cmdline"):
cmdline = bprocess["cmdline"]
else:
cmdline = ""
myrow.append(cmdline)
if bprocess.has_key("childproc_complete"):
getchildprocs(bprocess['childproc_complete'])
parser = build_cli_parser()
opts, args = parser.parse_args(sys.argv[1:])
if not opts.server or not opts.token or (not opts.procnamefile and not opts.procname) or not opts.percentless:
print "Missing required param."
sys.exit(-1)
cb = CbApi(opts.server, ssl_verify=opts.ssl_verify, token=opts.token)
if opts.procnamefile:
searchprocess = processsearchlist(opts.procnamefile)
else:
searchprocess = opts.procname.split(",")
for proc in searchprocess:
start = 0
data = cb.process_search("parent_name:%s" % (proc), rows=1, start=start)
facetterms = data['facets']
for term in reversed(facetterms['process_name']):
termratio = int(float(term['ratio']))
if int(opts.percentless) >= termratio:
start = 0
while True:
q = "parent_name:%s AND process_name:%s" % (proc,term['name'])
data = cb.process_search(q , rows=int(pagesize), start=start)
dest="ssl_verify",
help="Do not verify server SSL certificate.")
return parser
def processsearchlist(procnamefile):
with open(procnamefile) as tmp:
lines = filter(None, [line.strip() for line in tmp])
return lines
parser = build_cli_parser()
opts, args = parser.parse_args(sys.argv[1:])
if not opts.server or not opts.token or not opts.procsearchfile:
print "Missing required param."
sys.exit(-1)
cb = CbApi(opts.server, ssl_verify=opts.ssl_verify, token=opts.token)
searchprocess = processsearchlist(opts.procsearchfile)
for search in searchprocess:
data = cb.process_search(search, rows=1)
if 0 != (data['total_results']):
print "------------------------------"
print "Search Query | %s" % (search)
print "Resulting Processes | %s" % (data['total_results'])
facet = data['facets']
hosts = []
for term in facet['hostname']:
hosts.append("%s (%s%%)" % (term['name'], term['ratio']))
print "Resulting Hosts | " + "|".join(hosts)
myrow.append(bprocess["id"])
if bprocess.has_key("cmdline"):
cmdline = bprocess["cmdline"]
else:
cmdline = ""
myrow.append(cmdline)
if bprocess.has_key("childproc_complete"):
getchildprocs(bprocess['childproc_complete'])
parser = build_cli_parser()
opts, args = parser.parse_args(sys.argv[1:])
if not opts.server or not opts.token or not opts.query or not opts.percentless:
print "Missing required param."
sys.exit(-1)
cb = CbApi(opts.server, ssl_verify=opts.ssl_verify, token=opts.token)
start = 0
data = cb.process_search(opts.query, rows=1, start=start)
facetterms = data['facets']
pprint.pprint(facetterms)
'''
for term in reversed(facetterms['process_name']):
termratio = int(float(term['ratio']))
if int(opts.percentless) >= termratio:
start = 0
while True:
q = "parent_name:%s AND process_name:%s" % (proc,term['name'])
data = cb.process_search(q , rows=int(pagesize), start=start)
if 0 == len(data['results']):
class FileCollector:
def __init__(self):
config = load(open(r'conf.yaml', 'rb'))
self.address = config['address']
self.port = config['port']
self.banner = config['banner']
self.protection_url = config['protection']['url']
self.protection_token = config['protection']['token']
self.protection_headers = {'X-Auth-Token': self.protection_token}
self.protection_path = config['protection']['path']
self.protection_interval = config['protection']['interval']
self.protection_counter = config['protection']['counter']
self.response_url = config['response']['url']
self.response_token = config['response']['token']
self.response_path = config['response']['path']
self.archives = config['archives']
self.logs = config['logs']
self.password = config['password']
#~ self.root = getLogger("cbapi")
self.queue = Queue()
self.check_dirs()
setup_logging()
#~ basicConfig(filename=datetime.now().strftime(r'logs/FileCollector_%H_%M_%d_%m_%Y.log'), level=INFO)
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
self.cb = CbApi(self.response_url,
token=self.response_token,
ssl_verify=False)
def check_dirs(self):
if not isdir(self.protection_path):
log_event("[+] Creating {} directory".format(self.protection_path))
makedirs(self.protection_path)
if not isdir(self.response_path):
log_event("[+] Creating {} directory".format(self.response_path))
makedirs(self.response_path)
if not isdir(self.archives):
log_event("[+] Creating {} directory".format(self.archives))
makedirs(self.archives)
if not isdir(self.logs):
log_event("[+] Creating {} directory".format(self.logs))
makedirs(self.logs)
def start_service(self):
log_event(self.banner)
log_event("[*] Connecting to {}".format(self.protection_url))
log_event("[*] Connecting to {}".format(self.response_url))
self.listener = Listener((self.address, self.port))
while True:
try:
self.connection = self.listener.accept()
while True:
self.get_md5()
log_event("[*] Processing {}.".format(self.md5))
if self.get_fileCatalogData():
if not self.isQueued():
self.queue_file()
elif self.download_binary_cbr()['code']:
continue
else:
continue
except EOFError as eof:
log_event("[-] Lost connection to client.", True)
self.archive_and_encrypt()
def get_md5(self):
self.md5 = self.connection.recv()
def get_fileCatalogData(self):
self.dl_req = requests.get("{}{}{}".format(
self.protection_url, 'api/bit9platform/v1/fileCatalog?q=md5:',
self.md5),
headers=self.protection_headers,
verify=False)
if self.dl_req.ok:
self.fc_id = self.get_id()
if self.fc_id:
log_event("[+] Requested file found.[id] = {}".format(
self.fc_id))
return True
return False
else:
log_event("[-] Could not fetch File Catalog Data", True)
self.connection.send("[-] Could not fetch File Catalog Data")
return False
def isQueued(self):
log_event("[+] Checking for queued downloads.")
self.fileName = self.md5.lower()
isqueued = requests.get("{}{}{}".format(
self.protection_url,
'api/bit9platform/v1/fileCatalog?q=uploadStatus:0&limit=0&q=fileName:',
self.fileName),
headers=self.protection_headers,
verify=False)
if isqueued.ok:
log_event("[+] File is queued for uploading")
return True
return False
def queue_file(self):
log_event("[+] Queuing file for upload")
compId = self.dl_req.json()[0]['computerId']
self.data = {
'computerId': compId,
'fileCatalogId': self.fc_id,
'priority': 2,
'uploadStatus': 0
}
self.upload_req = requests.post("{}{}".format(
self.protection_url, 'api/bit9platform/v1/fileUpload'),
headers=self.protection_headers,
data=self.data,
verify=False)
if self.upload_req.ok:
self.check_fileQueue()
else:
log_event("[-] Requested file not found.", True)
self.connection.send("[-] Requested file not found.")
def check_fileQueue(self):
self.fileId = self.upload_req.json()['id']
log_event("[+] File is now queued. [id] = {}".format(self.fileId))
check_queue = requests.post("{}{}".format(
self.protection_url, 'api/bit9platform/v1/fileUpload'),
headers=self.protection_headers,
data=self.data,
verify=False)
ctr = 1
while check_queue.json()['uploadStatus'] is not 3:
log_event("[+] Sleeping for {} seconds.".format(
self.protection_interval))
sleep(self.protection_interval)
check_queue = requests.post("{}{}".format(
self.protection_url, 'api/bit9platform/v1/fileUpload'),
headers=self.protection_headers,
data=self.data,
verify=False)
#Sleep until 5 minutes has passed
ctr = ctr + 1
if ctr is self.protection_counter:
log_event("[-] File is still queued. Check the queue later.",
True)
self.connection.send(
"[-] File is still queued. Check the queue later.")
break
if check_queue.json()['uploadStatus'] is 3:
self.download_binary_cbp()
def get_id(self):
if isinstance(self.dl_req.json(), list):
if len(self.dl_req.json()) > 0:
return self.dl_req.json()[0]['id']
else:
log_event(
"[-] No File Catalog data found for {}.".format(self.md5),
True)
self.connection.send(
"[-] No File Catalog data found for {}.".format(self.md5))
return None
else:
return self.dl_req.json()['id']
def archive_and_encrypt(self):
try:
log_event("[*] Archiving downloaded file/s.")
prog_path = r"C:\Program Files\7-Zip\7z.exe"
if isfile(prog_path):
archive_name = datetime.now().strftime('%H_%M_%d_%m_%Y')
isZipped = check_call([
prog_path, 'a', '-p{}'.format(self.password), '-y',
r'{}/archive-{}.zip'.format(self.archives, archive_name),
'./{}/*'.format(self.protection_path), './{}/*'.format(
self.response_path)
],
stdout=open(devnull, 'w'))
if isZipped != 0:
raise CalledProcessError(cmd=prog_path,
returncode=1,
output='Failed to archive file/s')
else:
log_event('[+] File/s have been archived [{}.zip].'.format(
archive_name))
for file_response in listdir(self.response_path):
tmp_path = join(self.response_path, file_response)
remove(tmp_path)
for file_protection in listdir(self.protection_path):
tmp_path = join(self.protection_path, file_protection)
remove(tmp_path)
else:
raise WindowsError('[-] 7zip is missing in your system.')
except CalledProcessError as cpe:
log_event(cpe, True)
except WindowsError as we:
log_event(we, True)
def download_binary_cbr(self):
temp_text = {'msg': "[+] Download successful", 'code': 1}
try:
filename = join(self.response_path, self.md5.lower())
if not isfile(filename):
binary = self.cb.binary(self.md5)
dump = open(filename, "wb")
dump.write(binary)
dump.close()
else:
temp_text = {'msg': "[-] File exists.", 'code': 1}
except (ObjectNotFoundError, HTTPError):
temp_text = {'msg': "[-] File was not found", 'code': 0}
log_event(temp_text['msg'], (temp_text['code'] is 0))
return temp_text
def download_binary_cbp(self):
log_event("[+] Downloading sample")
dl = requests.get("{}{}".format(
self.protection_url,
'api/bit9platform/v1/fileUpload/{}?downloadFile=true'.format(
self.fileId)),
headers=self.headers,
verify=False,
stream=True)
chunk_size = 1024
size = int(dl.headers['Content-Length']) / chunk_size
i = 0
if dl.ok:
pbar = ProgressBar(widgets=[Percentage(), Bar()],
maxval=size).start()
with open("{}/{}".format(self.protection_path, self.fileName),
'wb') as f:
for chunk in dl.iter_content(chunk_size):
f.write(chunk)
pbar.update(i)
i += 1
#~ log_event("[+] Download complete")
pbar.finish()
f.close()
log_event("[+] Download complete.")
self.connection.send("[+] Download complete.")
else:
#~ print("[-] Unable to download file")
log_event("[-] Unable to download file.", True)
self.connection.send("[-] Unable to download file.")
help="File containing a line for each query you would like to run")
parser.add_option("-n", "--no-ssl-verify", action="store_false", default=True, dest="ssl_verify",
help="Do not verify server SSL certificate.")
return parser
def processsearchlist(procnamefile):
with open(procnamefile) as tmp:
lines = filter(None, [line.strip() for line in tmp])
return lines
parser = build_cli_parser()
opts, args = parser.parse_args(sys.argv[1:])
if not opts.server or not opts.token or not opts.procsearchfile:
print "Missing required param."
sys.exit(-1)
cb = CbApi(opts.server, ssl_verify=opts.ssl_verify, token=opts.token)
searchprocess = processsearchlist(opts.procsearchfile)
for search in searchprocess:
data = cb.process_search(search, rows=1)
if 0 != (data['total_results']):
print "------------------------------"
print "Search Query | %s" % (search)
print "Resulting Processes | %s" % (data['total_results'])
facet=data['facets']
hosts =[]
for term in facet['hostname']:
hosts.append("%s (%s%%)" % (term['name'], term['ratio']))
print "Resulting Hosts | "+"|".join(hosts)