class CBQuery(object):
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
def report(self, hostname, result):
# return the events associated with this process segment
# this will include netconns, as well as modloads, filemods, etc.
events = self.cb.process_events(result["id"], result["segment_id"])
proc = events["process"]
# for convenience, use locals for some process metadata fields
process_name = result.get("process_name", "<unknown>")
user_name = result.get("username", "<unknown>")
process_md5 = result.get("process_md5", "<unknown>")
# the search criteria (netconn_count:[1 to *]) should ensure that
# all results have at least one netconn
if proc.has_key("netconn_complete"):
# examine each netconn in turn
for netconn in proc["netconn_complete"]:
# split the netconn event into component parts
# note that the port is the remote port in the case of outbound
# netconns, and local port in the case of inbound netconns
ts, ip, port, proto, domain, dir = netconn.split("|")
# get the dotted-quad string representation of the ip
str_ip = socket.inet_ntoa(struct.pack("!i", int(ip)))
# the underlying data model provides the protocol number
# convert this to human-readable strings (tcp or udp)
if "6" == proto:
proto = "tcp"
elif "17" == proto:
proto = "udp"
# the underlying data model provides a boolean indication as to
# if this is an inbound or outbound network connection
if "true" == dir:
dir = "out"
else:
dir = "in"
# pring the record, using pipes as a delimiter
print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s)" % (hostname, ts, process_name, user_name, process_md5, proto, str_ip, port, dir, domain)
def check(self, hostname):
# print a legend
print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s)" % ("hostname", "timestamp", "process name", "username", "process md5", "protocol", "ip", "port", "direction", "domain")
# build the query string
q = "netconn_count:[1 to *] AND hostname:%s" % (hostname)
# begin with the first result - we'll perform the search in pages
# the default page size is 10 (10 reslts)
start = 0
# loop over the entire result set
while True:
# get the next page of results
procs = self.cb.process_search(q, start=start)
# if there are no results, we are done paging
if len(procs["results"]) == 0:
break
# examine each result individually
# each result represents a single process segment
for result in procs["results"]:
self.report(hostname, result)
# move forward to the next page
start = start + 10
class CBQuery(object):
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
def report(self, ioc, type, procs, detail=False):
for result in procs["results"]:
# print the results to stdout. you could do anything here -
# log to syslog, send a SMS, fire off a siren and strobe light, etc.
print
print "Found %s IOC for %s in:" % (type, ioc)
print
print "\tPath: %s" % result["path"]
print "\tHostname: %s" % result["hostname"]
print "\tStarted: %s" % result["start"]
print "\tLast Updated: %s" % result["last_update"]
print "\tDetails: %s/#analyze/%s/%s" % (self.cb_url, result["id"], result["segment_id"])
print
if detail:
self.report_detail(ioc, type, result)
def report_detail(self, ioc, type, result):
events = self.cb.process_events(result["id"], result["segment_id"])
proc = events["process"]
if type == "domain" and proc.has_key("netconn_complete"):
for netconn in proc["netconn_complete"]:
ts, ip, port, proto, domain, dir = netconn.split("|")
if ioc in domain:
str_ip = socket.inet_ntoa(struct.pack("!i", int(ip)))
print "%s\t%s (%s:%s)" % (ts, domain, str_ip, port)
elif type == "ipaddr" and proc.has_key("netconn_complete"):
for netconn in proc["netconn_complete"]:
ts, ip, port, proto, domain, direction = netconn.split("|")
packed_ip = struct.unpack("!i", socket.inet_aton(ioc))[0]
#import code; code.interact(local=locals())
if packed_ip == int(ip):
str_ip = socket.inet_ntoa(struct.pack("!i", int(ip)))
print "%s\t%s (%s:%s)" % (ts, domain, str_ip, port)
elif type == "md5" and proc.has_key("modload_complete"):
for modload in proc["modload_complete"]:
ts, md5, path = modload.split("|")
if ioc in md5:
print "%s\t%s %s" % (ts, md5, path)
if result["process_md5"] == ioc:
print "%s\t%s %s" % (result["start"], result["process_md5"], result["path"])
def check(self, iocs, type, detail=False):
# for each ioc, do a search for (type):(ioc)
# e.g,
# domain:bigfish.com
# md5:ce7a81ceccfa03e5e0dfd0d9a7f41466
#
# additionally, if a cron interval is specified, limit searches
# to processes updated in the last CRON_INTERVAL period
#
# note - this is a very inefficient way to do this, since you test only one
# IOC per request - you could build a large OR clause together with a few hundred
# to efficiently and quickly check 1000s of IOCs, at the cost of increased complexity
# when you discover a hit.
#
# note 2 - with a list of flat indicators, what you really want is a CB feed
# see http://github.com/carbonblack/cbfeeds
#
for ioc in iocs:
if CRON_INTERVAL:
q = "%s:%s and last_update:-%s" % (type, ioc, CRON_INTERVAL)
else:
q = "%s:%s" % (type, ioc)
print q
procs = self.cb.process_search(q)
# if there are _any_ hits, give us the details.
# then check the next ioc
if len(procs["results"]) > 0:
self.report(ioc, type, procs, detail)
else:
sys.stdout.write(".")
sys.stdout.flush()
class CBQuery(object):
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
def report(self, result, search_filename):
# return the events associated with this process segment
# this will include netconns, as well as modloads, filemods, etc.
#
events = self.cb.process_events(result["id"], result["segment_id"])
proc = events["process"]
# for convenience, use locals for some process metadata fields
#
host_name = result.get("hostname", "<unknown>")
process_name = result.get("process_name", "<unknown>")
user_name = result.get("username", "<unknown>")
if proc.has_key("filemod_complete"):
# examine each filemod in turn
#
for filemod in proc["filemod_complete"]:
# filemods are of the form:
# 1|2014-06-19 15:40:05.446|c:\dir\filename||
#
parts = filemod.split('|')
action, ts, filename, filemd5, filetype = parts[:5]
# the _document as a whole_ matched the query
# that doesn't mean each _indvidual filemod_ within the document matched
# the user-specified filename to search for
#
# iterate over each filemod and determine if the path matches
# what was specified
#
# make sense? hope so!
#
if search_filename.lower() not in filename.lower():
continue
if "1" == action:
action = "creation"
elif "2" == action or "8" == action:
action = "modification"
elif "4" == action:
action = "deletion"
print "%s|%s|%s|%s|%s|%s" % (host_name, ts, filename, process_name, user_name, action)
def check(self, filename):
# build the query string
#
q = "filemod:%s" % (filename)
# begin with the first result - we'll perform the search in pages
# the default page size is 10 (10 reslts)
#
start = 0
# loop over the entire result set
#
while True:
# get the next page of results
#
procs = self.cb.process_search(q, start=start)
# if there are no results, we are done paging
#
if len(procs["results"]) == 0:
break
# examine each result individually
# each result represents a single process segment
#
for result in procs["results"]:
self.report(result, filename)
# move forward to the next page
#
start = start + 10
class CBQuery(object):
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
def report(self, hostname, result):
# return the events associated with this process segment
# this will include netconns, as well as modloads, filemods, etc.
events = self.cb.process_events(result["id"], result["segment_id"])
proc = events["process"]
# for convenience, use locals for some process metadata fields
process_name = result.get("process_name", "<unknown>")
user_name = result.get("username", "<unknown>")
process_md5 = result.get("process_md5", "<unknown>")
# the search criteria (netconn_count:[1 to *]) should ensure that
# all results have at least one netconn
if proc.has_key("netconn_complete"):
# examine each netconn in turn
for netconn in proc["netconn_complete"]:
# split the netconn event into component parts
# note that the port is the remote port in the case of outbound
# netconns, and local port in the case of inbound netconns
ts, ip, port, proto, domain, dir = netconn.split("|")
# get the dotted-quad string representation of the ip
str_ip = socket.inet_ntoa(struct.pack("!i", int(ip)))
# the underlying data model provides the protocol number
# convert this to human-readable strings (tcp or udp)
if "6" == proto:
proto = "tcp"
elif "17" == proto:
proto = "udp"
# the underlying data model provides a boolean indication as to
# if this is an inbound or outbound network connection
if "true" == dir:
dir = "out"
else:
dir = "in"
# pring the record, using pipes as a delimiter
print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s)" % (
hostname, ts, process_name, user_name, process_md5, proto,
str_ip, port, dir, domain)
def check(self, hostname):
# print a legend
print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s)" % (
"hostname", "timestamp", "process name", "username", "process md5",
"protocol", "ip", "port", "direction", "domain")
# build the query string
q = "netconn_count:[1 to *] AND hostname:%s" % (hostname)
# begin with the first result - we'll perform the search in pages
# the default page size is 10 (10 reslts)
start = 0
# loop over the entire result set
while True:
# get the next page of results
procs = self.cb.process_search(q, start=start)
# if there are no results, we are done paging
if len(procs["results"]) == 0:
break
# examine each result individually
# each result represents a single process segment
for result in procs["results"]:
self.report(hostname, result)
# move forward to the next page
start = start + 10
class CBQuery(object):
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
# set up some stat tracking
self.stats = {}
self.stats['total_processes'] = 0
self.stats['total_netconns'] = 0
self.stats['matching_netconns'] = 0
self.stats['output_errors'] = 0
self.stats['process_errors'] = 0
def getStats(self):
"""
return the current statistics
"""
return self.stats
def outputNetConn(self, proc, netconn):
"""
output a single netconn event from a process document
the caller is responsible for ensuring that the document
meets start time and subnet criteria
"""
# for convenience, use locals for some process metadata fields
hostname = proc.get("hostname", "<unknown>")
process_name = proc.get("process_name", "<unknown>")
user_name = proc.get("username", "<unknown>")
process_md5 = proc.get("process_md5", "<unknown>")
cmdline = proc.get("cmdline", "<unknown>")
path = proc.get("path", "<unknown>")
procstarttime = proc.get("start", "<unknown>")
proclastupdate = proc.get("last_update", "<unknown>")
# split the netconn into component parts
ts, ip, port, proto, domain, dir = netconn.split("|")
# get the dotted-quad string representation of the ip
str_ip = socket.inet_ntoa(struct.pack("!i", int(ip)))
# the underlying data model provides the protocol number
# convert this to human-readable strings (tcp or udp)
if "6" == proto:
proto = "tcp"
elif "17" == proto:
proto = "udp"
# the underlying data model provides a boolean indication as to
# if this is an inbound or outbound network connection
if "true" == dir:
dir = "out"
else:
dir = "in"
# print the record, using pipes as a delimiter
print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|" % (procstarttime,proclastupdate,hostname, user_name, proto, str_ip, port, dir, domain, process_name, process_md5, path, cmdline)
def addressInNetwork(self, ip, cidr):
# the ip can be the emtpy string ('') in cases where the connection
# is made via a web proxy. in these cases the sensor cannot report
# the true remote IP as DNS resolution happens on the web proxy (and
# not the endpoint)
if '' == ip:
return False
try:
net = cidr.split('/')[0]
bits = cidr.split('/')[1]
if int(ip) > 0:
ipaddr = struct.unpack('<L', socket.inet_aton(ip))[0]
else:
ipaddr = struct.unpack('<L', socket.inet_aton(".".join(map(lambda n: str(int(ip)>>n & 0xFF), [24,16,8,0]))))[0]
netaddr = struct.unpack('<L', socket.inet_aton(net))[0]
netmask = ((1L << int(bits)) - 1)
return ipaddr & netmask == netaddr & netmask
except:
return False
def report(self, result, subnet):
# return the events associated with this process segment
# this will include netconns, as well as modloads, filemods, etc.
events = self.cb.process_events(result["id"], result["segment_id"])
proc = events["process"]
# the search criteria (netconn_count:[1 to *]) should ensure that
# all results have at least one netconn
if proc.has_key("netconn_complete"):
# examine each netconn in turn
for netconn in proc["netconn_complete"]:
# update the count of total netconns
self.stats['total_netconns'] = self.stats['total_netconns'] + 1
# split the netconn event into component parts
# note that the port is the remote port in the case of outbound
# netconns, and local port in the case of inbound netconns
#
# for the purpose of this example script, eat any errors
ts, ip, port, proto, domain, dir = netconn.split("|")
if self.addressInNetwork(ip, subnet):
try:
self.stats['matching_netconns'] = self.stats['matching_netconns'] + 1
self.outputNetConn(proc, netconn)
except:
self.stats['output_errors'] = self.stats['output_errors'] + 1
pass
def strip_to_int(ip):
"""
convert a dotted-quad string IP to the corresponding int32
"""
return struct.unpack('<L', socket.inet_aton(ip))[0]
def check(self, subnet, datetype, begin, end):
# print a legend
print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|" % ("ProcStartTime", "ProcUpdateTime","hostname", "username", "protocol", "ip", "port", "direction", "domain", "process name", "process md5", "process path", "cmdline")
# build the query string
if not end and not begin:
q = "ipaddr:%s" % (subnet,)
else:
if not end: end = "*"
if not begin: begin = "*"
q = "ipaddr:%s %s:[%s TO %s]" % (subnet, datetype, begin, end)
# begin with the first result - we'll perform the search in pages
# the default page size is 10 (10 results)
start = 0
# loop over the entire result set, paging as required
while True:
# get the next page of results
procs = self.cb.process_search(q, start=start)
# track the total number of matching results
self.stats['total_processes'] = procs['total_results']
# if there are no results, we are done paging
if len(procs["results"]) == 0:
break
# examine each result individually
# each result represents a single segment of a single process
#
# for the purposes of this example script, eat any errors
#
for result in procs["results"]:
try:
self.report(result, subnet)
except Exception, e:
self.stats['process_errors'] = self.stats['process_errors'] + 1
# move forward to the next page
start = start + 10
class CBQuery(object):
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
def report(self, result, search_filename):
# return the events associated with this process segment
# this will include netconns, as well as modloads, filemods, etc.
#
events = self.cb.process_events(result["id"], result["segment_id"])
proc = events["process"]
# for convenience, use locals for some process metadata fields
#
host_name = result.get("hostname", "<unknown>")
process_name = result.get("process_name", "<unknown>")
user_name = result.get("username", "<unknown>")
if proc.has_key("filemod_complete"):
# examine each filemod in turn
#
for filemod in proc["filemod_complete"]:
# filemods are of the form:
# 1|2014-06-19 15:40:05.446|c:\dir\filename||
#
action, ts, filename, filemd5, filetype = filemod.split('|')
# the _document as a whole_ matched the query
# that doesn't mean each _indvidual filemod_ within the document matched
# the user-specified filename to search for
#
# iterate over each filemod and determine if the path matches
# what was specified
#
# make sense? hope so!
#
if search_filename.lower() not in filename.lower():
continue
if "1" == action:
action = "creation"
elif "2" == action or "8" == action:
action = "modification"
elif "4" == action:
action = "deletion"
print "%s|%s|%s|%s|%s|%s" % (host_name, ts, filename,
process_name, user_name, action)
def check(self, filename):
# build the query string
#
q = "filemod:%s" % (filename)
# begin with the first result - we'll perform the search in pages
# the default page size is 10 (10 reslts)
#
start = 0
# loop over the entire result set
#
while True:
# get the next page of results
#
procs = self.cb.process_search(q, start=start)
# if there are no results, we are done paging
#
if len(procs["results"]) == 0:
break
# examine each result individually
# each result represents a single process segment
#
for result in procs["results"]:
self.report(result, filename)
# move forward to the next page
#
start = start + 10
class CBQuery(object):
def __init__(self, url, token, ssl_verify):
self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
self.cb_url = url
# set up some stat tracking
self.stats = {}
self.stats['total_processes'] = 0
self.stats['total_netconns'] = 0
self.stats['matching_netconns'] = 0
self.stats['output_errors'] = 0
self.stats['process_errors'] = 0
def getStats(self):
"""
return the current statistics
"""
return self.stats
def outputNetConn(self, proc, netconn):
"""
output a single netconn event from a process document
the caller is responsible for ensuring that the document
meets start time and subnet criteria
"""
# for convenience, use locals for some process metadata fields
hostname = proc.get("hostname", "<unknown>")
process_name = proc.get("process_name", "<unknown>")
user_name = proc.get("username", "<unknown>")
process_md5 = proc.get("process_md5", "<unknown>")
cmdline = proc.get("cmdline", "<unknown>")
path = proc.get("path", "<unknown>")
procstarttime = proc.get("start", "<unknown>")
proclastupdate = proc.get("last_update", "<unknown>")
# split the netconn into component parts
ts, ip, port, proto, domain, dir = netconn.split("|")
# get the dotted-quad string representation of the ip
str_ip = socket.inet_ntoa(struct.pack("!i", int(ip)))
# the underlying data model provides the protocol number
# convert this to human-readable strings (tcp or udp)
if "6" == proto:
proto = "tcp"
elif "17" == proto:
proto = "udp"
# the underlying data model provides a boolean indication as to
# if this is an inbound or outbound network connection
if "true" == dir:
dir = "out"
else:
dir = "in"
# print the record, using pipes as a delimiter
print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|" % (
procstarttime, proclastupdate, hostname, user_name, proto, str_ip,
port, dir, domain, process_name, process_md5, path, cmdline)
def addressInNetwork(self, ip, cidr):
# the ip can be the emtpy string ('') in cases where the connection
# is made via a web proxy. in these cases the sensor cannot report
# the true remote IP as DNS resolution happens on the web proxy (and
# not the endpoint)
if '' == ip:
return False
try:
net = cidr.split('/')[0]
bits = cidr.split('/')[1]
if int(ip) > 0:
ipaddr = struct.unpack('<L', socket.inet_aton(ip))[0]
else:
ipaddr = struct.unpack(
'<L',
socket.inet_aton(".".join(
map(lambda n: str(int(ip) >> n & 0xFF),
[24, 16, 8, 0]))))[0]
netaddr = struct.unpack('<L', socket.inet_aton(net))[0]
netmask = ((1L << int(bits)) - 1)
return ipaddr & netmask == netaddr & netmask
except:
return False
def report(self, result, subnet):
# return the events associated with this process segment
# this will include netconns, as well as modloads, filemods, etc.
events = self.cb.process_events(result["id"], result["segment_id"])
proc = events["process"]
# the search criteria (netconn_count:[1 to *]) should ensure that
# all results have at least one netconn
if proc.has_key("netconn_complete"):
# examine each netconn in turn
for netconn in proc["netconn_complete"]:
# update the count of total netconns
self.stats['total_netconns'] = self.stats['total_netconns'] + 1
# split the netconn event into component parts
# note that the port is the remote port in the case of outbound
# netconns, and local port in the case of inbound netconns
#
# for the purpose of this example script, eat any errors
ts, ip, port, proto, domain, dir = netconn.split("|")
if self.addressInNetwork(ip, subnet):
try:
self.stats['matching_netconns'] = self.stats[
'matching_netconns'] + 1
self.outputNetConn(proc, netconn)
except:
self.stats[
'output_errors'] = self.stats['output_errors'] + 1
pass
def strip_to_int(ip):
"""
convert a dotted-quad string IP to the corresponding int32
"""
return struct.unpack('<L', socket.inet_aton(ip))[0]
def check(self, subnet, datetype, begin, end):
# print a legend
print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|" % (
"ProcStartTime", "ProcUpdateTime", "hostname", "username",
"protocol", "ip", "port", "direction", "domain", "process name",
"process md5", "process path", "cmdline")
# build the query string
if not end and not begin:
q = "ipaddr:%s" % (subnet, )
else:
if not end: end = "*"
if not begin: begin = "*"
q = "ipaddr:%s %s:[%s TO %s]" % (subnet, datetype, begin, end)
# begin with the first result - we'll perform the search in pages
# the default page size is 10 (10 results)
start = 0
# loop over the entire result set, paging as required
while True:
# get the next page of results
procs = self.cb.process_search(q, start=start)
# track the total number of matching results
self.stats['total_processes'] = procs['total_results']
# if there are no results, we are done paging
if len(procs["results"]) == 0:
break
# examine each result individually
# each result represents a single segment of a single process
#
# for the purposes of this example script, eat any errors
#
for result in procs["results"]:
try:
self.report(result, subnet)
except Exception, e:
self.stats[
'process_errors'] = self.stats['process_errors'] + 1
# move forward to the next page
start = start + 10
