Esempio n. 1
0
 def __init__(self):
     config = load(open(r'conf.yaml', 'rb'))
     self.address = config['address']
     self.port = config['port']
     self.banner = config['banner']
     self.protection_url = config['protection']['url']
     self.protection_token = config['protection']['token']
     self.protection_headers = {'X-Auth-Token': self.protection_token}
     self.protection_path = config['protection']['path']
     self.protection_interval = config['protection']['interval']
     self.protection_counter = config['protection']['counter']
     self.response_url = config['response']['url']
     self.response_token = config['response']['token']
     self.response_path = config['response']['path']
     self.archives = config['archives']
     self.logs = config['logs']
     self.password = config['password']
     #~ self.root = getLogger("cbapi")
     self.queue = Queue()
     self.check_dirs()
     setup_logging()
     #~ basicConfig(filename=datetime.now().strftime(r'logs/FileCollector_%H_%M_%d_%m_%Y.log'), level=INFO)
     requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
     self.cb = CbApi(self.response_url,
                     token=self.response_token,
                     ssl_verify=False)
Esempio n. 2
0
    def __init__(self,
                 config_file_path,
                 debug_mode=False,
                 import_dir='',
                 export_dir=''):

        #
        # parse config file and save off the information we need
        #
        config_dict = parse_config(config_file_path)

        self.server_url = config_dict.get('server_url', 'https://127.0.0.1')
        self.api_token = config_dict.get('api_token', '')
        self.sites = config_dict.get('sites', [])
        self.debug = config_dict.get('debug', False)
        self.export_dir = export_dir
        self.import_dir = import_dir

        self.http_proxy_url = config_dict.get('http_proxy_url', None)
        self.https_proxy_url = config_dict.get('https_proxy_url', None)

        if self.export_dir and not os.path.exists(self.export_dir):
            os.mkdir(self.export_dir)

        #
        # Test Cb Response connectivity
        #
        try:
            self.cb = CbApi(server=self.server_url,
                            token=self.api_token,
                            ssl_verify=False)
            self.cb.feed_enum()
        except:
            logger.error(traceback.format_exc())
            sys.exit(-1)
Esempio n. 3
0
    def prepare(self):
        configuration_dict = splunk.clilib.cli_common.getConfStanza(
            'carbonblack', 'cbserver')

        cb_server = configuration_dict['cburl']
        token = configuration_dict['cbapikey']

        self.cb = CbApi(cb_server, token=token, ssl_verify=False)
    def __init__(self, url, token, ssl_verify):
        self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
        self.cb_url = url

        # set up some stat tracking
        self.stats = {}
        self.stats['total_processes'] = 0
        self.stats['total_netconns'] = 0
        self.stats['matching_netconns'] = 0
        self.stats['output_errors'] = 0
        self.stats['process_errors'] = 0
    def __init__(self, hosts_mapping):
        configuration_dict = splunk.clilib.cli_common.getConfStanza('carbonblack', 'cbserver')

        self.cb_server = configuration_dict['cburl']
        self.token = configuration_dict['cbapikey']

        self.cb = CbApi(self.cb_server, token=self.token, ssl_verify=False)

        self.logger = logging.getLogger(__name__)
        self.logger.setLevel(logging.DEBUG)

        self.hosts_mapping = hosts_mapping
Esempio n. 6
0
    def __init__(self, config_file_path, debug_mode=False, import_dir='', export_dir=''):

        #
        # parse config file and save off the information we need
        #
        config_dict = parse_config(config_file_path)

        self.server_url = config_dict.get('server_url', 'https://127.0.0.1')
        self.api_token = config_dict.get('api_token', '')
        self.sites = config_dict.get('sites', [])
        self.debug = config_dict.get('debug', False)
        self.export_dir = export_dir
        self.import_dir = import_dir

        if self.export_dir and not os.path.exists(self.export_dir):
            os.mkdir(self.export_dir)

        #
        # Test Cb Response connectivity
        #
        try:
            self.cb = CbApi(server=self.server_url, token=self.api_token, ssl_verify=False)
            self.cb.feed_enum()
        except:
            logger.error(traceback.format_exc())
            sys.exit(-1)
class BinarySearchCommand(GeneratingCommand):
    """Generates a binary search result from Carbon Black from a given MD5 or search query
        | binarysearch ${md5}
    """

    query = Option(name="query", require=True)
    field_names = ['digsig_publisher', 'digsig_result', 'digsig_sign_time', 'host_count', 'is_executable_image',
                   'last_seen', 'original_filename', 'os_type', 'product_name', 'product_version', 'md5']

    def prepare(self):
        configuration_dict = splunk.clilib.cli_common.getConfStanza('carbonblack', 'cbserver')

        self.cb_server = configuration_dict['cburl']
        token = configuration_dict['cbapikey']

        self.cb = CbApi(self.cb_server, token=token, ssl_verify=False)

    def generate(self):
        for bindata in self.cb.binary_search_iter(self.query):
            self.logger.info("yielding binary %s" % bindata["md5"])
            rawdata = dict((field_name, bindata.get(field_name, "")) for field_name in self.field_names)
        try:
            rawdata
            synthevent = {'sourcetype': 'bit9:carbonblack:json', '_time': time.time(), 'source': self.cb_server,
                          '_raw': rawdata}
            yield synthevent
        except Exception:
            synthevent = {'sourcetype': 'bit9:carbonblack:json', '_time': time.time(), 'source': self.cb_server,
                          '_raw': '{"Error":"MD5 not found"}'}
            yield synthevent
    def prepare(self):
        configuration_dict = splunk.clilib.cli_common.getConfStanza('carbonblack', 'cbserver')

        self.cb_server = configuration_dict['cburl']
        token = configuration_dict['cbapikey']

        self.cb = CbApi(self.cb_server, token=token, ssl_verify=False)
 def __init__(self, cb_server, token):
     self.cb = CbApi(cb_server, token=token, ssl_verify=False)
     self.worker_queue = Queue.Queue(maxsize=10)
     self.output_queue = Queue.Queue()
     self.logger = logging.getLogger(__name__)
     self.logger.setLevel(logging.DEBUG)
     logging.basicConfig()
Esempio n. 10
0
class ProcessSearchCommand(GeneratingCommand):
    """Generates a process search result from Carbon Black from a given IP or search query
        | processsearch query=${ip}
    """

    query = Option(name="query", require=True)

    field_names = [
        'cmdline', 'comms_ip', 'hostname', 'id', 'interface_ip', 'last_update',
        'os_type', 'parent_md5', 'parent_name', 'parent_pid',
        'parent_unique_id', 'path', 'process_md5', 'process_name',
        'process_pid', 'regmod_count', 'segment_id', 'sensor_id', 'start',
        'unique_id', 'username', 'childproc_count', 'crossproc_count',
        'modload_count', 'netconn_count', 'filemod_count', 'group', 'host_type'
    ]

    def prepare(self):

        configuration_dict = splunk.clilib.cli_common.getConfStanza(
            'carbonblack', 'cbserver')

        self.cb_server = configuration_dict['cburl']
        self.token = configuration_dict['cbapikey']

        self.cb = CbApi(self.cb_server, token=self.token, ssl_verify=False)

    def generate(self):
        self.logger.info("query %s" % self.query)
        i = 0
        for bindata in self.cb.process_search_iter(self.query):
            i += 1
            if i > 1000:
                # TODO: let's stop at 1,000 results for now?
                self.finish()
                return

            temp = dict((field_name, bindata[field_name])
                        for field_name in self.field_names)
            temp['sourcetype'] = 'bit9:carbonblack:json'

            #
            # Sometimes we have seen 'start' be equal to -1
            #
            try:
                temp['_time'] = int(
                    time.mktime(
                        dateutil.parser.parse(bindata['start']).timetuple()))
            except Exception as e:
                self.logger.exception('parsing bindata["start"] %s' %
                                      bindata['start'])
                temp['_time'] = 0

            temp['link_process'] = self.cb_server + '/#/analyze/' + bindata[
                'id'] + "/1"
            temp['source'] = 'cbapi'
            temp['_raw'] = json.dumps(temp)
            yield temp

            if i % 10 == 0:
                self.flush()
Esempio n. 11
0
class BinarySearchCommand(GeneratingCommand):
    """Generates a binary search result from Carbon Black from a given MD5 or search query
        | binarysearch ${md5}
    """

    query = Option(name="query", require=True)
    field_names = [
        'digsig_publisher', 'digsig_result', 'digsig_sign_time', 'host_count',
        'is_executable_image', 'last_seen', 'original_filename', 'os_type',
        'product_name', 'product_version', 'md5'
    ]

    def prepare(self):
        configuration_dict = splunk.clilib.cli_common.getConfStanza(
            'carbonblack', 'cbserver')

        cb_server = configuration_dict['cburl']
        token = configuration_dict['cbapikey']

        self.cb = CbApi(cb_server, token=token, ssl_verify=False)

    def generate(self):
        for bindata in self.cb.binary_search_iter(self.query):
            self.logger.info("yielding binary %s" % bindata["md5"])
            yield dict((field_name, bindata.get(field_name, ""))
                       for field_name in self.field_names)
Esempio n. 12
0
    def __init__(self, url, token, ssl_verify):
        self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
        self.cb_url = url

        # set up some stat tracking
        self.stats = {}
        self.stats['total_processes'] = 0
        self.stats['total_netconns'] = 0
        self.stats['matching_netconns'] = 0
        self.stats['output_errors'] = 0
        self.stats['process_errors'] = 0
Esempio n. 13
0
    def validate_form(self, form):
        if not super(CbModelView, self).validate_form(form):
            return False

        if not hasattr(form, 'url') or not hasattr(form, 'admin_key'):
            return True

        cb_server = form.url.data
        cb_token = form.admin_key.data

        # validate that we can connect to the cb server using the token provided
        c = CbApi(cb_server, token=cb_token, ssl_verify=False)
        try:
            info = c.info()
            return True
        except:
            import traceback
            traceback.print_exc()
            for field in [form.url, form.admin_key]:
                field.errors.append("Could not contact a Cb server at this URL and token")
            return False
Esempio n. 14
0
    def on_model_change(self, form, model, is_created):
        if is_created:
            (already_exist, failed_users, success_users) = bulk_synchronize(
                CbApi(model.url, token=model.admin_key, ssl_verify=False)
            )

            statuses = []
            if len(already_exist):
                statuses.append("Users {0:s} already existed".format(", ".join(already_exist)))
            if len(failed_users):
                statuses.append("Failed to add users {0:s}".format(", ".join(failed_users)))

            flash(". ".join(statuses))

        else:
            # all we do here is to update the SAML SP configuration
            load_metadata(model.id, model.saml_sp_config)
def main_helper(description, main, custom_required=None, custom_optional=None):
    """

    :param description:
    :param main:
    :return:
    """

    default_required = [("-c", "--cburl", "store", None, "server_url",
                         "CB server's URL.  e.g., http://127.0.0.1 "),
                        ("-a", "--apitoken", "store", None, "token",
                         "API Token for Carbon Black server")]

    default_optional = [("-n", "--no-ssl-verify", "store_false", True,
                         "ssl_verify", "Do not verify server SSL certificate.")
                        ]

    if not custom_required:
        custom_required = []

    if not custom_optional:
        custom_optional = []

    required = default_required + custom_required
    optional = default_optional + custom_optional

    parser = build_cli_parser(description, required + optional)
    opts, args = parser.parse_args()
    for opt in required:
        if not getattr(opts, opt[4]):
            print
            print "** Missing required parameter '%s' **" % opt[4]
            print
            parser.print_help()
            print
            sys.exit(-1)

    args = {}
    for opt in required + optional:
        name = opt[4]
        args[name] = getattr(opts, name)

    cb = CbApi(opts.server_url, ssl_verify=opts.ssl_verify, token=opts.token)
    main(cb, args)
Esempio n. 16
0
class CBQuery(object):
    def __init__(self, url, token, ssl_verify):
        self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
        self.cb_url = url

    def report(self, hostname, user_dictionary):
        print ""
        print "%s | %s : %s" % ("Hostname", "Process Count", "Username")
        print "--------------------------------------------"
 
	for key,value in user_dictionary.items():
		print "%s | %s = %s" % (hostname, value, key)

    def check(self, hostname):
        # print a legend
	print ""
	print "USER REPORT FOR %s:" % (hostname)
	print "--------------------------------------------"

        # build the query string
        q = "hostname:%s" % (hostname)
      
	#define dictionary
	user_dictionary = dict()
 
	# loop over the entire result set
        for result in self.cb.process_search_iter(q):
		user_name = result.get("username", "<unknown>")

		if user_name not in user_dictionary.keys():
			print "NEW USER found on %s : %s" % (hostname, user_name)
			user_dictionary[user_name] = 1
		else:
			user_dictionary[user_name] = user_dictionary[user_name] + 1

	self.report(hostname, user_dictionary)
class CBQuery(object):
    def __init__(self, url, token, ssl_verify):
        self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
        self.cb_url = url

    def report(self, hostname, user_dictionary):
        print ""
        print "%s | %s : %s" % ("Hostname", "Process Count", "Username")
        print "--------------------------------------------"

        for key, value in user_dictionary.items():
            print "%s | %s = %s" % (hostname, value, key)

    def check(self, hostname):
        # print a legend
        print ""
        print "USER REPORT FOR %s:" % (hostname)
        print "--------------------------------------------"

        # build the query string
        q = "hostname:%s" % (hostname)

        #define dictionary
        user_dictionary = dict()

        # loop over the entire result set
        for result in self.cb.process_search_iter(q):
            user_name = result.get("username", "<unknown>")

            if user_name not in user_dictionary.keys():
                print "NEW USER found on %s : %s" % (hostname, user_name)
                user_dictionary[user_name] = 1
            else:
                user_dictionary[user_name] = user_dictionary[user_name] + 1

        self.report(hostname, user_dictionary)
class CBQuery(object):
    def __init__(self, url, token, ssl_verify):
        self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
        self.cb_url = url

    def report(self, rundll_query, dll_dictionary, search_match_count):
        # CALLED BY: self.report(regex, regex_match_dictionary, search_match_count)

        print "--------------------------------------------"
        print "%s Command Line Matches:" % (search_match_count)
        print "%s : %s" % ("Search Match Count", "Command Line Match")
        print "--------------------------------------------"

        #ordered_dll_dictionary = collections.OrderedDict(sorted(dll_dictionary.items()))
        ordered_dll_dictionary = sorted(dll_dictionary.items(),
                                        key=operator.itemgetter(1))
        for value in ordered_dll_dictionary:
            print "%s : %s" % (value[1], value[0])

    def check(self, regex, ignore_case, group_reference_to_match, count_flag,
              matches_only_flag):
        # CALLED BY: cb.check(opts.regex, opts.ignore_case, opts.group_reference_to_match, opts.count_flag, opts.matches_only_flag)

        # print a legend
        print ""
        print "Displaying Report for Commandline regular expression matches"
        print ""
        print "Command Line Strings Matching REGEX: %s" % (regex)
        print "============================================================"
        print ""

        # build the query string
        q = "cmdline:*"

        #define dictionary
        regex_match_dictionary = dict()
        search_match_count = 0

        #define regexp
        # check if we need to ignore case, if so, update regexp
        if ignore_case:
            regexp = re.compile(regex, re.IGNORECASE)
        else:
            regexp = re.compile(regex)

        for result in self.cb.process_search_iter(q):
            cmdline = result.get("cmdline", "<unknown>")
            # print "CMD: %s" % (cmdline,)

            #SEARCH FOR REGEX IN STRING!!
            if matches_only_flag:
                # print "-----MATCHES ONLY"
                search_match_result = regexp.match(cmdline)
            else:
                # print "-----EVERYTHING"
                search_match_result = regexp.search(cmdline)

            if search_match_result is not None:
                # print "cmdline: %s" % (cmdline)
                # print "result: %s" % (search_match_result)
                # print "------------------------------------"

                # Iterate TOTAL Search Match Count
                search_match_count = search_match_count + 1

                # On Match, add to dictionary
                # 1st Check group_reference_to_match flag to see if we need to add a specific Group Reference or just the entire Command Line as the regex match
                if group_reference_to_match:
                    # print "cmdline: %s" % (cmdline)
                    # print"matching GROUP: %s" % (group_reference_to_match)
                    # print"search_match_result: %s" % (search_match_result)
                    regex_match_group_reference = search_match_result.group(
                        int(group_reference_to_match))
                    if regex_match_group_reference not in regex_match_dictionary.keys(
                    ):
                        print "%s" % (regex_match_group_reference)
                        regex_match_dictionary[regex_match_group_reference] = 1
                    else:
                        regex_match_dictionary[
                            regex_match_group_reference] = regex_match_dictionary[
                                regex_match_group_reference] + 1
                else:
                    if cmdline not in regex_match_dictionary.keys():
                        print "%s" % (cmdline)
                        regex_match_dictionary[cmdline] = 1
                    else:
                        regex_match_dictionary[
                            cmdline] = regex_match_dictionary[cmdline] + 1

        self.report(regex, regex_match_dictionary, search_match_count)
Esempio n. 19
0
class CBQuery(object):
    def __init__(self, url, token, ssl_verify):
        self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
        self.cb_url = url

    def report(self, hostname, result):

        # return the events associated with this process segment
        # this will include netconns, as well as modloads, filemods, etc.
        events = self.cb.process_events(result["id"], result["segment_id"])

        proc = events["process"]

        # for convenience, use locals for some process metadata fields
        process_name = result.get("process_name", "<unknown>")
        user_name = result.get("username", "<unknown>")
        process_md5 = result.get("process_md5", "<unknown>")

        # the search criteria (netconn_count:[1 to *]) should ensure that
        # all results have at least one netconn
        if proc.has_key("netconn_complete"):

            # examine each netconn in turn
            for netconn in proc["netconn_complete"]:

                # split the netconn event into component parts
                # note that the port is the remote port in the case of outbound
                # netconns, and local port in the case of inbound netconns
                ts, ip, port, proto, domain, dir = netconn.split("|")

                # get the dotted-quad string representation of the ip
                str_ip = socket.inet_ntoa(struct.pack("!i", int(ip)))

                # the underlying data model provides the protocol number
                # convert this to human-readable strings (tcp or udp)
                if "6" == proto:
                    proto = "tcp"
                elif "17" == proto:
                    proto = "udp"

                # the underlying data model provides a boolean indication as to
                # if this is an inbound or outbound network connection
                if "true" == dir:
                    dir = "out"
                else:
                    dir = "in"

                # pring the record, using pipes as a delimiter
                print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s)" % (
                    hostname, ts, process_name, user_name, process_md5, proto,
                    str_ip, port, dir, domain)

    def check(self, hostname):

        # print a legend
        print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s)" % (
            "hostname", "timestamp", "process name", "username", "process md5",
            "protocol", "ip", "port", "direction", "domain")

        # build the query string
        q = "netconn_count:[1 to *] AND hostname:%s" % (hostname)

        # begin with the first result - we'll perform the search in pages
        # the default page size is 10 (10 reslts)
        start = 0

        # loop over the entire result set
        while True:

            # get the next page of results
            procs = self.cb.process_search(q, start=start)

            # if there are no results, we are done paging
            if len(procs["results"]) == 0:
                break

            # examine each result individually
            # each result represents a single process segment
            for result in procs["results"]:
                self.report(hostname, result)

            # move forward to the next page
            start = start + 10
Esempio n. 20
0
 def connect_cb(self):
     return CbApi(self.url, token=self.admin_key, ssl_verify=False)
Esempio n. 21
0
class CBQuery(object):
    def __init__(self, url, token, ssl_verify):
        self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
        self.cb_url = url

        # set up some stat tracking
        self.stats = {}
        self.stats['total_processes'] = 0
        self.stats['total_netconns'] = 0
        self.stats['matching_netconns'] = 0
        self.stats['output_errors'] = 0
        self.stats['process_errors'] = 0

    def getStats(self):
        """
        return the current statistics
        """
        return self.stats

    def outputNetConn(self, proc, netconn):
        """
        output a single netconn event from a process document
        the caller is responsible for ensuring that the document
        meets start time and subnet criteria
        """

        # for convenience, use locals for some process metadata fields
        hostname = proc.get("hostname", "<unknown>")
        process_name = proc.get("process_name", "<unknown>")
        user_name = proc.get("username", "<unknown>")
        process_md5 = proc.get("process_md5", "<unknown>")
        cmdline = proc.get("cmdline", "<unknown>")
        path = proc.get("path", "<unknown>")
        procstarttime = proc.get("start", "<unknown>")
        proclastupdate = proc.get("last_update", "<unknown>")

        # split the netconn into component parts
        ts, ip, port, proto, domain, dir = netconn.split("|")

        # get the dotted-quad string representation of the ip
        str_ip = socket.inet_ntoa(struct.pack("!i", int(ip)))
                    
        # the underlying data model provides the protocol number
        # convert this to human-readable strings (tcp or udp)
        if "6" == proto:
            proto = "tcp"
        elif "17" == proto:
            proto = "udp"
                  
        # the underlying data model provides a boolean indication as to
        # if this is an inbound or outbound network connection 
        if "true" == dir:
            dir = "out"
        else:
           dir = "in" 

        # print the record, using pipes as a delimiter
        print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|" % (procstarttime,proclastupdate,hostname, user_name, proto, str_ip, port, dir, domain, process_name, process_md5, path, cmdline)

    def addressInNetwork(self, ip, cidr):

        # the ip can be the emtpy string ('') in cases where the connection
        # is made via a web proxy.  in these cases the sensor cannot report
        # the true remote IP as DNS resolution happens on the web proxy (and
        # not the endpoint)
        if '' == ip:
            return False

        try:
            net = cidr.split('/')[0]
            bits = cidr.split('/')[1]

            if int(ip) > 0: 
                ipaddr = struct.unpack('<L', socket.inet_aton(ip))[0]
            else:
                ipaddr = struct.unpack('<L', socket.inet_aton(".".join(map(lambda n: str(int(ip)>>n & 0xFF), [24,16,8,0]))))[0]
            netaddr = struct.unpack('<L', socket.inet_aton(net))[0]
            netmask = ((1L << int(bits)) - 1)
                
            return ipaddr & netmask == netaddr & netmask

        except:
            return False

    def report(self, result, subnet):

        # return the events associated with this process segment
        # this will include netconns, as well as modloads, filemods, etc.
        events = self.cb.process_events(result["id"], result["segment_id"])
        
        proc = events["process"]

        # the search criteria (netconn_count:[1 to *]) should ensure that
        # all results have at least one netconn
        if proc.has_key("netconn_complete"):

            # examine each netconn in turn
            for netconn in proc["netconn_complete"]:
        
                # update the count of total netconns
                self.stats['total_netconns'] = self.stats['total_netconns'] + 1
        
                # split the netconn event into component parts
                # note that the port is the remote port in the case of outbound
                # netconns, and local port in the case of inbound netconns
                #
                # for the purpose of this example script, eat any errors
                ts, ip, port, proto, domain, dir = netconn.split("|")
                if self.addressInNetwork(ip, subnet): 
                    try:
                        self.stats['matching_netconns'] = self.stats['matching_netconns'] + 1
                        self.outputNetConn(proc, netconn)
                    except:
                        self.stats['output_errors'] = self.stats['output_errors'] + 1
                        pass 

    def strip_to_int(ip):
        """
        convert a dotted-quad string IP to the corresponding int32
        """
        return struct.unpack('<L', socket.inet_aton(ip))[0]

    def check(self, subnet, datetype, begin, end):

        # print a legend
        print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|" % ("ProcStartTime", "ProcUpdateTime","hostname", "username", "protocol", "ip", "port", "direction", "domain",  "process name",  "process md5", "process path", "cmdline")

        # build the query string
        if not end and not begin:
            q = "ipaddr:%s" % (subnet,) 
        else:
            if not end: end = "*"
            if not begin: begin = "*"
            q = "ipaddr:%s %s:[%s TO %s]" % (subnet, datetype, begin, end)

        # begin with the first result - we'll perform the search in pages 
        # the default page size is 10 (10 results)
        start = 0

        # loop over the entire result set, paging as required
        while True:

            # get the next page of results 
            procs = self.cb.process_search(q, start=start)
            
            # track the total number of matching results
            self.stats['total_processes'] = procs['total_results']

            # if there are no results, we are done paging 
            if len(procs["results"]) == 0:
                break

            # examine each result individually
            # each result represents a single segment of a single process
            #
            # for the purposes of this example script, eat any errors
            #
            for result in procs["results"]:
                try:
                    self.report(result, subnet)
                except Exception, e:
                    self.stats['process_errors'] = self.stats['process_errors'] + 1

            # move forward to the next page 
            start = start + 10
def get_legacy_cbapi(splunk_service):
    cb_server, token = get_creds(splunk_service)
    return CbApi(cb_server, ssl_verify=False, token=token)
Esempio n. 23
0
class CBQuery(object):
    def __init__(self, url, token, ssl_verify):
        self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
        self.cb_url = url

    def report(self, result, search_filename):

        # return the events associated with this process segment
        # this will include netconns, as well as modloads, filemods, etc.
        #
        events = self.cb.process_events(result["id"], result["segment_id"])

        proc = events["process"]

        # for convenience, use locals for some process metadata fields
        #
        host_name = result.get("hostname", "<unknown>")
        process_name = result.get("process_name", "<unknown>")
        user_name = result.get("username", "<unknown>")

        if proc.has_key("filemod_complete"):

            # examine each filemod in turn
            #
            for filemod in proc["filemod_complete"]:

                # filemods are of the form:
                #   1|2014-06-19 15:40:05.446|c:\dir\filename||
                #
                action, ts, filename, filemd5, filetype = filemod.split('|')

                # the _document as a whole_ matched the query
                # that doesn't mean each _indvidual filemod_ within the document matched
                # the user-specified filename to search for
                #
                # iterate over each filemod and determine if the path matches
                # what was specified
                #
                # make sense?  hope so!
                #
                if search_filename.lower() not in filename.lower():
                    continue

                if "1" == action:
                    action = "creation"
                elif "2" == action or "8" == action:
                    action = "modification"
                elif "4" == action:
                    action = "deletion"

                print "%s|%s|%s|%s|%s|%s" % (host_name, ts, filename,
                                             process_name, user_name, action)

    def check(self, filename):

        # build the query string
        #
        q = "filemod:%s" % (filename)

        # begin with the first result - we'll perform the search in pages
        # the default page size is 10 (10 reslts)
        #
        start = 0

        # loop over the entire result set
        #
        while True:

            # get the next page of results
            #
            procs = self.cb.process_search(q, start=start)

            # if there are no results, we are done paging
            #
            if len(procs["results"]) == 0:
                break

            # examine each result individually
            # each result represents a single process segment
            #
            for result in procs["results"]:
                self.report(result, filename)

            # move forward to the next page
            #
            start = start + 10
Esempio n. 24
0
class Device(object):
    path = "carbonblack.endpoint"

    def __init__(self, hosts_mapping):
        configuration_dict = splunk.clilib.cli_common.getConfStanza('carbonblack', 'cbserver')

        self.cb_server = configuration_dict['cburl']
        self.token = configuration_dict['cbapikey']

        self.cb = CbApi(self.cb_server, token=self.token, ssl_verify=False)

        self.logger = logging.getLogger(__name__)
        self.logger.setLevel(logging.DEBUG)

        self.hosts_mapping = hosts_mapping

    def ban_hash(self, md5):
        """
        Performs a POST to the Carbon Black Server API for blacklisting an MD5 hash
        :param md5:
        :return:
        """

        print "blacklisting md5:%s" % (md5)

        headers = {'X-AUTH-TOKEN': self.token}

        data = {"md5hash": md5,
                "text": "Blacklist From Splunk",
                "last_ban_time": 0,
                "ban_count": 0,
                "last_ban_host": 0,
                "enabled": True}

        print "connecting to: %s/api/v1/banning/blacklist..." % (self.cb_server)

        r = requests.post("%s/api/v1/banning/blacklist" % (self.cb_server),
                          headers=headers,
                          data=json.dumps(data),
                          verify=False)

        if r.status_code == 409:
            print "This md5 hash is already blacklisted"
        elif r.status_code == 200:
            print "Carbon Black Server API Success"
        else:
            print "CarbonBlack Server API returned an error: %d" % (r.status_code)
            print "Be sure to check the Carbon Black API token"

    def get_sensor_id_from_ip(self, ip):
        filters = {}
        sensors = self.cb.sensors(filters)

        for sensor in sensors:
            src_ip = filter(bool, sensor.get('network_adapters', '').split('|'))
            for ip_address in src_ip:
                if unicode(ip, "utf-8") == ip_address.split(',')[0]:
                    return sensor.get('id', None)
        return None

    def pre_action(self, action_type, data):
        if action_type in ['isolate', 'flush']:
            src_ip = data.get('src_ip', None) or data.get('local_ip', None)
            if not src_ip:
                raise PrerequisiteFailedError("No source IP address provided")
            sensor_id = self.get_sensor_id_from_ip(src_ip)
            if not sensor_id:
                raise PrerequisiteFailedError("Cannot find sensor associated with source IP address %s" % src_ip)

            return sensor_id
        elif action_type in ['killproc']:
            proc_id = data.get('process_guid', None)
            if not proc_id:
                raise PrerequisiteFailedError("No Process GUID provided")
            if not isinstance(proc_id, six.string_types):
                raise PrerequisiteFailedError("Process GUID not valid: must be a string")
            if len(proc_id.split("-")) < 5:
                raise PrerequisiteFailedError("Process GUID not valid: must be a GUID")
            return proc_id
        elif action_type in ['banhash']:
            #
            # Pull out md5 from 'data'
            #
            md5 = data.get('md5', None)
            if not md5:
                #
                # Error out if we can't
                #
                raise PrerequisiteFailedError("Error: Unable to get an MD5 hash from parameters")
            return md5

        return None

    def submit_action(self, settings, data):
        """
        This gets called when the user executes a search
        :param settings:
        :param data:
        :return:
        """

        action_type = settings.get('action_type', '')
        try:
            sensor_id = self.pre_action(action_type, data)
        except PrerequisiteFailedError as e:
            # TODO: how do we signal failure back to ARF ARFARFARF
            self.logger.error(e.message)

        # TODO: return success

    def flush_action(self, sensor_id):
        print "Flushing sensor id: %s" % sensor_id
        #
        # We will always flush the sensor that triggered the action, so that we get the most up-to-date
        # information into the Cb console.
        #
        flusher = FlushAction(self.cb, self.logger)
        flusher.action(sensor_id)

    def isolate_action(self, sensor_id):
        isolator = IsolateAction(self.cb, self.logger)
        isolator.action(sensor_id)

    def kill_action(self, process_id):
        killer = KillProcessAction(self.cb, self.logger)
        killer.action(process_id)

    def run_action(self, settings, data):
        """
        This gets called when the user clicks the validate button
        :param settings:
        :param data:
        :return:
        """
        action_type = settings.get('action_type', '')
        # get sensor ID if required
        try:
            action_argument = self.pre_action(action_type, data)
        except PrerequisiteFailedError as e:
            self.logger.error(e.message)
        else:
            if action_type == 'banhash':
                self.ban_hash(action_argument)
            elif action_type == 'isolate':
                self.isolate_action(action_argument)
            elif action_type == 'flush':
                self.flush_action(action_argument)
            elif action_type == 'killproc':
                self.kill_action(action_argument)
Esempio n. 25
0
 def __init__(self, url, token, ssl_verify):
     self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
     self.cb_url = url
class CBQuery(object):
    def __init__(self, url, token, ssl_verify):
        self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
        self.cb_url = url

        # set up some stat tracking
        self.stats = {}
        self.stats['total_processes'] = 0
        self.stats['total_netconns'] = 0
        self.stats['matching_netconns'] = 0
        self.stats['output_errors'] = 0
        self.stats['process_errors'] = 0

    def getStats(self):
        """
        return the current statistics
        """
        return self.stats

    def outputNetConn(self, proc, netconn):
        """
        output a single netconn event from a process document
        the caller is responsible for ensuring that the document
        meets start time and subnet criteria
        """

        # for convenience, use locals for some process metadata fields
        hostname = proc.get("hostname", "<unknown>")
        process_name = proc.get("process_name", "<unknown>")
        user_name = proc.get("username", "<unknown>")
        process_md5 = proc.get("process_md5", "<unknown>")
        cmdline = proc.get("cmdline", "<unknown>")
        path = proc.get("path", "<unknown>")
        procstarttime = proc.get("start", "<unknown>")
        proclastupdate = proc.get("last_update", "<unknown>")

        # split the netconn into component parts
        ts, ip, port, proto, domain, dir = netconn.split("|")

        # get the dotted-quad string representation of the ip
        str_ip = socket.inet_ntoa(struct.pack("!i", int(ip)))

        # the underlying data model provides the protocol number
        # convert this to human-readable strings (tcp or udp)
        if "6" == proto:
            proto = "tcp"
        elif "17" == proto:
            proto = "udp"

        # the underlying data model provides a boolean indication as to
        # if this is an inbound or outbound network connection
        if "true" == dir:
            dir = "out"
        else:
            dir = "in"

        # print the record, using pipes as a delimiter
        print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|" % (
            procstarttime, proclastupdate, hostname, user_name, proto, str_ip,
            port, dir, domain, process_name, process_md5, path, cmdline)

    def addressInNetwork(self, ip, cidr):

        # the ip can be the emtpy string ('') in cases where the connection
        # is made via a web proxy.  in these cases the sensor cannot report
        # the true remote IP as DNS resolution happens on the web proxy (and
        # not the endpoint)
        if '' == ip:
            return False

        try:
            net = cidr.split('/')[0]
            bits = cidr.split('/')[1]

            if int(ip) > 0:
                ipaddr = struct.unpack('<L', socket.inet_aton(ip))[0]
            else:
                ipaddr = struct.unpack(
                    '<L',
                    socket.inet_aton(".".join(
                        map(lambda n: str(int(ip) >> n & 0xFF),
                            [24, 16, 8, 0]))))[0]
            netaddr = struct.unpack('<L', socket.inet_aton(net))[0]
            netmask = ((1L << int(bits)) - 1)

            return ipaddr & netmask == netaddr & netmask

        except:
            return False

    def report(self, result, subnet):

        # return the events associated with this process segment
        # this will include netconns, as well as modloads, filemods, etc.
        events = self.cb.process_events(result["id"], result["segment_id"])

        proc = events["process"]

        # the search criteria (netconn_count:[1 to *]) should ensure that
        # all results have at least one netconn
        if proc.has_key("netconn_complete"):

            # examine each netconn in turn
            for netconn in proc["netconn_complete"]:

                # update the count of total netconns
                self.stats['total_netconns'] = self.stats['total_netconns'] + 1

                # split the netconn event into component parts
                # note that the port is the remote port in the case of outbound
                # netconns, and local port in the case of inbound netconns
                #
                # for the purpose of this example script, eat any errors
                ts, ip, port, proto, domain, dir = netconn.split("|")
                if self.addressInNetwork(ip, subnet):
                    try:
                        self.stats['matching_netconns'] = self.stats[
                            'matching_netconns'] + 1
                        self.outputNetConn(proc, netconn)
                    except:
                        self.stats[
                            'output_errors'] = self.stats['output_errors'] + 1
                        pass

    def strip_to_int(ip):
        """
        convert a dotted-quad string IP to the corresponding int32
        """
        return struct.unpack('<L', socket.inet_aton(ip))[0]

    def check(self, subnet, datetype, begin, end):

        # print a legend
        print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|" % (
            "ProcStartTime", "ProcUpdateTime", "hostname", "username",
            "protocol", "ip", "port", "direction", "domain", "process name",
            "process md5", "process path", "cmdline")

        # build the query string
        if not end and not begin:
            q = "ipaddr:%s" % (subnet, )
        else:
            if not end: end = "*"
            if not begin: begin = "*"
            q = "ipaddr:%s %s:[%s TO %s]" % (subnet, datetype, begin, end)

        # begin with the first result - we'll perform the search in pages
        # the default page size is 10 (10 results)
        start = 0

        # loop over the entire result set, paging as required
        while True:

            # get the next page of results
            procs = self.cb.process_search(q, start=start)

            # track the total number of matching results
            self.stats['total_processes'] = procs['total_results']

            # if there are no results, we are done paging
            if len(procs["results"]) == 0:
                break

            # examine each result individually
            # each result represents a single segment of a single process
            #
            # for the purposes of this example script, eat any errors
            #
            for result in procs["results"]:
                try:
                    self.report(result, subnet)
                except Exception, e:
                    self.stats[
                        'process_errors'] = self.stats['process_errors'] + 1

            # move forward to the next page
            start = start + 10
Esempio n. 27
0
class ProcessSearchCommand(GeneratingCommand):
    """Generates a process search result from Carbon Black from a given IP or search query
        | processsearch query=${ip}
    """

    query = Option(name="query", require=True)

    field_names = ['cmdline',
                   'comms_ip',
                   'hostname',
                   'id',
                   'interface_ip',
                   'last_update',
                   'os_type',
                   'parent_md5',
                   'parent_name',
                   'parent_pid',
                   'parent_unique_id',
                   'path',
                   'process_md5',
                   'process_name',
                   'process_pid',
                   'regmod_count',
                   'segment_id',
                   'sensor_id',
                   'start',
                   'unique_id',
                   'username',
                   'childproc_count',
                   'crossproc_count',
                   'modload_count',
                   'netconn_count',
                   'filemod_count',
                   'group',
                   'host_type']

    def prepare(self):

        configuration_dict = splunk.clilib.cli_common.getConfStanza('carbonblack', 'cbserver')

        self.cb_server = configuration_dict['cburl']
        self.token = configuration_dict['cbapikey']

        self.cb = CbApi(self.cb_server, token=self.token, ssl_verify=False)

    def generate(self):
        self.logger.info("query %s" % self.query)
        i = 0
        for bindata in self.cb.process_search_iter(self.query):
            i += 1
            if i > 1000:
                # TODO: let's stop at 1,000 results for now?
                self.finish()
                return

            temp = dict((field_name, bindata[field_name]) for field_name in self.field_names)
            temp['sourcetype'] = 'bit9:carbonblack:json'

            #
            # Sometimes we have seen 'start' be equal to -1
            #
            try:
                temp['_time'] = int(time.mktime(dateutil.parser.parse(bindata['start']).timetuple()))
            except Exception as e:
                self.logger.exception('parsing bindata["start"] %s' % bindata['start'])
                temp['_time'] = 0

            temp['link_process'] = self.cb_server + '/#/analyze/' + bindata['id'] + "/1"
            temp['source'] = 'cbapi'
            temp['_raw'] = json.dumps(temp)
            yield temp

            if i % 10 == 0:
                self.flush()
Esempio n. 28
0
class CBQuery(object):
    def __init__(self, url, token, ssl_verify):
        self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
        self.cb_url = url

    def report(self, hostname, result):
        
        # return the events associated with this process segment
        # this will include netconns, as well as modloads, filemods, etc.
        events = self.cb.process_events(result["id"], result["segment_id"])
        
        proc = events["process"]

        # for convenience, use locals for some process metadata fields
        process_name = result.get("process_name", "<unknown>")
        user_name = result.get("username", "<unknown>")
        process_md5 = result.get("process_md5", "<unknown>")

        # the search criteria (netconn_count:[1 to *]) should ensure that
        # all results have at least one netconn
        if proc.has_key("netconn_complete"):

            # examine each netconn in turn
            for netconn in proc["netconn_complete"]:

                # split the netconn event into component parts
                # note that the port is the remote port in the case of outbound
                # netconns, and local port in the case of inbound netconns
                ts, ip, port, proto, domain, dir = netconn.split("|")
                
                # get the dotted-quad string representation of the ip
                str_ip = socket.inet_ntoa(struct.pack("!i", int(ip)))
                
                # the underlying data model provides the protocol number
                # convert this to human-readable strings (tcp or udp)
                if "6" == proto:
                    proto = "tcp"
                elif "17" == proto:
                    proto = "udp"
               
                # the underlying data model provides a boolean indication as to
                # if this is an inbound or outbound network connection 
                if "true" == dir:
                    dir = "out"
                else:
                    dir = "in" 

                # pring the record, using pipes as a delimiter
                print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s)" % (hostname, ts, process_name, user_name, process_md5, proto, str_ip, port, dir, domain)

    def check(self, hostname):

        # print a legend
        print "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s)" % ("hostname", "timestamp", "process name", "username", "process md5", "protocol", "ip", "port", "direction", "domain")

        # build the query string
        q = "netconn_count:[1 to *] AND hostname:%s" % (hostname)
      
        # begin with the first result - we'll perform the search in pages 
        # the default page size is 10 (10 reslts)
        start = 0

        # loop over the entire result set
        while True:

            # get the next page of results 
            procs = self.cb.process_search(q, start=start)
      
            # if there are no results, we are done paging 
            if len(procs["results"]) == 0:
                break

            # examine each result individually
            # each result represents a single process segment
            for result in procs["results"]:
                self.report(hostname, result)

            # move forward to the next page 
            start = start + 10
Esempio n. 29
0
class CbTaxiiFeedConverter(object):
    def __init__(self, configpath, export_mode=False):

        self.export_mode = export_mode

        self.sites = []
        if self.export_mode:
            _logger.warn("CB Taxii %s running (EXPORT MODE)" % __version__)
        else:
            _logger.warn("CB Taxii %s running" % __version__)

        config = ConfigParser.ConfigParser()
        if not os.path.exists(configpath):
            _logger.error("Config File %s does not exist!" % configpath)
            print("Config File %s does not exist!" % configpath)
            sys.exit(-1)

        config.read(configpath)

        # SEE IF THERE's A DIFFERENT SERVER_PORT
        server_port = 443
        if config.has_section("cbconfig"):
            if config.has_option("cbconfig", "server_port"):
                server_port = config.getint("cbconfig", "server_port")

        for section in config.sections():
            # don't do cbconfig
            if section.lower() == 'cbconfig':
                continue

            site = config.get(section, "site")
            output_path = config.get(section, "output_path")
            icon_link = config.get(section, "icon_link")
            username = config.get(section, "username")
            password = config.get(section, "password")
            feeds_enable = config.getboolean(section, "feeds_enable")
            feeds_alerting = config.get(section, "feeds_alerting")

            ### OPTIONAL ARGUMENTS #######################################################
            if config.has_option(section, "start_date"):
                start_date = config.get(section, "start_date")
            else:
                start_date = "2015-01-01 00:00:00"

            if config.has_option(section, "use_https"):
                use_https=config.getboolean(section, "use_https")
            else:
                use_https = False

            cert_file = None
            key_file = None

            if config.has_option(section, "cert_file") and config.has_option(section, "key_file"):
                cert_file = config.get(section, "cert_file").strip()
                if cert_file == "":
                    cert_file = None
                elif not os.path.exists(cert_file):
                    _logger.error("Cert file supplied but doesn't exist: %s" % (cert_file))

                key_file = config.get(section, "key_file").strip()
                if key_file == "":
                    cert_file = None
                elif not os.path.exists(key_file):
                    _logger.error("Key file supplied but doesn't exist: %s" % (key_file))

            if config.has_option(section, "minutes_to_advance"):
                minutes_to_advance = int(config.get(section, "minutes_to_advance"))
            else:
                minutes_to_advance = 15


            if config.has_option(section, "enable_ip_ranges"):
                enable_ip_ranges = config.getboolean(section, "enable_ip_ranges")
            else:
                enable_ip_ranges = True

            _logger.info("Configured Site: %s Path: %s" % (site, output_path))

            self.sites.append({"site": site,
                               "output_path": output_path,
                               "username": username,
                               "password": password,
                               "icon_link": icon_link,
                               "feeds_enable": feeds_enable,
                               "feeds_alerting": feeds_alerting,
                               "enable_ip_ranges": enable_ip_ranges,
                               "start_date": start_date,
                               "use_https": use_https,
                               "key_file": key_file,
                               "cert_file": cert_file,
                               "minutes_to_advance": minutes_to_advance})

        self.api_token = lookup_admin_api_token()
        server_url = "https://127.0.0.1:%d/" % server_port
        _logger.info("Using Server URL: %s" % server_url)
        self.cb = CbApi(server_url, token=self.api_token, ssl_verify=False)
        try:
            # TEST CB CONNECTIVITY
            self.cb.feed_enum()
        except:
            e = traceback.format_exc()
            _logger.error("Unable to connect to CB using url: %s Error: %s" % (server_url, e))
            print("Unable to connect to CB using url: %s Error: %s" % (server_url, e))
            sys.exit(-1)

    @staticmethod
    def _message_to_reports(filepath, site, site_url, collection, enable_ip_ranges):
        context = etree.iterparse(filepath, tag='{http://stix.mitre.org/stix-1}STIX_Package')
        global _logger
        reports = fast_xml_iter(context, stix_element_to_reports, site, site_url, collection, enable_ip_ranges, _logger)
        return reports

    def _write_message_to_disk(self, message):
        fd,path = tempfile.mkstemp()
        #        os.write(fd, message)
        os.close(fd)
        f = file(path, 'wb')
        f.write(message)
        f.close()
        return path

    def _export_message_to_disk(self, feed_name, start_time, end_time, message):
        log_dir = "/var/run/cb/cbtaxii-export"
        if not os.path.exists(log_dir):
            os.mkdir(log_dir)
        path = "%s/%s-%s-%s.xml" % (log_dir, feed_name, start_time, end_time)
        path = path.replace(' ', '_')
        f = file(path, 'wb')
        f.write(message)
        f.close()
        return path


    def _import_collection(self, client, site, collection):
        collection_name = collection.get('collection_name', '')
        sanitized_feed_name = cleanup_string("%s%s" % (site.get('site'), collection_name))
        available = collection.get('available', False)
        collection_type = collection.get('collection_type', '').upper()
        _logger.info("%s,%s,%s,%s,%s" % (site.get('site'),
                                              collection_name,
                                              sanitized_feed_name,
                                              available,
                                              collection_type))

        if not available or collection_type != "DATA_FEED":
            return

        start_date_str = site.get('start_date')
        if not start_date_str or len(start_date_str) == 0:
            start_date_str = "2015-04-01 00:00:00"

        feed_helper = FeedHelper(site.get('output_path'), sanitized_feed_name, site.get('minutes_to_advance'), start_date_str, self.export_mode)

        _logger.info("Feed start time %s" % feed_helper.start_date)

        reports = []
        # CATCHUP -- TODO, move to a function??
        while True:
            these_reports = []
            tries = 0
            while tries < 5:
                try:
                    if feed_helper.start_date > feed_helper.end_date:
                        break

                    t1 = time.time()
                    message = client.retrieve_collection(collection_name, feed_helper.start_date, feed_helper.end_date)
                    t2 = time.time()

                    message_len = len(message)

                    if self.export_mode:
                        path = self._export_message_to_disk(sanitized_feed_name, feed_helper.start_date, feed_helper.end_date, message)
                        _logger.info("%s - %s - %s - %d (%f)- %s" % (feed_helper.start_date, feed_helper.end_date, collection_name, message_len, (t2-t1), path))
                        message = None
                    else:
                        filepath = self._write_message_to_disk(message)
                        message = None
                        site_url = "%s://%s" % ("https" if site.get('use_https') else "http", site.get('site'))
                        these_reports = self._message_to_reports(filepath, site.get('site'), site_url, collection_name, site.get('enable_ip_ranges'))
                        t3 = time.time()
                        os.remove(filepath)
                        count = len(these_reports)
                        _logger.info("%s - %s - %s - %d (%d)(%.2f)(%.2f)" % (feed_helper.start_date, feed_helper.end_date, collection_name, count, message_len, (t2-t1), (t3-t2)))
                    break
                except:
                    _logger.error("%s" % traceback.format_exc())
                    time.sleep(5)
                    tries += 1

            if tries == 5:
                _logger.error("Giving up for site %s, collection %s" % (site.get('site'), collection))
                return

            if not self.export_mode:
                reports.extend(these_reports)

            if not feed_helper.advance():
                break
        ########## end while (for iterating across time)

        _logger.info("COMPLETED %s,%s,%s,%s,%s (%d)" % (site.get('site'),
                                              collection_name,
                                              sanitized_feed_name,
                                              available,
                                              collection_type,
                                              len(reports)))

        if not self.export_mode:
            # TODO -- clean this up
            if len(reports) > 0:
                # load existing data and convert new data
                reports = feed_helper.load_existing_feed_data() + reports

                # convert feed info and reports to json
                data = build_feed_data(sanitized_feed_name,
                                       "%s %s" % (site.get('site'), collection_name),
                                       site.get('site'),
                                       site.get('icon_link'),
                                       reports)

                # SAVE THE DATA: write out the feed file and save the details for when we last queried it
                if feed_helper.write_feed(data):
                    feed_helper.save_details()

                # Actually add CB feed if necessary
                feed_id = self.cb.feed_get_id_by_name(sanitized_feed_name)
                if not feed_id:
                    data = self.cb.feed_add_from_url("file://" + feed_helper.path,
                                              site.get('feeds_enable'),
                                              False,
                                              False)

                    # FEED ALERTING!!
                    feed_id = data.get('id')
                    url = "https://127.0.0.1/api/v1/feed/%d/action" % feed_id
                    alert_types = site.get('feeds_alerting', '').split(',')
                    headers = {'X-Auth-Token' : self.api_token, "Accept" : "application/json"}
                    for alert in alert_types:
                        if alert.lower() == "syslog":
                            action_data = {"action_data": """{"email_recipients":[1]}""", "action_type": 1, "group_id": feed_id, "watchlist_id": ""}
                            resp = requests.post(url, headers=headers, data=json.dumps(action_data), verify=False)
                            if resp.status_code != 200:
                                _logger.warn("Error for syslog action (%d): %s" % (feed_id, resp.content))
                        elif alert.lower() == "cb":
                            action_data = {"action_data": """{"email_recipients":[1]}""", "action_type": 3, "group_id": feed_id, "watchlist_id": ""}
                            resp = requests.post(url, headers=headers, data=json.dumps(action_data), verify=False)
                            if resp.status_code != 200:
                                _logger.warn("Error for cb action (%d): %s" % (feed_id, resp.content))
            else: # no reports
                feed_helper.save_details()

    @staticmethod
    def perform_from_files(directory):
        global _logger
        _logger = create_stdout_log("cb-taxii", logging.DEBUG)
        files = os.listdir(directory)
        for filepath in files:
            if not filepath.endswith(".xml"):
                continue
            pieces = filepath.split('-')
            site = pieces[0]
            filepath = os.path.join(directory, filepath)
            these_reports = CbTaxiiFeedConverter._message_to_reports(filepath, site, site, site, True)
            for report in these_reports:
                iocs = report.get('iocs')
                if "dns" in iocs:
                    print "%s - %s" % (site, iocs['dns'])

                if "ipv4" in iocs:
                    print "%s - %s" % (site, iocs['ipv4'])

                if "query" in iocs:
                    print "%s - %s" % (site, iocs['query'])

                if "hash" in iocs:
                    print "%s - %s" % (site, iocs['hash'])

    def perform(self):
        """
        Loops through the sites supplied and adds each one if necessary.

        Then downloads new data and appends to existing feed file.
        """

        for site in self.sites:
            client = TaxiiClient(site.get('site'),
                                 site.get('username'),
                                 site.get('password'),
                                 site.get('use_https'),
                                 site.get('key_file'),
                                 site.get('cert_file'))
            try:
                collections = client.enumerate_collections(_logger)
                if len(collections) == 0:
                    _logger.warn("No collections returned!")
            except UnauthorizedException, e:
                _logger.error("Site: %s, Exception: %s" % (site.get('site'), e))
                continue

            for collection in collections:
                self._import_collection(client, site, collection)
Esempio n. 30
0
        cmdline = bprocess["cmdline"]
    else:
        cmdline = ""
    myrow.append(cmdline)
    if bprocess.has_key("childproc_complete"):
        getchildprocs(bprocess['childproc_complete'])


parser = build_cli_parser()
opts, args = parser.parse_args(sys.argv[1:])
if not opts.server or not opts.token or (
        not opts.procnamefile and not opts.procname) or not opts.percentless:
    print "Missing required param."
    sys.exit(-1)

cb = CbApi(opts.server, ssl_verify=opts.ssl_verify, token=opts.token)
if opts.procnamefile:
    searchprocess = processsearchlist(opts.procnamefile)
else:
    searchprocess = opts.procname.split(",")
for proc in searchprocess:
    start = 0
    data = cb.process_search("parent_name:%s" % (proc), rows=1, start=start)
    facetterms = data['facets']
    for term in reversed(facetterms['process_name']):
        termratio = int(float(term['ratio']))
        if int(opts.percentless) >= termratio:
            start = 0
            while True:
                q = "parent_name:%s AND process_name:%s" % (proc, term['name'])
                data = cb.process_search(q, rows=int(pagesize), start=start)
Esempio n. 31
0
    def __init__(self, configpath, export_mode=False):

        self.export_mode = export_mode

        self.sites = []
        if self.export_mode:
            _logger.warn("CB Taxii %s running (EXPORT MODE)" % __version__)
        else:
            _logger.warn("CB Taxii %s running" % __version__)

        config = ConfigParser.ConfigParser()
        if not os.path.exists(configpath):
            _logger.error("Config File %s does not exist!" % configpath)
            print("Config File %s does not exist!" % configpath)
            sys.exit(-1)

        config.read(configpath)

        # SEE IF THERE's A DIFFERENT SERVER_PORT
        server_port = 443
        if config.has_section("cbconfig"):
            if config.has_option("cbconfig", "server_port"):
                server_port = config.getint("cbconfig", "server_port")

        for section in config.sections():
            # don't do cbconfig
            if section.lower() == 'cbconfig':
                continue

            site = config.get(section, "site")
            output_path = config.get(section, "output_path")
            icon_link = config.get(section, "icon_link")
            username = config.get(section, "username")
            password = config.get(section, "password")
            feeds_enable = config.getboolean(section, "feeds_enable")
            feeds_alerting = config.get(section, "feeds_alerting")

            ### OPTIONAL ARGUMENTS #######################################################
            if config.has_option(section, "start_date"):
                start_date = config.get(section, "start_date")
            else:
                start_date = "2015-01-01 00:00:00"

            if config.has_option(section, "use_https"):
                use_https=config.getboolean(section, "use_https")
            else:
                use_https = False

            cert_file = None
            key_file = None

            if config.has_option(section, "cert_file") and config.has_option(section, "key_file"):
                cert_file = config.get(section, "cert_file").strip()
                if cert_file == "":
                    cert_file = None
                elif not os.path.exists(cert_file):
                    _logger.error("Cert file supplied but doesn't exist: %s" % (cert_file))

                key_file = config.get(section, "key_file").strip()
                if key_file == "":
                    cert_file = None
                elif not os.path.exists(key_file):
                    _logger.error("Key file supplied but doesn't exist: %s" % (key_file))

            if config.has_option(section, "minutes_to_advance"):
                minutes_to_advance = int(config.get(section, "minutes_to_advance"))
            else:
                minutes_to_advance = 15


            if config.has_option(section, "enable_ip_ranges"):
                enable_ip_ranges = config.getboolean(section, "enable_ip_ranges")
            else:
                enable_ip_ranges = True

            _logger.info("Configured Site: %s Path: %s" % (site, output_path))

            self.sites.append({"site": site,
                               "output_path": output_path,
                               "username": username,
                               "password": password,
                               "icon_link": icon_link,
                               "feeds_enable": feeds_enable,
                               "feeds_alerting": feeds_alerting,
                               "enable_ip_ranges": enable_ip_ranges,
                               "start_date": start_date,
                               "use_https": use_https,
                               "key_file": key_file,
                               "cert_file": cert_file,
                               "minutes_to_advance": minutes_to_advance})

        self.api_token = lookup_admin_api_token()
        server_url = "https://127.0.0.1:%d/" % server_port
        _logger.info("Using Server URL: %s" % server_url)
        self.cb = CbApi(server_url, token=self.api_token, ssl_verify=False)
        try:
            # TEST CB CONNECTIVITY
            self.cb.feed_enum()
        except:
            e = traceback.format_exc()
            _logger.error("Unable to connect to CB using url: %s Error: %s" % (server_url, e))
            print("Unable to connect to CB using url: %s Error: %s" % (server_url, e))
            sys.exit(-1)
Esempio n. 32
0
class CBQuery(object):
    def __init__(self, url, token, ssl_verify):
        self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
        self.cb_url = url

    def report(self, result, search_filename):
        
        # return the events associated with this process segment
        # this will include netconns, as well as modloads, filemods, etc.
        #
        events = self.cb.process_events(result["id"], result["segment_id"])
        
        proc = events["process"]

        # for convenience, use locals for some process metadata fields
        #
        host_name = result.get("hostname", "<unknown>")
        process_name = result.get("process_name", "<unknown>")
        user_name = result.get("username", "<unknown>")

        if proc.has_key("filemod_complete"):

            # examine each filemod in turn
            #
            for filemod in proc["filemod_complete"]:

                # filemods are of the form:
                #   1|2014-06-19 15:40:05.446|c:\dir\filename||
                #
                parts = filemod.split('|')
                action, ts, filename, filemd5, filetype = parts[:5]

                # the _document as a whole_ matched the query
                # that doesn't mean each _indvidual filemod_ within the document matched
                # the user-specified filename to search for
                #
                # iterate over each filemod and determine if the path matches
                # what was specified
                #
                # make sense?  hope so!
                #
                if search_filename.lower() not in filename.lower():
                    continue 

                if "1" == action:
                    action = "creation"
                elif "2" == action or "8" == action:
                    action = "modification"
                elif "4" == action:
                    action = "deletion"

                print "%s|%s|%s|%s|%s|%s" % (host_name, ts, filename, process_name, user_name, action) 

    def check(self, filename):

        # build the query string
        #
        q = "filemod:%s" % (filename)
      
        # begin with the first result - we'll perform the search in pages 
        # the default page size is 10 (10 reslts)
        #
        start = 0

        # loop over the entire result set
        #
        while True:

            # get the next page of results
            # 
            procs = self.cb.process_search(q, start=start)
      
            # if there are no results, we are done paging 
            #
            if len(procs["results"]) == 0:
                break

            # examine each result individually
            # each result represents a single process segment
            #
            for result in procs["results"]:
                self.report(result, filename)

            # move forward to the next page
            #
            start = start + 10
Esempio n. 33
0
class CbTaxiiFeedConverter(object):
    def __init__(self,
                 config_file_path,
                 debug_mode=False,
                 import_dir='',
                 export_dir=''):

        #
        # parse config file and save off the information we need
        #
        config_dict = parse_config(config_file_path)

        self.server_url = config_dict.get('server_url', 'https://127.0.0.1')
        self.api_token = config_dict.get('api_token', '')
        self.sites = config_dict.get('sites', [])
        self.debug = config_dict.get('debug', False)
        self.export_dir = export_dir
        self.import_dir = import_dir

        self.http_proxy_url = config_dict.get('http_proxy_url', None)
        self.https_proxy_url = config_dict.get('https_proxy_url', None)

        if self.export_dir and not os.path.exists(self.export_dir):
            os.mkdir(self.export_dir)

        #
        # Test Cb Response connectivity
        #
        try:
            self.cb = CbApi(server=self.server_url,
                            token=self.api_token,
                            ssl_verify=False)
            self.cb.feed_enum()
        except:
            logger.error(traceback.format_exc())
            sys.exit(-1)

    def write_to_temp_file(self, message):
        temp_file = tempfile.NamedTemporaryFile()
        temp_file.write(message)
        temp_file.flush()
        return temp_file, temp_file.name

    def read_from_xml(self):
        """
        Walk the import dir and return all filenames.  We are assuming all xml files
        :return:
        """
        f = []
        for (dirpath, dirnames, filenames) in os.walk(self.import_dir):
            f.extend(filenames)
            break
        return f

    def export_xml(self, feed_name, start_time, end_time, block_num, message):
        """
        :param feed_name:
        :param start_time:
        :param end_time:
        :param block_num:
        :param message:
        :return:
        """
        #
        # create a directory to store all content blocks
        #
        dir_name = "{}".format(feed_name).replace(' ', '_')
        full_dir_name = os.path.join(self.export_dir, dir_name)

        #
        # Make sure the directory exists
        #
        if not os.path.exists(os.path.join(self.export_dir, dir_name)):
            os.mkdir(full_dir_name)

        #
        # Actually write the file
        #
        file_name = "{}-{}-{}".format(start_time, end_time,
                                      block_num).replace(' ', "_")
        full_file_name = os.path.join(full_dir_name, file_name)

        with open(full_file_name, 'wb') as file_handle:
            file_handle.write(message)

    def _import_collection(self, client, site, collection, data_set=False):

        collection_name = collection.name
        sanitized_feed_name = cleanup_string(
            "%s%s" % (site.get('site'), collection_name))
        feed_summary = "%s %s" % (site.get('site'), collection_name)
        available = collection.available
        collection_type = collection.type
        default_score = site.get('default_score')
        logger.info("%s,%s,%s,%s,%s" %
                    (site.get('site'), collection_name, sanitized_feed_name,
                     available, collection_type))

        if not available:
            return False

        #
        # Sanity check on start date
        #
        start_date_str = site.get('start_date')
        if not start_date_str or len(start_date_str) == 0:
            start_date_str = "2017-01-01 00:00:00"

        #
        # Create a feed helper object
        #
        feed_helper = FeedHelper(site.get('output_path'), sanitized_feed_name,
                                 site.get('minutes_to_advance'),
                                 start_date_str)

        if not data_set:
            logger.info("Feed start time %s" % feed_helper.start_date)
        logger.info("polling Collection: {}...".format(collection.name))

        #
        # Build up the URI for polling
        #

        if not site.get('poll_path', ''):
            uri = None
        else:
            uri = ''
            if site.get('use_https'):
                uri += 'https://'
            else:
                uri += 'http://'

            uri += site.get('site')
            uri += site.get('poll_path')
            logger.info('Poll path: {}'.format(uri))

        reports = []
        while True:

            try:
                try:
                    content_blocks = client.poll(
                        uri=uri,
                        collection_name=collection.name,
                        begin_date=feed_helper.start_date,
                        end_date=feed_helper.end_date,
                        content_bindings=BINDING_CHOICES)

                except Exception as e:
                    logger.info(e.message)
                    content_blocks = []

                #
                # Iterate through all content_blocks
                #
                num_blocks = 0

                if not data_set:
                    logger.info("polling start_date: {}, end_date: {}".format(
                        feed_helper.start_date, feed_helper.end_date))
                for block in content_blocks:

                    #
                    # if in export mode then save off this content block
                    #
                    if self.export_dir:
                        self.export_xml(collection_name,
                                        feed_helper.start_date,
                                        feed_helper.end_date, num_blocks,
                                        block.content)

                    #
                    # This code accounts for a case found with ThreatCentral.io where the content is url encoded.
                    # etree.fromstring can parse this data.
                    #
                    try:
                        root = etree.fromstring(block.content)
                        content = root.find(
                            './/{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content'
                        )
                        if content is not None and len(content) == 0 and len(
                                list(content)) == 0:
                            #
                            # Content has no children.  So lets make sure we parse the xml text for content and re-add
                            # it as valid XML so we can parse
                            #
                            new_stix_package = etree.fromstring(
                                root.find(
                                    "{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content_Block/{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content"
                                ).text)
                            content.append(new_stix_package)

                        #
                        # Since we modified the xml, we need create a new xml message string to parse
                        #
                        message = etree.tostring(root)

                        #
                        # Write the content block to disk so we can parse with python stix
                        #
                        file_handle, file_path = self.write_to_temp_file(
                            message)

                        #
                        # Parse STIX data
                        #
                        stix_package = STIXPackage.from_xml(file_path)

                        #
                        # if it is a DATA_SET make feed_summary from the stix_header description
                        # NOTE: this is for RecordedFuture, also note that we only do this for data_sets.
                        #       to date I have only seen RecordedFuture use data_sets
                        #
                        if data_set and stix_package.stix_header and stix_package.stix_header.descriptions:
                            for desc in stix_package.stix_header.descriptions:
                                feed_summary = desc.value
                                break

                        #
                        # Get the timestamp of the STIX Package so we can use this in our feed
                        #
                        timestamp = total_seconds(stix_package.timestamp)

                        if stix_package.indicators:
                            for indicator in stix_package.indicators:
                                if not indicator or not indicator.observable:
                                    continue

                                if not indicator.timestamp:
                                    timestamp = 0
                                else:
                                    timestamp = int(
                                        (indicator.timestamp -
                                         datetime.datetime(1970, 1, 1).replace(
                                             tzinfo=dateutil.tz.tzutc())
                                         ).total_seconds())

                                reports.extend(
                                    cybox_parse_observable(
                                        indicator.observable, indicator,
                                        timestamp, default_score))

                        #
                        # Now lets find some data.  Iterate through all observables and parse
                        #
                        if stix_package.observables:
                            for observable in stix_package.observables:
                                if not observable:
                                    continue
                                #
                                # Cybox observable returns a list
                                #
                                reports.extend(
                                    cybox_parse_observable(
                                        observable, None, timestamp,
                                        default_score))

                        #
                        # Delete our temporary file
                        #
                        file_handle.close()

                        num_blocks += 1

                        #
                        # end for loop through content blocks
                        #

                    except Exception as e:
                        #logger.info(traceback.format_exc())
                        logger.info(e.message)
                        continue

                logger.info("content blocks read: {}".format(num_blocks))
                logger.info("current number of reports: {}".format(
                    len(reports)))

                #
                # DEBUG CODE
                #
                #if len(reports) > 10:
                #    break

                #
                # Attempt to advance the start time and end time
                #

            except Exception as e:
                logger.info(traceback.format_exc())

            #
            # If it is just a data_set, the data is unordered, so we can just break out of the while loop
            #
            if data_set:
                break

            if feed_helper.advance():
                continue
            else:
                break
            #
            # end While True
            #

        logger.info("Found {} new reports.".format(len(reports)))

        reports = feed_helper.load_existing_feed_data() + reports

        logger.info("Total number of reports: {}".format(len(reports)))

        data = build_feed_data(sanitized_feed_name,
                               "%s %s" % (site.get('site'), collection_name),
                               feed_summary, site.get('site'),
                               site.get('icon_link'), reports)

        if feed_helper.write_feed(data):
            feed_helper.save_details()

        #
        # Create Cb Response Feed if necessary
        #

        feed_id = self.cb.feed_get_id_by_name(sanitized_feed_name)
        if not feed_id:
            self.cb.feed_add_from_url("file://" + feed_helper.path,
                                      site.get('feeds_enable'), False, False)

    def perform(self):
        """
        :param self:
        :param enumerate_collections_only:
        :return:
        """
        for site in self.sites:

            client = create_client(site.get('site'),
                                   use_https=site.get('use_https'),
                                   discovery_path=site.get('discovery_path'))

            #
            # Set verify_ssl and ca_cert inside the client
            #
            client.set_auth(verify_ssl=site.get('ssl_verify'),
                            ca_cert=site.get('ca_cert'))

            #
            # Proxy Settings
            #
            proxy_dict = dict()

            if self.http_proxy_url:
                logger.info("Found HTTP Proxy: {}".format(self.http_proxy_url))
                proxy_dict['http'] = self.http_proxy_url

            if self.https_proxy_url:
                logger.info("Found HTTPS Proxy: {}".format(
                    self.https_proxy_url))
                proxy_dict['https'] = self.https_proxy_url

            if proxy_dict:
                client.set_proxies(proxy_dict)

            if site.get('username') or site.get('cert_file'):
                #
                # If a username is supplied use basic authentication
                #
                logger.info("Found Username in config, using basic auth...")
                client.set_auth(username=site.get('username'),
                                password=site.get('password'),
                                verify_ssl=site.get('ssl_verify'),
                                ca_cert=site.get('ca_cert'),
                                cert_file=site.get('cert_file'),
                                key_file=site.get('key_file'))

            if not site.get('collection_management_path', ''):
                collections = client.get_collections()
            else:
                uri = ''
                if site.get('use_https'):
                    uri += 'https://'
                else:
                    uri += 'http://'

                uri += site.get('site')
                uri += site.get('collection_management_path')
                logger.info('Collection Management Path: {}'.format(uri))

                collections = client.get_collections(uri=uri)

            for collection in collections:
                logger.info('Collection Name: {}, Collection Type: {}'.format(
                    collection.name, collection.type))

            if len(collections) == 0:
                logger.info('Unable to find any collections.  Exiting...')
                sys.exit(0)

            desired_collections = [
                x.strip() for x in site.get('collections').lower().split(',')
            ]

            want_all = False
            if '*' in desired_collections:
                want_all = True

            for collection in collections:
                if collection.type != 'DATA_FEED' and collection.type != 'DATA_SET':
                    continue

                if collection.type == 'DATA_SET':
                    data_set = True
                else:
                    data_set = False

                if want_all or collection.name.lower() in desired_collections:
                    self._import_collection(client, site, collection, data_set)
Esempio n. 34
0
class CbTaxiiFeedConverter(object):
    def __init__(self, config_file_path, debug_mode=False, import_dir='', export_dir=''):

        #
        # parse config file and save off the information we need
        #
        config_dict = parse_config(config_file_path)

        self.server_url = config_dict.get('server_url', 'https://127.0.0.1')
        self.api_token = config_dict.get('api_token', '')
        self.sites = config_dict.get('sites', [])
        self.debug = config_dict.get('debug', False)
        self.export_dir = export_dir
        self.import_dir = import_dir

        if self.export_dir and not os.path.exists(self.export_dir):
            os.mkdir(self.export_dir)

        #
        # Test Cb Response connectivity
        #
        try:
            self.cb = CbApi(server=self.server_url, token=self.api_token, ssl_verify=False)
            self.cb.feed_enum()
        except:
            logger.error(traceback.format_exc())
            sys.exit(-1)

    def write_to_temp_file(self, message):
        temp_file = tempfile.NamedTemporaryFile()
        temp_file.write(message)
        temp_file.flush()
        return temp_file, temp_file.name

    def read_from_xml(self):
        """
        Walk the import dir and return all filenames.  We are assuming all xml files
        :return:
        """
        f = []
        for (dirpath, dirnames, filenames) in os.walk(self.import_dir):
            f.extend(filenames)
            break
        return f

    def export_xml(self, feed_name, start_time, end_time, block_num, message):
        """
        :param feed_name:
        :param start_time:
        :param end_time:
        :param block_num:
        :param message:
        :return:
        """
        #
        # create a directory to store all content blocks
        #
        dir_name = "{}".format(feed_name).replace(' ', '_')
        full_dir_name = os.path.join(self.export_dir, dir_name)

        #
        # Make sure the directory exists
        #
        if not os.path.exists(os.path.join(self.export_dir, dir_name)):
            os.mkdir(full_dir_name)

        #
        # Actually write the file
        #
        file_name = "{}-{}-{}".format(start_time, end_time, block_num).replace(' ', "_")
        full_file_name = os.path.join(full_dir_name, file_name)

        with open(full_file_name, 'wb') as file_handle:
            file_handle.write(message)


    def _import_collection(self, client, site, collection):

        collection_name = collection.name
        sanitized_feed_name = cleanup_string("%s%s" % (site.get('site'), collection_name))
        available = collection.available
        collection_type = collection.type
        logger.info("%s,%s,%s,%s,%s" % (site.get('site'),
                                         collection_name,
                                         sanitized_feed_name,
                                         available,
                                         collection_type))

        #
        # We only care about DATA_FEED type
        #
        if not available or collection_type != "DATA_FEED":
            return False

        #
        # Sanity check on start date
        #
        start_date_str = site.get('start_date')
        if not start_date_str or len(start_date_str) == 0:
            start_date_str = "2016-12-01 00:00:00"


        #
        # Create a feed helper object
        #
        feed_helper = FeedHelper(
            site.get('output_path'),
            sanitized_feed_name,
            site.get('minutes_to_advance'),
            start_date_str)

        logger.info("Feed start time %s" % feed_helper.start_date)
        logger.info("polling Collection: {}...".format(collection.name))

        #
        # Build up the URI for polling
        #

        if not site.get('poll_path', ''):
            uri = None
        else:
            uri = ''
            if site.get('use_https'):
                uri += 'https://'
            else:
                uri += 'http://'

            uri += site.get('site')
            uri += site.get('poll_path')
            logger.info('Poll path: {}'.format(uri))

        reports = []
        while True:

            content_blocks = client.poll(uri=uri,
                                         collection_name=collection.name,
                                         begin_date=feed_helper.start_date,
                                         end_date=feed_helper.end_date,
                                         #content_bindings=BINDING_CHOICES)
                                         content_bindings=[CB_STIX_XML_12])

            #
            # Iterate through all content_blocks
            #
            num_blocks = 0
            for block in content_blocks:

                #
                # if in export mode then save off this content block
                #
                if self.export_dir:
                    self.export_xml(collection_name,
                                    feed_helper.start_date,
                                    feed_helper.end_date,
                                    num_blocks,
                                    block.content)

                #
                # This code accounts for a case found with ThreatCentral.io where the content is url encoded.
                # etree.fromstring can parse this data.
                #
                root = etree.fromstring(block.content)
                content = root.find('.//{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content')
                if content is not None and len(content) == 0 and len(list(content)) == 0:
                    #
                    # Content has no children.  So lets make sure we parse the xml text for content and re-add
                    # it as valid XML so we can parse
                    #
                    new_stix_package = etree.fromstring(root.find(
                        "{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content_Block/{http://taxii.mitre.org/messages/taxii_xml_binding-1.1}Content").text)
                    content.append(new_stix_package)

                #
                # Since we modified the xml, we need create a new xml message string to parse
                #
                message = etree.tostring(root)

                #
                # Write the content block to disk so we can parse with python stix
                #
                file_handle, file_path = self.write_to_temp_file(message)

                #
                # Parse STIX data
                #
                stix_package = STIXPackage.from_xml(file_path)

                #
                # Get the timestamp of the STIX Package so we can use this in our feed
                #
                timestamp = total_seconds(stix_package.timestamp)

                #
                # Now lets find some data.  Iterate through all observables and parse
                #
                if stix_package.observables:
                    for observable in stix_package.observables:
                        #
                        # Cybox observable returns a list
                        #
                        reports.extend(cybox_parse_observable(observable, timestamp))

                #
                # Delete our temporary file
                #
                file_handle.close()

                num_blocks += 1

                #
                # end for loop through content blocks
                #

            logger.info("content blocks read: {}".format(num_blocks))
            logger.info("current number of reports: {}".format(len(reports)))

            #
            # DEBUG CODE
            #
            #if len(reports) > 10:
            #    break

            #
            # Attempt to advance the start time and end time
            #
            if feed_helper.advance():
                continue
            else:
                break
            #
            # end While True
            #

        logger.info("Found {} new reports.".format(len(reports)))

        reports = feed_helper.load_existing_feed_data() + reports

        logger.info("Total number of reports: {}".format(len(reports)))

        data = build_feed_data(sanitized_feed_name,
                               "%s %s" % (site.get('site'), collection_name),
                               site.get('site'),
                               site.get('icon_link'),
                               reports)

        if feed_helper.write_feed(data):
            feed_helper.save_details()

        #
        # Create Cb Response Feed if necessary
        #
        feed_id = self.cb.feed_get_id_by_name(sanitized_feed_name)
        if not feed_id:
            data = self.cb.feed_add_from_url("file://" + feed_helper.path,
                                             site.get('feeds_enable'),
                                             False,
                                             False)


    def perform(self):
        """
        :param self:
        :param enumerate_collections_only:
        :return:
        """
        for site in self.sites:

            client = create_client(site.get('site'),
                                   use_https=site.get('use_https'),
                                   discovery_path=site.get('discovery_path'))

            if not site.get('collection_management_path', ''):
                collections = client.get_collections()
            else:
                uri = ''
                if site.get('use_https'):
                    uri += 'https://'
                else:
                    uri += 'http://'

                uri += site.get('site')
                uri += site.get('collection_management_path')
                logger.info('Collection Management Path: {}'.format(uri))

                collections = client.get_collections(uri=uri)

            for collection in collections:
                logger.info('Collection Name: {}, Collection Type: {}'.format(collection.name, collection.type))

            if len(collections) == 0:
                logger.info('Unable to find any collections.  Exiting...')
                sys.exit(0)

            desired_collections = site.get('collections').lower().split(',')

            want_all = False
            if '*' in desired_collections:
                want_all = True

            for collection in collections:
                if collection.type != 'DATA_FEED':
                    continue

                if want_all or collection.name.lower() in desired_collections:
                    self._import_collection(client, site, collection)
class CBQuery(object):
    def __init__(self, url, token, ssl_verify):
        self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
        self.cb_url = url

    def report(self, rundll_query, dll_dictionary, search_match_count):
	    # CALLED BY: self.report(regex, regex_match_dictionary, search_match_count)

        print "--------------------------------------------"
        print "%s Command Line Matches:" % (search_match_count)
        print "%s : %s" % ("Search Match Count", "Command Line Match")
        print "--------------------------------------------"
 
	    #ordered_dll_dictionary = collections.OrderedDict(sorted(dll_dictionary.items()))
        ordered_dll_dictionary = sorted(dll_dictionary.items(), key=operator.itemgetter(1))
        for value in ordered_dll_dictionary:
            print "%s : %s" % (value[1], value[0])

    def check(self, regex, ignore_case, group_reference_to_match, count_flag, matches_only_flag):
	    # CALLED BY: cb.check(opts.regex, opts.ignore_case, opts.group_reference_to_match, opts.count_flag, opts.matches_only_flag)

        # print a legend
    	print ""
        print "Displaying Report for Commandline regular expression matches"
        print ""
        print "Command Line Strings Matching REGEX: %s" % (regex)
        print "============================================================"
        print ""

        # build the query string
        q = "cmdline:*"

        #define dictionary
        regex_match_dictionary = dict()
        search_match_count = 0
        
        #define regexp
        # check if we need to ignore case, if so, update regexp
        if ignore_case:
            regexp = re.compile(regex, re.IGNORECASE)
        else:
            regexp = re.compile(regex)


        for result in self.cb.process_search_iter(q):
            cmdline = result.get("cmdline", "<unknown>")
            # print "CMD: %s" % (cmdline,)

            #SEARCH FOR REGEX IN STRING!!
            if matches_only_flag:
                # print "-----MATCHES ONLY"
                search_match_result = regexp.match(cmdline)
            else:
                # print "-----EVERYTHING"
                search_match_result = regexp.search(cmdline)

            if search_match_result is not None:
                # print "cmdline: %s" % (cmdline)
                # print "result: %s" % (search_match_result)
                # print "------------------------------------"

                # Iterate TOTAL Search Match Count
                search_match_count = search_match_count + 1

                # On Match, add to dictionary
                # 1st Check group_reference_to_match flag to see if we need to add a specific Group Reference or just the entire Command Line as the regex match
                if group_reference_to_match:
                    # print "cmdline: %s" % (cmdline)
                    # print"matching GROUP: %s" % (group_reference_to_match)
                    # print"search_match_result: %s" % (search_match_result)
                    regex_match_group_reference = search_match_result.group(int(group_reference_to_match))
                    if regex_match_group_reference not in regex_match_dictionary.keys():
                        print "%s" % (regex_match_group_reference)
                        regex_match_dictionary[regex_match_group_reference] = 1
                    else:
                        regex_match_dictionary[regex_match_group_reference] = regex_match_dictionary[regex_match_group_reference] + 1
                else:
                    if cmdline not in regex_match_dictionary.keys():
                        print "%s" % (cmdline)
                        regex_match_dictionary[cmdline] = 1
                    else:
                        regex_match_dictionary[cmdline] = regex_match_dictionary[cmdline] + 1

        self.report(regex, regex_match_dictionary, search_match_count)
Esempio n. 36
0
class CBQuery(object):
    def __init__(self, url, token, ssl_verify):
        self.cb = CbApi(url, token=token, ssl_verify=ssl_verify)
        self.cb_url = url

    def report(self, ioc, type, procs, detail=False):
        for result in procs["results"]:
            # print the results to stdout. you could do anything here - 
            # log to syslog, send a SMS, fire off a siren and strobe light, etc.
            print
            print "Found %s IOC for %s in:" % (type, ioc)
            print
            print "\tPath: %s"          % result["path"]
            print "\tHostname: %s"      % result["hostname"]
            print "\tStarted: %s"       % result["start"]
            print "\tLast Updated: %s"  % result["last_update"]
            print "\tDetails: %s/#analyze/%s/%s" % (self.cb_url, result["id"], result["segment_id"])
            print

            if detail:
                self.report_detail(ioc, type, result)

    def report_detail(self, ioc, type, result):
        events = self.cb.process_events(result["id"], result["segment_id"])
        proc = events["process"]

        if type == "domain" and proc.has_key("netconn_complete"):
            for netconn in proc["netconn_complete"]:
                ts, ip, port, proto, domain, dir = netconn.split("|")
                if ioc in domain:
                    str_ip = socket.inet_ntoa(struct.pack("!i", int(ip)))
                    print "%s\t%s (%s:%s)" % (ts, domain, str_ip, port)

        elif type == "ipaddr" and proc.has_key("netconn_complete"):
            for netconn in proc["netconn_complete"]:
                ts, ip, port, proto, domain, direction = netconn.split("|")
                packed_ip = struct.unpack("!i", socket.inet_aton(ioc))[0]
                #import code; code.interact(local=locals())
                if packed_ip == int(ip):
                    str_ip = socket.inet_ntoa(struct.pack("!i", int(ip)))
                    print "%s\t%s (%s:%s)" % (ts, domain, str_ip, port)

        elif type == "md5" and proc.has_key("modload_complete"):
            for modload in proc["modload_complete"]:
                ts, md5, path = modload.split("|")
                if ioc in md5:
                    print "%s\t%s %s" % (ts, md5, path)

            if result["process_md5"] == ioc:
                print "%s\t%s %s" % (result["start"], result["process_md5"], result["path"])

    def check(self, iocs, type, detail=False):
        # for each ioc, do a search for (type):(ioc)
        # e.g, 
        #   domain:bigfish.com
        #   md5:ce7a81ceccfa03e5e0dfd0d9a7f41466
        # 
        # additionally, if a cron interval is specified, limit searches
        # to processes updated in the last CRON_INTERVAL period
        # 
        # note - this is a very inefficient way to do this, since you test only one
        # IOC per request - you could build a large OR clause together with a few hundred
        # to efficiently and quickly check 1000s of IOCs, at the cost of increased complexity
        # when you discover a hit.
        #
        # note 2 - with a list of flat indicators, what you really want is a CB feed
        # see http://github.com/carbonblack/cbfeeds
        #
        for ioc in iocs:
            if CRON_INTERVAL:
                q = "%s:%s and last_update:-%s" % (type, ioc, CRON_INTERVAL)
            else:
                q = "%s:%s" % (type, ioc)
            print q
            procs = self.cb.process_search(q)

            # if there are _any_ hits, give us the details.
            # then check the next ioc
            if len(procs["results"]) > 0:
                self.report(ioc, type, procs, detail)
            else:
                sys.stdout.write(".")
                sys.stdout.flush()
Esempio n. 37
0
	myrow.append(bprocess["id"]) 
	if bprocess.has_key("cmdline"):
		cmdline = bprocess["cmdline"]
	else:
		cmdline = ""
	myrow.append(cmdline)
	if bprocess.has_key("childproc_complete"):
		getchildprocs(bprocess['childproc_complete'])

parser = build_cli_parser()
opts, args = parser.parse_args(sys.argv[1:])
if not opts.server or not opts.token or (not opts.procnamefile and not opts.procname) or not opts.percentless:
	print "Missing required param."
	sys.exit(-1)

cb = CbApi(opts.server, ssl_verify=opts.ssl_verify, token=opts.token)
if opts.procnamefile:
	searchprocess = processsearchlist(opts.procnamefile)
else:
	searchprocess = opts.procname.split(",")
for proc in searchprocess:
	start = 0
	data = cb.process_search("parent_name:%s" % (proc), rows=1, start=start)
	facetterms = data['facets']
	for term in reversed(facetterms['process_name']):
		termratio = int(float(term['ratio']))
		if int(opts.percentless) >= termratio:
			start = 0
			while True:
				q = "parent_name:%s AND process_name:%s" % (proc,term['name'])
				data = cb.process_search(q , rows=int(pagesize), start=start)
                      dest="ssl_verify",
                      help="Do not verify server SSL certificate.")
    return parser


def processsearchlist(procnamefile):
    with open(procnamefile) as tmp:
        lines = filter(None, [line.strip() for line in tmp])
    return lines


parser = build_cli_parser()
opts, args = parser.parse_args(sys.argv[1:])
if not opts.server or not opts.token or not opts.procsearchfile:
    print "Missing required param."
    sys.exit(-1)

cb = CbApi(opts.server, ssl_verify=opts.ssl_verify, token=opts.token)
searchprocess = processsearchlist(opts.procsearchfile)
for search in searchprocess:
    data = cb.process_search(search, rows=1)
    if 0 != (data['total_results']):
        print "------------------------------"
        print "Search Query | %s" % (search)
        print "Resulting Processes | %s" % (data['total_results'])
        facet = data['facets']
        hosts = []
        for term in facet['hostname']:
            hosts.append("%s (%s%%)" % (term['name'], term['ratio']))
        print "Resulting Hosts | " + "|".join(hosts)
Esempio n. 39
0
    myrow.append(bprocess["id"])
    if bprocess.has_key("cmdline"):
        cmdline = bprocess["cmdline"]
    else:
        cmdline = ""
    myrow.append(cmdline)
    if bprocess.has_key("childproc_complete"):
        getchildprocs(bprocess['childproc_complete'])

parser = build_cli_parser()
opts, args = parser.parse_args(sys.argv[1:])
if not opts.server or not opts.token or not opts.query or not opts.percentless:
    print "Missing required param."
    sys.exit(-1)

cb = CbApi(opts.server, ssl_verify=opts.ssl_verify, token=opts.token)


start = 0
data = cb.process_search(opts.query, rows=1, start=start)
facetterms = data['facets']
pprint.pprint(facetterms)
'''
for term in reversed(facetterms['process_name']):
    termratio = int(float(term['ratio']))
    if int(opts.percentless) >= termratio:
        start = 0
        while True:
            q = "parent_name:%s AND process_name:%s" % (proc,term['name'])
            data = cb.process_search(q , rows=int(pagesize), start=start)
            if 0 == len(data['results']):
Esempio n. 40
0
class FileCollector:
    def __init__(self):
        config = load(open(r'conf.yaml', 'rb'))
        self.address = config['address']
        self.port = config['port']
        self.banner = config['banner']
        self.protection_url = config['protection']['url']
        self.protection_token = config['protection']['token']
        self.protection_headers = {'X-Auth-Token': self.protection_token}
        self.protection_path = config['protection']['path']
        self.protection_interval = config['protection']['interval']
        self.protection_counter = config['protection']['counter']
        self.response_url = config['response']['url']
        self.response_token = config['response']['token']
        self.response_path = config['response']['path']
        self.archives = config['archives']
        self.logs = config['logs']
        self.password = config['password']
        #~ self.root = getLogger("cbapi")
        self.queue = Queue()
        self.check_dirs()
        setup_logging()
        #~ basicConfig(filename=datetime.now().strftime(r'logs/FileCollector_%H_%M_%d_%m_%Y.log'), level=INFO)
        requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
        self.cb = CbApi(self.response_url,
                        token=self.response_token,
                        ssl_verify=False)

    def check_dirs(self):
        if not isdir(self.protection_path):
            log_event("[+] Creating {} directory".format(self.protection_path))
            makedirs(self.protection_path)
        if not isdir(self.response_path):
            log_event("[+] Creating {} directory".format(self.response_path))
            makedirs(self.response_path)
        if not isdir(self.archives):
            log_event("[+] Creating {} directory".format(self.archives))
            makedirs(self.archives)
        if not isdir(self.logs):
            log_event("[+] Creating {} directory".format(self.logs))
            makedirs(self.logs)

    def start_service(self):
        log_event(self.banner)
        log_event("[*] Connecting to {}".format(self.protection_url))
        log_event("[*] Connecting to {}".format(self.response_url))
        self.listener = Listener((self.address, self.port))
        while True:
            try:
                self.connection = self.listener.accept()
                while True:
                    self.get_md5()
                    log_event("[*] Processing {}.".format(self.md5))
                    if self.get_fileCatalogData():
                        if not self.isQueued():
                            self.queue_file()
                    elif self.download_binary_cbr()['code']:
                        continue
                    else:
                        continue
            except EOFError as eof:
                log_event("[-] Lost connection to client.", True)
                self.archive_and_encrypt()

    def get_md5(self):
        self.md5 = self.connection.recv()

    def get_fileCatalogData(self):
        self.dl_req = requests.get("{}{}{}".format(
            self.protection_url, 'api/bit9platform/v1/fileCatalog?q=md5:',
            self.md5),
                                   headers=self.protection_headers,
                                   verify=False)
        if self.dl_req.ok:
            self.fc_id = self.get_id()
            if self.fc_id:
                log_event("[+] Requested file found.[id] = {}".format(
                    self.fc_id))
                return True
            return False
        else:
            log_event("[-] Could not fetch File Catalog Data", True)
            self.connection.send("[-] Could not fetch File Catalog Data")
            return False

    def isQueued(self):
        log_event("[+] Checking for queued downloads.")
        self.fileName = self.md5.lower()
        isqueued = requests.get("{}{}{}".format(
            self.protection_url,
            'api/bit9platform/v1/fileCatalog?q=uploadStatus:0&limit=0&q=fileName:',
            self.fileName),
                                headers=self.protection_headers,
                                verify=False)
        if isqueued.ok:
            log_event("[+] File is queued for uploading")
            return True
        return False

    def queue_file(self):
        log_event("[+] Queuing file for upload")
        compId = self.dl_req.json()[0]['computerId']
        self.data = {
            'computerId': compId,
            'fileCatalogId': self.fc_id,
            'priority': 2,
            'uploadStatus': 0
        }
        self.upload_req = requests.post("{}{}".format(
            self.protection_url, 'api/bit9platform/v1/fileUpload'),
                                        headers=self.protection_headers,
                                        data=self.data,
                                        verify=False)
        if self.upload_req.ok:
            self.check_fileQueue()
        else:
            log_event("[-] Requested file not found.", True)
            self.connection.send("[-] Requested file not found.")

    def check_fileQueue(self):
        self.fileId = self.upload_req.json()['id']
        log_event("[+] File is now queued. [id] = {}".format(self.fileId))
        check_queue = requests.post("{}{}".format(
            self.protection_url, 'api/bit9platform/v1/fileUpload'),
                                    headers=self.protection_headers,
                                    data=self.data,
                                    verify=False)
        ctr = 1
        while check_queue.json()['uploadStatus'] is not 3:
            log_event("[+] Sleeping for {} seconds.".format(
                self.protection_interval))
            sleep(self.protection_interval)
            check_queue = requests.post("{}{}".format(
                self.protection_url, 'api/bit9platform/v1/fileUpload'),
                                        headers=self.protection_headers,
                                        data=self.data,
                                        verify=False)
            #Sleep until 5 minutes has passed
            ctr = ctr + 1
            if ctr is self.protection_counter:
                log_event("[-] File is still queued. Check the queue later.",
                          True)
                self.connection.send(
                    "[-] File is still queued. Check the queue later.")
                break
        if check_queue.json()['uploadStatus'] is 3:
            self.download_binary_cbp()

    def get_id(self):
        if isinstance(self.dl_req.json(), list):
            if len(self.dl_req.json()) > 0:
                return self.dl_req.json()[0]['id']
            else:
                log_event(
                    "[-] No File Catalog data found for {}.".format(self.md5),
                    True)
                self.connection.send(
                    "[-] No File Catalog data found for {}.".format(self.md5))
                return None
        else:
            return self.dl_req.json()['id']

    def archive_and_encrypt(self):
        try:
            log_event("[*] Archiving downloaded file/s.")
            prog_path = r"C:\Program Files\7-Zip\7z.exe"
            if isfile(prog_path):
                archive_name = datetime.now().strftime('%H_%M_%d_%m_%Y')
                isZipped = check_call([
                    prog_path, 'a', '-p{}'.format(self.password), '-y',
                    r'{}/archive-{}.zip'.format(self.archives, archive_name),
                    './{}/*'.format(self.protection_path), './{}/*'.format(
                        self.response_path)
                ],
                                      stdout=open(devnull, 'w'))
                if isZipped != 0:
                    raise CalledProcessError(cmd=prog_path,
                                             returncode=1,
                                             output='Failed to archive file/s')
                else:
                    log_event('[+] File/s have been archived [{}.zip].'.format(
                        archive_name))
                for file_response in listdir(self.response_path):
                    tmp_path = join(self.response_path, file_response)
                    remove(tmp_path)
                for file_protection in listdir(self.protection_path):
                    tmp_path = join(self.protection_path, file_protection)
                    remove(tmp_path)
            else:
                raise WindowsError('[-] 7zip is missing in your system.')
        except CalledProcessError as cpe:
            log_event(cpe, True)
        except WindowsError as we:
            log_event(we, True)

    def download_binary_cbr(self):
        temp_text = {'msg': "[+] Download successful", 'code': 1}
        try:
            filename = join(self.response_path, self.md5.lower())
            if not isfile(filename):
                binary = self.cb.binary(self.md5)
                dump = open(filename, "wb")
                dump.write(binary)
                dump.close()
            else:
                temp_text = {'msg': "[-] File exists.", 'code': 1}
        except (ObjectNotFoundError, HTTPError):
            temp_text = {'msg': "[-] File was not found", 'code': 0}
        log_event(temp_text['msg'], (temp_text['code'] is 0))
        return temp_text

    def download_binary_cbp(self):
        log_event("[+] Downloading sample")
        dl = requests.get("{}{}".format(
            self.protection_url,
            'api/bit9platform/v1/fileUpload/{}?downloadFile=true'.format(
                self.fileId)),
                          headers=self.headers,
                          verify=False,
                          stream=True)
        chunk_size = 1024
        size = int(dl.headers['Content-Length']) / chunk_size
        i = 0
        if dl.ok:
            pbar = ProgressBar(widgets=[Percentage(), Bar()],
                               maxval=size).start()
            with open("{}/{}".format(self.protection_path, self.fileName),
                      'wb') as f:
                for chunk in dl.iter_content(chunk_size):
                    f.write(chunk)
                    pbar.update(i)
                    i += 1
                #~ log_event("[+] Download complete")
                pbar.finish()
                f.close()
                log_event("[+] Download complete.")
                self.connection.send("[+] Download complete.")

        else:
            #~ print("[-] Unable to download file")
            log_event("[-] Unable to download file.", True)
            self.connection.send("[-] Unable to download file.")
Esempio n. 41
0
                      help="File containing a line for each query you would like to run")
    parser.add_option("-n", "--no-ssl-verify", action="store_false", default=True, dest="ssl_verify",
                      help="Do not verify server SSL certificate.")
    return parser


def processsearchlist(procnamefile):
    with open(procnamefile) as tmp:
        lines = filter(None, [line.strip() for line in tmp])
    return lines

parser = build_cli_parser()
opts, args = parser.parse_args(sys.argv[1:])
if not opts.server or not opts.token or not opts.procsearchfile:
    print "Missing required param."
    sys.exit(-1)

cb = CbApi(opts.server, ssl_verify=opts.ssl_verify, token=opts.token)
searchprocess = processsearchlist(opts.procsearchfile)
for search in searchprocess:
    data = cb.process_search(search, rows=1)
    if 0 != (data['total_results']):
        print "------------------------------"
        print "Search Query | %s" % (search)
        print "Resulting Processes | %s" % (data['total_results'])
        facet=data['facets']
        hosts =[]
        for term in facet['hostname']:
            hosts.append("%s (%s%%)" % (term['name'], term['ratio']))
        print "Resulting Hosts | "+"|".join(hosts)