def create_feed(options):
   
    # generate the required feed information fields
    # based on command-line arguments
    # 
    feedinfo = {'name': options.name,
                'display_name': options.display_name,
                'provider_url': options.url,
                'summary': options.summary,
                'tech_data': options.techdata}
   
    # if an icon was provided, encode as base64 and
    # include in the feed information
    # 
    if options.icon:
        bytes = base64.b64encode(open(options.icon).read())
        feedinfo['icon'] = bytes 
        
    # build a CbFeedInfo instance
    # this does field validation
    #    
    feedinfo = CbFeedInfo(**feedinfo)
   
    # build a list of reports (always one report in this
    # case).  the single report will include all the IOCs  
    # 
    reports = build_reports(options)
   
    # build a CbFeed instance
    # this does field validation (including on the report data)
    # 
    feed = CbFeed(feedinfo, reports)

    return feed.dump()
Exemple #2
0
    def run(self):
        try:
            with file_lock(os.path.join(CbIntegrationBridge.WORKING_FILE_ROOT, '%s.pid' % self.integration_name)):
                reports = self.perform()

                if not reports:
                    return True

                if self.debug:
                    self.print_reports(reports)

                # Create the feed
                feed = CbFeed(self.feed_metadata, reports)
                if not feed.validate():
                    self.logger.error("Feed {0:s} did not validate, not updating".format(self.integration_name))
                    return False

                raw_feed_data = feed.dump()

                with NamedTemporaryFile(dir=self.working_file_root, delete=False) as fp:
                    fp.write(raw_feed_data)
                    self.logger.info("Creating {0:s} feed at {1:s}".format(self.integration_name, self.feed_file_name))
                    os.rename(fp.name, self.feed_file_name)

                c = connect_local_cbapi()
                feed_id = c.feed_get_id_by_name(self.integration_name)
                if not feed_id:
                    self.logger.info("Creating {0:s} feed for the first time".format(self.integration_name))
                    c.feed_add_from_url("file://" + self.feed_file_name, True, False, False)

                # force a synchronization
                c.feed_synchronize(self.integration_name)
        except AlreadyRunningError:
            self.logger.error("{0:s} is already running".format(self.integration_name))
Exemple #3
0
def create():
    nodes = get_tor_nodes()
    reports = build_reports(nodes)

    feedinfo = {'name': 'tor',
                'display_name': "Tor Exit Nodes",
                'provider_url': 'https://www.torproject.org/',
                'summary': "This feed is a list of Tor Node IP addresses, updated every 30 minutes.",
                'tech_data': "There are no requirements to share any data to receive this feed.",
                'icon': 'tor.png',
                'icon_small': 'tor.small.jpg',
                'category': 'Open Source',
                }

    # lazy way out to get right icon path.  sorry.
    old_cwd = os.getcwd()
    os.chdir(os.path.dirname(os.path.realpath(__file__)))

    feedinfo = CbFeedInfo(**feedinfo)
    feed = CbFeed(feedinfo, reports)
    created_feed = feed.dump()

    os.chdir(old_cwd)

    return created_feed
Exemple #4
0
def create(localcsv=None):
    if localcsv:
        lines = open(localcsv, "r").readlines()
         
    else:
        r = requests.get("http://www.malwaredomainlist.com/mdlcsv.php", stream=True)
        lines = r.text.split("\r\n")
    
    reports = reports_from_csv(lines)
    feedinfo = {'name': 'mdl',
                'display_name': "Malware Domain List",
                'provider_url': "http://www.malwaredomainlist.com/mdl.php",
                'summary': "Malware Domain List is a non-commercial community project to track domains used by malware." +
                           " This feed contains the most recent 180 days of entries.",
                'tech_data': "There are no requirements to share any data to receive this feed.",
                "icon": "mdl.png"
                }

    # lazy way out 
    old_cwd = os.getcwd()
    os.chdir(os.path.dirname(os.path.realpath(__file__)))

    feedinfo = CbFeedInfo(**feedinfo)
    feed = CbFeed(feedinfo, reports)
    bytes = feed.dump()

    os.chdir(old_cwd)

    return bytes
def create():
    nodes = get_et_fwips()
    reports = build_reports(nodes)

    feedinfo = {'name': 'Emerging_Threats',
                'display_name': "Emerging Threats IPs",
                'provider_url': 'https://www.emergingthreats.net/',
                'summary': "This feed is a list of IP addresses, from Emerging Threats public feed.",
                'tech_data': "There are no requirements to share any data to receive this feed.",
                'icon': 'et_image.jpg',
                'icon_small': 'et_image_small.jpg',
                'category': 'Open Source',
                }

    # lazy way out to get right icon path.  sorry.
    old_cwd = os.getcwd()
    os.chdir(os.path.dirname(os.path.realpath(__file__)))

    feedinfo = CbFeedInfo(**feedinfo)
    feed = CbFeed(feedinfo, reports)
    created_feed = feed.dump()

    os.chdir(old_cwd)

    return created_feed
Exemple #6
0
def create():
    reports = []
    reports.extend(get_zeus())
    reports.extend(get_palevo())
    reports.extend(get_spyeye())
   
    feedinfo = {'name': 'abusech',
                'display_name': "abuse.ch Malware Domains",
                'provider_url': "http://www.abuse.ch",
                'summary': "abuse.ch tracks C&C servers for Zeus, SpyEye and Palevo malware. " + 
                           "This feed combines the three domain names blocklists.",
                'tech_data': "There are no requirements to share any data to receive this feed.",
                "icon": "abuse.ch.jpg"
                }

    # the lazy way to the icon 
    old_cwd = os.getcwd()
    os.chdir(os.path.dirname(os.path.realpath(__file__)))

    feedinfo = CbFeedInfo(**feedinfo)
    feed = CbFeed(feedinfo, reports)
    bytes = feed.dump()

    os.chdir(old_cwd)

    return bytes
def build_feed_data(feed_name: str, display_name: str, feed_summary: str, site: str, icon_link: str,
                    reports: List[Dict[str, Any]]) -> str:
    """
    Return a feed definition as a JSON string definition.

    :param feed_name: the short name of the feed
    :param display_name: the display name of the feed
    :param feed_summary: the feed summary
    :param site: the site name
    :param icon_link: path to the icon source
    :param reports:  List of gathered reports
    :return: feed as JSON string
    """

    feedinfo = {'name': feed_name,
                'display_name': display_name,
                'provider_url': 'http://' + site,
                'summary': feed_summary,
                'tech_data': "There are no requirements to share any data to receive this feed.",
                }

    # handle optionals
    if icon_link:
        feedinfo['icon'] = icon_link

    feedinfo = CbFeedInfo(**feedinfo)

    reports = remove_duplicate_reports(reports)

    feed = CbFeed(feedinfo, reports)
    return feed.dump()
Exemple #8
0
def create_feed(options):

    # generate the required feed information fields
    # based on command-line arguments
    #
    feedinfo = {
        'name': options.name,
        'display_name': options.display_name,
        'provider_url': options.url,
        'summary': options.summary,
        'tech_data': options.techdata
    }

    # if an icon was provided, encode as base64 and
    # include in the feed information
    #
    if options.icon:
        b64bytes = base64.b64encode(open(options.icon).read()).decode("utf-8")
        feedinfo['icon'] = b64bytes

    # if a small icon was provided, encode as base64 and
    # include in the feed information
    #
    if options.small_icon:
        b64bytes = base64.b64encode(open(
            options.small_icon).read()).decode("utf-8")
        feedinfo['icon_small'] = b64bytes

    # if a feed category was provided, include it in the feed information
    #
    if options.category:
        feedinfo['category'] = options.category

    # build a CbFeedInfo instance
    # this does field validation
    #
    feedinfo = CbFeedInfo(**feedinfo)

    # build a list of reports (always one report in this
    # case).  the single report will include all the IOCs
    #
    reports = build_reports(options)

    # build a CbFeed instance
    # this does field validation (including on the report data)
    #
    feed = CbFeed(feedinfo, reports)

    return feed.dump()
def create():
    nodes = get_tor_nodes()
    reports = build_reports(nodes)
    
    feedinfo = {'name': 'tor',
                'display_name': "Tor Exit Nodes",
                'provider_url': 'https://www.torproject.org/',
                'summary': "This feed is a list of Tor Node IP addresses, updated every 30 minutes.",
                'tech_data': "There are no requirements to share any data to receive this feed.",
                'icon': 'images/tor.png'}
            
    feedinfo = CbFeedInfo(**feedinfo)
    feed = CbFeed(feedinfo, reports)

    return feed.dump()
Exemple #10
0
def create():
    nodes = get_tor_nodes()
    reports = build_reports(nodes)
    
    feedinfo = {'name': 'tor',
                'display_name': "Tor Exit Nodes",
                'provider_url': 'https://www.torproject.org/',
                'summary': "This feed is a list of Tor Node IP addresses, updated every 30 minutes.",
                'tech_data': "There are no requirements to share any data to receive this feed.",
                'icon': 'images/tor.png'}
            
    feedinfo = CbFeedInfo(**feedinfo)
    feed = CbFeed(feedinfo, reports)

    return feed.dump()
Exemple #11
0
def create(input_source):
    reports = build_reports(input_source)

    # ****************************
    # TODO - you probably want to change these values to reflect your 
    # local input source
    feedinfo = {'name': 'stiximport',
                'display_name': "STIX Package Import",
                'provider_url': 'http://stix.mitre.org',
                'summary': "This feed was imported from stix package(s) at %s" % input_source,
                'tech_data': "There are no requirements to share any data to receive this feed.",
                'icon': 'images/stix.gif'
                }

    feedinfo = CbFeedInfo(**feedinfo)
    feed = CbFeed(feedinfo, reports)
    return feed.dump()
def build_feed_data(feed_name, feed_description, site, icon_link, reports):
    """
    :return:feed as bytes to be written out
    """
    feedinfo = {'name': feed_name,
                'display_name': feed_description,
                'provider_url': 'http://' + site,
                'summary': "TAXII Feed %s" % feed_description,
                'tech_data': "There are no requirements to share any data to receive this feed.",
                'icon': icon_link
                }

    feedinfo = CbFeedInfo(**feedinfo)

    reports = remove_duplicate_reports(reports)

    feed = CbFeed(feedinfo, reports)
    return feed.dump()
Exemple #13
0
    def run(self):
        try:
            with file_lock(
                    os.path.join(CbIntegrationBridge.WORKING_FILE_ROOT,
                                 '%s.pid' % self.integration_name)):
                reports = self.perform()

                if not reports:
                    return True

                if self.debug:
                    self.print_reports(reports)

                # Create the feed
                feed = CbFeed(self.feed_metadata, reports)
                if not feed.validate():
                    self.logger.error(
                        "Feed {0:s} did not validate, not updating".format(
                            self.integration_name))
                    return False

                raw_feed_data = feed.dump()

                with NamedTemporaryFile(dir=self.working_file_root,
                                        delete=False) as fp:
                    fp.write(raw_feed_data)
                    self.logger.info("Creating {0:s} feed at {1:s}".format(
                        self.integration_name, self.feed_file_name))
                    os.rename(fp.name, self.feed_file_name)

                c = connect_local_cbapi()
                feed_id = c.feed_get_id_by_name(self.integration_name)
                if not feed_id:
                    self.logger.info(
                        "Creating {0:s} feed for the first time".format(
                            self.integration_name))
                    c.feed_add_from_url("file://" + self.feed_file_name, True,
                                        False, False)

                # force a synchronization
                c.feed_synchronize(self.integration_name)
        except AlreadyRunningError:
            self.logger.error("{0:s} is already running".format(
                self.integration_name))
Exemple #14
0
def create(input_source):
    reports = build_reports(input_source)

    # ****************************
    # TODO - you probably want to change these values to reflect your
    # local input source
    feedinfo = {
        'name': 'stiximport',
        'display_name': "STIX Package Import",
        'provider_url': 'http://stix.mitre.org',
        'summary':
        "This feed was imported from stix package(s) at %s" % input_source,
        'tech_data':
        "There are no requirements to share any data to receive this feed.",
        'icon': 'images/stix.gif'
    }

    feedinfo = CbFeedInfo(**feedinfo)
    feed = CbFeed(feedinfo, reports)
    return feed.dump()
Exemple #15
0
def build_feed_data(feed_name, display_name, feed_summary, site, icon_link,
                    reports):
    """
    :return:feed as bytes to be written out
    """
    feedinfo = {
        'name': feed_name,
        'display_name': display_name,
        'provider_url': 'http://' + site,
        'summary': feed_summary,
        'tech_data':
        "There are no requirements to share any data to receive this feed.",
        'icon': icon_link
    }

    feedinfo = CbFeedInfo(**feedinfo)

    reports = remove_duplicate_reports(reports)

    feed = CbFeed(feedinfo, reports)
    return feed.dump()
def create():
    reports = []
    tmp = get_zeus()
    if tmp:
        reports.extend(tmp)
    tmp = get_feodo()
    if tmp:
        reports.extend(tmp)
    tmp = get_ransomware()
    if tmp:
        reports.extend(tmp)

    feedinfo = {
        'name': 'abusech',
        'display_name': "abuse.ch Malware Domains",
        'provider_url': "http://www.abuse.ch",
        'summary':
        "abuse.ch tracks C&C servers for Zeus and Palevo malware. " +
        "This feed combines the two domain names blocklists.",
        'tech_data':
        "There are no requirements to share any data to receive this feed.",
        'icon': "abuse.ch.jpg",
        'icon_small': "abuse.ch.small.jpg",
        'category': "Open Source"
    }

    # the lazy way to the icon
    old_cwd = os.getcwd()
    os.chdir(os.path.dirname(os.path.realpath(__file__)))

    feedinfo = CbFeedInfo(**feedinfo)
    feed = CbFeed(feedinfo, reports)
    feed_bytes = feed.dump()

    os.chdir(old_cwd)

    return feed_bytes
Exemple #17
0
def create(config_file, existing_csv=None, reports_to_skip=[]):
    # parse the configuration file
    # this configuration file includes the keys needed to talk to the
    # iSight report server, etc.
    #
    #print "-> Parsing iSight configuration..."
    cfg = isight_config.ISightConfig(config_file)

    # instantiate a local iSight API object
    #
    #print "-> Instantiating an iSight API object..."
    api = isight_api.ISightAPI(cfg.iSightRemoteImportUrl,
                               cfg.iSightRemoteImportUsername,
                               cfg.iSightRemoteImportPassword,
                               cfg.iSightRemoteImportPublicKey,
                               cfg.iSightRemoteImportPrivateKey)

    if not existing_csv:
        # query the iSight report server for raw CSV report data
        # query 'back' the specified number of days
        #
        #print "-> Querying iSight server for last %d days of reports..." % (cfg.iSightRemoteImportDaysBack)
        #
        # @todo iSIGHT has a new-and-improved REST API which could be used instead of this legacy API
        #
        raw_report_data = api.get_i_and_w(cfg.iSightRemoteImportDaysBack)

        # save off the raw report data for future reference
        #
        #print "-> Saving iSight report data to iSight.csv..."
        f = open('iSight.csv', 'w')
        f.write(raw_report_data)
        f.close()
    else: 
        raw_report_data = open(existing_csv, "r").read()

    # convert the raw report data into something more managable
    # in particular, a list of dictionaries, with each dictionary describing a report
    # this helper routine accounts for the fact that report data is spread across
    # multiple lines of the raw CSV blob
    #
    results = isight_helpers.isight_csv_to_iocs_dict([raw_report_data])

    # set up a dictionary for basic stat tracking
    #
    stats = {'md5' : {'total' : 0, 'max' : 0},
             'ipaddr' : {'total' : 0, 'max' : 0}, 
             'domain' : {'total' : 0, 'max' : 0}}

    for report_id in results.keys():
        stats['md5']['total'] += len(results[report_id]['md5'])
        if len(results[report_id]['md5']) > stats['md5']['max']:
            stats['md5']['max'] = len(results[report_id]['md5'])
        stats['ipaddr']['total'] += len(results[report_id]['ipaddr'])
        if len(results[report_id]['ipaddr']) > stats['ipaddr']['max']:
            stats['ipaddr']['max'] = len(results[report_id]['ipaddr'])
        stats['domain']['total'] += len(results[report_id]['domain'])
        if len(results[report_id]['domain']) > stats['domain']['max']:
            stats['domain']['max'] = len(results[report_id]['domain'])
    
    #print "  -> Total Reports:                                  %d" % (len(results.keys()))
    #print "  -> ----------------------------------------------- ---"
    #print "  -> Maximum number of MD5s in one report:           %d" % (stats['md5']['max'])
    #print "  -> Total MD5s across all reports:                  %d" % (stats['md5']['total'])
    #print "  -> Maximum number of IPv4 addresses in one report: %d" % (stats['ipaddr']['max'])
    #print "  -> Total IPv4 addresses in all reports:            %d" % (stats['ipaddr']['total'])
    #print "  -> Maximum number of DNS names in one report:      %d" % (stats['domain']['max'])
    #print "  -> Total DNS names in all reports:                 %d" % (stats['domain']['total'])

    # generate the feed data from the raw iSight report data
    #
    #print "-> Generating feed data..."
    reports = generate_reports(results, api)
    
    # shim to skip entire reports
    reports = [report for report in reports if report.data['id'] not in reports_to_skip]

    # generate the feed metadata (feed information)
    # this is a static description of the feed itself
    #

    # lazy way out 
    cwd_old = os.getcwd()
    os.chdir(os.path.dirname(os.path.realpath(__file__)))

    #print "-> Generating feed metadata..."
    feedinfo = generate_feed_information()

    # write out feed document
    #
    feed = CbFeed(feedinfo, reports)

    #print "-> Writing out completed feed document..."
    return feed.dump()

    os.chdir(cwd_old)
def perform(configpath, export_mode):
    if not os.path.exists(configpath):
        raise FatalError("Config File %s does not exist!" % configpath)

    config = ConfigParser.RawConfigParser(defaults=
        {
            'iSightRemoteImportUrl': 'https://api.isightpartners.com',
            'iSightRemoteImportDaysBack': 80,
            'iSightDefaultScore': 50,
            'iSightFeedName': 'isightconnector',
            'iSightGetReports': 'false'
        }
    )
    config.read(configpath)

    if not config.has_section('cb-isight'):
        raise FatalError("Config File must have cb-isight section")

    for option in ['iSightRemoteImportPublicKey', 'iSightRemoteImportPrivateKey', 'carbonblack_server_token']:
        if not config.has_option('cb-isight', option):
            raise FatalError("Config file not complete: missing option {0:s}".format(option))

    api_key = config.get("cb-isight", "iSightRemoteImportPublicKey")
    sec_key = config.get("cb-isight", "iSightRemoteImportPrivateKey")
    days_back = config.getint("cb-isight", "iSightRemoteImportDaysBack")
    api_route = config.get("cb-isight", "iSightRemoteImportUrl")
    default_score = config.get("cb-isight", "iSightDefaultScore")
    feed_name = config.get("cb-isight", "iSightFeedName")
    read_reports = config.getboolean("cb-isight", "iSightGetReports")

    if config.has_option('cb-isight', 'https_proxy'):
        os.environ['HTTPS_PROXY'] = config.get('cb-isight', 'https_proxy')
        os.environ['no_proxy'] = '127.0.0.1,localhost'

    isight_bridge = Bridge(api_route, api_key, sec_key)
    if read_reports:
        reports = isight_bridge.perform(days_back)
    else:
        reports = isight_bridge.perform_iocs(days_back)

    cb_reports = []
    for report in reports:
        cb_id = 'isight-%s' % report['id']
        feed_entry = dict((k, report[k]) for k in ('title', 'link', 'iocs', 'timestamp'))
        feed_entry['id'] = cb_id
        feed_entry['score'] = default_score
        cb_reports.append(feed_entry)

    feed = CbFeed(generate_feed_metadata(feed_name), cb_reports)
    raw_feed_data = feed.dump()

    if export_mode:
        print raw_feed_data
    else:
        with NamedTemporaryFile(dir=CB_ISIGHT_ROOT, delete=False) as fp:
            fp.write(raw_feed_data)
        destination_filename = os.path.join(CB_ISIGHT_ROOT, 'isight_feed.json')
        _logger.info("Creating iSIGHT feed at {0:s}".format(destination_filename))
        os.rename(fp.name, destination_filename)
        os.chmod(destination_filename, 0755)

        if config.has_option('cb-isight', 'carbonblack_server_url'):
            local_cb_server = config.get('cb-isight', 'carbonblack_server_url')
        else:
            local_cb_server = 'https://127.0.0.1'

        c = connect_local_cbapi(local_cb_server, config.get('cb-isight', 'carbonblack_server_token'))
        feed_id = c.feed_get_id_by_name(feed_name)
        if not feed_id:
            _logger.info("Creating iSIGHT feed for the first time")
            c.feed_add_from_url("file://" + CB_ISIGHT_ROOT + '/isight_feed.json', True, False, False)

        # force a synchronization
        c.feed_synchronize(feed_name)
def create(config_file, existing_csv=None, reports_to_skip=[]):
    # parse the configuration file
    # this configuration file includes the keys needed to talk to the
    # iSight report server, etc.
    #
    #print "-> Parsing iSight configuration..."
    cfg = isight_config.ISightConfig(config_file)

    # instantiate a local iSight API object
    #
    #print "-> Instantiating an iSight API object..."
    api = isight_api.ISightAPI(cfg.iSightRemoteImportUrl,
                               cfg.iSightRemoteImportUsername,
                               cfg.iSightRemoteImportPassword,
                               cfg.iSightRemoteImportPublicKey,
                               cfg.iSightRemoteImportPrivateKey)

    if not existing_csv:
        # query the iSight report server for raw CSV report data
        # query 'back' the specified number of days
        #
        #print "-> Querying iSight server for last %d days of reports..." % (cfg.iSightRemoteImportDaysBack)
        #
        # @todo iSIGHT has a new-and-improved REST API which could be used instead of this legacy API
        #
        raw_report_data = api.get_i_and_w(cfg.iSightRemoteImportDaysBack)

        # save off the raw report data for future reference
        #
        #print "-> Saving iSight report data to iSight.csv..."
        f = open('iSight.csv', 'w')
        f.write(raw_report_data)
        f.close()
    else:
        raw_report_data = open(existing_csv, "r").read()

    # convert the raw report data into something more managable
    # in particular, a list of dictionaries, with each dictionary describing a report
    # this helper routine accounts for the fact that report data is spread across
    # multiple lines of the raw CSV blob
    #
    results = isight_helpers.isight_csv_to_iocs_dict([raw_report_data])

    # set up a dictionary for basic stat tracking
    #
    stats = {
        'md5': {
            'total': 0,
            'max': 0
        },
        'ipaddr': {
            'total': 0,
            'max': 0
        },
        'domain': {
            'total': 0,
            'max': 0
        }
    }

    for report_id in results.keys():
        stats['md5']['total'] += len(results[report_id]['md5'])
        if len(results[report_id]['md5']) > stats['md5']['max']:
            stats['md5']['max'] = len(results[report_id]['md5'])
        stats['ipaddr']['total'] += len(results[report_id]['ipaddr'])
        if len(results[report_id]['ipaddr']) > stats['ipaddr']['max']:
            stats['ipaddr']['max'] = len(results[report_id]['ipaddr'])
        stats['domain']['total'] += len(results[report_id]['domain'])
        if len(results[report_id]['domain']) > stats['domain']['max']:
            stats['domain']['max'] = len(results[report_id]['domain'])

    #print "  -> Total Reports:                                  %d" % (len(results.keys()))
    #print "  -> ----------------------------------------------- ---"
    #print "  -> Maximum number of MD5s in one report:           %d" % (stats['md5']['max'])
    #print "  -> Total MD5s across all reports:                  %d" % (stats['md5']['total'])
    #print "  -> Maximum number of IPv4 addresses in one report: %d" % (stats['ipaddr']['max'])
    #print "  -> Total IPv4 addresses in all reports:            %d" % (stats['ipaddr']['total'])
    #print "  -> Maximum number of DNS names in one report:      %d" % (stats['domain']['max'])
    #print "  -> Total DNS names in all reports:                 %d" % (stats['domain']['total'])

    # generate the feed data from the raw iSight report data
    #
    #print "-> Generating feed data..."
    reports = generate_reports(results, api)

    # shim to skip entire reports
    reports = [
        report for report in reports
        if report.data['id'] not in reports_to_skip
    ]

    # generate the feed metadata (feed information)
    # this is a static description of the feed itself
    #

    # lazy way out
    cwd_old = os.getcwd()
    os.chdir(os.path.dirname(os.path.realpath(__file__)))

    #print "-> Generating feed metadata..."
    feedinfo = generate_feed_information()

    # write out feed document
    #
    feed = CbFeed(feedinfo, reports)

    #print "-> Writing out completed feed document..."
    return feed.dump()

    os.chdir(cwd_old)