def create(): reports = [] reports.extend(get_zeus()) reports.extend(get_palevo()) reports.extend(get_spyeye()) feedinfo = {'name': 'abusech', 'display_name': "abuse.ch Malware Domains", 'provider_url': "http://www.abuse.ch", 'summary': "abuse.ch tracks C&C servers for Zeus, SpyEye and Palevo malware. " + "This feed combines the three domain names blocklists.", 'tech_data': "There are no requirements to share any data to receive this feed.", "icon": "abuse.ch.jpg" } # the lazy way to the icon old_cwd = os.getcwd() os.chdir(os.path.dirname(os.path.realpath(__file__))) feedinfo = CbFeedInfo(**feedinfo) feed = CbFeed(feedinfo, reports) bytes = feed.dump() os.chdir(old_cwd) return bytes
def build_feed_data(feed_name: str, display_name: str, feed_summary: str, site: str, icon_link: str, reports: List[Dict[str, Any]]) -> str: """ Return a feed definition as a JSON string definition. :param feed_name: the short name of the feed :param display_name: the display name of the feed :param feed_summary: the feed summary :param site: the site name :param icon_link: path to the icon source :param reports: List of gathered reports :return: feed as JSON string """ feedinfo = {'name': feed_name, 'display_name': display_name, 'provider_url': 'http://' + site, 'summary': feed_summary, 'tech_data': "There are no requirements to share any data to receive this feed.", } # handle optionals if icon_link: feedinfo['icon'] = icon_link feedinfo = CbFeedInfo(**feedinfo) reports = remove_duplicate_reports(reports) feed = CbFeed(feedinfo, reports) return feed.dump()
def create(): nodes = get_tor_nodes() reports = build_reports(nodes) feedinfo = {'name': 'tor', 'display_name': "Tor Exit Nodes", 'provider_url': 'https://www.torproject.org/', 'summary': "This feed is a list of Tor Node IP addresses, updated every 30 minutes.", 'tech_data': "There are no requirements to share any data to receive this feed.", 'icon': 'tor.png', 'icon_small': 'tor.small.jpg', 'category': 'Open Source', } # lazy way out to get right icon path. sorry. old_cwd = os.getcwd() os.chdir(os.path.dirname(os.path.realpath(__file__))) feedinfo = CbFeedInfo(**feedinfo) feed = CbFeed(feedinfo, reports) created_feed = feed.dump() os.chdir(old_cwd) return created_feed
def create(localcsv=None): if localcsv: lines = open(localcsv, "r").readlines() else: r = requests.get("http://www.malwaredomainlist.com/mdlcsv.php", stream=True) lines = r.text.split("\r\n") reports = reports_from_csv(lines) feedinfo = {'name': 'mdl', 'display_name': "Malware Domain List", 'provider_url': "http://www.malwaredomainlist.com/mdl.php", 'summary': "Malware Domain List is a non-commercial community project to track domains used by malware." + " This feed contains the most recent 180 days of entries.", 'tech_data': "There are no requirements to share any data to receive this feed.", "icon": "mdl.png" } # lazy way out old_cwd = os.getcwd() os.chdir(os.path.dirname(os.path.realpath(__file__))) feedinfo = CbFeedInfo(**feedinfo) feed = CbFeed(feedinfo, reports) bytes = feed.dump() os.chdir(old_cwd) return bytes
def create_feed(options): # generate the required feed information fields # based on command-line arguments # feedinfo = { 'name': options.name, 'display_name': options.display_name, 'provider_url': options.url, 'summary': options.summary, 'tech_data': options.techdata } # if an icon was provided, encode as base64 and # include in the feed information # if options.icon: b64bytes = base64.b64encode(open(options.icon).read()).decode("utf-8") feedinfo['icon'] = b64bytes # if a small icon was provided, encode as base64 and # include in the feed information # if options.small_icon: b64bytes = base64.b64encode(open( options.small_icon).read()).decode("utf-8") feedinfo['icon_small'] = b64bytes # if a feed category was provided, include it in the feed information # if options.category: feedinfo['category'] = options.category # build a CbFeedInfo instance # this does field validation # feedinfo = CbFeedInfo(**feedinfo) # build a list of reports (always one report in this # case). the single report will include all the IOCs # reports = build_reports(options) # build a CbFeed instance # this does field validation (including on the report data) # feed = CbFeed(feedinfo, reports) return feed.dump()
def create(): nodes = get_tor_nodes() reports = build_reports(nodes) feedinfo = {'name': 'tor', 'display_name': "Tor Exit Nodes", 'provider_url': 'https://www.torproject.org/', 'summary': "This feed is a list of Tor Node IP addresses, updated every 30 minutes.", 'tech_data': "There are no requirements to share any data to receive this feed.", 'icon': 'images/tor.png'} feedinfo = CbFeedInfo(**feedinfo) feed = CbFeed(feedinfo, reports) return feed.dump()
def run(self): try: with file_lock( os.path.join(CbIntegrationBridge.WORKING_FILE_ROOT, '%s.pid' % self.integration_name)): reports = self.perform() if not reports: return True if self.debug: self.print_reports(reports) # Create the feed feed = CbFeed(self.feed_metadata, reports) if not feed.validate(): self.logger.error( "Feed {0:s} did not validate, not updating".format( self.integration_name)) return False raw_feed_data = feed.dump() with NamedTemporaryFile(dir=self.working_file_root, delete=False) as fp: fp.write(raw_feed_data) self.logger.info("Creating {0:s} feed at {1:s}".format( self.integration_name, self.feed_file_name)) os.rename(fp.name, self.feed_file_name) c = connect_local_cbapi() feed_id = c.feed_get_id_by_name(self.integration_name) if not feed_id: self.logger.info( "Creating {0:s} feed for the first time".format( self.integration_name)) c.feed_add_from_url("file://" + self.feed_file_name, True, False, False) # force a synchronization c.feed_synchronize(self.integration_name) except AlreadyRunningError: self.logger.error("{0:s} is already running".format( self.integration_name))
def create(input_source): reports = build_reports(input_source) # **************************** # TODO - you probably want to change these values to reflect your # local input source feedinfo = { 'name': 'stiximport', 'display_name': "STIX Package Import", 'provider_url': 'http://stix.mitre.org', 'summary': "This feed was imported from stix package(s) at %s" % input_source, 'tech_data': "There are no requirements to share any data to receive this feed.", 'icon': 'images/stix.gif' } feedinfo = CbFeedInfo(**feedinfo) feed = CbFeed(feedinfo, reports) return feed.dump()
def build_feed_data(feed_name, display_name, feed_summary, site, icon_link, reports): """ :return:feed as bytes to be written out """ feedinfo = { 'name': feed_name, 'display_name': display_name, 'provider_url': 'http://' + site, 'summary': feed_summary, 'tech_data': "There are no requirements to share any data to receive this feed.", 'icon': icon_link } feedinfo = CbFeedInfo(**feedinfo) reports = remove_duplicate_reports(reports) feed = CbFeed(feedinfo, reports) return feed.dump()
def create(): reports = [] tmp = get_zeus() if tmp: reports.extend(tmp) tmp = get_feodo() if tmp: reports.extend(tmp) tmp = get_ransomware() if tmp: reports.extend(tmp) feedinfo = { 'name': 'abusech', 'display_name': "abuse.ch Malware Domains", 'provider_url': "http://www.abuse.ch", 'summary': "abuse.ch tracks C&C servers for Zeus and Palevo malware. " + "This feed combines the two domain names blocklists.", 'tech_data': "There are no requirements to share any data to receive this feed.", 'icon': "abuse.ch.jpg", 'icon_small': "abuse.ch.small.jpg", 'category': "Open Source" } # the lazy way to the icon old_cwd = os.getcwd() os.chdir(os.path.dirname(os.path.realpath(__file__))) feedinfo = CbFeedInfo(**feedinfo) feed = CbFeed(feedinfo, reports) feed_bytes = feed.dump() os.chdir(old_cwd) return feed_bytes
def create(config_file, existing_csv=None, reports_to_skip=[]): # parse the configuration file # this configuration file includes the keys needed to talk to the # iSight report server, etc. # #print "-> Parsing iSight configuration..." cfg = isight_config.ISightConfig(config_file) # instantiate a local iSight API object # #print "-> Instantiating an iSight API object..." api = isight_api.ISightAPI(cfg.iSightRemoteImportUrl, cfg.iSightRemoteImportUsername, cfg.iSightRemoteImportPassword, cfg.iSightRemoteImportPublicKey, cfg.iSightRemoteImportPrivateKey) if not existing_csv: # query the iSight report server for raw CSV report data # query 'back' the specified number of days # #print "-> Querying iSight server for last %d days of reports..." % (cfg.iSightRemoteImportDaysBack) # # @todo iSIGHT has a new-and-improved REST API which could be used instead of this legacy API # raw_report_data = api.get_i_and_w(cfg.iSightRemoteImportDaysBack) # save off the raw report data for future reference # #print "-> Saving iSight report data to iSight.csv..." f = open('iSight.csv', 'w') f.write(raw_report_data) f.close() else: raw_report_data = open(existing_csv, "r").read() # convert the raw report data into something more managable # in particular, a list of dictionaries, with each dictionary describing a report # this helper routine accounts for the fact that report data is spread across # multiple lines of the raw CSV blob # results = isight_helpers.isight_csv_to_iocs_dict([raw_report_data]) # set up a dictionary for basic stat tracking # stats = { 'md5': { 'total': 0, 'max': 0 }, 'ipaddr': { 'total': 0, 'max': 0 }, 'domain': { 'total': 0, 'max': 0 } } for report_id in results.keys(): stats['md5']['total'] += len(results[report_id]['md5']) if len(results[report_id]['md5']) > stats['md5']['max']: stats['md5']['max'] = len(results[report_id]['md5']) stats['ipaddr']['total'] += len(results[report_id]['ipaddr']) if len(results[report_id]['ipaddr']) > stats['ipaddr']['max']: stats['ipaddr']['max'] = len(results[report_id]['ipaddr']) stats['domain']['total'] += len(results[report_id]['domain']) if len(results[report_id]['domain']) > stats['domain']['max']: stats['domain']['max'] = len(results[report_id]['domain']) #print " -> Total Reports: %d" % (len(results.keys())) #print " -> ----------------------------------------------- ---" #print " -> Maximum number of MD5s in one report: %d" % (stats['md5']['max']) #print " -> Total MD5s across all reports: %d" % (stats['md5']['total']) #print " -> Maximum number of IPv4 addresses in one report: %d" % (stats['ipaddr']['max']) #print " -> Total IPv4 addresses in all reports: %d" % (stats['ipaddr']['total']) #print " -> Maximum number of DNS names in one report: %d" % (stats['domain']['max']) #print " -> Total DNS names in all reports: %d" % (stats['domain']['total']) # generate the feed data from the raw iSight report data # #print "-> Generating feed data..." reports = generate_reports(results, api) # shim to skip entire reports reports = [ report for report in reports if report.data['id'] not in reports_to_skip ] # generate the feed metadata (feed information) # this is a static description of the feed itself # # lazy way out cwd_old = os.getcwd() os.chdir(os.path.dirname(os.path.realpath(__file__))) #print "-> Generating feed metadata..." feedinfo = generate_feed_information() # write out feed document # feed = CbFeed(feedinfo, reports) #print "-> Writing out completed feed document..." return feed.dump() os.chdir(cwd_old)