def load_feeds(self, rule, feed=None): if isinstance(rule, str) and os.path.isdir(rule): for f in sorted(os.listdir(rule)): if f.startswith('.'): continue if os.path.isdir(f): continue self.logger.info("processing {0}/{1}".format(rule, f)) try: r = Rule(path=os.path.join(rule, f)) except RuleUnsupported as e: logger.error(e) continue for feed in r.feeds: yield r, feed else: self.logger.info("processing {0}".format(rule)) if isinstance(rule, str): try: rule = Rule(path=rule) except RuleUnsupported as e: logger.error(e) return if feed: # replace the feeds dict with the single feed # raises KeyError if it doesn't exist rule.feeds = {feed: rule.feeds[feed]} for f in rule.feeds: yield rule, f
def load_parser(self, rule, feed, limit=None, data=None, filters=None): if isinstance(rule, str): rule = Rule(rule) fetch = Fetcher(rule, feed, data=data, no_fetch=self.no_fetch, verify_ssl=self.verify_ssl, limit=limit) self.last_cache = fetch.cache parser_name = rule.feeds[feed].get('parser') or rule.parser or PARSER_DEFAULT if not parser_name: from csirtg_smrt.utils.zcontent import get_type try: parser_name = get_type(self.last_cache) except Exception as e: logger.error(e) if not parser_name: parser_name = PARSER_DEFAULT plugin_path = os.path.join(os.path.dirname(__file__), 'parser') if getattr(sys, 'frozen', False): plugin_path = os.path.join(sys._MEIPASS, plugin_path) parser = load_plugin(plugin_path, parser_name) if parser is None: self.logger.info('trying z{}'.format(parser_name)) parser = load_plugin(csirtg_smrt.parser.__path__[0], 'z{}'.format(parser_name)) if parser is None: raise SystemError('Unable to load parser: {}'.format(parser_name)) self.logger.debug("loading parser: {}".format(parser)) return parser(self.client, fetch, rule, feed, limit=limit, filters=filters, fireball=self.fireball)
import py.test from csirtg_smrt import Smrt from csirtg_smrt.rule import Rule from csirtg_smrt.constants import REMOTE_ADDR from csirtg_smrt.constants import PYVERSION rule = 'test/zemail/zemail.yml' rule = Rule(path=rule) rule.fetcher = 'stdin' s = Smrt(REMOTE_ADDR, 1234, client='dummy') def test_zemail(): feed = 'abuse' with open('test/zemail/single_plain_01.eml') as f: data = f.read() x = list(s.process(rule, feed=feed, data=data)) assert len(x) > 0 assert x[0].indicator == 'http://www.socialservices.cn/detail.php?id=9'
import py.test from csirtg_smrt import Smrt from csirtg_smrt.rule import Rule from csirtg_smrt.constants import REMOTE_ADDR from pprint import pprint rule = 'test/vxvault/vxvault.yml' rule = Rule(path=rule) rule.fetcher = 'file' s = Smrt(REMOTE_ADDR, 1234, client='dummy') def test_vxvault_urls(): rule.feeds['urls']['remote'] = 'test/vxvault/feed.txt' x = s.process(rule, feed="urls") x = list(x) assert len(x) > 0 urls = set() tags = set() for xx in x: urls.add(xx.indicator) tags.add(xx.tags[0]) assert 'http://jeansowghtqq.com/85.exe' in urls assert 'malware' in tags
import py.test from csirtg_smrt import Smrt from csirtg_smrt.rule import Rule from csirtg_smrt.constants import REMOTE_ADDR from pprint import pprint rule = 'test/spamhaus/spamhaus.yml' rule = Rule(path=rule) rule.fetcher = 'file' s = Smrt(REMOTE_ADDR, 1234, client='dummy') def test_spamhaus_drop(): rule.feeds['drop']['remote'] = 'test/spamhaus/drop.txt' x = s.process(rule, feed="drop") x = list(x) assert len(list(x)) > 0 def test_spamhaus_edrop(): rule.feeds['edrop']['remote'] = 'test/spamhaus/edrop.txt' x = s.process(rule, feed="edrop") x = list(x) assert len(x) > 0
import os DISABLE_TESTS = True if os.environ.get('CSIRTG_SMRT_STIX_TEST'): if os.environ['CSIRTG_SMRT_STIX_TEST'] == '1': DISABLE_TESTS = False if not DISABLE_TESTS: try: from stix.core import STIXPackage except ImportError: raise ImportError( 'STIX not installed by default, install using `pip install stix`') rule = 'test/stix/test.yml' rule = Rule(path=rule) s = Smrt(REMOTE_ADDR, 1234, client='dummy') @pytest.mark.skipif(DISABLE_TESTS, reason='need to set CSIRTG_SMRT_STIX_TEST=1 to run') def test_stix(): x = s.process(rule, feed='fqdn') x = list(x) assert len(x) > 0 assert len(x[0].indicator) > 4 indicators = set() for xx in x: indicators.add(xx.indicator)