def load_feeds(self, rule, feed=None):
if isinstance(rule, str) and os.path.isdir(rule):
for f in sorted(os.listdir(rule)):
if f.startswith('.'):
continue
if os.path.isdir(f):
continue
self.logger.info("processing {0}/{1}".format(rule, f))
try:
r = Rule(path=os.path.join(rule, f))
except RuleUnsupported as e:
logger.error(e)
continue
for feed in r.feeds:
yield r, feed
else:
self.logger.info("processing {0}".format(rule))
if isinstance(rule, str):
try:
rule = Rule(path=rule)
except RuleUnsupported as e:
logger.error(e)
return
if feed:
# replace the feeds dict with the single feed
# raises KeyError if it doesn't exist
rule.feeds = {feed: rule.feeds[feed]}
for f in rule.feeds:
yield rule, f
def load_feeds(self, rule, feed=None):
if isinstance(rule, str) and os.path.isdir(rule):
for f in sorted(os.listdir(rule)):
if f.startswith('.'):
continue
if os.path.isdir(f):
continue
self.logger.info("processing {0}/{1}".format(rule, f))
try:
r = Rule(path=os.path.join(rule, f))
except RuleUnsupported as e:
logger.error(e)
continue
for feed in r.feeds:
yield r, feed
else:
self.logger.info("processing {0}".format(rule))
if isinstance(rule, str):
try:
rule = Rule(path=rule)
except RuleUnsupported as e:
logger.error(e)
return
if feed:
# replace the feeds dict with the single feed
# raises KeyError if it doesn't exist
rule.feeds = {feed: rule.feeds[feed]}
for f in rule.feeds:
yield rule, f
def load_parser(self, rule, feed, limit=None, data=None, filters=None):
if isinstance(rule, str):
rule = Rule(rule)
fetch = Fetcher(rule, feed, data=data, no_fetch=self.no_fetch, verify_ssl=self.verify_ssl, limit=limit)
self.last_cache = fetch.cache
parser_name = rule.feeds[feed].get('parser') or rule.parser or PARSER_DEFAULT
if not parser_name:
from csirtg_smrt.utils.zcontent import get_type
try:
parser_name = get_type(self.last_cache)
except Exception as e:
logger.error(e)
if not parser_name:
parser_name = PARSER_DEFAULT
plugin_path = os.path.join(os.path.dirname(__file__), 'parser')
if getattr(sys, 'frozen', False):
plugin_path = os.path.join(sys._MEIPASS, plugin_path)
parser = load_plugin(plugin_path, parser_name)
if parser is None:
self.logger.info('trying z{}'.format(parser_name))
parser = load_plugin(csirtg_smrt.parser.__path__[0], 'z{}'.format(parser_name))
if parser is None:
raise SystemError('Unable to load parser: {}'.format(parser_name))
self.logger.debug("loading parser: {}".format(parser))
return parser(self.client, fetch, rule, feed, limit=limit, filters=filters, fireball=self.fireball)
import py.test
from csirtg_smrt import Smrt
from csirtg_smrt.rule import Rule
from csirtg_smrt.constants import REMOTE_ADDR
from csirtg_smrt.constants import PYVERSION
rule = 'test/zemail/zemail.yml'
rule = Rule(path=rule)
rule.fetcher = 'stdin'
s = Smrt(REMOTE_ADDR, 1234, client='dummy')
def test_zemail():
feed = 'abuse'
with open('test/zemail/single_plain_01.eml') as f:
data = f.read()
x = list(s.process(rule, feed=feed, data=data))
assert len(x) > 0
assert x[0].indicator == 'http://www.socialservices.cn/detail.php?id=9'
import py.test
from csirtg_smrt import Smrt
from csirtg_smrt.rule import Rule
from csirtg_smrt.constants import REMOTE_ADDR
from pprint import pprint
rule = 'test/vxvault/vxvault.yml'
rule = Rule(path=rule)
rule.fetcher = 'file'
s = Smrt(REMOTE_ADDR, 1234, client='dummy')
def test_vxvault_urls():
rule.feeds['urls']['remote'] = 'test/vxvault/feed.txt'
x = s.process(rule, feed="urls")
x = list(x)
assert len(x) > 0
urls = set()
tags = set()
for xx in x:
urls.add(xx.indicator)
tags.add(xx.tags[0])
assert 'http://jeansowghtqq.com/85.exe' in urls
assert 'malware' in tags
import py.test
from csirtg_smrt import Smrt
from csirtg_smrt.rule import Rule
from csirtg_smrt.constants import REMOTE_ADDR
from pprint import pprint
rule = 'test/spamhaus/spamhaus.yml'
rule = Rule(path=rule)
rule.fetcher = 'file'
s = Smrt(REMOTE_ADDR, 1234, client='dummy')
def test_spamhaus_drop():
rule.feeds['drop']['remote'] = 'test/spamhaus/drop.txt'
x = s.process(rule, feed="drop")
x = list(x)
assert len(list(x)) > 0
def test_spamhaus_edrop():
rule.feeds['edrop']['remote'] = 'test/spamhaus/edrop.txt'
x = s.process(rule, feed="edrop")
x = list(x)
assert len(x) > 0
import os
DISABLE_TESTS = True
if os.environ.get('CSIRTG_SMRT_STIX_TEST'):
if os.environ['CSIRTG_SMRT_STIX_TEST'] == '1':
DISABLE_TESTS = False
if not DISABLE_TESTS:
try:
from stix.core import STIXPackage
except ImportError:
raise ImportError(
'STIX not installed by default, install using `pip install stix`')
rule = 'test/stix/test.yml'
rule = Rule(path=rule)
s = Smrt(REMOTE_ADDR, 1234, client='dummy')
@pytest.mark.skipif(DISABLE_TESTS,
reason='need to set CSIRTG_SMRT_STIX_TEST=1 to run')
def test_stix():
x = s.process(rule, feed='fqdn')
x = list(x)
assert len(x) > 0
assert len(x[0].indicator) > 4
indicators = set()
for xx in x:
indicators.add(xx.indicator)
