def test_syslog():
with Smrt(remote='localhost:514', client='syslog') as s:
assert type(s) is Smrt
rule, feed = next(s.load_feeds('test/smrt/rules/csirtg.yml', feed='port-scanners'))
x = list(s.process(rule, feed))
assert len(x) > 0
def test_smrt_base():
with Smrt(REMOTE_ADDR, 1234, client='dummy') as s:
assert type(s) is Smrt
for r, f in s.load_feeds('test/smrt/rules'):
x = list(s.process(r, f))
assert len(x) > 0
x = []
for r, f in s.load_feeds('test/smrt/rules/csirtg.yml'):
x = list(s.process(r, f))
assert len(x) > 0
x = []
for r, f in s.load_feeds('test/smrt/rules/csirtg.yml',
feed='port-scanners'):
x = list(s.process(r, f))
assert len(x) > 0
x = []
try:
r, f = next(
s.load_feeds('test/smrt/rules/csirtg.yml',
feed='port-scanners2'))
except KeyError:
pass
assert len(x) == 0
def test_smrt_defang():
with Smrt(None, None, client='dummy') as s:
assert type(s) is Smrt
x = []
for r, f in s.load_feeds('test/smrt/rules/csirtg_defang.yml'):
x = list(s.process(r, f))
assert len(x) > 0
def test_syslog():
with Smrt(remote='localhost:514', client='syslog') as s:
assert type(s) is Smrt
pprint(s)
x = s.process('test/smrt/rules/csirtg.yml', feed='port-scanners')
assert len(x) > 0
def test_zyre():
with Smrt(remote=None, client='zyre') as s:
assert type(s) is Smrt
x = s.process('test/smrt/rules/csirtg.yml',
feed='port-scanners',
limit=2)
assert len(x) > 0
def test_smrt_line_filter():
with Smrt(None, None, client='dummy') as s:
r, f = next(
s.load_feeds('test/smrt/rules/csirtg.yml', feed='port-scanners'))
r.line_filter = '109.111.134.64'
n = list(s.process(r, f))
assert len(n) == 1
assert n[0].indicator == '109.111.134.64'
def test_smrt_csv_quoted():
with Smrt(None, None, client='dummy') as s:
assert type(s) is Smrt
x = []
for r, f in s.load_feeds('test/smrt/rules/csv_quoted.yml', feed='test'):
x = list(s.process(r, f))
assert len(x) > 0
assert x[0].description == '1.2.3, aaabbbcccddd'
assert x[1].description == '1,2,3'
def test_smrt_archiver_both():
tmpfile = tempfile.mktemp()
archiver = Archiver(dbfile=tmpfile)
rule = 'test/smrt/rules/archiver.yml'
feed = 'both'
with Smrt(REMOTE_ADDR, 1234, client='stdout', archiver=archiver) as s:
assert type(s) is Smrt
for r, f in s.load_feeds(rule, feed=feed):
x = list(s.process(r, f))
assert len(x) > 0
f = {i.indicator: i.__dict__() for i in x}
assert f['216.243.31.2'][
'lasttime'] == '2016-03-23T20:22:27.000000Z'
with Smrt(REMOTE_ADDR, 1234, client='stdout', archiver=archiver) as s:
assert type(s) is Smrt
for r, f in s.load_feeds(rule, feed=feed):
x = list(s.process(r, f))
assert len(x) == 0
def test_smrt_archiver_neither():
tmpfile = tempfile.mktemp()
archiver = Archiver(dbfile=tmpfile)
rule = 'test/smrt/rules/archiver.yml'
feed = 'neither'
with Smrt(REMOTE_ADDR, 1234, client='stdout', archiver=archiver) as s:
assert type(s) is Smrt
for r, f in s.load_feeds(rule, feed=feed):
x = list(s.process(r, f))
assert len(x) > 0
f = {i.indicator: i.__dict__() for i in x}
assert f['216.243.31.2'].get('lasttime') is None
with Smrt(REMOTE_ADDR, 1234, client='stdout', archiver=archiver) as s:
assert type(s) is Smrt
for r, f in s.load_feeds(rule, feed=feed):
x = list(s.process(r, f))
assert len(x) == 0
def test_smrt_elastcisearch():
with Smrt(remote=REMOTE, client='elasticsearch') as s:
assert type(s) is Smrt
x = s.process('test/smrt/rules/csirtg.yml', feed='port-scanners')
assert len(x) > 0
x = s.process('test/smrt/rules/csirtg.yml', feed='port-scanners')
assert len(x) > 0
# cleanup
es = connections.get_connection()
cli = elasticsearch.client.IndicesClient(es)
cli.delete(index='indicators-*')
def test_smrt():
with Smrt(REMOTE_ADDR, 1234, client='dummy') as s:
assert type(s) is Smrt
x = s.process('test/smrt/rules')
assert len(x) > 0
x = s.process('test/smrt/rules/csirtg.yml')
assert len(x) > 0
x = s.process('test/smrt/rules/csirtg.yml', feed='port-scanners')
assert len(x) > 0
x = s.process('test/smrt/rules/csirtg.yml', feed='port-scanners2')
assert len(x) == 0
def test_smrt_rule_paths():
with Smrt(REMOTE_ADDR, 1234, client='dummy') as s:
r, f = next(
s.load_feeds('test/smrt/rules/csirtg.yml', feed='port-scanners'))
assert f is not None
assert r is not None
r, f = (None, None)
try:
r, f = next(
s.load_feeds('test/smrt/rules/csirtg.yml~',
feed='port-scanners'))
except StopIteration:
pass
assert f is None
def test_client_csirtg():
with Smrt(client='csirtg',
username=USERNAME,
feed='csirtg_smrt_test',
token=TOKEN) as s:
assert type(s) is Smrt
# create test feed
cli = CSIRTGClient(token=TOKEN)
f = Feed(cli).new(USERNAME, FEED)
assert f
assert f['user'] == USERNAME
x = s.process('test/smrt/rules/csirtg.yml', feed='port-scanners')
assert len(x) > 0
# remove test feed
f = Feed(cli).remove(USERNAME, FEED)
assert f == 200
def test_smrt_remote_regex():
with Smrt(None, None, client='dummy') as s:
assert type(s) is Smrt
x = []
for r, f in s.load_feeds('test/smrt/remote_regex.yml',
feed='port-scanners'):
x = list(s.process(r, f))
assert len(x) > 0
x = []
for r, f in s.load_feeds('test/smrt/remote_regex.yml',
feed='port-scanners-fail'):
try:
x = list(s.process(r, f))
except RuntimeError as e:
pass
assert len(x) == 0
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CSIRTG_RUNTIME_PATH
example usage:
$ csirtg-cef -f /var/log/foo.log
$ ZYRE_GROUP=honeynet csirtg-cef -d -f /var/log/foo.log --client zyre
$ csirtg-cef -f /var/log/foo.log --client csirtg --user wes --feed scanners -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg-cef',
parents=[p],
)
p.add_argument('--no-verify-ssl',
help='turn TLS/SSL verification OFF',
action='store_true')
p.add_argument('-f', '--file')
p.add_argument('--client', default='stdout')
p.add_argument('--user')
p.add_argument('--feed')
p.add_argument('--format', default='csv')
p.add_argument('--tags',
help='specify indicator tags [default %(default)s',
default='scanner')
p.add_argument('--provider',
help='specify provider [default %(default)s]',
default=PROVIDER)
p.add_argument('--tail-docker')
args = p.parse_args()
if not args.provider:
raise RuntimeError('Missing --provider flag')
if not args.file:
raise RuntimeError('Missing --file flag')
# setup logging
setup_logging(args)
logger.debug('starting on: {}'.format(args.file))
verify_ssl = True
if args.no_verify_ssl:
verify_ssl = False
f = open(args.file)
from csirtg_smrt import Smrt
s = Smrt(client=args.client,
username=args.user,
feed=args.feed,
verify_ssl=verify_ssl)
try:
for line in tailer.follow(f):
i = parse_line(line)
if not i:
logger.debug('skipping line')
continue
i = Indicator(**i)
logger.debug(i)
i.provider = args.provider
i.tags = args.tags
if args.client == 'stdout':
print(FORMATS[args.format](data=[i]))
else:
s.client.indicators_create(i)
logger.info('indicator created: {}'.format(i.indicator))
except KeyboardInterrupt:
logger.info('SIGINT caught... stopping')
if args.client != 'stdout':
s.client.stop()
logger.info('exiting...')
def test_smrt_splunk():
with Smrt(remote=REMOTE, client='splunk') as s:
assert type(s) is Smrt
x = s.process('test/smrt/rules/csirtg.yml', feed='port-scanners')
assert len(x) > 0
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CSIRTG_RUNTIME_PATH
example usage:
$ csirtg-ufw -f /var/log/ufw.log
$ ZYRE_GROUP=honeynet csirtg-ufw -d -f /var/log/ufw.log --client zyre
$ csirtg-ufw -f /var/log/ufw.log --client csirtg --user wes --feed scanners -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg-ufw',
parents=[p],
)
p.add_argument('--no-verify-ssl', help='turn TLS/SSL verification OFF', action='store_true')
p.add_argument('-f', '--file', default=FILENAME)
p.add_argument('--client', default='stdout')
p.add_argument('--user')
p.add_argument('--feed')
p.add_argument('--format', default='csv')
p.add_argument('--provider', help='specify provider [default %(default)s]', default=PROVIDER)
p.add_argument('--ignore-client-errors', help='skip when client errors out (eg: HTTP 5XX, etc)', action='store_true')
p.add_argument('--aggregate', help='specify how many seconds to aggregate batches before sending to client '
'[default %(default)s]', default=60)
args = p.parse_args()
if not args.provider:
raise RuntimeError('Missing --provider flag')
# setup logging
setup_logging(args)
logger.debug('starting on: {}'.format(args.file))
verify_ssl = True
if args.no_verify_ssl:
verify_ssl = False
from csirtg_smrt import Smrt
s = Smrt(client=args.client, username=args.user, feed=args.feed, verify_ssl=verify_ssl)
bucket = set()
last_t = round_time(round=int(args.aggregate))
try:
for line in tail(args.file):
if 'csirtg-ufw' in line:
continue
if '[UFW BLOCK]' not in line:
continue
if ' SYN ' not in line:
continue
logger.debug(line)
try:
i = parse_line(line)
except AttributeError:
logger.debug("line not matched: \n{}".format(line))
continue
i = Indicator(**i)
i.provider = args.provider
u_indicator = ':'.join([i.indicator,'/'.join([i.portlist,i.protocol])])
if args.aggregate:
t = round_time(dt=datetime.now(), round=int(args.aggregate))
if t != last_t:
bucket = set()
last_t = t
if u_indicator in bucket:
logger.info('skipping send {}'.format(u_indicator))
continue
bucket.add(u_indicator)
if args.client == 'stdout':
print(FORMATS[args.format](data=[i]))
continue
try:
s.client.indicators_create(i)
logger.info('indicator created: {}'.format(u_indicator))
except Exception as e:
logger.error(e)
if args.ignore_client_errors:
pass
except KeyboardInterrupt:
logger.info('SIGINT caught... stopping')
if args.client != 'stdout':
try:
s.client.stop()
except AttributeError:
pass
logger.info('exiting...')
import py.test
from csirtg_smrt import Smrt
from csirtg_smrt.rule import Rule
from csirtg_smrt.constants import REMOTE_ADDR
from csirtg_smrt.constants import PYVERSION
rule = 'test/zemail/zemail.yml'
rule = Rule(path=rule)
rule.fetcher = 'stdin'
s = Smrt(REMOTE_ADDR, 1234, client='dummy')
def test_zemail():
feed = 'abuse'
with open('test/zemail/single_plain_01.eml') as f:
data = f.read()
x = list(s.process(rule, feed=feed, data=data))
assert len(x) > 0
assert x[0].indicator == 'http://www.socialservices.cn/detail.php?id=9'
def main():
p = get_argument_parser()
p = ArgumentParser(
description=textwrap.dedent('''\
Env Variables:
CSIRTG_RUNTIME_PATH
example usage:
$ csirtg-cef -f /var/log/foo.log
$ ZYRE_GROUP=honeynet csirtg-cef -d -f /var/log/foo.log --client zyre
$ csirtg-cef -f /var/log/foo.log --client csirtg --user wes --feed scanners -d
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg-cef',
parents=[p],
)
p.add_argument('--no-verify-ssl', help='turn TLS/SSL verification OFF', action='store_true')
p.add_argument('-f', '--file')
p.add_argument('--client', default='stdout')
p.add_argument('--user')
p.add_argument('--feed')
p.add_argument('--format', default='csv')
p.add_argument('--tags', help='specify indicator tags [default %(default)s', default='scanner')
p.add_argument('--provider', help='specify provider [default %(default)s]', default=PROVIDER)
p.add_argument('--aggregate', help='specify how many seconds to aggregate batches before sending to client '
'[default %(default)s]', default=60)
p.add_argument('--tail-docker')
args = p.parse_args()
# setup logging
setup_logging(args)
verify_ssl = True
if args.no_verify_ssl:
verify_ssl = False
if args.file:
logger.debug('starting on: {}'.format(args.file))
data_source = tail(args.file)
elif args.tail_docker:
logger.debug('starting on container: {}'.format(args.tail_docker))
#data_source = subprocess.Popen(["docker", "logs", "-f", "--tail", "0", args.tail_docker], bufsize=1, stdout=subprocess.PIPE).stdout
client = docker.from_env(version='auto')
container = client.containers.get(args.tail_docker)
data_source = container.logs(stream=True, follow=True, tail=0)
else:
logger.error('Missing --file or --tail-docker flag')
raise SystemExit
logger.info('sending data as: %s' % args.provider)
s = Smrt(client=args.client, username=args.user, feed=args.feed, verify_ssl=verify_ssl)
bucket = set()
last_t = round_time(round=int(args.aggregate))
try:
for line in data_source:
i = parse_line(line)
if not i:
logger.debug('skipping line')
continue
i = Indicator(**i)
logger.debug(i)
i.provider = args.provider
i.tags = args.tags
if args.aggregate:
t = round_time(dt=datetime.now(), round=int(args.aggregate))
if t != last_t:
bucket = set()
last_t = t
if i.indicator in bucket:
logger.info('skipping send {}'.format(i.indicator))
continue
bucket.add(i.indicator)
if args.client == 'stdout':
print(FORMATS[args.format](data=[i]))
else:
try:
s.client.indicators_create(i)
logger.info('indicator created: {}'.format(i.indicator))
except Exception as e:
logger.error(e)
except Exception as e:
logger.error(e)
except KeyboardInterrupt:
logger.info('SIGINT caught... stopping')
if args.client != 'stdout':
s.client.stop()
logger.info('exiting...')