def test_csirtg_darknet():
feed = 'csirtgadgets/darknet'
r, f = next(load_rules(rule, feed))
r.feeds[feed]['remote'] = 'test/csirtg/feed.txt'
cli = Client(r, f)
s = FM(client='stdout')
parser_name = get_type(cli.cache)
x = list(s.process(r, f, parser_name, cli))
x = list(x)
assert len(x) > 0
ips = set()
tags = set()
for xx in x:
ips.add(xx.indicator)
tags.add(xx.tags[0])
assert '109.111.134.64' in ips
assert 'scanner' in tags
def test_malwaredomains_urlshorteners():
indicators = set()
tags = set()
from csirtg_fm.clients.http import Client
cli = Client(rule, 'registrars')
cli._cache_decode()
cli.cache = 'test/malwaredomains/bulk_registrars.txt'
parser_name = get_type(cli.cache)
assert parser_name == 'pattern'
for i in s.process(rule, 'registrars', parser_name, cli, limit=250):
if not i:
continue
assert parse_timestamp(i.reported_at).year > 1980
# assert parse_timestamp(i.last_at).year > 1980
# assert parse_timestamp(i.first_at).year > 1980
indicators.add(i.indicator)
tags.add(i.tags[0])
assert 'registrar' in tags
assert 'us.pn' in indicators
def test_malwaredomains_botnet():
indicators = set()
tags = set()
from csirtg_fm.clients.http import Client
cli = Client(rule, 'botnet')
cli._cache_decode()
cli.cache = 'test/malwaredomains/domains.txt'
parser_name = get_type(cli.cache)
assert parser_name == 'tsv'
for i in s.process(rule, 'botnet', parser_name, cli, limit=250):
if not i:
continue
assert parse_timestamp(i.reported_at).year > 1980
# assert parse_timestamp(i.last_at).year > 1980
# assert parse_timestamp(i.first_at).year > 1980
indicators.add(i.indicator)
tags.add(i.tags[0])
assert 'botnet' in tags
assert 'attack_page' not in tags
assert '9virgins.com' in indicators
def test_malc0de_malware():
from csirtg_fm.clients.http import Client
cli = Client(rule, 'malware')
parser_name = get_type(cli.cache)
assert parser_name == 'rss'
indicators = set()
for i in s.process(rule, 'malware', parser_name, cli, indicators=[]):
if not i:
continue
indicators.add(i.indicator)
pprint(indicators)
assert '71941a88f8c895e405dd5cf665f1ef0c' in indicators
def test_openphish():
indicators = []
from csirtg_fm.clients.http import Client
cli = Client(rule, 'urls')
parser_name = get_type(cli.cache)
assert parser_name == 'csv'
for i in s.process(rule, 'urls', parser_name, cli, limit=25,
indicators=[]):
if not i:
continue
indicators.append(i)
assert len(indicators) > 0
assert len(indicators[0].indicator) > 4
def test_malc0de_urls():
from csirtg_fm.clients.http import Client
cli = Client(rule, 'urls')
parser_name = get_type(cli.cache)
assert parser_name == 'rss'
indicators = set()
for i in s.process(rule, 'urls', parser_name, cli, indicators=[]):
if not i:
continue
indicators.add(i.indicator)
pprint(indicators)
assert len(indicators) > 0
assert len(indicators.pop()) > 4
assert 'http://url.goosai.com/down/ufffdufffd?ufffdufffdufffd?ufffdufffdbreakprisonsearchv2.7u03afu06f0ufffdufffdufffdufffdufffdufffdat25_35027.exe' in indicators
def test_abuse_ch_urlhaus():
indicators = set()
tags = set()
from csirtg_fm.clients.http import Client
cli = Client(rule, 'urlhaus')
parser_name = get_type(cli.cache)
assert parser_name == 'csv'
for i in s.process(rule, 'urlhaus', parser_name, cli, limit=250):
if not i:
continue
indicators.add(i.indicator)
tags.add(i.tags[0])
assert 'http://business.imuta.ng/default/us/summit-companies-invoice-12648214' in indicators
assert 'http://mshcoop.com/download/en/scan' in indicators
assert 'exploit' in tags
def test_phishtank_urls():
indicators = set()
tags = set()
from csirtg_fm.clients.http import Client
cli = Client(rule, 'urls')
cli._cache_decode()
cli.cache = 'test/phishtank/feed.json'
parser_name = get_type(cli.cache)
assert parser_name == 'json'
for i in s.process(rule, 'urls', parser_name, cli):
if not i:
continue
assert parse_timestamp(i.reported_at).year > 1980
assert parse_timestamp(i.last_at).year > 1980
assert parse_timestamp(i.first_at).year > 1980
indicators.add(i.indicator)
tags.add(i.tags[0])
assert 'http://charlesleonardconstruction.com/irs/confim/index.html' in indicators
def _run_fm(args, **kwargs):
data = kwargs.get('data')
verify_ssl = True
if args.no_verify_ssl:
verify_ssl = False
archiver = NOOPArchiver()
if args.remember:
archiver = Archiver(dbfile=args.remember_path)
goback = args.goback
if goback:
goback = arrow.utcnow().replace(days=-int(goback))
logger.info('starting run...')
s = FM(archiver=archiver, client=args.client, goback=goback, ml=args.ml,
skip_invalid=args.skip_invalid)
fetch = True
if args.no_fetch:
fetch = False
data = []
indicators = []
for r, f in load_rules(args.rule, feed=args.feed):
if not f:
print("\n")
print('Feed not found: %s' % args.feed)
print("\n")
raise SystemExit
# detect which client we should be using
if '/' in f:
if 'csirtgadgets' in f:
parser_name = 'csirtg'
cli = None
if not os.getenv('CSIRTG_TOKEN'):
logger.info('')
logger.info('CSIRTG_TOKEN var not set in ENV, skipping %s' % f)
logger.info('Sign up for a Free account: https://csirtg.io')
logger.info('')
continue
limit = int(args.limit)
if limit > 500:
limit = 500
if r.limit and int(r.limit) < limit:
limit = int(r.limit)
try:
for i in s.fetch_csirtg(f, limit=limit):
data.append(i)
except Exception as e:
logger.error(e)
continue
elif 'apwg' in f:
parser_name = 'apwg'
cli = None
limit = int(args.limit)
if limit > 500:
limit = 500
if r.limit and int(r.limit) < limit:
limit = int(r.limit)
for i in s.fetch_apwg(f, limit=limit):
data.append(i)
else:
from .clients.http import Client
cli = Client(r, f, verify_ssl=verify_ssl)
# fetch the feeds
cli.fetch(fetch=fetch)
# decode the content and load the parser
try:
logger.debug('testing parser: %s' % cli.cache)
parser_name = get_type(cli.cache)
logger.debug('detected parser: %s' % parser_name)
except Exception as e:
logger.debug(e)
if r.feeds[f].get('pattern'):
parser_name = 'pattern'
if not parser_name:
parser_name = r.feeds[f].get('parser') or r.parser or 'pattern'
# process the indicators by passing the parser and file handle [or data]
logger.info('processing: {} - {}:{}'.format(args.rule, r.provider, f))
try:
for i in s.process(r, f, parser_name, cli, limit=args.limit,
indicators=data):
if not i:
continue
indicators.append(i)
except Exception as e:
logger.error(e)
import traceback
traceback.print_exc()
if args.client == 'stdout':
for l in FORMATS[args.format](data=indicators, cols=args.fields.split(',')):
print(l)
logger.info('cleaning up')
archiver.cleanup()
archiver.clear_memcache()
logger.info('finished run')
if args.service:
logger.info('sleeping...')
