def test_csirtg_darknet():
feed = 'csirtgadgets/darknet'
r, f = next(load_rules(rule, feed))
r.feeds[feed]['remote'] = 'test/csirtg/feed.txt'
cli = Client(r, f)
s = FM(client='stdout')
parser_name = get_type(cli.cache)
x = list(s.process(r, f, parser_name, cli))
x = list(x)
assert len(x) > 0
ips = set()
tags = set()
for xx in x:
ips.add(xx.indicator)
tags.add(xx.tags[0])
assert '109.111.134.64' in ips
assert 'scanner' in tags
def _run_fm(args, **kwargs):
data = kwargs.get('data')
verify_ssl = True
if args.no_verify_ssl:
verify_ssl = False
archiver = NOOPArchiver()
if args.remember:
archiver = Archiver(dbfile=args.remember_path)
goback = args.goback
if goback:
goback = arrow.utcnow().replace(days=-int(goback))
logger.info('starting run...')
s = FM(archiver=archiver, client=args.client, goback=goback, ml=args.ml,
skip_invalid=args.skip_invalid)
fetch = True
if args.no_fetch:
fetch = False
data = []
indicators = []
for r, f in load_rules(args.rule, feed=args.feed):
if not f:
print("\n")
print('Feed not found: %s' % args.feed)
print("\n")
raise SystemExit
# detect which client we should be using
if '/' in f:
if 'csirtgadgets' in f:
parser_name = 'csirtg'
cli = None
if not os.getenv('CSIRTG_TOKEN'):
logger.info('')
logger.info('CSIRTG_TOKEN var not set in ENV, skipping %s' % f)
logger.info('Sign up for a Free account: https://csirtg.io')
logger.info('')
continue
limit = int(args.limit)
if limit > 500:
limit = 500
if r.limit and int(r.limit) < limit:
limit = int(r.limit)
try:
for i in s.fetch_csirtg(f, limit=limit):
data.append(i)
except Exception as e:
logger.error(e)
continue
elif 'apwg' in f:
parser_name = 'apwg'
cli = None
limit = int(args.limit)
if limit > 500:
limit = 500
if r.limit and int(r.limit) < limit:
limit = int(r.limit)
for i in s.fetch_apwg(f, limit=limit):
data.append(i)
else:
from .clients.http import Client
cli = Client(r, f, verify_ssl=verify_ssl)
# fetch the feeds
cli.fetch(fetch=fetch)
# decode the content and load the parser
try:
logger.debug('testing parser: %s' % cli.cache)
parser_name = get_type(cli.cache)
logger.debug('detected parser: %s' % parser_name)
except Exception as e:
logger.debug(e)
if r.feeds[f].get('pattern'):
parser_name = 'pattern'
if not parser_name:
parser_name = r.feeds[f].get('parser') or r.parser or 'pattern'
# process the indicators by passing the parser and file handle [or data]
logger.info('processing: {} - {}:{}'.format(args.rule, r.provider, f))
try:
for i in s.process(r, f, parser_name, cli, limit=args.limit,
indicators=data):
if not i:
continue
indicators.append(i)
except Exception as e:
logger.error(e)
import traceback
traceback.print_exc()
if args.client == 'stdout':
for l in FORMATS[args.format](data=indicators, cols=args.fields.split(',')):
print(l)
logger.info('cleaning up')
archiver.cleanup()
archiver.clear_memcache()
logger.info('finished run')
if args.service:
logger.info('sleeping...')
from csirtg_fm import FM
from csirtg_fm.content import get_type
from csirtg_fm.utils import decode
from csirtg_dt import get as parse_timestamp
rule = 'test/phishtank/phishtank.yml'
s = FM()
def test_phishtank_urls():
indicators = set()
tags = set()
from csirtg_fm.clients.http import Client
cli = Client(rule, 'urls')
cli.cache = decode(cli.cache)
cli.cache = 'test/phishtank/feed.json'
parser_name = get_type(cli.cache)
assert parser_name == 'json'
for i in s.process(rule, 'urls', parser_name, cli):
if not i:
continue
assert parse_timestamp(i.reported_at).year > 1980
assert parse_timestamp(i.last_at).year > 1980
assert parse_timestamp(i.first_at).year > 1980
indicators.add(i.indicator)
tags.add(i.tags[0])