def test_csirtg_darknet(): feed = 'csirtgadgets/darknet' r, f = next(load_rules(rule, feed)) r.feeds[feed]['remote'] = 'test/csirtg/feed.txt' cli = Client(r, f) s = FM(client='stdout') parser_name = get_type(cli.cache) x = list(s.process(r, f, parser_name, cli)) x = list(x) assert len(x) > 0 ips = set() tags = set() for xx in x: ips.add(xx.indicator) tags.add(xx.tags[0]) assert '109.111.134.64' in ips assert 'scanner' in tags
def test_malwaredomains_urlshorteners(): indicators = set() tags = set() from csirtg_fm.clients.http import Client cli = Client(rule, 'registrars') cli._cache_decode() cli.cache = 'test/malwaredomains/bulk_registrars.txt' parser_name = get_type(cli.cache) assert parser_name == 'pattern' for i in s.process(rule, 'registrars', parser_name, cli, limit=250): if not i: continue assert parse_timestamp(i.reported_at).year > 1980 # assert parse_timestamp(i.last_at).year > 1980 # assert parse_timestamp(i.first_at).year > 1980 indicators.add(i.indicator) tags.add(i.tags[0]) assert 'registrar' in tags assert 'us.pn' in indicators
def test_malwaredomains_botnet(): indicators = set() tags = set() from csirtg_fm.clients.http import Client cli = Client(rule, 'botnet') cli._cache_decode() cli.cache = 'test/malwaredomains/domains.txt' parser_name = get_type(cli.cache) assert parser_name == 'tsv' for i in s.process(rule, 'botnet', parser_name, cli, limit=250): if not i: continue assert parse_timestamp(i.reported_at).year > 1980 # assert parse_timestamp(i.last_at).year > 1980 # assert parse_timestamp(i.first_at).year > 1980 indicators.add(i.indicator) tags.add(i.tags[0]) assert 'botnet' in tags assert 'attack_page' not in tags assert '9virgins.com' in indicators
def test_malc0de_malware(): from csirtg_fm.clients.http import Client cli = Client(rule, 'malware') parser_name = get_type(cli.cache) assert parser_name == 'rss' indicators = set() for i in s.process(rule, 'malware', parser_name, cli, indicators=[]): if not i: continue indicators.add(i.indicator) pprint(indicators) assert '71941a88f8c895e405dd5cf665f1ef0c' in indicators
def test_openphish(): indicators = [] from csirtg_fm.clients.http import Client cli = Client(rule, 'urls') parser_name = get_type(cli.cache) assert parser_name == 'csv' for i in s.process(rule, 'urls', parser_name, cli, limit=25, indicators=[]): if not i: continue indicators.append(i) assert len(indicators) > 0 assert len(indicators[0].indicator) > 4
def test_malc0de_urls(): from csirtg_fm.clients.http import Client cli = Client(rule, 'urls') parser_name = get_type(cli.cache) assert parser_name == 'rss' indicators = set() for i in s.process(rule, 'urls', parser_name, cli, indicators=[]): if not i: continue indicators.add(i.indicator) pprint(indicators) assert len(indicators) > 0 assert len(indicators.pop()) > 4 assert 'http://url.goosai.com/down/ufffdufffd?ufffdufffdufffd?ufffdufffdbreakprisonsearchv2.7u03afu06f0ufffdufffdufffdufffdufffdufffdat25_35027.exe' in indicators
def test_abuse_ch_urlhaus(): indicators = set() tags = set() from csirtg_fm.clients.http import Client cli = Client(rule, 'urlhaus') parser_name = get_type(cli.cache) assert parser_name == 'csv' for i in s.process(rule, 'urlhaus', parser_name, cli, limit=250): if not i: continue indicators.add(i.indicator) tags.add(i.tags[0]) assert 'http://business.imuta.ng/default/us/summit-companies-invoice-12648214' in indicators assert 'http://mshcoop.com/download/en/scan' in indicators assert 'exploit' in tags
def test_phishtank_urls(): indicators = set() tags = set() from csirtg_fm.clients.http import Client cli = Client(rule, 'urls') cli._cache_decode() cli.cache = 'test/phishtank/feed.json' parser_name = get_type(cli.cache) assert parser_name == 'json' for i in s.process(rule, 'urls', parser_name, cli): if not i: continue assert parse_timestamp(i.reported_at).year > 1980 assert parse_timestamp(i.last_at).year > 1980 assert parse_timestamp(i.first_at).year > 1980 indicators.add(i.indicator) tags.add(i.tags[0]) assert 'http://charlesleonardconstruction.com/irs/confim/index.html' in indicators
def _run_fm(args, **kwargs): data = kwargs.get('data') verify_ssl = True if args.no_verify_ssl: verify_ssl = False archiver = NOOPArchiver() if args.remember: archiver = Archiver(dbfile=args.remember_path) goback = args.goback if goback: goback = arrow.utcnow().replace(days=-int(goback)) logger.info('starting run...') s = FM(archiver=archiver, client=args.client, goback=goback, ml=args.ml, skip_invalid=args.skip_invalid) fetch = True if args.no_fetch: fetch = False data = [] indicators = [] for r, f in load_rules(args.rule, feed=args.feed): if not f: print("\n") print('Feed not found: %s' % args.feed) print("\n") raise SystemExit # detect which client we should be using if '/' in f: if 'csirtgadgets' in f: parser_name = 'csirtg' cli = None if not os.getenv('CSIRTG_TOKEN'): logger.info('') logger.info('CSIRTG_TOKEN var not set in ENV, skipping %s' % f) logger.info('Sign up for a Free account: https://csirtg.io') logger.info('') continue limit = int(args.limit) if limit > 500: limit = 500 if r.limit and int(r.limit) < limit: limit = int(r.limit) try: for i in s.fetch_csirtg(f, limit=limit): data.append(i) except Exception as e: logger.error(e) continue elif 'apwg' in f: parser_name = 'apwg' cli = None limit = int(args.limit) if limit > 500: limit = 500 if r.limit and int(r.limit) < limit: limit = int(r.limit) for i in s.fetch_apwg(f, limit=limit): data.append(i) else: from .clients.http import Client cli = Client(r, f, verify_ssl=verify_ssl) # fetch the feeds cli.fetch(fetch=fetch) # decode the content and load the parser try: logger.debug('testing parser: %s' % cli.cache) parser_name = get_type(cli.cache) logger.debug('detected parser: %s' % parser_name) except Exception as e: logger.debug(e) if r.feeds[f].get('pattern'): parser_name = 'pattern' if not parser_name: parser_name = r.feeds[f].get('parser') or r.parser or 'pattern' # process the indicators by passing the parser and file handle [or data] logger.info('processing: {} - {}:{}'.format(args.rule, r.provider, f)) try: for i in s.process(r, f, parser_name, cli, limit=args.limit, indicators=data): if not i: continue indicators.append(i) except Exception as e: logger.error(e) import traceback traceback.print_exc() if args.client == 'stdout': for l in FORMATS[args.format](data=indicators, cols=args.fields.split(',')): print(l) logger.info('cleaning up') archiver.cleanup() archiver.clear_memcache() logger.info('finished run') if args.service: logger.info('sleeping...')