def test_csirtg_darknet():
feed = 'csirtgadgets/darknet'
r, f, _ = next(load_rules(rule, feed))
r['feeds'][feed]['remote'] = 'test/csirtg/feed.txt'
cli = Client(r, f)
s = FM(client='stdout')
parser_name = get_type(cli.cache)
x = list(s.process(r, f, parser_name, cli))
x = list(x)
assert len(x) > 0
ips = set()
tags = set()
for xx in x:
ips.add(xx.indicator)
tags.add(xx.tags[0])
assert '109.111.134.64' in ips
assert 'scanner' in tags
def __init__(self, rule, feed, **kwargs):
self.feed = feed
self.rule = rule
self.cache = kwargs.get('cache', FM_CACHE)
self.timeout = FETCHER_TIMEOUT
self.verify_ssl = kwargs.get('verify_ssl', True)
self.handle = requests.session()
self.handle.headers['User-Agent'] = \
f"csirtg-fm/{VERSION} (csirtgadgets.com)"
self.handle.headers['Accept'] = 'application/json'
if isinstance(self.rule, str):
from csirtg_fm.utils.rules import load_rules
self.rule, f, p = next(load_rules(self.rule, feed))
if not self.rule.get('defaults'):
self.rule['defaults'] = {}
self.provider = self.rule.get('provider')
self._init_remote(feed)
self._init_provider()
self._init_paths(feed)
if self.username:
self.auth = (self.username, self.password)
def process(self, rule, feed, parser_name, cli, limit=None, indicators=[]):
if isinstance(rule, str):
rule, _, _ = next(load_rules(rule, feed))
if rule['feeds'][feed].get('limit') and limit == 25:
limit = rule['feeds'][feed].get('limit')
if parser_name not in ['csirtg', 'apwg']:
# detect and load the parser
plugin_path = os.path.join(os.path.dirname(__file__), 'parsers')
parser = load_plugin(plugin_path, parser_name)
parser = parser.Plugin(rule=rule,
feed=feed,
cache=cli.cache,
limit=limit)
# bring up the pipeline
indicators = parser.process(skip_invalid=self.skip_invalid)
indicators = (i for i in indicators if self.is_valid(i))
indicators = (Indicator(**i) for i in indicators)
indicators = (format_keys(i) for i in indicators)
# check to see if the indicator is too old
if self.goback:
indicators = (i for i in indicators if not self.is_old(i))
if limit:
indicators = itertools.islice(indicators, int(limit))
indicators = (i for i in indicators if not self.is_archived(i))
indicators_batches = chunk(indicators, int(FIREBALL_SIZE))
for batch in indicators_batches:
# send batch
if self.client and self.client != 'stdout':
logger.info('sending: %i' % len(batch))
self.client.indicators_create(batch)
# archive
self.archiver.begin()
for i in batch:
yield i
self.archiver.create(i)
# commit
self.archiver.commit()
def test_abuse_ch_urlhaus():
indicators = set()
tags = set()
from csirtg_fm.clients.http import Client
r = load_rules(rule, 'urlhaus')
cli = Client(rule, 'urlhaus')
parser_name = get_type(cli.cache)
assert parser_name == 'csv'
for i in s.process(rule, 'urlhaus', parser_name, cli, limit=250):
if not i:
continue
indicators.add(i.indicator)
tags.add(i.tags[0])
assert 'http://business.imuta.ng/default/us/summit-companies-invoice-12648214' in indicators
assert 'http://mshcoop.com/download/en/scan' in indicators
assert 'exploit' in tags
def _run_fm(args, **kwargs):
data = kwargs.get('data')
verify_ssl = True
if args.no_verify_ssl:
verify_ssl = False
archiver = NOOPArchiver()
if args.remember:
archiver = Archiver(dbfile=args.remember_path)
goback = args.goback
if goback:
goback = arrow.utcnow().shift(days=-int(goback))
logger.info('starting run...')
s = FM(archiver=archiver, client=args.client, goback=goback,
skip_invalid=args.skip_invalid)
fetch = True
if args.no_fetch:
fetch = False
data = []
indicators = []
for r, f, ff in load_rules(args.rule, feed=args.feed):
if not f:
print("\n")
print('Feed not found: %s' % args.feed)
print("\n")
raise SystemExit
# detect which client we should be using
if '/' in f:
data = []
if 'csirtgadgets' in f:
parser_name = 'csirtg'
cli = None
if not os.getenv('CSIRTG_TOKEN'):
logger.info('')
logger.info('CSIRTG_TOKEN var not set in ENV, skipping %s' % f)
logger.info('Sign up for a Free account: https://csirtg.io')
logger.info('')
continue
limit = int(args.limit)
if limit > 500:
limit = 500
if r.get('limit') and int(r['limit']) < limit:
limit = int(r['limit'])
try:
for i in fetch_csirtg(f, limit=limit):
data.append(i)
except Exception as e:
logger.error(e)
continue
elif 'apwg' in f:
parser_name = 'apwg'
cli = None
limit = int(args.limit)
if limit > 500:
limit = 500
if r.get('limit') and int(r['limit']) < limit:
limit = int(r['limit'])
try:
for i in fetch_apwg(f, limit=limit):
data.append(i)
except Exception as e:
logger.error(e, exc_info=True)
continue
else:
from .clients.http import Client
cli = Client(r, f, verify_ssl=verify_ssl)
logger.info(f"processing: {ff} - {f}")
# fetch the feeds
try:
cli.fetch(fetch=fetch)
except Exception as e:
logger.error(e, exc_info=True)
continue
# decode the content and load the parser
try:
logger.debug('testing parser: %s' % cli.cache)
parser_name = get_type(cli.cache)
logger.debug('detected parser: %s' % parser_name)
except Exception as e:
logger.debug(e)
if r['feeds'][f].get('pattern'):
logger.debug("overriding parser with pattern..")
parser_name = 'pattern'
if not parser_name:
parser_name = r['feeds'][f].get('parser') or r.get('parser') \
or 'pattern'
try:
for i in s.process(r, f, parser_name, cli, limit=args.limit,
indicators=data):
if not i:
continue
indicators.append(i)
except Exception as e:
logger.error(e)
import traceback
traceback.print_exc()
if args.client == 'stdout':
for l in FORMATS[args.format](data=indicators,
cols=args.fields.split(',')):
print(l)
logger.info('cleaning up')
count = archiver.cleanup()
logger.info('purged %i records' % count)
archiver.clear_memcache()
logger.info('finished run')
if args.service:
logger.info('sleeping...')