def test_obs_url():
o = Indicator('http://example.org')
assert o.is_private() is False
assert o.indicator == 'http://example.org'
assert o.itype is not 'fqdn'
assert o.itype is 'url'
def text_to_list(text, known_only=True):
separator = find_seperator(text)
t_tokens = top_tokens(text)
top = set()
for t in range(0, 9):
top.add(t_tokens[t])
if known_only:
if separator not in KNOWN_SEPERATORS:
pprint(top)
raise SystemError('separator not in known list: {}'.format(separator))
ret = []
for l in text.split("\n"):
if l == '':
continue
if l.startswith('#') or l.startswith(';'):
continue
cols = l.split(separator)
cols = [x.strip() for x in cols]
indicator = Indicator()
for e in cols:
if e:
try:
i = resolve_itype(e)
if i:
indicator.indicator = e
indicator.itype = i
except NotImplementedError:
pass
try:
ts = arrow.get(e)
if ts:
indicator.lasttime = ts.datetime
except (arrow.parser.ParserError, UnicodeDecodeError):
pass
if e in top:
indicator.tags = [e]
if indicator.itype and indicator.indicator:
ret.append(indicator)
return ret
def process(self):
defaults = self._defaults()
map = self.rule.feeds[self.feed]['map']
values = self.rule.feeds[self.feed]['values']
data = []
for l in self.fetcher.process():
i = copy.deepcopy(defaults)
l = json.loads(l)
for e in l:
i = {}
for x, c in enumerate(map):
i[values[x]] = e[c]
try:
self.logger.debug(i)
i = normalize_itype(i)
i = Indicator(**i)
r = self.client.submit(i)
data.append(r)
except NotImplementedError as e:
self.logger.error(e)
self.logger.info('skipping: {}'.format(i['indicator']))
return data
def process(self):
if self.rule.feeds[self.feed].get('values'):
cols = self.rule.feeds[self.feed].get('values')
else:
cols = self.rule.defaults['values']
defaults = self._defaults()
if isinstance(cols, str):
cols = cols.split(',')
rv = []
for l in self.fetcher.process():
#self.logger.debug(l)
#pprint(l)
if self.ignore(l): # comment or skip
continue
try:
m = self.pattern.search(l).groups()
#self.logger.debug(m)
if isinstance(m, str):
m = [m]
except ValueError:
continue
except AttributeError:
continue
if len(cols):
i = copy.deepcopy(defaults)
for idx, col in enumerate(cols):
if col:
i[col] = m[idx]
i.pop("values", None)
i.pop("pattern", None)
self.logger.debug(i)
try:
i = Indicator(**i)
except NotImplementedError as e:
self.logger.error(e)
self.logger.info('skipping: {}'.format(i['indicator']))
else:
self.logger.debug(i)
r = self.client.submit(i)
rv.append(r)
if self.limit:
self.limit -= 1
if self.limit == 0:
self.logger.debug('limit reached...')
break
return rv
def test_urls_ok():
data = [
'http://192.168.1.1/1.html', 'http://www41.xzmnt.com',
'http://get.ahoybest.com/n/3.6.16/12205897/microsoft lync server 2010.exe'
]
for d in data:
d = Indicator(d)
assert d.itype is 'url'
def test_storage_sqlite():
dbfile = tempfile.mktemp()
with Storage(store='sqlite', dbfile=dbfile) as s:
ob = [
Indicator(indicator='example.com',
tags='botnet',
provider='csirtgadgets.org').__dict__,
Indicator(indicator='example2.com',
tags='malware',
provider='csirtgadgets.org').__dict__
]
x = s.handle_submission('1234', ob)
assert x > 0
x = s.handle_search('1234', {'indicator': 'example.com'})
os.unlink(dbfile)
def process(self):
defaults = self._defaults()
patterns = copy.deepcopy(self.rule.feeds[self.feed]['pattern'])
for p in patterns:
patterns[p]['pattern'] = re.compile(patterns[p]['pattern'])
feed = []
for l in self.fetcher.process():
feed.append(l)
feed = "\n".join(feed)
try:
feed = feedparser.parse(feed)
except Exception as e:
self.logger.error('Error parsing feed: {}'.format(e))
self.logger.error(defaults['remote'])
raise e
rv = []
for e in feed.entries:
i = copy.deepcopy(defaults)
for k in e:
if k == 'summary' and patterns.get('description'):
try:
m = patterns['description']['pattern'].search(
e[k]).groups()
except AttributeError:
continue
for idx, c in enumerate(patterns['description']['values']):
i[c] = m[idx]
elif patterns.get(k):
try:
m = patterns[k]['pattern'].search(e[k]).groups()
except AttributeError:
continue
for idx, c in enumerate(patterns[k]['values']):
i[c] = m[idx]
if not i.get('indicator'):
self.logger.error('missing indicator: {}'.format(e[k]))
continue
try:
i = normalize_itype(i)
i = Indicator(**i)
self.logger.debug(i)
r = self.client.submit(i)
rv.append(r)
except NotImplementedError as e:
self.logger.error(e)
self.logger.info('skipping: {}'.format(i['indicator']))
return rv
def handle_message(self, s, e):
self.logger.info('handling message...')
m = s.recv_multipart()
m = json.loads(m[0])
self.logger.debug(m)
m = Indicator(**m)
for p in self.plugins:
p.process(m, self.router)
def test_urls_not_ok():
data = [
'http://wp-content/plugins/tinymce-advanced/mce/emoticons/img/Yahoo-login/yahoo.html'
]
for d in data:
try:
d = Indicator(d)
from pprint import pprint
pprint(d)
except NotImplementedError:
pass
else:
raise NotImplementedError
def handle_search(self, token, data):
# need to send searches through the _submission pipe
data = json.loads(data)
if data.get('indicator'):
i = Indicator(indicator=data['indicator'],
tlp='green',
confidence=5,
tags='search')
r = self.handle_submission(token, str(i))
if r:
self.logger.info('search logged')
data = json.dumps(data)
self.storage.send_multipart(['search', token, data])
return self.storage.recv()
def handle_submission(self, token, data):
# this needs to be threaded out, badly.
data = json.loads(data)
i = Indicator(**data)
for g in self.gatherers:
i = g.process(i)
data = str(i)
if i.confidence >= MIN_CONFIDENCE:
if self.p2p:
self.logger.info('sending to peers...')
self.p2p.send(data.encode('utf-8'))
self.hunters.send(data)
self.storage.send_multipart(['submission', token, data])
m = self.storage.recv()
return m
def text_to_list(text, known_only=True):
separator = find_seperator(text)
t_tokens = top_tokens(text)
top = set()
for t in range(0, 9):
top.add(t_tokens[t])
if known_only:
if separator not in KNOWN_SEPERATORS:
pprint(top)
raise SystemError(
'separator not in known list: {}'.format(separator))
ret = []
for l in text.split("\n"):
if l == '':
continue
if l.startswith('#') or l.startswith(';'):
continue
cols = l.split(separator)
cols = [x.strip() for x in cols]
indicator = Indicator()
for e in cols:
if e:
try:
i = resolve_itype(e)
if i:
indicator.indicator = e
indicator.itype = i
except NotImplementedError:
pass
try:
ts = arrow.get(e)
if ts:
indicator.lasttime = ts.datetime
except (arrow.parser.ParserError, UnicodeDecodeError):
pass
if e in top:
indicator.tags = [e]
if indicator.itype and indicator.indicator:
ret.append(indicator)
return ret
def process(self):
defaults = self._defaults()
cols = defaults['values']
rv = []
for l in self.fetcher.process():
if l == '' or self.is_comment(l):
continue
l = l.replace('\"', '')
m = self.pattern.split(l)
if len(cols):
obs = {}
for k, v in defaults.items():
obs[k] = v
for idx, col in enumerate(cols):
if col is not None:
obs[col] = m[idx]
obs.pop("values", None)
try:
obs = Indicator(**obs)
except NotImplementedError as e:
self.logger.error(e)
self.logger.info('skipping: {}'.format(obs['indicator']))
else:
r = self.client.submit(obs)
rv.append(r)
if self.limit:
self.limit -= 1
if self.limit == 0:
self.logger.debug('limit reached...')
break
self.logger.debug('done...')
return rv
def _kv_to_indicator(self, kv):
return Indicator(**kv)
def obs():
return Indicator(indicator='example.com',
tags=['botnet'],
provider='csirtgadgets.org')
def _not(data):
for d in data:
d = Indicator(d)
assert d.itype != 'url'
def test_ipv4_ok():
data = ['192.168.1.0/24', '192.168.1.1', '255.255.255.255']
for d in data:
assert Indicator(indicator=d).itype is 'ipv4'
def test_fqdn_ok():
data = ['example.org', '1.2.3.4.com', 'xn----jtbbmekqknepg3a.xn--p1ai']
for d in data:
d = Indicator(d)
assert d.itype is 'fqdn'
def test_obs_ipv4():
o = Indicator('192.168.1.1')
assert o.is_private()
assert o.indicator == '192.168.1.1'
assert o.itype == 'ipv4'
def test_obs_fqdn():
o = Indicator('example.org')
assert o.is_private() is False
assert o.indicator == 'example.org'
assert o.itype == 'fqdn'
def test_ipv4_private():
data = ['128.205.1.0/24', '2001:1608:10:147::21', '2001:4860::8888/64']
for d in data:
assert not Indicator(indicator=d).is_private()
assert Indicator('192.168.1.1').is_private()
def _not(data):
for d in data:
d = Indicator(d)
assert d.itype is not 'ipv4'
