def process(self): defaults = self._defaults() map = self.rule.feeds[self.feed]['map'] values = self.rule.feeds[self.feed]['values'] data = [] for l in self.fetcher.process(): i = copy.deepcopy(defaults) l = json.loads(l) for e in l: i = {} for x, c in enumerate(map): i[values[x]] = e[c] try: self.logger.debug(i) i = normalize_itype(i) i = Indicator(**i) r = self.client.submit(i) data.append(r) except NotImplementedError as e: self.logger.error(e) self.logger.info('skipping: {}'.format(i['indicator'])) return data
def test_obs_url(): o = Indicator('http://example.org') assert o.is_private() is False assert o.indicator == 'http://example.org' assert o.itype is not 'fqdn' assert o.itype is 'url'
def process(self): if self.rule.feeds[self.feed].get('values'): cols = self.rule.feeds[self.feed].get('values') else: cols = self.rule.defaults['values'] defaults = self._defaults() if isinstance(cols, str): cols = cols.split(',') rv = [] for l in self.fetcher.process(): #self.logger.debug(l) #pprint(l) if self.ignore(l): # comment or skip continue try: m = self.pattern.search(l).groups() #self.logger.debug(m) if isinstance(m, str): m = [m] except ValueError: continue except AttributeError: continue if len(cols): i = copy.deepcopy(defaults) for idx, col in enumerate(cols): if col: i[col] = m[idx] i.pop("values", None) i.pop("pattern", None) self.logger.debug(i) try: i = Indicator(**i) except NotImplementedError as e: self.logger.error(e) self.logger.info('skipping: {}'.format(i['indicator'])) else: self.logger.debug(i) r = self.client.submit(i) rv.append(r) if self.limit: self.limit -= 1 if self.limit == 0: self.logger.debug('limit reached...') break return rv
def test_urls_ok(): data = [ 'http://192.168.1.1/1.html', 'http://www41.xzmnt.com', 'http://get.ahoybest.com/n/3.6.16/12205897/microsoft lync server 2010.exe' ] for d in data: d = Indicator(d) assert d.itype is 'url'
def test_storage_sqlite(): dbfile = tempfile.mktemp() with Storage(store='sqlite', dbfile=dbfile) as s: ob = [ Indicator(indicator='example.com', tags='botnet', provider='csirtgadgets.org').__dict__, Indicator(indicator='example2.com', tags='malware', provider='csirtgadgets.org').__dict__ ] x = s.handle_submission('1234', ob) assert x > 0 x = s.handle_search('1234', {'indicator': 'example.com'}) os.unlink(dbfile)
def process(self): defaults = self._defaults() patterns = copy.deepcopy(self.rule.feeds[self.feed]['pattern']) for p in patterns: patterns[p]['pattern'] = re.compile(patterns[p]['pattern']) feed = [] for l in self.fetcher.process(): feed.append(l) feed = "\n".join(feed) try: feed = feedparser.parse(feed) except Exception as e: self.logger.error('Error parsing feed: {}'.format(e)) self.logger.error(defaults['remote']) raise e rv = [] for e in feed.entries: i = copy.deepcopy(defaults) for k in e: if k == 'summary' and patterns.get('description'): try: m = patterns['description']['pattern'].search( e[k]).groups() except AttributeError: continue for idx, c in enumerate(patterns['description']['values']): i[c] = m[idx] elif patterns.get(k): try: m = patterns[k]['pattern'].search(e[k]).groups() except AttributeError: continue for idx, c in enumerate(patterns[k]['values']): i[c] = m[idx] if not i.get('indicator'): self.logger.error('missing indicator: {}'.format(e[k])) continue try: i = normalize_itype(i) i = Indicator(**i) self.logger.debug(i) r = self.client.submit(i) rv.append(r) except NotImplementedError as e: self.logger.error(e) self.logger.info('skipping: {}'.format(i['indicator'])) return rv
def handle_message(self, s, e): self.logger.info('handling message...') m = s.recv_multipart() m = json.loads(m[0]) self.logger.debug(m) m = Indicator(**m) for p in self.plugins: p.process(m, self.router)
def test_urls_not_ok(): data = [ 'http://wp-content/plugins/tinymce-advanced/mce/emoticons/img/Yahoo-login/yahoo.html' ] for d in data: try: d = Indicator(d) from pprint import pprint pprint(d) except NotImplementedError: pass else: raise NotImplementedError
def text_to_list(text, known_only=True): separator = find_seperator(text) t_tokens = top_tokens(text) top = set() for t in range(0, 9): top.add(t_tokens[t]) if known_only: if separator not in KNOWN_SEPERATORS: pprint(top) raise SystemError( 'separator not in known list: {}'.format(separator)) ret = [] for l in text.split("\n"): if l == '': continue if l.startswith('#') or l.startswith(';'): continue cols = l.split(separator) cols = [x.strip() for x in cols] indicator = Indicator() for e in cols: if e: try: i = resolve_itype(e) if i: indicator.indicator = e indicator.itype = i except NotImplementedError: pass try: ts = arrow.get(e) if ts: indicator.lasttime = ts.datetime except (arrow.parser.ParserError, UnicodeDecodeError): pass if e in top: indicator.tags = [e] if indicator.itype and indicator.indicator: ret.append(indicator) return ret
def handle_search(self, token, data): # need to send searches through the _submission pipe data = json.loads(data) if data.get('indicator'): i = Indicator(indicator=data['indicator'], tlp='green', confidence=5, tags='search') r = self.handle_submission(token, str(i)) if r: self.logger.info('search logged') data = json.dumps(data) self.storage.send_multipart(['search', token, data]) return self.storage.recv()
def handle_submission(self, token, data): # this needs to be threaded out, badly. data = json.loads(data) i = Indicator(**data) for g in self.gatherers: i = g.process(i) data = str(i) if i.confidence >= MIN_CONFIDENCE: if self.p2p: self.logger.info('sending to peers...') self.p2p.send(data.encode('utf-8')) self.hunters.send(data) self.storage.send_multipart(['submission', token, data]) m = self.storage.recv() return m
def process(self): defaults = self._defaults() cols = defaults['values'] rv = [] for l in self.fetcher.process(): if l == '' or self.is_comment(l): continue l = l.replace('\"', '') m = self.pattern.split(l) if len(cols): obs = {} for k, v in defaults.items(): obs[k] = v for idx, col in enumerate(cols): if col is not None: obs[col] = m[idx] obs.pop("values", None) try: obs = Indicator(**obs) except NotImplementedError as e: self.logger.error(e) self.logger.info('skipping: {}'.format(obs['indicator'])) else: r = self.client.submit(obs) rv.append(r) if self.limit: self.limit -= 1 if self.limit == 0: self.logger.debug('limit reached...') break self.logger.debug('done...') return rv
def _kv_to_indicator(self, kv): return Indicator(**kv)
def obs(): return Indicator(indicator='example.com', tags=['botnet'], provider='csirtgadgets.org')
def _not(data): for d in data: d = Indicator(d) assert d.itype != 'url'
def test_ipv4_ok(): data = ['192.168.1.0/24', '192.168.1.1', '255.255.255.255'] for d in data: assert Indicator(indicator=d).itype is 'ipv4'
def test_fqdn_ok(): data = ['example.org', '1.2.3.4.com', 'xn----jtbbmekqknepg3a.xn--p1ai'] for d in data: d = Indicator(d) assert d.itype is 'fqdn'
def test_obs_ipv4(): o = Indicator('192.168.1.1') assert o.is_private() assert o.indicator == '192.168.1.1' assert o.itype == 'ipv4'
def test_obs_fqdn(): o = Indicator('example.org') assert o.is_private() is False assert o.indicator == 'example.org' assert o.itype == 'fqdn'
def test_ipv4_private(): data = ['128.205.1.0/24', '2001:1608:10:147::21', '2001:4860::8888/64'] for d in data: assert not Indicator(indicator=d).is_private() assert Indicator('192.168.1.1').is_private()
def _not(data): for d in data: d = Indicator(d) assert d.itype is not 'ipv4'