def write(self, e): sid = e['session'] peerIP = e['src_ip'] ts = e['timestamp'] today = str(datetime.now().date()) logger.info('today is %s' % today) if not self.context.get(today): logger.info('resetting context for %s' % today) self.context = {} self.context[today] = {} if not self.context[today].get(peerIP): self.context[today][peerIP] = [] i = { 'user': self.user, 'feed': self.feed, 'indicator': peerIP, 'portlist': self.port, 'protocol': 'tcp', 'tags': 'scanner,ssh', 'firsttime': ts, 'lasttime': ts } ret = Indicator(self.client, i).submit() logger.info('logged to csirtg %s ' % ret['indicator']['location']) else: pprint(self.context) self.context[today][peerIP].append(sid)
def _log_indicator(indicator, content): global context global day today = arrow.utcnow().format('YYYY-MM-DD') if today != day: context = set() day = today if indicator in context: return context.add(indicator) i = { "user": USER, "feed": FEED, "indicator": indicator, "tags": ['httpd', 'scanner'], "description": 'unauthorized probe', "portlist": "80", "content": content, } logger.debug(json.dumps(i, indent=4)) if TOKEN != '': try: ret = Indicator(i).submit() logger.debug(ret) except Exception as e: logger.error(e)
def test_indicator_search_url(client): INDICATOR = 'http://www.example.com/test/index.html' # create feed and test created feed f = Feed(client).new(USER, FEED, description='build search test feed') assert f['created_at'] # create test and submit test indicator i = Indicator( client, { 'user': USER, 'feed': FEED, 'indicator': INDICATOR, 'comment': 'this is a test comment' }) r = i.submit() # test creating the indicator assert r['indicator'] == INDICATOR assert r['itype'] == 'uri' assert r['created_at'] # search for indicator s = Search(client) r = s.search(INDICATOR, 10) for record in r: if record['feed'] == 'live-test-feed': assert record['indicator'] == INDICATOR # delete test feed f = Feed(client).remove(USER, FEED) assert f == 200
def indicators_create(self, data): if not isinstance(data, list): data = [data] indicators = [] for x in data: d = {} if isinstance(x, csirtg_indicator.Indicator): d = x.__dict__() else: d = x d['feed'] = self.feed d['user'] = self.user i = Indicator( self.handle, d ) rv = i.submit() indicators.append(rv) assert len(indicators) > 0 return indicators
def test_indicator_search_ipv6(client): INDICATOR = '2001:4860:4860::8888' # create feed and test created feed f = Feed(client).new(USER, FEED, description='build search test feed') assert f['created_at'] # create test and submit test indicator i = Indicator( client, { 'user': USER, 'feed': FEED, 'indicator': INDICATOR, 'comment': 'this is a test comment' }) r = i.submit() # test creating the indicator assert r['indicator']['indicator'] == INDICATOR assert r['indicator']['itype'] == 'ipv6' assert r['indicator']['created_at'] # search for indicator s = Search(client) r = s.search(INDICATOR, 10) for record in r['feed']['indicators']: if record['indicator']['feed'] == 'live-test-feed': assert record['indicator']['indicator'] == INDICATOR # delete test feed f = Feed(client).remove(USER, FEED) assert f == 200
def indicators_create(self, data): d = data.__dict__() d['feed'] = self.feed d['user'] = self.user i = Indicator( self.handle, d ) rv = i.submit() assert rv
def write(self, e): sid = e['session'] peerIP = e['src_ip'] ts = e['timestamp'] system = e['system'] if system not in [ 'cowrie.ssh.factory.CowrieSSHFactory', 'cowrie.telnet.transport.HoneyPotTelnetFactory' ]: logger.debug('skipping {}'.format(system)) return today = str(datetime.now().date()) if not self.context.get(today): logger.debug('resetting context for %s' % today) self.context = {} self.context[today] = set() key = ','.join([peerIP, system]) if key in self.context[today]: logger.debug('skipping {}'.format(key)) return self.context[today].add(key) tags = 'scanner,ssh' port = 22 if e['system'] == 'cowrie.telnet.transport.HoneyPotTelnetFactory': tags = 'scanner,telnet' port = 23 i = { 'user': self.user, 'feed': self.feed, 'indicator': peerIP, 'portlist': port, 'protocol': 'tcp', 'tags': tags, 'firsttime': ts, 'lasttime': ts, 'description': self.description } ret = Indicator(self.client, i).submit() logger.info('logged to csirtg %s ' % ret['indicator']['location'])
def indicators_create(self, data): if not isinstance(data, list): data = [data] indicators = [] for x in data: d = x.__dict__() d['feed'] = self.feed d['user'] = self.user i = Indicator(d) rv = i.submit() indicators.append(rv) assert len(indicators) > 0 return indicators
def write(self, e): peerIP = e["src_ip"] ts = e["timestamp"] system = e.get("system", None) if system not in [ "cowrie.ssh.factory.CowrieSSHFactory", "cowrie.telnet.transport.HoneyPotTelnetFactory", ]: return today = str(datetime.now().date()) if not self.context.get(today): self.context = {} self.context[today] = set() key = ",".join([peerIP, system]) if key in self.context[today]: return self.context[today].add(key) tags = "scanner,ssh" port = 22 if e["system"] == "cowrie.telnet.transport.HoneyPotTelnetFactory": tags = "scanner,telnet" port = 23 i = { "user": self.user, "feed": self.feed, "indicator": peerIP, "portlist": port, "protocol": "tcp", "tags": tags, "firsttime": ts, "lasttime": ts, "description": self.description, } ret = Indicator(self.client, i).submit() log.msg("logged to csirtg %s " % ret["location"])
def on_message(self, ws, message): m = json.loads(message) if self.on_admin_message(m): return m = json.loads(m['message']) if m['provider'] == "%s/%s" % (USER, FEED): return today = datetime.today().strftime('%Y-%m-%d') if self.today != today: self.context = {} self.today = today if m['indicator'] not in self.context: self.context[m['indicator']] = {} self.context[m['indicator']]['providers'] = set() self.context[m['indicator']]['tags'] = set() self.context[m['indicator']]['lasttime'] = m.get( 'lasttime', datetime.today()) self.context[m['indicator']]['providers'].add(m['provider']) for t in m['tags'].split(','): self.context[m['indicator']]['tags'].add(t) if len(self.context[m['indicator']]['providers']) >= TRIGGER: h = { 'description': 'correlated', 'tags': ','.join(list(self.context[m['indicator']]['tags'])), 'indicator': m['indicator'], 'feed': FEED, 'user': USER, 'lasttime': self.context[m['indicator']]['lasttime'], } ret = Indicator(self.cli, h).submit() logger.info('posted: {0}'.format(ret['location']))
def test_indicator_attachment_jar(client): f = Feed(client).new(USER, FEED, description='test build feed') assert f['created_at'] i = Indicator( client, { 'user': USER, 'feed': FEED, 'attachment': 'samples/malware.jar', 'comment': 'asdfasdfasdf' }) r = i.submit() assert r['indicator'] == '8ab0079d8e80c2e166b3b12364c89255d79c9f75' assert r['attachments'][0]['attachment'] assert r['attachments'][0]['filesize'] assert r['attachments'][0]['created_at'] # delete test feed f = Feed(client).remove(USER, FEED) assert f == 200
def test_indicator_attachment_zip(client): f = Feed(client).new(USER, FEED, description='test build feed') assert f['created_at'] i = Indicator( client, { 'user': USER, 'feed': FEED, 'attachment': 'samples/malware.jar.zip', 'comment': 'asdfasdfasdf' }) r = i.submit() assert r['indicator'] == 'f31e226048d9bd45513e691a50a4b83893397235' assert r['attachments'][0]['attachment'] assert r['attachments'][0]['filesize'] assert r['attachments'][0]['created_at'] # delete test feed f = Feed(client).remove(USER, FEED) assert f == 200
def test_indicator_attachment_txt(client): f = Feed(client).new(USER, FEED, description='test build feed') assert f['created_at'] i = Indicator( client, { 'user': USER, 'feed': FEED, 'attachment': 'samples/message.eml', 'comment': 'asdfasdfasdf' }) r = i.submit() assert r['indicator'] == '2f9496a6331b2e75e5208b93d144e8fe484b316a' assert r['attachments'][0]['attachment'] assert r['attachments'][0]['filesize'] assert r['attachments'][0]['created_at'] # delete test feed f = Feed(client).remove(USER, FEED) assert f == 200
def test_indicator_attachment_pdf(client): f = Feed(client).new(USER, FEED, description='test build feed') assert f['created_at'] i = Indicator( client, { 'user': USER, 'feed': FEED, 'attachment': 'samples/hello_world.pdf', 'comment': 'asdfasdfasdf' }) r = i.submit() assert r['indicator'] == 'cc9881dc27a8d3e410cdf7e667ff5efa5cbfdaed' assert r['attachments'][0]['attachment'] assert r['attachments'][0]['filesize'] assert r['attachments'][0]['created_at'] # delete test feed f = Feed(client).remove(USER, FEED) assert f == 200
def test_indicator_attachment_doc(client): f = Feed(client).new(USER, FEED, description='test build feed') assert f['created_at'] i = Indicator( client, { 'user': USER, 'feed': FEED, 'attachment': 'samples/business_relationship.doc', 'comment': 'asdfasdfasdf' }) r = i.submit() assert r['indicator'] == 'f0ee0d5a1279fbdd93a9c5b9a1377894113f0ec0' assert r['attachments'][0]['attachment'] assert r['attachments'][0]['filesize'] assert r['attachments'][0]['created_at'] # delete test feed f = Feed(client).remove(USER, FEED) assert f == 200
def test_indicator_attachment_docx(client): f = Feed(client).new(USER, FEED, description='test build feed') assert f['created_at'] i = Indicator( client, { 'user': USER, 'feed': FEED, 'attachment': 'samples/c..docx', 'comment': 'asdfasdfasdf' }) r = i.submit() assert r['indicator'] == 'c2642e519c7f325300ed250710b4f815ac542c1d' assert r['attachments'][0]['attachment'] assert r['attachments'][0]['filesize'] assert r['attachments'][0]['created_at'] # delete test feed f = Feed(client).remove(USER, FEED) assert f == 200
def main(): parser = ArgumentParser(description=textwrap.dedent('''\ example usage: $ csirtg --search example.com $ csirtg --user csirtgadgets --feeds $ csirtg --user csirtgadgets --feed uce-urls $ csirtg --user csirtgadgets --new --feed scanners --description 'a feed of port scanners' $ csirtg --user csirtgadgets --feed scanners --new --indicator 1.1.1.1 --tags scanner --comment 'this is a port scanner' $ csirtg --user csirtgadgets --feed uce-attachments --new --attachment 'fax.zip' --description 'file attached in uce email' '''), formatter_class=RawDescriptionHelpFormatter, prog='csirtg') parser.add_argument("-v", "--verbose", action="count", help="set verbosity level [default: %(default)s]") parser.add_argument('-d', '--debug', action="store_true") parser.add_argument('--token', help="specify token", default=TOKEN) parser.add_argument('-l', '--limit', help="specify results limit [default: %(default)s]", default=LIMIT) parser.add_argument('--remote', help="remote api location [default: %(default)s]", default=REMOTE) parser.add_argument('--timeout', help='connection timeout [default: %(default)s]', default=TIMEOUT) parser.add_argument('-C', '--config', help="configuration file [default: %(default)s]", default=os.path.expanduser("~/.csirtg.yml")) # env var parser.add_argument('--format', help="specify an output format [default: %(default)s]", default='table') # actions parser.add_argument('-q', '--search', help="search for an indicator") parser.add_argument('--feeds', action="store_true", help="show a list of feeds (per user)") parser.add_argument('--new', action='store_true', help="create a new feed or indicator") # vars parser.add_argument('--user', help="specify a user") parser.add_argument('--feed', help="specify feed name") parser.add_argument('--indicator', dest='indicator', help="specify an indicator [eg: 1.2.3.4, example.com, " "http://example.com/1.html") parser.add_argument('--tags', help="specify tags") parser.add_argument('--comment', help="enter a comment for the indicator") parser.add_argument('--description', help="specify a feed description") parser.add_argument('--portlist', help="specify a portlist [eg: 22,23-35,443]") parser.add_argument('--protocol', help="specify TCP, UDP or ICMP") parser.add_argument( '--firsttime', help="timestamp when first seen [eg: 2015-11-23T00:00:00Z]") parser.add_argument( '--lasttime', help= "timestamp when last seen [eg: 2015-11-24T00:00:00Z], treated as 'greater " "than'") parser.add_argument('--attachment', help="specify an attachment [eg: /path/to/file]") parser.add_argument('--attachment-name', help="specify the attachment filename") parser.add_argument('--no-verify-ssl', help='Turn TLS verification OFF', action="store_true") # Process arguments args = parser.parse_args() setup_logging(args) logger = logging.getLogger(__name__) ## TODO -- yconf ? options = vars(args) o = read_config(args) options.update(o) verify_ssl = True if options.get('no_verify_ssl'): verify_ssl = False cli = Client(remote=options['remote'], token=options['token'], verify_ssl=verify_ssl) if options.get('search'): logger.info("Searching for: {0}".format(options.get('search'))) ret = Search(cli).search(options.get('search'), limit=options['limit']) format = format_factory(options['format']) format(ret).write() logger.info("Done") elif options.get('feeds'): logger.info("Searching feeds for user: {0}".format(options['user'])) feeds = Feed(cli).index(options['user']) from prettytable import PrettyTable cols = ['name', 'description', 'license', 'updated_at'] t = PrettyTable(cols) for f in feeds: r = [] for c in cols: y = f['feed'].get(c) if c == 'license': y = y['name'] y = str(y) y = (y[:30] + '..') if len(y) > 30 else y r.append(y) t.add_row(r) print(str(t)) logger.info("Done") elif options.get('feed') and options.get('new') and ( not options.get('indicator') and not options.get('attachment')): if not options.get('user'): parser.error('--user is required') logger.info("Creating feed {0} for user {1}".format( options['feed'], options['user'])) feed = Feed(cli).new(options['user'], options['feed'], description=options['description']) from prettytable import PrettyTable cols = ['name', 'description', 'license', 'updated_at'] t = PrettyTable(cols) r = [] for c in cols: y = feed.get(c) if c == 'license': y = y['name'] y = str(y) y = (y[:30] + '..') if len(y) > 30 else y r.append(y) t.add_row(r) print(str(t)) logger.info("Done") elif options.get('feed') and not options.get('new'): if not options.get('user'): parser.error('--user is required') logger.info("Fetching feed {0} for user {1}".format( options['feed'], options['user'])) data = Feed(cli).show( options['user'], options['feed'], limit=options['limit'], lasttime=options['lasttime'], ) if data['feed'].get('indicators'): format = format_factory(options['format']) format(data).write() logger.info("Done") # submit new indicator elif options.get('feed') and ( options.get('indicator') or (options.get('attachment') and not options.get('indicator')) ) and options.get('new') and options.get('user'): try: logger.info("Creating indicator in feed {0} for user {1}".format( options['feed'], options['user'])) ret = Indicator(cli, options).submit() logger.info('posted: {0}'.format(ret['indicator']['location'])) ret = {'feed': {'indicators': [ret]}} format = format_factory(options['format']) format(ret).write() logger.info("Done") except RuntimeError as e: logger.error(e)
def main(): parser = ArgumentParser(description=textwrap.dedent('''\ example usage: $ csirtg --search example.com $ csirtg --user csirtgadgets --feeds $ csirtg --user csirtgadgets --feed uce-urls $ csirtg --user csirtgadgets --feed-new scanners --description 'a feed of port scanners' $ csirtg --user csirtgadgets --feed scanners --indicator-new 1.1.1.1 --tags scanner --comment 'this is a port scanner' $ csirtg --user csirtgadgets --feed uce-attachments --new --attachment 'fax.zip' --description 'file attached in uce email' '''), formatter_class=RawDescriptionHelpFormatter, prog='csirtg') parser.add_argument("-v", "--verbose", action="count", help="set verbosity level [default: %(default)s]") parser.add_argument('-d', '--debug', action="store_true") parser.add_argument('--token', help="specify token", default=TOKEN) parser.add_argument('-l', '--limit', help="specify results limit [default: %(default)s]", default=LIMIT) parser.add_argument('--remote', help="remote api location [default: %(default)s]", default=REMOTE) parser.add_argument('--timeout', help='connection timeout [default: %(default)s]', default=TIMEOUT) parser.add_argument('-C', '--config', help="configuration file [default: %(default)s]", default=os.path.expanduser("~/.csirtg.yml")) # env var parser.add_argument('--format', help="specify an output format [default: %(default)s]", default='table') parser.add_argument('--columns', help='specify output columns [default %(default)s]', default=','.join(COLUMNS)) # actions parser.add_argument('-q', '--search', help="search for an indicator") parser.add_argument('--feeds', action="store_true", help="show a list of feeds (per user)") parser.add_argument('--new', action='store_true', help="create a new feed or indicator") parser.add_argument('--predict', help="test an indicator against the prediction api") parser.add_argument('--predict-stdin', action="store_true") # vars parser.add_argument('--user', help="specify a user") parser.add_argument('--feed', help="specify feed name") parser.add_argument('--feed-new', help='help specify a new feed') parser.add_argument('--indicator', dest='indicator', help="specify an indicator [eg: 1.2.3.4, example.com, " "http://example.com/1.html") parser.add_argument('--indicator-new', help='create a new indicator') parser.add_argument('--tags', help="specify tags") parser.add_argument('--comment', help="enter a comment for the indicator") parser.add_argument('--description', help="specify a feed description") parser.add_argument('--portlist', help="specify a portlist [eg: 22,23-35,443]") parser.add_argument('--protocol', help="specify TCP, UDP or ICMP") parser.add_argument( '--firsttime', help="timestamp when first seen [eg: 2015-11-23T00:00:00Z]") parser.add_argument( '--lasttime', help= "timestamp when last seen [eg: 2015-11-24T00:00:00Z], treated as 'greater " "than'") parser.add_argument('--attachment', help="specify an attachment [eg: /path/to/file]") parser.add_argument('--attachment-name', help="specify the attachment filename") parser.add_argument('--no-verify-ssl', help='Turn TLS verification OFF', action="store_true") parser.add_argument('--sinkhole', action='store_true') parser.add_argument( '--threatmodel', '--tm', help='predict using a threatmodel (eg: wes/phishing-urls)') # Process arguments args = parser.parse_args() setup_logging(args) logger = logging.getLogger(__name__) if args.verbose: logger.setLevel(logging.INFO) if args.debug: logger.setLevel(logging.DEBUG) options = vars(args) o = read_config(args) options.update(o) verify_ssl = True if options.get('no_verify_ssl'): verify_ssl = False cli = Client(remote=options['remote'], token=options['token'], verify_ssl=verify_ssl) if options.get('sinkhole'): import sys if not sys.stdin.isatty(): stdin = sys.stdin.read() else: logger.error("No data passed via STDIN") raise SystemExit from csirtgsdk.sinkhole import Sinkhole ret = Sinkhole(cli).post(stdin) if ret.get('status') == 'unauthorized': logger.error('unauthorized') else: if logger.getEffectiveLevel() == logging.DEBUG: print("Predictions: \n") for p in ret.get('predictions', []): print("\t%s" % p) raise SystemExit if options.get('threatmodel'): try: from csirtgsdk.threatmodel import Threatmodel except: print("") print( "Missing some extra machine learning libraries required to make this work.." ) print("$ pip install numpy keras pandas tensorflow") print("") raise SystemExit user, name = options['threatmodel'].split('/') m = Threatmodel(cli).show(user, name, options['search']) print("Prediction: %f" % (m * 100)) raise SystemExit if options.get('predict') or options.get('predict_stdin'): import sys if options.get('predict'): logger.info("Searching for: {0}".format(options.get('predict'))) predict = options['predict'] ret = Predict(cli).get(predict) print("Prediction Score: %s - %s" % (ret, predict)) else: import sys if not sys.stdin.isatty(): for predict in sys.stdin.read().split("\n"): if predict == '': continue ret = Predict(cli).get(predict) print("Prediction Score: %s - %s" % (ret, predict)) else: logger.error("No data passed via STDIN") raise SystemExit raise SystemExit if options.get('search'): logger.info("Searching for: {0}".format(options.get('search'))) ret = Search(cli).search(options.get('search'), limit=options['limit']) if isinstance(ret, dict): ret = [ret] print(FORMATS[options.get('format')](data=ret, cols=args.columns.split(','))) raise SystemExit if options.get('indicator_new') or options.get('attachment'): options['indicator'] = options['indicator_new'] logger.info("Creating indicator in feed {0} for user {1}".format( options['feed'], options['user'])) ret = Indicator(cli, options).submit() logger.info('posted: {0}'.format(ret['location'])) if isinstance(ret, dict): ret = [ret] print(FORMATS[options.get('format')](data=ret, cols=args.columns.split(','))) raise SystemExit if options.get('feeds'): logger.info("Searching feeds for user: {0}".format(options['user'])) f = Feed(cli) feeds = f.index(options['user']) for l in f.get_lines(feeds): print(l) raise SystemExit if options.get('feed_new'): if not options.get('user'): parser.error('--user is required') if not options.get('description'): parser.error('--description is required') logger.info("Creating feed {0} for user {1}".format( options['feed_new'], options['user'])) f = Feed(cli) feed = f.new(options['user'], options['feed_new'], description=options['description']) for l in f.get_lines(feed): print(l) raise SystemExit if options.get('feed'): if not options.get('user'): parser.error('--user is required') logger.info("Fetching feed {0} for user {1}".format( options['feed'], options['user'])) data = Feed(cli).show( options['user'], options['feed'], limit=options['limit'], lasttime=options['lasttime'], ) if data.get('indicators'): print(FORMATS[options.get('format')](data=data['indicators'], cols=args.columns.split(',')))