def test_indicator_search_url(client):
INDICATOR = 'http://www.example.com/test/index.html'
# create feed and test created feed
f = Feed(client).new(USER, FEED, description='build search test feed')
assert f['created_at']
# create test and submit test indicator
i = Indicator(
client, {
'user': USER,
'feed': FEED,
'indicator': INDICATOR,
'comment': 'this is a test comment'
})
r = i.submit()
# test creating the indicator
assert r['indicator'] == INDICATOR
assert r['itype'] == 'uri'
assert r['created_at']
# search for indicator
s = Search(client)
r = s.search(INDICATOR, 10)
for record in r:
if record['feed'] == 'live-test-feed':
assert record['indicator'] == INDICATOR
# delete test feed
f = Feed(client).remove(USER, FEED)
assert f == 200
def indicators_create(self, data):
if not isinstance(data, list):
data = [data]
indicators = []
for x in data:
d = {}
if isinstance(x, csirtg_indicator.Indicator):
d = x.__dict__()
else:
d = x
d['feed'] = self.feed
d['user'] = self.user
i = Indicator(
self.handle,
d
)
rv = i.submit()
indicators.append(rv)
assert len(indicators) > 0
return indicators
def test_indicator_search_ipv6(client):
INDICATOR = '2001:4860:4860::8888'
# create feed and test created feed
f = Feed(client).new(USER, FEED, description='build search test feed')
assert f['created_at']
# create test and submit test indicator
i = Indicator(
client, {
'user': USER,
'feed': FEED,
'indicator': INDICATOR,
'comment': 'this is a test comment'
})
r = i.submit()
# test creating the indicator
assert r['indicator']['indicator'] == INDICATOR
assert r['indicator']['itype'] == 'ipv6'
assert r['indicator']['created_at']
# search for indicator
s = Search(client)
r = s.search(INDICATOR, 10)
for record in r['feed']['indicators']:
if record['indicator']['feed'] == 'live-test-feed':
assert record['indicator']['indicator'] == INDICATOR
# delete test feed
f = Feed(client).remove(USER, FEED)
assert f == 200
def indicators_create(self, data):
d = data.__dict__()
d['feed'] = self.feed
d['user'] = self.user
i = Indicator(
self.handle,
d
)
rv = i.submit()
assert rv
def write(self, e):
sid = e['session']
peerIP = e['src_ip']
ts = e['timestamp']
today = str(datetime.now().date())
logger.info('today is %s' % today)
if not self.context.get(today):
logger.info('resetting context for %s' % today)
self.context = {}
self.context[today] = {}
if not self.context[today].get(peerIP):
self.context[today][peerIP] = []
i = {
'user': self.user,
'feed': self.feed,
'indicator': peerIP,
'portlist': self.port,
'protocol': 'tcp',
'tags': 'scanner,ssh',
'firsttime': ts,
'lasttime': ts
}
ret = Indicator(self.client, i).submit()
logger.info('logged to csirtg %s ' % ret['indicator']['location'])
else:
pprint(self.context)
self.context[today][peerIP].append(sid)
def _log_indicator(indicator, content):
global context
global day
today = arrow.utcnow().format('YYYY-MM-DD')
if today != day:
context = set()
day = today
if indicator in context:
return
context.add(indicator)
i = {
"user": USER,
"feed": FEED,
"indicator": indicator,
"tags": ['httpd', 'scanner'],
"description": 'unauthorized probe',
"portlist": "80",
"content": content,
}
logger.debug(json.dumps(i, indent=4))
if TOKEN != '':
try:
ret = Indicator(i).submit()
logger.debug(ret)
except Exception as e:
logger.error(e)
def indicators_create(self, data):
if not isinstance(data, list):
data = [data]
indicators = []
for x in data:
d = x.__dict__()
d['feed'] = self.feed
d['user'] = self.user
i = Indicator(d)
rv = i.submit()
indicators.append(rv)
assert len(indicators) > 0
return indicators
def test_indicator_attachment_docx(client):
f = Feed(client).new(USER, FEED, description='test build feed')
assert f['created_at']
i = Indicator(
client, {
'user': USER,
'feed': FEED,
'attachment': 'samples/c..docx',
'comment': 'asdfasdfasdf'
})
r = i.submit()
assert r['indicator'] == 'c2642e519c7f325300ed250710b4f815ac542c1d'
assert r['attachments'][0]['attachment']
assert r['attachments'][0]['filesize']
assert r['attachments'][0]['created_at']
# delete test feed
f = Feed(client).remove(USER, FEED)
assert f == 200
def test_indicator_attachment_zip(client):
f = Feed(client).new(USER, FEED, description='test build feed')
assert f['created_at']
i = Indicator(
client, {
'user': USER,
'feed': FEED,
'attachment': 'samples/malware.jar.zip',
'comment': 'asdfasdfasdf'
})
r = i.submit()
assert r['indicator'] == 'f31e226048d9bd45513e691a50a4b83893397235'
assert r['attachments'][0]['attachment']
assert r['attachments'][0]['filesize']
assert r['attachments'][0]['created_at']
# delete test feed
f = Feed(client).remove(USER, FEED)
assert f == 200
def test_indicator_attachment_txt(client):
f = Feed(client).new(USER, FEED, description='test build feed')
assert f['created_at']
i = Indicator(
client, {
'user': USER,
'feed': FEED,
'attachment': 'samples/message.eml',
'comment': 'asdfasdfasdf'
})
r = i.submit()
assert r['indicator'] == '2f9496a6331b2e75e5208b93d144e8fe484b316a'
assert r['attachments'][0]['attachment']
assert r['attachments'][0]['filesize']
assert r['attachments'][0]['created_at']
# delete test feed
f = Feed(client).remove(USER, FEED)
assert f == 200
def test_indicator_attachment_pdf(client):
f = Feed(client).new(USER, FEED, description='test build feed')
assert f['created_at']
i = Indicator(
client, {
'user': USER,
'feed': FEED,
'attachment': 'samples/hello_world.pdf',
'comment': 'asdfasdfasdf'
})
r = i.submit()
assert r['indicator'] == 'cc9881dc27a8d3e410cdf7e667ff5efa5cbfdaed'
assert r['attachments'][0]['attachment']
assert r['attachments'][0]['filesize']
assert r['attachments'][0]['created_at']
# delete test feed
f = Feed(client).remove(USER, FEED)
assert f == 200
def test_indicator_attachment_doc(client):
f = Feed(client).new(USER, FEED, description='test build feed')
assert f['created_at']
i = Indicator(
client, {
'user': USER,
'feed': FEED,
'attachment': 'samples/business_relationship.doc',
'comment': 'asdfasdfasdf'
})
r = i.submit()
assert r['indicator'] == 'f0ee0d5a1279fbdd93a9c5b9a1377894113f0ec0'
assert r['attachments'][0]['attachment']
assert r['attachments'][0]['filesize']
assert r['attachments'][0]['created_at']
# delete test feed
f = Feed(client).remove(USER, FEED)
assert f == 200
def test_indicator_attachment_jar(client):
f = Feed(client).new(USER, FEED, description='test build feed')
assert f['created_at']
i = Indicator(
client, {
'user': USER,
'feed': FEED,
'attachment': 'samples/malware.jar',
'comment': 'asdfasdfasdf'
})
r = i.submit()
assert r['indicator'] == '8ab0079d8e80c2e166b3b12364c89255d79c9f75'
assert r['attachments'][0]['attachment']
assert r['attachments'][0]['filesize']
assert r['attachments'][0]['created_at']
# delete test feed
f = Feed(client).remove(USER, FEED)
assert f == 200
def indicators_create(self, data):
if not isinstance(data, list):
data = [data]
indicators = []
for x in data:
d = {}
if isinstance(x, csirtg_indicator.Indicator):
d = x.__dict__()
else:
d = x
d['feed'] = self.feed
d['user'] = self.user
i = Indicator(d)
rv = i.submit()
indicators.append(rv)
assert len(indicators) > 0
return indicators
def write(self, e):
sid = e['session']
peerIP = e['src_ip']
ts = e['timestamp']
system = e['system']
if system not in [
'cowrie.ssh.factory.CowrieSSHFactory',
'cowrie.telnet.transport.HoneyPotTelnetFactory'
]:
logger.debug('skipping {}'.format(system))
return
today = str(datetime.now().date())
if not self.context.get(today):
logger.debug('resetting context for %s' % today)
self.context = {}
self.context[today] = set()
key = ','.join([peerIP, system])
if key in self.context[today]:
logger.debug('skipping {}'.format(key))
return
self.context[today].add(key)
tags = 'scanner,ssh'
port = 22
if e['system'] == 'cowrie.telnet.transport.HoneyPotTelnetFactory':
tags = 'scanner,telnet'
port = 23
i = {
'user': self.user,
'feed': self.feed,
'indicator': peerIP,
'portlist': port,
'protocol': 'tcp',
'tags': tags,
'firsttime': ts,
'lasttime': ts,
'description': self.description
}
ret = Indicator(self.client, i).submit()
logger.info('logged to csirtg %s ' % ret['indicator']['location'])
def write(self, e):
peerIP = e["src_ip"]
ts = e["timestamp"]
system = e.get("system", None)
if system not in [
"cowrie.ssh.factory.CowrieSSHFactory",
"cowrie.telnet.transport.HoneyPotTelnetFactory",
]:
return
today = str(datetime.now().date())
if not self.context.get(today):
self.context = {}
self.context[today] = set()
key = ",".join([peerIP, system])
if key in self.context[today]:
return
self.context[today].add(key)
tags = "scanner,ssh"
port = 22
if e["system"] == "cowrie.telnet.transport.HoneyPotTelnetFactory":
tags = "scanner,telnet"
port = 23
i = {
"user": self.user,
"feed": self.feed,
"indicator": peerIP,
"portlist": port,
"protocol": "tcp",
"tags": tags,
"firsttime": ts,
"lasttime": ts,
"description": self.description,
}
ret = Indicator(self.client, i).submit()
log.msg("logged to csirtg %s " % ret["location"])
def on_message(self, ws, message):
m = json.loads(message)
if self.on_admin_message(m):
return
m = json.loads(m['message'])
if m['provider'] == "%s/%s" % (USER, FEED):
return
today = datetime.today().strftime('%Y-%m-%d')
if self.today != today:
self.context = {}
self.today = today
if m['indicator'] not in self.context:
self.context[m['indicator']] = {}
self.context[m['indicator']]['providers'] = set()
self.context[m['indicator']]['tags'] = set()
self.context[m['indicator']]['lasttime'] = m.get(
'lasttime', datetime.today())
self.context[m['indicator']]['providers'].add(m['provider'])
for t in m['tags'].split(','):
self.context[m['indicator']]['tags'].add(t)
if len(self.context[m['indicator']]['providers']) >= TRIGGER:
h = {
'description': 'correlated',
'tags':
','.join(list(self.context[m['indicator']]['tags'])),
'indicator': m['indicator'],
'feed': FEED,
'user': USER,
'lasttime': self.context[m['indicator']]['lasttime'],
}
ret = Indicator(self.cli, h).submit()
logger.info('posted: {0}'.format(ret['location']))
def main():
parser = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ csirtg --search example.com
$ csirtg --user csirtgadgets --feeds
$ csirtg --user csirtgadgets --feed uce-urls
$ csirtg --user csirtgadgets --new --feed scanners --description 'a feed of port scanners'
$ csirtg --user csirtgadgets --feed scanners --new --indicator 1.1.1.1 --tags scanner --comment
'this is a port scanner'
$ csirtg --user csirtgadgets --feed uce-attachments --new --attachment 'fax.zip'
--description 'file attached in uce email'
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg')
parser.add_argument("-v",
"--verbose",
action="count",
help="set verbosity level [default: %(default)s]")
parser.add_argument('-d', '--debug', action="store_true")
parser.add_argument('--token', help="specify token", default=TOKEN)
parser.add_argument('-l',
'--limit',
help="specify results limit [default: %(default)s]",
default=LIMIT)
parser.add_argument('--remote',
help="remote api location [default: %(default)s]",
default=REMOTE)
parser.add_argument('--timeout',
help='connection timeout [default: %(default)s]',
default=TIMEOUT)
parser.add_argument('-C',
'--config',
help="configuration file [default: %(default)s]",
default=os.path.expanduser("~/.csirtg.yml")) # env var
parser.add_argument('--format',
help="specify an output format [default: %(default)s]",
default='table')
# actions
parser.add_argument('-q', '--search', help="search for an indicator")
parser.add_argument('--feeds',
action="store_true",
help="show a list of feeds (per user)")
parser.add_argument('--new',
action='store_true',
help="create a new feed or indicator")
# vars
parser.add_argument('--user', help="specify a user")
parser.add_argument('--feed', help="specify feed name")
parser.add_argument('--indicator',
dest='indicator',
help="specify an indicator [eg: 1.2.3.4, example.com, "
"http://example.com/1.html")
parser.add_argument('--tags', help="specify tags")
parser.add_argument('--comment', help="enter a comment for the indicator")
parser.add_argument('--description', help="specify a feed description")
parser.add_argument('--portlist',
help="specify a portlist [eg: 22,23-35,443]")
parser.add_argument('--protocol', help="specify TCP, UDP or ICMP")
parser.add_argument(
'--firsttime',
help="timestamp when first seen [eg: 2015-11-23T00:00:00Z]")
parser.add_argument(
'--lasttime',
help=
"timestamp when last seen [eg: 2015-11-24T00:00:00Z], treated as 'greater "
"than'")
parser.add_argument('--attachment',
help="specify an attachment [eg: /path/to/file]")
parser.add_argument('--attachment-name',
help="specify the attachment filename")
parser.add_argument('--no-verify-ssl',
help='Turn TLS verification OFF',
action="store_true")
# Process arguments
args = parser.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
## TODO -- yconf ?
options = vars(args)
o = read_config(args)
options.update(o)
verify_ssl = True
if options.get('no_verify_ssl'):
verify_ssl = False
cli = Client(remote=options['remote'],
token=options['token'],
verify_ssl=verify_ssl)
if options.get('search'):
logger.info("Searching for: {0}".format(options.get('search')))
ret = Search(cli).search(options.get('search'), limit=options['limit'])
format = format_factory(options['format'])
format(ret).write()
logger.info("Done")
elif options.get('feeds'):
logger.info("Searching feeds for user: {0}".format(options['user']))
feeds = Feed(cli).index(options['user'])
from prettytable import PrettyTable
cols = ['name', 'description', 'license', 'updated_at']
t = PrettyTable(cols)
for f in feeds:
r = []
for c in cols:
y = f['feed'].get(c)
if c == 'license':
y = y['name']
y = str(y)
y = (y[:30] + '..') if len(y) > 30 else y
r.append(y)
t.add_row(r)
print(str(t))
logger.info("Done")
elif options.get('feed') and options.get('new') and (
not options.get('indicator') and not options.get('attachment')):
if not options.get('user'):
parser.error('--user is required')
logger.info("Creating feed {0} for user {1}".format(
options['feed'], options['user']))
feed = Feed(cli).new(options['user'],
options['feed'],
description=options['description'])
from prettytable import PrettyTable
cols = ['name', 'description', 'license', 'updated_at']
t = PrettyTable(cols)
r = []
for c in cols:
y = feed.get(c)
if c == 'license':
y = y['name']
y = str(y)
y = (y[:30] + '..') if len(y) > 30 else y
r.append(y)
t.add_row(r)
print(str(t))
logger.info("Done")
elif options.get('feed') and not options.get('new'):
if not options.get('user'):
parser.error('--user is required')
logger.info("Fetching feed {0} for user {1}".format(
options['feed'], options['user']))
data = Feed(cli).show(
options['user'],
options['feed'],
limit=options['limit'],
lasttime=options['lasttime'],
)
if data['feed'].get('indicators'):
format = format_factory(options['format'])
format(data).write()
logger.info("Done")
# submit new indicator
elif options.get('feed') and (
options.get('indicator') or
(options.get('attachment') and not options.get('indicator'))
) and options.get('new') and options.get('user'):
try:
logger.info("Creating indicator in feed {0} for user {1}".format(
options['feed'], options['user']))
ret = Indicator(cli, options).submit()
logger.info('posted: {0}'.format(ret['indicator']['location']))
ret = {'feed': {'indicators': [ret]}}
format = format_factory(options['format'])
format(ret).write()
logger.info("Done")
except RuntimeError as e:
logger.error(e)
def main():
parser = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ csirtg --search example.com
$ csirtg --user csirtgadgets --feeds
$ csirtg --user csirtgadgets --feed uce-urls
$ csirtg --user csirtgadgets --feed-new scanners --description 'a feed of port scanners'
$ csirtg --user csirtgadgets --feed scanners --indicator-new 1.1.1.1 --tags scanner --comment
'this is a port scanner'
$ csirtg --user csirtgadgets --feed uce-attachments --new --attachment 'fax.zip'
--description 'file attached in uce email'
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg')
parser.add_argument("-v",
"--verbose",
action="count",
help="set verbosity level [default: %(default)s]")
parser.add_argument('-d', '--debug', action="store_true")
parser.add_argument('--token', help="specify token", default=TOKEN)
parser.add_argument('-l',
'--limit',
help="specify results limit [default: %(default)s]",
default=LIMIT)
parser.add_argument('--remote',
help="remote api location [default: %(default)s]",
default=REMOTE)
parser.add_argument('--timeout',
help='connection timeout [default: %(default)s]',
default=TIMEOUT)
parser.add_argument('-C',
'--config',
help="configuration file [default: %(default)s]",
default=os.path.expanduser("~/.csirtg.yml")) # env var
parser.add_argument('--format',
help="specify an output format [default: %(default)s]",
default='table')
parser.add_argument('--columns',
help='specify output columns [default %(default)s]',
default=','.join(COLUMNS))
# actions
parser.add_argument('-q', '--search', help="search for an indicator")
parser.add_argument('--feeds',
action="store_true",
help="show a list of feeds (per user)")
parser.add_argument('--new',
action='store_true',
help="create a new feed or indicator")
parser.add_argument('--predict',
help="test an indicator against the prediction api")
parser.add_argument('--predict-stdin', action="store_true")
# vars
parser.add_argument('--user', help="specify a user")
parser.add_argument('--feed', help="specify feed name")
parser.add_argument('--feed-new', help='help specify a new feed')
parser.add_argument('--indicator',
dest='indicator',
help="specify an indicator [eg: 1.2.3.4, example.com, "
"http://example.com/1.html")
parser.add_argument('--indicator-new', help='create a new indicator')
parser.add_argument('--tags', help="specify tags")
parser.add_argument('--comment', help="enter a comment for the indicator")
parser.add_argument('--description', help="specify a feed description")
parser.add_argument('--portlist',
help="specify a portlist [eg: 22,23-35,443]")
parser.add_argument('--protocol', help="specify TCP, UDP or ICMP")
parser.add_argument(
'--firsttime',
help="timestamp when first seen [eg: 2015-11-23T00:00:00Z]")
parser.add_argument(
'--lasttime',
help=
"timestamp when last seen [eg: 2015-11-24T00:00:00Z], treated as 'greater "
"than'")
parser.add_argument('--attachment',
help="specify an attachment [eg: /path/to/file]")
parser.add_argument('--attachment-name',
help="specify the attachment filename")
parser.add_argument('--no-verify-ssl',
help='Turn TLS verification OFF',
action="store_true")
parser.add_argument('--sinkhole', action='store_true')
parser.add_argument(
'--threatmodel',
'--tm',
help='predict using a threatmodel (eg: wes/phishing-urls)')
# Process arguments
args = parser.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
if args.verbose:
logger.setLevel(logging.INFO)
if args.debug:
logger.setLevel(logging.DEBUG)
options = vars(args)
o = read_config(args)
options.update(o)
verify_ssl = True
if options.get('no_verify_ssl'):
verify_ssl = False
cli = Client(remote=options['remote'],
token=options['token'],
verify_ssl=verify_ssl)
if options.get('sinkhole'):
import sys
if not sys.stdin.isatty():
stdin = sys.stdin.read()
else:
logger.error("No data passed via STDIN")
raise SystemExit
from csirtgsdk.sinkhole import Sinkhole
ret = Sinkhole(cli).post(stdin)
if ret.get('status') == 'unauthorized':
logger.error('unauthorized')
else:
if logger.getEffectiveLevel() == logging.DEBUG:
print("Predictions: \n")
for p in ret.get('predictions', []):
print("\t%s" % p)
raise SystemExit
if options.get('threatmodel'):
try:
from csirtgsdk.threatmodel import Threatmodel
except:
print("")
print(
"Missing some extra machine learning libraries required to make this work.."
)
print("$ pip install numpy keras pandas tensorflow")
print("")
raise SystemExit
user, name = options['threatmodel'].split('/')
m = Threatmodel(cli).show(user, name, options['search'])
print("Prediction: %f" % (m * 100))
raise SystemExit
if options.get('predict') or options.get('predict_stdin'):
import sys
if options.get('predict'):
logger.info("Searching for: {0}".format(options.get('predict')))
predict = options['predict']
ret = Predict(cli).get(predict)
print("Prediction Score: %s - %s" % (ret, predict))
else:
import sys
if not sys.stdin.isatty():
for predict in sys.stdin.read().split("\n"):
if predict == '':
continue
ret = Predict(cli).get(predict)
print("Prediction Score: %s - %s" % (ret, predict))
else:
logger.error("No data passed via STDIN")
raise SystemExit
raise SystemExit
if options.get('search'):
logger.info("Searching for: {0}".format(options.get('search')))
ret = Search(cli).search(options.get('search'), limit=options['limit'])
if isinstance(ret, dict):
ret = [ret]
print(FORMATS[options.get('format')](data=ret,
cols=args.columns.split(',')))
raise SystemExit
if options.get('indicator_new') or options.get('attachment'):
options['indicator'] = options['indicator_new']
logger.info("Creating indicator in feed {0} for user {1}".format(
options['feed'], options['user']))
ret = Indicator(cli, options).submit()
logger.info('posted: {0}'.format(ret['location']))
if isinstance(ret, dict):
ret = [ret]
print(FORMATS[options.get('format')](data=ret,
cols=args.columns.split(',')))
raise SystemExit
if options.get('feeds'):
logger.info("Searching feeds for user: {0}".format(options['user']))
f = Feed(cli)
feeds = f.index(options['user'])
for l in f.get_lines(feeds):
print(l)
raise SystemExit
if options.get('feed_new'):
if not options.get('user'):
parser.error('--user is required')
if not options.get('description'):
parser.error('--description is required')
logger.info("Creating feed {0} for user {1}".format(
options['feed_new'], options['user']))
f = Feed(cli)
feed = f.new(options['user'],
options['feed_new'],
description=options['description'])
for l in f.get_lines(feed):
print(l)
raise SystemExit
if options.get('feed'):
if not options.get('user'):
parser.error('--user is required')
logger.info("Fetching feed {0} for user {1}".format(
options['feed'], options['user']))
data = Feed(cli).show(
options['user'],
options['feed'],
limit=options['limit'],
lasttime=options['lasttime'],
)
if data.get('indicators'):
print(FORMATS[options.get('format')](data=data['indicators'],
cols=args.columns.split(',')))
