def test_indicator_search_ipv6(client):
INDICATOR = '2001:4860:4860::8888'
# create feed and test created feed
f = Feed(client).new(USER, FEED, description='build search test feed')
assert f['created_at']
# create test and submit test indicator
i = Indicator(
client, {
'user': USER,
'feed': FEED,
'indicator': INDICATOR,
'comment': 'this is a test comment'
})
r = i.submit()
# test creating the indicator
assert r['indicator']['indicator'] == INDICATOR
assert r['indicator']['itype'] == 'ipv6'
assert r['indicator']['created_at']
# search for indicator
s = Search(client)
r = s.search(INDICATOR, 10)
for record in r['feed']['indicators']:
if record['indicator']['feed'] == 'live-test-feed':
assert record['indicator']['indicator'] == INDICATOR
# delete test feed
f = Feed(client).remove(USER, FEED)
assert f == 200
def test_indicator_search_url(client):
INDICATOR = 'http://www.example.com/test/index.html'
# create feed and test created feed
f = Feed(client).new(USER, FEED, description='build search test feed')
assert f['created_at']
# create test and submit test indicator
i = Indicator(
client, {
'user': USER,
'feed': FEED,
'indicator': INDICATOR,
'comment': 'this is a test comment'
})
r = i.submit()
# test creating the indicator
assert r['indicator'] == INDICATOR
assert r['itype'] == 'uri'
assert r['created_at']
# search for indicator
s = Search(client)
r = s.search(INDICATOR, 10)
for record in r:
if record['feed'] == 'live-test-feed':
assert record['indicator'] == INDICATOR
# delete test feed
f = Feed(client).remove(USER, FEED)
assert f == 200
def test_create_feed(client):
# make sure feed isnt left over from a previous run
f = Feed(client).remove(USER, 'CI_BUILD_TEST')
f = Feed(client).new(USER, 'CI_BUILD_TEST', description='test build feed')
assert f['created_at']
f = Feed(client).remove(USER, 'CI_BUILD_TEST')
assert f == 200
def pull(self):
lineCount = 0
CSIRTGThreat = dict()
csirtgAll = dict()
csirtgIndicatorDict = dict()
indicatorCounter = 0
remote = 'https://csirtg.io/api'
remote2 = 'https://csirtg.io/api/users/csirtgadgets/feeds'
token = '3c6e9294747b06a841eadc8b8b2e73be'
token = '1c277786ae04719eb71d9e5d8e9c98be'
verify_ssl = True
user = '******'
feed = 'uce-urls'
count = 0
dataDict = dict()
# Initiate client object
cli = Client(remote=remote, token=token, verify_ssl=verify_ssl)
# Return a list of feeds (per user)
ret = Feed(cli).index(user)
# pprint the returned data structure
#pprint(ret)
try:
for feedItem in ret:
feed = feedItem['name']
ret = Feed(cli).show(user, feed, limit=None)
count += 1
dataDict[feed] = ret.copy()
print(" - Getting feed item: " + str(count) +
feedItem['name'])
#if count == 5:
# break
#pprint(dataDict)
print("copying all feeds into JSON")
js = json.dumps(dataDict)
fp = open("CSIRTG.JSON", "w")
fp.write(js)
fp.close()
self.readFile(feedItem['name'])
except:
print("ERROR: CSIRTG : too many requests in an hour")
def test_client_csirtg():
with Smrt(client='csirtg',
username=USERNAME,
feed='csirtg_smrt_test',
token=TOKEN) as s:
assert type(s) is Smrt
# create test feed
cli = CSIRTGClient(token=TOKEN)
f = Feed(cli).new(USERNAME, FEED)
assert f
assert f['user'] == USERNAME
x = s.process('test/smrt/rules/csirtg.yml', feed='port-scanners')
assert len(x) > 0
# remove test feed
f = Feed(cli).remove(USERNAME, FEED)
assert f == 200
def fetch(self, user, feed, limit=50):
data = Feed(self.handle).show(user, feed, limit=limit)
data = data['indicators']
for i in data:
i['provider'] = 'csirtg.io'
i['reference'] = 'https://csirtg.io/users/%s/feeds/%s' % (user,
feed)
i['tlp'] = i.get('tlp', 'green')
return data
def test_indicator_attachment_docx(client):
f = Feed(client).new(USER, FEED, description='test build feed')
assert f['created_at']
i = Indicator(
client, {
'user': USER,
'feed': FEED,
'attachment': 'samples/c..docx',
'comment': 'asdfasdfasdf'
})
r = i.submit()
assert r['indicator'] == 'c2642e519c7f325300ed250710b4f815ac542c1d'
assert r['attachments'][0]['attachment']
assert r['attachments'][0]['filesize']
assert r['attachments'][0]['created_at']
# delete test feed
f = Feed(client).remove(USER, FEED)
assert f == 200
def test_indicator_attachment_jar(client):
f = Feed(client).new(USER, FEED, description='test build feed')
assert f['created_at']
i = Indicator(
client, {
'user': USER,
'feed': FEED,
'attachment': 'samples/malware.jar',
'comment': 'asdfasdfasdf'
})
r = i.submit()
assert r['indicator'] == '8ab0079d8e80c2e166b3b12364c89255d79c9f75'
assert r['attachments'][0]['attachment']
assert r['attachments'][0]['filesize']
assert r['attachments'][0]['created_at']
# delete test feed
f = Feed(client).remove(USER, FEED)
assert f == 200
def test_indicator_attachment_zip(client):
f = Feed(client).new(USER, FEED, description='test build feed')
assert f['created_at']
i = Indicator(
client, {
'user': USER,
'feed': FEED,
'attachment': 'samples/malware.jar.zip',
'comment': 'asdfasdfasdf'
})
r = i.submit()
assert r['indicator'] == 'f31e226048d9bd45513e691a50a4b83893397235'
assert r['attachments'][0]['attachment']
assert r['attachments'][0]['filesize']
assert r['attachments'][0]['created_at']
# delete test feed
f = Feed(client).remove(USER, FEED)
assert f == 200
def test_indicator_attachment_txt(client):
f = Feed(client).new(USER, FEED, description='test build feed')
assert f['created_at']
i = Indicator(
client, {
'user': USER,
'feed': FEED,
'attachment': 'samples/message.eml',
'comment': 'asdfasdfasdf'
})
r = i.submit()
assert r['indicator'] == '2f9496a6331b2e75e5208b93d144e8fe484b316a'
assert r['attachments'][0]['attachment']
assert r['attachments'][0]['filesize']
assert r['attachments'][0]['created_at']
# delete test feed
f = Feed(client).remove(USER, FEED)
assert f == 200
def test_indicator_attachment_pdf(client):
f = Feed(client).new(USER, FEED, description='test build feed')
assert f['created_at']
i = Indicator(
client, {
'user': USER,
'feed': FEED,
'attachment': 'samples/hello_world.pdf',
'comment': 'asdfasdfasdf'
})
r = i.submit()
assert r['indicator'] == 'cc9881dc27a8d3e410cdf7e667ff5efa5cbfdaed'
assert r['attachments'][0]['attachment']
assert r['attachments'][0]['filesize']
assert r['attachments'][0]['created_at']
# delete test feed
f = Feed(client).remove(USER, FEED)
assert f == 200
def test_indicator_attachment_doc(client):
f = Feed(client).new(USER, FEED, description='test build feed')
assert f['created_at']
i = Indicator(
client, {
'user': USER,
'feed': FEED,
'attachment': 'samples/business_relationship.doc',
'comment': 'asdfasdfasdf'
})
r = i.submit()
assert r['indicator'] == 'f0ee0d5a1279fbdd93a9c5b9a1377894113f0ec0'
assert r['attachments'][0]['attachment']
assert r['attachments'][0]['filesize']
assert r['attachments'][0]['created_at']
# delete test feed
f = Feed(client).remove(USER, FEED)
assert f == 200
def test_feed(client):
f = Feed(client)
assert f.client
def main():
parser = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ csirtg --search example.com
$ csirtg --user csirtgadgets --feeds
$ csirtg --user csirtgadgets --feed uce-urls
$ csirtg --user csirtgadgets --feed-new scanners --description 'a feed of port scanners'
$ csirtg --user csirtgadgets --feed scanners --indicator-new 1.1.1.1 --tags scanner --comment
'this is a port scanner'
$ csirtg --user csirtgadgets --feed uce-attachments --new --attachment 'fax.zip'
--description 'file attached in uce email'
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg')
parser.add_argument("-v",
"--verbose",
action="count",
help="set verbosity level [default: %(default)s]")
parser.add_argument('-d', '--debug', action="store_true")
parser.add_argument('--token', help="specify token", default=TOKEN)
parser.add_argument('-l',
'--limit',
help="specify results limit [default: %(default)s]",
default=LIMIT)
parser.add_argument('--remote',
help="remote api location [default: %(default)s]",
default=REMOTE)
parser.add_argument('--timeout',
help='connection timeout [default: %(default)s]',
default=TIMEOUT)
parser.add_argument('-C',
'--config',
help="configuration file [default: %(default)s]",
default=os.path.expanduser("~/.csirtg.yml")) # env var
parser.add_argument('--format',
help="specify an output format [default: %(default)s]",
default='table')
parser.add_argument('--columns',
help='specify output columns [default %(default)s]',
default=','.join(COLUMNS))
# actions
parser.add_argument('-q', '--search', help="search for an indicator")
parser.add_argument('--feeds',
action="store_true",
help="show a list of feeds (per user)")
parser.add_argument('--new',
action='store_true',
help="create a new feed or indicator")
parser.add_argument('--predict',
help="test an indicator against the prediction api")
parser.add_argument('--predict-stdin', action="store_true")
# vars
parser.add_argument('--user', help="specify a user")
parser.add_argument('--feed', help="specify feed name")
parser.add_argument('--feed-new', help='help specify a new feed')
parser.add_argument('--indicator',
dest='indicator',
help="specify an indicator [eg: 1.2.3.4, example.com, "
"http://example.com/1.html")
parser.add_argument('--indicator-new', help='create a new indicator')
parser.add_argument('--tags', help="specify tags")
parser.add_argument('--comment', help="enter a comment for the indicator")
parser.add_argument('--description', help="specify a feed description")
parser.add_argument('--portlist',
help="specify a portlist [eg: 22,23-35,443]")
parser.add_argument('--protocol', help="specify TCP, UDP or ICMP")
parser.add_argument(
'--firsttime',
help="timestamp when first seen [eg: 2015-11-23T00:00:00Z]")
parser.add_argument(
'--lasttime',
help=
"timestamp when last seen [eg: 2015-11-24T00:00:00Z], treated as 'greater "
"than'")
parser.add_argument('--attachment',
help="specify an attachment [eg: /path/to/file]")
parser.add_argument('--attachment-name',
help="specify the attachment filename")
parser.add_argument('--no-verify-ssl',
help='Turn TLS verification OFF',
action="store_true")
parser.add_argument('--sinkhole', action='store_true')
parser.add_argument(
'--threatmodel',
'--tm',
help='predict using a threatmodel (eg: wes/phishing-urls)')
# Process arguments
args = parser.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
if args.verbose:
logger.setLevel(logging.INFO)
if args.debug:
logger.setLevel(logging.DEBUG)
options = vars(args)
o = read_config(args)
options.update(o)
verify_ssl = True
if options.get('no_verify_ssl'):
verify_ssl = False
cli = Client(remote=options['remote'],
token=options['token'],
verify_ssl=verify_ssl)
if options.get('sinkhole'):
import sys
if not sys.stdin.isatty():
stdin = sys.stdin.read()
else:
logger.error("No data passed via STDIN")
raise SystemExit
from csirtgsdk.sinkhole import Sinkhole
ret = Sinkhole(cli).post(stdin)
if ret.get('status') == 'unauthorized':
logger.error('unauthorized')
else:
if logger.getEffectiveLevel() == logging.DEBUG:
print("Predictions: \n")
for p in ret.get('predictions', []):
print("\t%s" % p)
raise SystemExit
if options.get('threatmodel'):
try:
from csirtgsdk.threatmodel import Threatmodel
except:
print("")
print(
"Missing some extra machine learning libraries required to make this work.."
)
print("$ pip install numpy keras pandas tensorflow")
print("")
raise SystemExit
user, name = options['threatmodel'].split('/')
m = Threatmodel(cli).show(user, name, options['search'])
print("Prediction: %f" % (m * 100))
raise SystemExit
if options.get('predict') or options.get('predict_stdin'):
import sys
if options.get('predict'):
logger.info("Searching for: {0}".format(options.get('predict')))
predict = options['predict']
ret = Predict(cli).get(predict)
print("Prediction Score: %s - %s" % (ret, predict))
else:
import sys
if not sys.stdin.isatty():
for predict in sys.stdin.read().split("\n"):
if predict == '':
continue
ret = Predict(cli).get(predict)
print("Prediction Score: %s - %s" % (ret, predict))
else:
logger.error("No data passed via STDIN")
raise SystemExit
raise SystemExit
if options.get('search'):
logger.info("Searching for: {0}".format(options.get('search')))
ret = Search(cli).search(options.get('search'), limit=options['limit'])
if isinstance(ret, dict):
ret = [ret]
print(FORMATS[options.get('format')](data=ret,
cols=args.columns.split(',')))
raise SystemExit
if options.get('indicator_new') or options.get('attachment'):
options['indicator'] = options['indicator_new']
logger.info("Creating indicator in feed {0} for user {1}".format(
options['feed'], options['user']))
ret = Indicator(cli, options).submit()
logger.info('posted: {0}'.format(ret['location']))
if isinstance(ret, dict):
ret = [ret]
print(FORMATS[options.get('format')](data=ret,
cols=args.columns.split(',')))
raise SystemExit
if options.get('feeds'):
logger.info("Searching feeds for user: {0}".format(options['user']))
f = Feed(cli)
feeds = f.index(options['user'])
for l in f.get_lines(feeds):
print(l)
raise SystemExit
if options.get('feed_new'):
if not options.get('user'):
parser.error('--user is required')
if not options.get('description'):
parser.error('--description is required')
logger.info("Creating feed {0} for user {1}".format(
options['feed_new'], options['user']))
f = Feed(cli)
feed = f.new(options['user'],
options['feed_new'],
description=options['description'])
for l in f.get_lines(feed):
print(l)
raise SystemExit
if options.get('feed'):
if not options.get('user'):
parser.error('--user is required')
logger.info("Fetching feed {0} for user {1}".format(
options['feed'], options['user']))
data = Feed(cli).show(
options['user'],
options['feed'],
limit=options['limit'],
lasttime=options['lasttime'],
)
if data.get('indicators'):
print(FORMATS[options.get('format')](data=data['indicators'],
cols=args.columns.split(',')))
def main():
parser = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ csirtg --search example.com
$ csirtg --user csirtgadgets --feeds
$ csirtg --user csirtgadgets --feed uce-urls
$ csirtg --user csirtgadgets --new --feed scanners --description 'a feed of port scanners'
$ csirtg --user csirtgadgets --feed scanners --new --indicator 1.1.1.1 --tags scanner --comment
'this is a port scanner'
$ csirtg --user csirtgadgets --feed uce-attachments --new --attachment 'fax.zip'
--description 'file attached in uce email'
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg')
parser.add_argument("-v",
"--verbose",
action="count",
help="set verbosity level [default: %(default)s]")
parser.add_argument('-d', '--debug', action="store_true")
parser.add_argument('--token', help="specify token", default=TOKEN)
parser.add_argument('-l',
'--limit',
help="specify results limit [default: %(default)s]",
default=LIMIT)
parser.add_argument('--remote',
help="remote api location [default: %(default)s]",
default=REMOTE)
parser.add_argument('--timeout',
help='connection timeout [default: %(default)s]',
default=TIMEOUT)
parser.add_argument('-C',
'--config',
help="configuration file [default: %(default)s]",
default=os.path.expanduser("~/.csirtg.yml")) # env var
parser.add_argument('--format',
help="specify an output format [default: %(default)s]",
default='table')
# actions
parser.add_argument('-q', '--search', help="search for an indicator")
parser.add_argument('--feeds',
action="store_true",
help="show a list of feeds (per user)")
parser.add_argument('--new',
action='store_true',
help="create a new feed or indicator")
# vars
parser.add_argument('--user', help="specify a user")
parser.add_argument('--feed', help="specify feed name")
parser.add_argument('--indicator',
dest='indicator',
help="specify an indicator [eg: 1.2.3.4, example.com, "
"http://example.com/1.html")
parser.add_argument('--tags', help="specify tags")
parser.add_argument('--comment', help="enter a comment for the indicator")
parser.add_argument('--description', help="specify a feed description")
parser.add_argument('--portlist',
help="specify a portlist [eg: 22,23-35,443]")
parser.add_argument('--protocol', help="specify TCP, UDP or ICMP")
parser.add_argument(
'--firsttime',
help="timestamp when first seen [eg: 2015-11-23T00:00:00Z]")
parser.add_argument(
'--lasttime',
help=
"timestamp when last seen [eg: 2015-11-24T00:00:00Z], treated as 'greater "
"than'")
parser.add_argument('--attachment',
help="specify an attachment [eg: /path/to/file]")
parser.add_argument('--attachment-name',
help="specify the attachment filename")
parser.add_argument('--no-verify-ssl',
help='Turn TLS verification OFF',
action="store_true")
# Process arguments
args = parser.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
## TODO -- yconf ?
options = vars(args)
o = read_config(args)
options.update(o)
verify_ssl = True
if options.get('no_verify_ssl'):
verify_ssl = False
cli = Client(remote=options['remote'],
token=options['token'],
verify_ssl=verify_ssl)
if options.get('search'):
logger.info("Searching for: {0}".format(options.get('search')))
ret = Search(cli).search(options.get('search'), limit=options['limit'])
format = format_factory(options['format'])
format(ret).write()
logger.info("Done")
elif options.get('feeds'):
logger.info("Searching feeds for user: {0}".format(options['user']))
feeds = Feed(cli).index(options['user'])
from prettytable import PrettyTable
cols = ['name', 'description', 'license', 'updated_at']
t = PrettyTable(cols)
for f in feeds:
r = []
for c in cols:
y = f['feed'].get(c)
if c == 'license':
y = y['name']
y = str(y)
y = (y[:30] + '..') if len(y) > 30 else y
r.append(y)
t.add_row(r)
print(str(t))
logger.info("Done")
elif options.get('feed') and options.get('new') and (
not options.get('indicator') and not options.get('attachment')):
if not options.get('user'):
parser.error('--user is required')
logger.info("Creating feed {0} for user {1}".format(
options['feed'], options['user']))
feed = Feed(cli).new(options['user'],
options['feed'],
description=options['description'])
from prettytable import PrettyTable
cols = ['name', 'description', 'license', 'updated_at']
t = PrettyTable(cols)
r = []
for c in cols:
y = feed.get(c)
if c == 'license':
y = y['name']
y = str(y)
y = (y[:30] + '..') if len(y) > 30 else y
r.append(y)
t.add_row(r)
print(str(t))
logger.info("Done")
elif options.get('feed') and not options.get('new'):
if not options.get('user'):
parser.error('--user is required')
logger.info("Fetching feed {0} for user {1}".format(
options['feed'], options['user']))
data = Feed(cli).show(
options['user'],
options['feed'],
limit=options['limit'],
lasttime=options['lasttime'],
)
if data['feed'].get('indicators'):
format = format_factory(options['format'])
format(data).write()
logger.info("Done")
# submit new indicator
elif options.get('feed') and (
options.get('indicator') or
(options.get('attachment') and not options.get('indicator'))
) and options.get('new') and options.get('user'):
try:
logger.info("Creating indicator in feed {0} for user {1}".format(
options['feed'], options['user']))
ret = Indicator(cli, options).submit()
logger.info('posted: {0}'.format(ret['indicator']['location']))
ret = {'feed': {'indicators': [ret]}}
format = format_factory(options['format'])
format(ret).write()
logger.info("Done")
except RuntimeError as e:
logger.error(e)