def test_indicator_search_ipv6(client):
INDICATOR = '2001:4860:4860::8888'
# create feed and test created feed
f = Feed(client).new(USER, FEED, description='build search test feed')
assert f['created_at']
# create test and submit test indicator
i = Indicator(
client, {
'user': USER,
'feed': FEED,
'indicator': INDICATOR,
'comment': 'this is a test comment'
})
r = i.submit()
# test creating the indicator
assert r['indicator']['indicator'] == INDICATOR
assert r['indicator']['itype'] == 'ipv6'
assert r['indicator']['created_at']
# search for indicator
s = Search(client)
r = s.search(INDICATOR, 10)
for record in r['feed']['indicators']:
if record['indicator']['feed'] == 'live-test-feed':
assert record['indicator']['indicator'] == INDICATOR
# delete test feed
f = Feed(client).remove(USER, FEED)
assert f == 200
def test_indicator_search_url(client):
INDICATOR = 'http://www.example.com/test/index.html'
# create feed and test created feed
f = Feed(client).new(USER, FEED, description='build search test feed')
assert f['created_at']
# create test and submit test indicator
i = Indicator(
client, {
'user': USER,
'feed': FEED,
'indicator': INDICATOR,
'comment': 'this is a test comment'
})
r = i.submit()
# test creating the indicator
assert r['indicator'] == INDICATOR
assert r['itype'] == 'uri'
assert r['created_at']
# search for indicator
s = Search(client)
r = s.search(INDICATOR, 10)
for record in r:
if record['feed'] == 'live-test-feed':
assert record['indicator'] == INDICATOR
# delete test feed
f = Feed(client).remove(USER, FEED)
assert f == 200
def run(self):
try:
client = Client(remote='https://csirtg.io/api', token=self.api_key)
search = Search(client)
data = search.search(self.artifact['name'])
if len(data['feed']['indicators']) > 0:
self.artifact['data']['csirtg'] = search.search(self.artifact['name'])
except:
pass
def run(self):
try:
client = Client(remote='https://csirtg.io/api', token=self.api_key)
search = Search(client)
data = search.search(self.artifact['name'])
if len(data['feed']['indicators']) > 0:
self.artifact['data']['csirtg'] = search.search(
self.artifact['name'])
except:
pass
def run(self):
try:
client = Client(remote='https://csirtg.io/api', token=self.api_key)
search = Search(client)
data = search.search(self.artifact['name'])
if len(data['feed']['indicators']) > 0:
self.artifact['data']['csirtg'] = search.search(
self.artifact['name'])
except Exception as err:
warning('Caught exception in module (%s)' % str(err))
def csirtg(self):
try:
from csirtgsdk.search import Search
except ImportError:
print('')
print('The csirtg function requires the csirtgsdk')
print('$ pip install csirtgsdk')
print('$ export CSIRTG_TOKEN=1234...')
print('')
raise SystemExit
return Search().search(self.indicator, limit=5)
def test_search(client):
f = Search(client)
assert f.client
def main():
parser = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ csirtg --search example.com
$ csirtg --user csirtgadgets --feeds
$ csirtg --user csirtgadgets --feed uce-urls
$ csirtg --user csirtgadgets --feed-new scanners --description 'a feed of port scanners'
$ csirtg --user csirtgadgets --feed scanners --indicator-new 1.1.1.1 --tags scanner --comment
'this is a port scanner'
$ csirtg --user csirtgadgets --feed uce-attachments --new --attachment 'fax.zip'
--description 'file attached in uce email'
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg')
parser.add_argument("-v",
"--verbose",
action="count",
help="set verbosity level [default: %(default)s]")
parser.add_argument('-d', '--debug', action="store_true")
parser.add_argument('--token', help="specify token", default=TOKEN)
parser.add_argument('-l',
'--limit',
help="specify results limit [default: %(default)s]",
default=LIMIT)
parser.add_argument('--remote',
help="remote api location [default: %(default)s]",
default=REMOTE)
parser.add_argument('--timeout',
help='connection timeout [default: %(default)s]',
default=TIMEOUT)
parser.add_argument('-C',
'--config',
help="configuration file [default: %(default)s]",
default=os.path.expanduser("~/.csirtg.yml")) # env var
parser.add_argument('--format',
help="specify an output format [default: %(default)s]",
default='table')
parser.add_argument('--columns',
help='specify output columns [default %(default)s]',
default=','.join(COLUMNS))
# actions
parser.add_argument('-q', '--search', help="search for an indicator")
parser.add_argument('--feeds',
action="store_true",
help="show a list of feeds (per user)")
parser.add_argument('--new',
action='store_true',
help="create a new feed or indicator")
parser.add_argument('--predict',
help="test an indicator against the prediction api")
parser.add_argument('--predict-stdin', action="store_true")
# vars
parser.add_argument('--user', help="specify a user")
parser.add_argument('--feed', help="specify feed name")
parser.add_argument('--feed-new', help='help specify a new feed')
parser.add_argument('--indicator',
dest='indicator',
help="specify an indicator [eg: 1.2.3.4, example.com, "
"http://example.com/1.html")
parser.add_argument('--indicator-new', help='create a new indicator')
parser.add_argument('--tags', help="specify tags")
parser.add_argument('--comment', help="enter a comment for the indicator")
parser.add_argument('--description', help="specify a feed description")
parser.add_argument('--portlist',
help="specify a portlist [eg: 22,23-35,443]")
parser.add_argument('--protocol', help="specify TCP, UDP or ICMP")
parser.add_argument(
'--firsttime',
help="timestamp when first seen [eg: 2015-11-23T00:00:00Z]")
parser.add_argument(
'--lasttime',
help=
"timestamp when last seen [eg: 2015-11-24T00:00:00Z], treated as 'greater "
"than'")
parser.add_argument('--attachment',
help="specify an attachment [eg: /path/to/file]")
parser.add_argument('--attachment-name',
help="specify the attachment filename")
parser.add_argument('--no-verify-ssl',
help='Turn TLS verification OFF',
action="store_true")
parser.add_argument('--sinkhole', action='store_true')
parser.add_argument(
'--threatmodel',
'--tm',
help='predict using a threatmodel (eg: wes/phishing-urls)')
# Process arguments
args = parser.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
if args.verbose:
logger.setLevel(logging.INFO)
if args.debug:
logger.setLevel(logging.DEBUG)
options = vars(args)
o = read_config(args)
options.update(o)
verify_ssl = True
if options.get('no_verify_ssl'):
verify_ssl = False
cli = Client(remote=options['remote'],
token=options['token'],
verify_ssl=verify_ssl)
if options.get('sinkhole'):
import sys
if not sys.stdin.isatty():
stdin = sys.stdin.read()
else:
logger.error("No data passed via STDIN")
raise SystemExit
from csirtgsdk.sinkhole import Sinkhole
ret = Sinkhole(cli).post(stdin)
if ret.get('status') == 'unauthorized':
logger.error('unauthorized')
else:
if logger.getEffectiveLevel() == logging.DEBUG:
print("Predictions: \n")
for p in ret.get('predictions', []):
print("\t%s" % p)
raise SystemExit
if options.get('threatmodel'):
try:
from csirtgsdk.threatmodel import Threatmodel
except:
print("")
print(
"Missing some extra machine learning libraries required to make this work.."
)
print("$ pip install numpy keras pandas tensorflow")
print("")
raise SystemExit
user, name = options['threatmodel'].split('/')
m = Threatmodel(cli).show(user, name, options['search'])
print("Prediction: %f" % (m * 100))
raise SystemExit
if options.get('predict') or options.get('predict_stdin'):
import sys
if options.get('predict'):
logger.info("Searching for: {0}".format(options.get('predict')))
predict = options['predict']
ret = Predict(cli).get(predict)
print("Prediction Score: %s - %s" % (ret, predict))
else:
import sys
if not sys.stdin.isatty():
for predict in sys.stdin.read().split("\n"):
if predict == '':
continue
ret = Predict(cli).get(predict)
print("Prediction Score: %s - %s" % (ret, predict))
else:
logger.error("No data passed via STDIN")
raise SystemExit
raise SystemExit
if options.get('search'):
logger.info("Searching for: {0}".format(options.get('search')))
ret = Search(cli).search(options.get('search'), limit=options['limit'])
if isinstance(ret, dict):
ret = [ret]
print(FORMATS[options.get('format')](data=ret,
cols=args.columns.split(',')))
raise SystemExit
if options.get('indicator_new') or options.get('attachment'):
options['indicator'] = options['indicator_new']
logger.info("Creating indicator in feed {0} for user {1}".format(
options['feed'], options['user']))
ret = Indicator(cli, options).submit()
logger.info('posted: {0}'.format(ret['location']))
if isinstance(ret, dict):
ret = [ret]
print(FORMATS[options.get('format')](data=ret,
cols=args.columns.split(',')))
raise SystemExit
if options.get('feeds'):
logger.info("Searching feeds for user: {0}".format(options['user']))
f = Feed(cli)
feeds = f.index(options['user'])
for l in f.get_lines(feeds):
print(l)
raise SystemExit
if options.get('feed_new'):
if not options.get('user'):
parser.error('--user is required')
if not options.get('description'):
parser.error('--description is required')
logger.info("Creating feed {0} for user {1}".format(
options['feed_new'], options['user']))
f = Feed(cli)
feed = f.new(options['user'],
options['feed_new'],
description=options['description'])
for l in f.get_lines(feed):
print(l)
raise SystemExit
if options.get('feed'):
if not options.get('user'):
parser.error('--user is required')
logger.info("Fetching feed {0} for user {1}".format(
options['feed'], options['user']))
data = Feed(cli).show(
options['user'],
options['feed'],
limit=options['limit'],
lasttime=options['lasttime'],
)
if data.get('indicators'):
print(FORMATS[options.get('format')](data=data['indicators'],
cols=args.columns.split(',')))
def main():
parser = ArgumentParser(description=textwrap.dedent('''\
example usage:
$ csirtg --search example.com
$ csirtg --user csirtgadgets --feeds
$ csirtg --user csirtgadgets --feed uce-urls
$ csirtg --user csirtgadgets --new --feed scanners --description 'a feed of port scanners'
$ csirtg --user csirtgadgets --feed scanners --new --indicator 1.1.1.1 --tags scanner --comment
'this is a port scanner'
$ csirtg --user csirtgadgets --feed uce-attachments --new --attachment 'fax.zip'
--description 'file attached in uce email'
'''),
formatter_class=RawDescriptionHelpFormatter,
prog='csirtg')
parser.add_argument("-v",
"--verbose",
action="count",
help="set verbosity level [default: %(default)s]")
parser.add_argument('-d', '--debug', action="store_true")
parser.add_argument('--token', help="specify token", default=TOKEN)
parser.add_argument('-l',
'--limit',
help="specify results limit [default: %(default)s]",
default=LIMIT)
parser.add_argument('--remote',
help="remote api location [default: %(default)s]",
default=REMOTE)
parser.add_argument('--timeout',
help='connection timeout [default: %(default)s]',
default=TIMEOUT)
parser.add_argument('-C',
'--config',
help="configuration file [default: %(default)s]",
default=os.path.expanduser("~/.csirtg.yml")) # env var
parser.add_argument('--format',
help="specify an output format [default: %(default)s]",
default='table')
# actions
parser.add_argument('-q', '--search', help="search for an indicator")
parser.add_argument('--feeds',
action="store_true",
help="show a list of feeds (per user)")
parser.add_argument('--new',
action='store_true',
help="create a new feed or indicator")
# vars
parser.add_argument('--user', help="specify a user")
parser.add_argument('--feed', help="specify feed name")
parser.add_argument('--indicator',
dest='indicator',
help="specify an indicator [eg: 1.2.3.4, example.com, "
"http://example.com/1.html")
parser.add_argument('--tags', help="specify tags")
parser.add_argument('--comment', help="enter a comment for the indicator")
parser.add_argument('--description', help="specify a feed description")
parser.add_argument('--portlist',
help="specify a portlist [eg: 22,23-35,443]")
parser.add_argument('--protocol', help="specify TCP, UDP or ICMP")
parser.add_argument(
'--firsttime',
help="timestamp when first seen [eg: 2015-11-23T00:00:00Z]")
parser.add_argument(
'--lasttime',
help=
"timestamp when last seen [eg: 2015-11-24T00:00:00Z], treated as 'greater "
"than'")
parser.add_argument('--attachment',
help="specify an attachment [eg: /path/to/file]")
parser.add_argument('--attachment-name',
help="specify the attachment filename")
parser.add_argument('--no-verify-ssl',
help='Turn TLS verification OFF',
action="store_true")
# Process arguments
args = parser.parse_args()
setup_logging(args)
logger = logging.getLogger(__name__)
## TODO -- yconf ?
options = vars(args)
o = read_config(args)
options.update(o)
verify_ssl = True
if options.get('no_verify_ssl'):
verify_ssl = False
cli = Client(remote=options['remote'],
token=options['token'],
verify_ssl=verify_ssl)
if options.get('search'):
logger.info("Searching for: {0}".format(options.get('search')))
ret = Search(cli).search(options.get('search'), limit=options['limit'])
format = format_factory(options['format'])
format(ret).write()
logger.info("Done")
elif options.get('feeds'):
logger.info("Searching feeds for user: {0}".format(options['user']))
feeds = Feed(cli).index(options['user'])
from prettytable import PrettyTable
cols = ['name', 'description', 'license', 'updated_at']
t = PrettyTable(cols)
for f in feeds:
r = []
for c in cols:
y = f['feed'].get(c)
if c == 'license':
y = y['name']
y = str(y)
y = (y[:30] + '..') if len(y) > 30 else y
r.append(y)
t.add_row(r)
print(str(t))
logger.info("Done")
elif options.get('feed') and options.get('new') and (
not options.get('indicator') and not options.get('attachment')):
if not options.get('user'):
parser.error('--user is required')
logger.info("Creating feed {0} for user {1}".format(
options['feed'], options['user']))
feed = Feed(cli).new(options['user'],
options['feed'],
description=options['description'])
from prettytable import PrettyTable
cols = ['name', 'description', 'license', 'updated_at']
t = PrettyTable(cols)
r = []
for c in cols:
y = feed.get(c)
if c == 'license':
y = y['name']
y = str(y)
y = (y[:30] + '..') if len(y) > 30 else y
r.append(y)
t.add_row(r)
print(str(t))
logger.info("Done")
elif options.get('feed') and not options.get('new'):
if not options.get('user'):
parser.error('--user is required')
logger.info("Fetching feed {0} for user {1}".format(
options['feed'], options['user']))
data = Feed(cli).show(
options['user'],
options['feed'],
limit=options['limit'],
lasttime=options['lasttime'],
)
if data['feed'].get('indicators'):
format = format_factory(options['format'])
format(data).write()
logger.info("Done")
# submit new indicator
elif options.get('feed') and (
options.get('indicator') or
(options.get('attachment') and not options.get('indicator'))
) and options.get('new') and options.get('user'):
try:
logger.info("Creating indicator in feed {0} for user {1}".format(
options['feed'], options['user']))
ret = Indicator(cli, options).submit()
logger.info('posted: {0}'.format(ret['indicator']['location']))
ret = {'feed': {'indicators': [ret]}}
format = format_factory(options['format'])
format(ret).write()
logger.info("Done")
except RuntimeError as e:
logger.error(e)