def get_configs():
config = read_cfg(args.config)
# Allow inline or external Warden config
wconfig = config.get("warden", "warden_client.cfg")
if isinstance(wconfig, basestring):
wconfig = read_cfg(wconfig)
fconfig = config.get(args.func, {})
return wconfig, fconfig
def get_configs():
config = read_cfg(args.config)
# Allow inline or external Warden config
wconfig = config.get("warden", "warden_client.cfg")
if isinstance(wconfig, basestring):
wconfig = read_cfg(wconfig)
fconfig = config.get(args.func, {})
return wconfig, fconfig
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--count')
args = parser.parse_args()
count = 100
if args.count:
count = int(args.count)
wclient = Client(**read_cfg("warden_client_tester.cfg"))
# Also inline arguments are possible:
# wclient = Client(
# url = 'https://warden.example.com/warden3',
# keyfile = '/opt/warden3/etc/key.pem',
# certfile = '/opt/warden3/etc/cert.pem',
# cafile = '/opt/warden3/etc/tcs-ca-bundle.pem',
# timeout=10,
# errlog={"level": "debug"},
# filelog={"level": "debug"},
# idstore="MyClient.id",
# name="cz.example.warden.test")
#info = wclient.getDebug()
#wclient.logger.debug(info)
#info = wclient.getInfo()
#wclient.logger.info(info)
#wclient.logger.debug("Sending %d event(s)" % count)
start = time()
ret = wclient.sendEvents([gen_random_idea(client_name=wclient.name) for i in range(count)])
ret['time'] = (time()-start)
wclient.logger.info(ret)
def Run(module_name, module_desc, req_type, req_format, conv_func, arg_parser = None):
"""Run the main loop of the reporter module called `module_name` with `module_desc` (used in help).
The module requires data format of `req_type` type and `req_format` specifier - these must be given by author of the module.
`conv_func(rec, args)` is a callback function that must translate given incoming alert `rec` (typically in UniRec according to `req_type`) into IDEA message. `args` contains CLI arguments parsed by ArgumentParser. `conv_func` must return dict().
"""
global trap
# *** Parse command-line arguments ***
if arg_parser is None:
arg_parser = argparse.ArgumentParser()
arg_parser.formatter_class = argparse.RawDescriptionHelpFormatter
# Set description
arg_parser.description = str.format(desc_template,
name=module_name,
type={pytrap.FMT_RAW:'raw', pytrap.FMT_UNIREC:'UniRec', pytrap.FMT_JSON:'JSON'}.get(req_type,'???'),
fmt=req_format,
original_desc = module_desc+"\n\n " if module_desc else "",
)
# Add arguments defining outputs
# TRAP output
arg_parser.add_argument('-T', '--trap', action='store_true',
help='Enable output via TRAP interface (JSON type with format id "IDEA"). Parameters are set using "-i" option as usual.')
# Config file
arg_parser.add_argument('-c', '--config', metavar="FILE", default="./config.yaml", type=str,
help='Specify YAML config file path which to load.')
arg_parser.add_argument('-d', '--dry', action='store_true',
help="""Do not run, just print loaded config.""")
# Warden3 output
arg_parser.add_argument('-W', '--warden', metavar="CONFIG_FILE",
help='Send IDEA messages to Warden server. Load configuration of Warden client from CONFIG_FILE.')
# Other options
arg_parser.add_argument('-n', '--name', metavar='NODE_NAME',
help='Name of the node, filled into "Node.Name" element of the IDEA message. Required argument.')
arg_parser.add_argument('-v', '--verbose', metavar='VERBOSE_LEVEL', default=3, type=int,
help="""Enable verbose mode (may be used by some modules, common part doesn't print anything).\nLevel 1 logs everything, level 5 only critical errors. Level 0 doesn't log.""")
# TRAP parameters
trap_args = arg_parser.add_argument_group('Common TRAP parameters')
trap_args.add_argument('-i', metavar="IFC_SPEC", required=True,
help='See http://nemea.liberouter.org/trap-ifcspec/ for more information.')
# Parse arguments
args = arg_parser.parse_args()
# Set log level
logging.basicConfig(level=(args.verbose*10), format=FORMAT)
# Check if node name is set if Warden output is enabled
if args.name is None:
#if args.warden:
# sys.stderr.write(module_name+": Error: Node name must be specified if Warden output is used (set param --name).\n")
# exit(1)
logger.warning("Node name is not specified.")
# *** Initialize TRAP ***
logger.info("Trap arguments: %s", args.i)
trap.init(["-i", args.i], 1, 1 if args.trap else 0)
#trap.setVerboseLevel(3)
signal.signal(signal.SIGINT, signal_h)
# Set required input format
trap.setRequiredFmt(0, req_type, req_format)
if args.trap:
trap.setDataFmt(0, pytrap.FMT_JSON, "IDEA")
# *** Create output handles/clients/etc ***
wardenclient = None
if args.warden:
try:
import warden_client
except:
logger.error("There is no available warden_client python module. Install it or remove '--warden' from the module's arguments.")
sys.exit(1)
config = warden_client.read_cfg(args.warden)
config['name'] = args.name
wardenclient = warden_client.Client(**config)
# Initialize configuration
config = Config.Config(args.config, trap = trap, warden = wardenclient)
if not args.dry:
# *** Main loop ***
URInputTmplt = None
if req_type == pytrap.FMT_UNIREC and req_format != "":
URInputTmplt = pytrap.UnirecTemplate(req_format) # TRAP expects us to have predefined template for required set of fields
rec = URInputTmplt
stop = False
while not stop:
logger.info("Starting receiving")
# *** Read data from input interface ***
try:
data = trap.recv()
except pytrap.FormatMismatch:
logger.error("Input data format mismatch in receiving from TRAP interface")
break
except pytrap.FormatChanged as e:
# Get negotiated input data format
(fmttype, fmtspec) = trap.getDataFmt(0)
# If data type is UniRec, create UniRec template
if fmttype == pytrap.FMT_UNIREC:
URInputTmplt = pytrap.UnirecTemplate(fmtspec)
else:
URInputTmplt = None
rec = URInputTmplt
data = e.data
except pytrap.Terminated:
break
# Check for "end-of-stream" record
if len(data) <= 1:
if args.trap:
# If we have output, send "end-of-stream" record and exit
trap.send(0, b"0")
break
# Assert that if UniRec input is required, input template is set
assert(req_type != pytrap.FMT_UNIREC or URInputTmplt is not None)
# Convert raw input data to UniRec object (if UniRec input is expected)
if req_type == pytrap.FMT_UNIREC:
rec.setData(data)
elif req_type == pytrap.FMT_JSON:
rec = json.loads(str(data))
else: # TRAP_FMT_RAW
rec = data
# *** Convert input record to IDEA ***
# Pass the input record to conversion function to create IDEA message
idea = conv_func(rec, args)
if idea is None:
# Record can't be converted - skip it
continue
if args.name is not None:
idea['Node'][0]['Name'] = args.name
# *** Send IDEA to outputs ***
# Perform rule matching and action running on the idea message
try:
config.match(idea)
except pytrap.Terminated:
logger.error("PyTrap was terminated")
break
except DropMsg:
logger.info("Message was dropped by Drop action.")
continue
except Exception as e:
logger.error(str(e))
break
else:
# DRY argument given, just print config and exit
print(config)
if wardenclient:
wardenclient.close()
trap.finalize()
def main():
wclient = Client(**read_cfg("warden_client.cfg"))
# Also inline arguments are possible:
# wclient = Client(
# url = 'https://warden.example.com/warden3',
# keyfile = '/opt/warden3/etc/key.pem',
# certfile = '/opt/warden3/etc/cert.pem',
# cafile = '/opt/warden3/etc/tcs-ca-bundle.pem',
# timeout=10,
# errlog={"level": "debug"},
# filelog={"level": "debug"},
# idstore="MyClient.id",
# name="cz.example.warden.test")
print("=== Debug ===")
info = wclient.getDebug()
pprint(info)
# All methods return something.
# If you want to catch possible errors (for example implement some
# form of persistent retry, or save failed events for later, you may
# check for Error instance and act based on contained info.
# If you want just to be informed, this is not necessary, just
# configure logging correctly and check logs.
if isinstance(info, Error):
print(info)
print("=== Server info ===")
info = wclient.getInfo()
print("=== Sending 10 event(s) ===")
start = time()
ret = wclient.sendEvents(
[gen_random_idea(client_name=wclient.name) for i in range(10)])
print(ret)
print("Time: %f" % (time() - start))
print("=== Getting 10 events ===")
start = time()
# cat = ['Availability', 'Abusive.Spam','Attempt.Login']
# cat = ['Attempt', 'Information','Fraud.Scam','Malware.Virus']
# cat = ['Fraud', 'Abusive.Spam']
# nocat = ['Availability', 'Information', 'Fraud.Scam']
cat = []
nocat = []
#tag = ['Log', 'Data']
#notag = ['Flow', 'Datagram']
tag = []
notag = []
#group = ['cz.tul.ward.kippo','cz.vsb.buldog.kippo']
#nogroup = ['cz.zcu.civ.afrodita','cz.vutbr.net.bee.hpscan']
group = []
nogroup = []
ret = wclient.getEvents(count=10,
cat=cat,
nocat=nocat,
tag=tag,
notag=notag,
group=group,
nogroup=nogroup)
print("Time: %f" % (time() - start))
print("Got %i events" % len(ret))
for e in ret:
print(e.get("Category"),
e.get("Node")[0].get("Type"),
e.get("Node")[0].get("Name"))
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
from warden_client import read_cfg, format_timestamp
from twisted.conch.telnet import TelnetTransport, StatefulTelnetProtocol
from twisted.internet.protocol import ServerFactory
from twisted.internet import reactor
from time import sleep
import json
import warden_utils_flab as w3u
import os
hconfig = read_cfg('telnetd.cfg')
logger = w3u.getLogger(hconfig['logfile'])
class telnetd(StatefulTelnetProtocol, object):
DEF_USER = "******"
DEF_PASS = "******"
DEF_AUTOTEST_USER = "******"
DEF_AUTOTEST_PASS = "******"
MAX_LOGIN_COUNT = 3
LOGIN_DELAY = 1
PROMPT = "/ # "
NEWLINE = "\n"
MSG_PRELOGIN = "******"
MSG_BADPASS = "******"
state = 'User'
logcount = 0
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2011-2015 Cesnet z.s.p.o
# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
from warden_client import Client, Error, read_cfg, format_timestamp
import json
from time import time, gmtime, strftime, sleep
from uuid import uuid4
import sqlite3
import warden_utils_flab as w3u
aconfig = read_cfg('warden_client_glastopf.cfg')
wconfig = read_cfg('warden_client.cfg')
aclient_name = aconfig['name']
wconfig['name'] = aclient_name
aanonymised = aconfig['anonymised']
aanonymised_net = aconfig['target_net']
aanonymised = aanonymised if (aanonymised_net != '0.0.0.0/0') or (aanonymised_net == 'omit') else '0.0.0.0/0'
awin = aconfig['awin'] * 60
aconattempts = aconfig['con_attempts']
aretryinterval = aconfig['con_retry_interval']
adbfile = aconfig['dbfile']
wclient = Client(**wconfig)
def gen_event_idea_gl(detect_time, src_ip, src_port, request_url, request_raw, pattern, filename):
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
from warden_client import Client, Error, read_cfg, format_timestamp
from time import time, gmtime, strftime
from math import trunc
from uuid import uuid4
import json
import string
import os
import sys
import warden_utils_flab as w3u
aconfig = read_cfg('warden_client_uchotcp.cfg')
wconfig = read_cfg('warden_client.cfg')
aclient_name = aconfig['name']
wconfig['name'] = aclient_name
aanonymised = aconfig['anonymised']
aanonymised_net = aconfig['target_net']
aanonymised = aanonymised if (aanonymised_net != '0.0.0.0/0') or (aanonymised_net == 'omit') else '0.0.0.0/0'
wclient = Client(**wconfig)
def gen_event_idea_uchotcp(detect_time, src_ip, src_port, dst_ip, dst_port, proto, decoded, smart, data):
event = {
"Format": "IDEA0",
"ID": str(uuid4()),
"DetectTime": detect_time,
"Category": ["Recon.Scanning", "Test"],
"Note": "Uchotcp event",
"ConnCount": 1,
#!/usr/bin/python
from warden_client import read_cfg, format_timestamp
from jinja2 import Environment, FileSystemLoader
import SimpleHTTPServer, SocketServer, logging, sys
import json
import os
import sys
import re
import base64
import warden_utils_flab as w3u
import mimetypes
hconfig = read_cfg('uchoweb.cfg')
content_base = os.path.join(os.getcwd(), "content/")
templates_base = os.path.join(os.getcwd(), "templates/")
port = hconfig.get('port', 8081)
personality = hconfig.get('personality', 'Apache Tomcat/7.0.56')
logger = w3u.getLogger(hconfig['logfile'])
class ServerHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
server_version = personality
sys_version = ""
def doAbort(self):
j2_env = Environment(loader=FileSystemLoader(templates_base), trim_blocks=True)
output = j2_env.get_template('404').render(path=self.path)
self.send_response(404)
self.end_headers()
def main():
wclient = Client(**read_cfg("warden_client.cfg"))
# Also inline arguments are possible:
# wclient = Client(
# url = 'https://warden.example.com/warden3',
# keyfile = '/opt/warden3/etc/key.pem',
# certfile = '/opt/warden3/etc/cert.pem',
# cafile = '/opt/warden3/etc/tcs-ca-bundle.pem',
# timeout=10,
# errlog={"level": "debug"},
# filelog={"level": "debug"},
# idstore="MyClient.id",
# name="cz.example.warden.test")
print "=== Debug ==="
info = wclient.getDebug()
pprint(info)
# All methods return something.
# If you want to catch possible errors (for example implement some
# form of persistent retry, or save failed events for later, you may
# check for Error instance and act based on contained info.
# If you want just to be informed, this is not necessary, just
# configure logging correctly and check logs.
if isinstance(info, Error):
print info
print "=== Server info ==="
info = wclient.getInfo()
print "=== Sending 10 event(s) ==="
start = time()
ret = wclient.sendEvents([gen_random_idea(client_name=wclient.name) for i in range(10)])
print ret
print "Time: %f" % (time()-start)
print "=== Getting 10 events ==="
start = time()
# cat = ['Availability', 'Abusive.Spam','Attempt.Login']
# cat = ['Attempt', 'Information','Fraud.Scam','Malware.Virus']
# cat = ['Fraud', 'Abusive.Spam']
# nocat = ['Availability', 'Information', 'Fraud.Scam']
cat = []
nocat = []
#tag = ['Log', 'Data']
#notag = ['Flow', 'Datagram']
tag = []
notag = []
#group = ['cz.tul.ward.kippo','cz.vsb.buldog.kippo']
#nogroup = ['cz.zcu.civ.afrodita','cz.vutbr.net.bee.hpscan']
group = []
nogroup = []
ret = wclient.getEvents(count=10, cat=cat, nocat=nocat, tag=tag, notag=notag, group=group, nogroup=nogroup)
print "Time: %f" % (time()-start)
print "Got %i events" % len(ret)
for e in ret:
print e.get("Category"), e.get("Node")[0].get("Type"), e.get("Node")[0].get("Name")
def Run(module_name,
module_desc,
req_type,
req_format,
conv_func,
arg_parser=None):
""" TODO doc
"""
# *** Parse command-line arguments ***
if arg_parser is None:
arg_parser = argparse.ArgumentParser()
arg_parser.formatter_class = argparse.RawDescriptionHelpFormatter
# Set description
arg_parser.description = str.format(
desc_template,
name=module_name,
type={
pytrap.FMT_RAW: 'raw',
pytrap.FMT_UNIREC: 'UniRec',
pytrap.FMT_JSON: 'JSON'
}.get(req_type, '???'),
fmt=req_format,
original_desc=module_desc + "\n\n " if module_desc else "",
)
# Add arguments defining outputs
# TRAP output
arg_parser.add_argument(
'--trap',
action='store_true',
help=
'Enable output via TRAP interface (JSON type with format id "IDEA"). Parameters are set using "-i" option as usual.'
)
# File output
arg_parser.add_argument(
'--file',
metavar="FILE",
type=str,
help=
'Enable output to file (each IDEA message printed to new line in JSON format). Set to "-" to use standard output.'
)
arg_parser.add_argument(
'--file-indent',
metavar="N",
type=int,
help='Pretty-format JSON in output file using N spaces for indentation.'
)
arg_parser.add_argument('--file-append',
action='store_true',
help='Append to file instead of overwrite.')
# MongoDB output
arg_parser.add_argument(
'--mongodb',
metavar="DBNAME",
help='Enable output to MongoDB. Connect to database named DBNAME.')
arg_parser.add_argument(
'--mongodb-coll',
metavar="COLL",
default='alerts',
help='Put IDEA messages into collection named COLL (default: "alerts").'
)
arg_parser.add_argument(
'--mongodb-host',
metavar="HOSTNAME",
default='localhost',
help='Connect to MongoDB running on HOSTNAME (default: "localhost").')
arg_parser.add_argument(
'--mongodb-port',
metavar="PORT",
type=int,
default=27017,
help='Connect to MongoDB running on port number PORT (default: 27017).'
)
# Warden3 output
arg_parser.add_argument(
'--warden',
metavar="CONFIG_FILE",
help=
'Send IDEA messages to Warden server. Load configuration of Warden client from CONFIG_FILE.'
)
# Other options
arg_parser.add_argument(
'-n',
'--name',
metavar='NODE_NAME',
help=
'Name of the node, filled into "Node.Name" element of the IDEA message. Required if Warden output is used, recommended otherwise.'
)
arg_parser.add_argument(
'--test',
action='store_true',
help='Add "Test" to "Category" before sending a message to output(s).')
arg_parser.add_argument(
'-v',
'--verbose',
action='store_true',
help=
"Enable verbose mode (may be used by some modules, common part donesn't print anything"
)
arg_parser.add_argument(
'--srcwhitelist-file',
metavar="FILE",
type=str,
help=
"File with addresses/subnets in format: <ip address>/<mask>,<data>\\n \n where /<mask>,<data> is optional, <data> is a user-specific optional content. Whitelist is applied to SRC_IP field. If SRC_IP from the alert is on whitelist, the alert IS NOT reported."
)
arg_parser.add_argument(
'--dstwhitelist-file',
metavar="FILE",
type=str,
help=
"File with addresses/subnets, whitelist is applied on DST_IP, see --srcwhitelist-file help."
)
# TRAP parameters
trap_args = arg_parser.add_argument_group('Common TRAP parameters')
trap_args.add_argument(
'-i',
metavar="IFC_SPEC",
required=True,
help='TODO (ideally this section should be added by TRAP')
# Parse arguments
args = arg_parser.parse_args()
# Check if at least one output is enabled
if not (args.file or args.trap or args.mongodb or args.warden):
sys.stderr.write(module_name +
": Error: At least one output must be selected\n")
exit(1)
# Check if node name is set if Warden output is enabled
if args.name is None:
if args.warden:
sys.stderr.write(
module_name +
": Error: Node name must be specified if Warden output is used (set param --name).\n"
)
exit(1)
else:
sys.stderr.write(module_name +
": Warning: Node name is not specified.\n")
# *** Initialize TRAP ***
trap = pytrap.TrapCtx()
trap.init(["-i", args.i], 1, 1 if args.trap else 0)
#trap.setVerboseLevel(3)
#trap.registerDefaultSignalHandler()
# Set required input format
trap.setRequiredFmt(0, req_type, req_format)
# If TRAP output is enabled, set output format (JSON, format id "IDEA")
if args.trap:
trap.setDataFmt(0, pytrap.FMT_JSON, "IDEA")
# *** Create output handles/clients/etc ***
filehandle = None
mongoclient = None
mongocoll = None
wardenclient = None
if args.file:
if args.file == '-':
filehandle = sys.stdout
else:
filehandle = open(args.file, "a" if args.file_append else "w")
if args.mongodb:
import pymongo
mongoclient = pymongo.MongoClient(args.mongodb_host, args.mongodb_port)
mongocoll = mongoclient[args.mongodb][args.mongodb_coll]
if args.warden:
import warden_client
try:
config = warden_client.read_cfg(args.warden)
except ValueError as e:
sys.stderr.write(
"{0}: Failed to load Warden config file '{1}'\n{2}\n".format(
module_name, args.warden, e))
exit(1)
config['name'] = args.name
wardenclient = warden_client.Client(**config)
# Check if a whitelist is set, parse the file and prepare context for binary search
import ip_prefix_search
if args.srcwhitelist_file:
if 'ipaddr SRC_IP' in req_format.split(","):
srcwhitelist = ip_prefix_search.IPPSContext.fromFile(
args.srcwhitelist_file)
else:
srcwhitelist = None
if args.dstwhitelist_file:
if 'ipaddr DST_IP' in req_format.split(","):
dstwhitelist = ip_prefix_search.IPPSContext.fromFile(
args.dstwhitelist_file)
else:
dstwhitelist = None
# *** Main loop ***
URInputTmplt = None
if req_type == pytrap.FMT_UNIREC and req_format != "":
URInputTmplt = pytrap.UnirecTemplate(
req_format
) # TRAP expects us to have predefined template for required set of fields
rec = URInputTmplt
stop = False
while not stop:
# *** Read data from input interface ***
try:
data = trap.recv()
except pytrap.FormatMismatch:
sys.stderr.write(
module_name + ": Error: input data format mismatch\n"
) #Required: "+str((req_type,req_format))+"\nReceived: "+str(trap.get_data_fmt(trap.IFC_INPUT, 0))+"\n")
break
except pytrap.FormatChanged as e:
# TODO: This should be handled by trap.recv transparently
# Get negotiated input data format
(fmttype, fmtspec) = trap.getDataFmt(0)
# If data type is UniRec, create UniRec template
if fmttype == pytrap.FMT_UNIREC:
URInputTmplt = pytrap.UnirecTemplate(fmtspec)
else:
URInputTmplt = None
rec = URInputTmplt
data = e.data
except pytrap.Terminated:
break
# Check for "end-of-stream" record
if len(data) <= 1:
# If we have output, send "end-of-stream" record and exit
if args.trap:
trap.send(0, b"0")
break
# Assert that if UniRec input is required, input template is set
assert (req_type != pytrap.FMT_UNIREC or URInputTmplt is not None)
# Convert raw input data to UniRec object (if UniRec input is expected)
if req_type == pytrap.FMT_UNIREC:
rec.setData(data)
elif req_type == pytrap.FMT_JSON:
rec = json.loads(data)
else: # TRAP_FMT_RAW
rec = data
# Check whitelists
if srcwhitelist and srcwhitelist.ip_search(rec.SRC_IP):
continue
if dstwhitelist and dstwhitelist.ip_search(rec.DST_IP):
continue
# *** Convert input record to IDEA ***
# Pass the input record to conversion function to create IDEA message
idea = conv_func(rec, args)
if idea is None:
continue # Record can't be converted - skip it (notice should be printed by the conv function)
if args.name is not None:
idea['Node'][0]['Name'] = args.name
if args.test:
idea['Category'].append('Test')
# *** Send IDEA to outputs ***
# File output
if filehandle:
filehandle.write(json.dumps(idea, indent=args.file_indent) + '\n')
# TRAP output
if args.trap:
try:
trap.send(json.dumps(idea), 0)
except pytrap.TimeoutError:
# skip this message
pass
except pytrap.Terminated:
# don't exit immediately, first finish sending to other outputs
stop = True
# MongoDB output
if mongocoll:
# We need to change IDEA message here, but we may need it unchanged
# later -> copy it (shallow copy is sufficient)
idea2 = idea.copy()
# Convert timestamps from string to Date format
idea2['DetectTime'] = datetime.strptime(idea2['DetectTime'],
"%Y-%m-%dT%H:%M:%SZ")
for i in ['CreateTime', 'EventTime', 'CeaseTime']:
if idea2.has_key(i):
idea2[i] = datetime.strptime(idea2[i],
"%Y-%m-%dT%H:%M:%SZ")
try:
mongocoll.insert(idea2)
except pymongo.errors.AutoReconnect:
sys.stderr.write(module_name +
": Error: MongoDB connection failure.\n")
stop = True
# Warden output
if wardenclient:
wardenclient.sendEvents([idea])
# *** Cleanup ***
if filehandle and filehandle != sys.stdout:
filehandle.close()
if mongoclient:
mongoclient.close()
if wardenclient:
wardenclient.close()
trap.finalize()
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
from warden_client import Client, Error, read_cfg, format_timestamp
from time import time, gmtime, strftime
from math import trunc
from uuid import uuid4
import json
import string
import os
import sys
import warden_utils_flab as w3u
aconfig = read_cfg('warden_client_telnetd.cfg')
wconfig = read_cfg('warden_client.cfg')
aclient_name = aconfig['name']
wconfig['name'] = aclient_name
aanonymised = aconfig['anonymised']
aanonymised_net = aconfig['target_net']
aanonymised = aanonymised if (aanonymised_net != '0.0.0.0/0') or (aanonymised_net == 'omit') else '0.0.0.0/0'
wclient = Client(**wconfig)
def gen_event_idea_telnetd(detect_time, src_ip, src_port, dst_ip, dst_port, proto, category, data):
event = {
"Format": "IDEA0",
"ID": str(uuid4()),
"DetectTime": detect_time,
"Category": [category, "Test"],
"Note": "telnetd event",
"ConnCount": 1,
finally:
sock.shutdown(socket.SHUT_RDWR)
sock.close()
return len(ret)
if __name__ == "__main__":
signal.signal(signal.SIGTERM , handler)
if sys.stdout.name == '<stdout>':
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
if sys.stderr.name == '<stderr>':
sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0)
aconfig = read_cfg(DEFAULT_ACONFIG)
wconfig = read_cfg(aconfig.get('warden', DEFAULT_WCONFIG))
aname = aconfig.get('name', DEFAULT_NAME)
wconfig['name'] = aname
wclient = Client(**wconfig)
while True:
try:
#fetch until queue drain and have a rest for while
while (fetch_and_send(wclient) != 0):
pass
sleep(60)
except KeyboardInterrupt as e:
break
except Exception as e:
print e
Error(message="Error saving event",
exc=sys.exc_info(),
file=dest_file,
event_ids=[event.get("ID")]).log(wclient.logger)
count_err += 1
wclient.logger.info("warden_archiver: received %d, errors %d" %
(count_ok, count_err))
if __name__ == "__main__":
# Parse arguments
args = argparser.parse_args()
# Create Warden client
wclient = Client(**read_cfg(args.warden_config))
wclient.logger.info("Warden archiver started")
# Ensure the archive directory exists
os.makedirs(args.archive_dir, exist_ok=True)
# Setup signal handlers
for (signum, handler) in signals.items():
signal.signal(signum, handler)
# Run receiver
receiver(config, wclient, args.archive_dir)
wclient.logger.info("Warden archiver stopped")
#!/usr/bin/python
from twisted.internet.error import CannotListenError
from twisted.internet.protocol import DatagramProtocol
from twisted.internet import reactor
from warden_client import read_cfg, format_timestamp
from time import time
import re
import json
import string
import logging
import warden_utils_flab as w3u
import scapy.all
hconfig = read_cfg('uchoudp.cfg')
skipports = hconfig.get('port_skip', [])
logger = w3u.getLogger(hconfig['logfile'])
class UchoUDP(DatagramProtocol):
proto = 'udp'
dst_ip = '0.0.0.0'
iface = 'eth0'
def __init__(self):
self.dst_ip = w3u.get_ip_address(self.iface)
def datagramReceived(self, data, (host, port)):
if re.match("autotest.*", data):
self.transport.write(data, (host, port))
def Run(module_name, module_desc, req_type, req_format, conv_func, arg_parser = None):
""" TODO doc
"""
# *** Parse command-line arguments ***
if arg_parser is None:
arg_parser = argparse.ArgumentParser()
arg_parser.formatter_class = argparse.RawDescriptionHelpFormatter
# Set description
arg_parser.description = str.format(desc_template,
name=module_name,
type={trap.TRAP_FMT_RAW:'raw', trap.TRAP_FMT_UNIREC:'UniRec', trap.TRAP_FMT_JSON:'JSON'}.get(req_type,'???'),
fmt=req_format,
original_desc = module_desc+"\n\n " if module_desc else "",
)
# Add arguments defining outputs
# TRAP output
arg_parser.add_argument('--trap', action='store_true',
help='Enable output via TRAP interface (JSON type with format id "IDEA"). Parameters are set using "-i" option as usual.')
# File output
arg_parser.add_argument('--file', metavar="FILE", type=str,
help='Enable output to file (each IDEA message printed to new line in JSON format). Set to "-" to use standard output.')
arg_parser.add_argument('--file-indent', metavar="N", type=int,
help='Pretty-format JSON in output file using N spaces for indentation.')
arg_parser.add_argument('--file-append', action='store_true',
help='Append to file instead of overwrite.')
# MongoDB output
arg_parser.add_argument('--mongodb', metavar="DBNAME",
help='Enable output to MongoDB. Connect to database named DBNAME.')
arg_parser.add_argument('--mongodb-coll', metavar="COLL", default='alerts',
help='Put IDEA messages into collection named COLL (default: "alerts").')
arg_parser.add_argument('--mongodb-host', metavar="HOSTNAME", default='localhost',
help='Connect to MongoDB running on HOSTNAME (default: "localhost").')
arg_parser.add_argument('--mongodb-port', metavar="PORT", type=int, default=27017,
help='Connect to MongoDB running on port number PORT (default: 27017).')
# Warden3 output
arg_parser.add_argument('--warden', metavar="CONFIG_FILE",
help='Send IDEA messages to Warden server. Load configuration of Warden client from CONFIG_FILE.')
# Other options
arg_parser.add_argument('-n', '--name', metavar='NODE_NAME',
help='Name of the node, filled into "Node.Name" element of the IDEA message. Required if Warden output is used, recommended otherwise.')
arg_parser.add_argument('--test', action='store_true',
help='Add "Test" to "Category" before sending a message to output(s).')
arg_parser.add_argument('-v', '--verbose', action='store_true',
help="Enable verbose mode (may be used by some modules, common part donesn't print anything")
# TRAP parameters
trap_args = arg_parser.add_argument_group('Common TRAP parameters')
trap_args.add_argument('-i', metavar="IFC_SPEC", required=True,
help='TODO (ideally this section should be added by TRAP')
# Parse arguments
args = arg_parser.parse_args()
# Check if at least one output is enabled
if not (args.file or args.trap or args.mongodb or args.warden):
sys.stderr.write(module_name+": Error: At least one output must be selected\n")
exit(1)
# Check if node name is set if Warden output is enabled
if args.name is None:
if args.warden:
sys.stderr.write(module_name+": Error: Node name must be specified if Warden output is used (set param --name).\n")
exit(1)
else:
sys.stderr.write(module_name+": Warning: Node name is not specified.\n")
# *** Initialize TRAP ***
module_info = trap.CreateModuleInfo(
module_name, # Module name
"", # Description
1, # Number of input interfaces
1 if args.trap else 0, # Number of output interfaces
None # optionParser
)
ifc_spec = trap.parseParams(['-i', args.i], module_info)
trap.init(module_info, ifc_spec)
trap.registerDefaultSignalHandler()
# Set required input format
trap.set_required_fmt(0, req_type, req_format)
# If TRAP output is enabled, set output format (JSON, format id "IDEA")
if args.trap:
trap.set_data_fmt(0, trap.TRAP_FMT_JSON, "IDEA")
# *** Create output handles/clients/etc ***
filehandle = None
mongoclient = None
mongocoll = None
wardenclient = None
if args.file:
if args.file == '-':
filehandle = sys.stdout
else:
filehandle = open(args.file, "a" if args.file_append else "w")
if args.mongodb:
import pymongo
mongoclient = pymongo.MongoClient(args.mongodb_host, args.mongodb_port)
mongocoll = mongoclient[args.mongodb][args.mongodb_coll]
if args.warden:
import warden_client
wardenclient = warden_client.Client(**warden_client.read_cfg(args.warden))
# *** Main loop ***
URInputTmplt = None
if req_type == trap.TRAP_FMT_UNIREC and req_format != "":
URInputTmplt = unirec.CreateTemplate("URInputTmplt", req_format) # TRAP expects us to have predefined template for required set of fields
while not trap.stop:
# *** Read data from input interface ***
try:
data = trap.recv(0)
except trap.EFMTMismatch:
sys.stderr.write(module_name+": Error: input data format mismatch\n")#Required: "+str((req_type,req_format))+"\nReceived: "+str(trap.get_data_fmt(trap.IFC_INPUT, 0))+"\n")
break
except trap.EFMTChanged as e:
# TODO: This should be handled by trap.recv transparently
# Get negotiated input data format
(fmttype, fmtspec) = trap.get_data_fmt(trap.IFC_INPUT, 0)
# If data type is UniRec, create UniRec template
#print "Creating template", fmtspec
if fmttype == trap.TRAP_FMT_UNIREC:
URInputTmplt = unirec.CreateTemplate("URInputTmplt", fmtspec)
else:
URInputTmplt = None
data = e.data
except trap.ETerminated:
break
# Check for "end-of-stream" record
if len(data) <= 1:
# If we have output, send "end-of-stream" record and exit
if args.trap:
trap.send(0, "0")
break
# Assert that if UniRec input is required, input template is set
assert(req_type != trap.TRAP_FMT_UNIREC or URInputTmplt is not None)
# Convert raw input data to UniRec object (if UniRec input is expected)
if req_type == trap.TRAP_FMT_UNIREC:
rec = URInputTmplt(data)
elif req_type == trap.TRAP_FMT_JSON:
rec = json.loads(data)
else: # TRAP_FMT_RAW
rec = data
# *** Convert input record to IDEA ***
# Pass the input record to conversion function to create IDEA message
idea = conv_func(rec, args)
if idea is None:
continue # Record can't be converted - skip it (notice should be printed by the conv function)
if args.name is not None:
idea['Node'][0]['Name'] = args.name
if args.test:
idea['Category'].append('Test')
# *** Send IDEA to outputs ***
# File output
if filehandle:
filehandle.write(json.dumps(idea, indent=args.file_indent)+'\n')
# TRAP output
if args.trap:
try:
trap.send(0, json.dumps(idea))
except trap.ETerminated:
# don't exit immediately, first finish sending to other outputs
trap.stop = 1
# MongoDB output
if mongocoll:
# Convert timestamps from string to Date format
idea['DetectTime'] = datetime.strptime(idea['DetectTime'], "%Y-%m-%dT%H:%M:%SZ")
for i in [ 'CreateTime', 'EventTime', 'CeaseTime' ]:
if idea.has_key(i):
idea[i] = datetime.strptime(idea[i], "%Y-%m-%dT%H:%M:%SZ")
try:
mongocoll.insert(idea)
except pymongo.errors.AutoReconnect:
sys.stderr.write(module_name+": Error: MongoDB connection failure.\n")
trap.stop = 1
# Warden output
if wardenclient:
wardenclient.sendEvents([idea])
# *** Cleanup ***
if filehandle and filehandle != sys.stdout:
filehandle.close()
if mongoclient:
mongoclient.close()
if wardenclient:
wardenclient.close()
trap.finalize()
def Run(module_name,
module_desc,
req_type,
req_format,
conv_func,
arg_parser=None):
""" TODO doc
"""
# *** Parse command-line arguments ***
if arg_parser is None:
arg_parser = argparse.ArgumentParser()
arg_parser.formatter_class = argparse.RawDescriptionHelpFormatter
# Set description
arg_parser.description = str.format(
desc_template,
name=module_name,
type={
trap.TRAP_FMT_RAW: 'raw',
trap.TRAP_FMT_UNIREC: 'UniRec',
trap.TRAP_FMT_JSON: 'JSON'
}.get(req_type, '???'),
fmt=req_format,
original_desc=module_desc + "\n\n " if module_desc else "",
)
# Add arguments defining outputs
# TRAP output
arg_parser.add_argument(
'--trap',
action='store_true',
help=
'Enable output via TRAP interface (JSON type with format id "IDEA"). Parameters are set using "-i" option as usual.'
)
# File output
arg_parser.add_argument(
'--file',
metavar="FILE",
type=str,
help=
'Enable output to file (each IDEA message printed to new line in JSON format). Set to "-" to use standard output.'
)
arg_parser.add_argument(
'--file-indent',
metavar="N",
type=int,
help='Pretty-format JSON in output file using N spaces for indentation.'
)
arg_parser.add_argument('--file-append',
action='store_true',
help='Append to file instead of overwrite.')
# MongoDB output
arg_parser.add_argument(
'--mongodb',
metavar="DBNAME",
help='Enable output to MongoDB. Connect to database named DBNAME.')
arg_parser.add_argument(
'--mongodb-coll',
metavar="COLL",
default='alerts',
help='Put IDEA messages into collection named COLL (default: "alerts").'
)
arg_parser.add_argument(
'--mongodb-host',
metavar="HOSTNAME",
default='localhost',
help='Connect to MongoDB running on HOSTNAME (default: "localhost").')
arg_parser.add_argument(
'--mongodb-port',
metavar="PORT",
type=int,
default=27017,
help='Connect to MongoDB running on port number PORT (default: 27017).'
)
# Warden3 output
arg_parser.add_argument(
'--warden',
metavar="CONFIG_FILE",
help=
'Send IDEA messages to Warden server. Load configuration of Warden client from CONFIG_FILE.'
)
# Other options
arg_parser.add_argument(
'-n',
'--name',
metavar='NODE_NAME',
help=
'Name of the node, filled into "Node.Name" element of the IDEA message. Required if Warden output is used, recommended otherwise.'
)
arg_parser.add_argument(
'--test',
action='store_true',
help='Add "Test" to "Category" before sending a message to output(s).')
arg_parser.add_argument(
'-v',
'--verbose',
action='store_true',
help=
"Enable verbose mode (may be used by some modules, common part donesn't print anything"
)
# TRAP parameters
trap_args = arg_parser.add_argument_group('Common TRAP parameters')
trap_args.add_argument(
'-i',
metavar="IFC_SPEC",
required=True,
help='TODO (ideally this section should be added by TRAP')
# Parse arguments
args = arg_parser.parse_args()
# Check if at least one output is enabled
if not (args.file or args.trap or args.mongodb or args.warden):
sys.stderr.write(module_name +
": Error: At least one output must be selected\n")
exit(1)
# Check if node name is set if Warden output is enabled
if args.name is None:
if args.warden:
sys.stderr.write(
module_name +
": Error: Node name must be specified if Warden output is used (set param --name).\n"
)
exit(1)
else:
sys.stderr.write(module_name +
": Warning: Node name is not specified.\n")
# *** Initialize TRAP ***
module_info = trap.CreateModuleInfo(
module_name, # Module name
"", # Description
1, # Number of input interfaces
1 if args.trap else 0, # Number of output interfaces
None # optionParser
)
ifc_spec = trap.parseParams(['-i', args.i], module_info)
trap.init(module_info, ifc_spec)
trap.registerDefaultSignalHandler()
# Set required input format
trap.set_required_fmt(0, req_type, req_format)
# If TRAP output is enabled, set output format (JSON, format id "IDEA")
if args.trap:
trap.set_data_fmt(0, trap.TRAP_FMT_JSON, "IDEA")
# *** Create output handles/clients/etc ***
filehandle = None
mongoclient = None
mongocoll = None
wardenclient = None
if args.file:
if args.file == '-':
filehandle = sys.stdout
else:
filehandle = open(args.file, "a" if args.file_append else "w")
if args.mongodb:
import pymongo
mongoclient = pymongo.MongoClient(args.mongodb_host, args.mongodb_port)
mongocoll = mongoclient[args.mongodb][args.mongodb_coll]
if args.warden:
import warden_client
wardenclient = warden_client.Client(
**warden_client.read_cfg(args.warden))
# *** Main loop ***
URInputTmplt = None
if req_type == trap.TRAP_FMT_UNIREC and req_format != "":
URInputTmplt = unirec.CreateTemplate(
"URInputTmplt", req_format
) # TRAP expects us to have predefined template for required set of fields
while not trap.stop:
# *** Read data from input interface ***
try:
data = trap.recv(0)
except trap.EFMTMismatch:
sys.stderr.write(
module_name + ": Error: input data format mismatch\n"
) #Required: "+str((req_type,req_format))+"\nReceived: "+str(trap.get_data_fmt(trap.IFC_INPUT, 0))+"\n")
break
except trap.EFMTChanged as e:
# TODO: This should be handled by trap.recv transparently
# Get negotiated input data format
(fmttype, fmtspec) = trap.get_data_fmt(trap.IFC_INPUT, 0)
# If data type is UniRec, create UniRec template
#print "Creating template", fmtspec
if fmttype == trap.TRAP_FMT_UNIREC:
URInputTmplt = unirec.CreateTemplate("URInputTmplt", fmtspec)
else:
URInputTmplt = None
data = e.data
except trap.ETerminated:
break
# Check for "end-of-stream" record
if len(data) <= 1:
# If we have output, send "end-of-stream" record and exit
if args.trap:
trap.send(0, "0")
break
# Assert that if UniRec input is required, input template is set
assert (req_type != trap.TRAP_FMT_UNIREC or URInputTmplt is not None)
# Convert raw input data to UniRec object (if UniRec input is expected)
if req_type == trap.TRAP_FMT_UNIREC:
rec = URInputTmplt(data)
elif req_type == trap.TRAP_FMT_JSON:
rec = json.loads(data)
else: # TRAP_FMT_RAW
rec = data
# *** Convert input record to IDEA ***
# Pass the input record to conversion function to create IDEA message
idea = conv_func(rec, args)
if idea is None:
continue # Record can't be converted - skip it (notice should be printed by the conv function)
if args.name is not None:
idea['Node'][0]['Name'] = args.name
if args.test:
idea['Category'].append('Test')
# *** Send IDEA to outputs ***
# File output
if filehandle:
filehandle.write(json.dumps(idea, indent=args.file_indent) + '\n')
# TRAP output
if args.trap:
try:
trap.send(0, json.dumps(idea))
except trap.ETerminated:
# don't exit immediately, first finish sending to other outputs
trap.stop = 1
# MongoDB output
if mongocoll:
# Convert timestamps from string to Date format
idea['DetectTime'] = datetime.strptime(idea['DetectTime'],
"%Y-%m-%dT%H:%M:%SZ")
for i in ['CreateTime', 'EventTime', 'CeaseTime']:
if idea.has_key(i):
idea[i] = datetime.strptime(idea[i], "%Y-%m-%dT%H:%M:%SZ")
try:
mongocoll.insert(idea)
except pymongo.errors.AutoReconnect:
sys.stderr.write(module_name +
": Error: MongoDB connection failure.\n")
trap.stop = 1
# Warden output
if wardenclient:
wardenclient.sendEvents([idea])
# *** Cleanup ***
if filehandle and filehandle != sys.stdout:
filehandle.close()
if mongoclient:
mongoclient.close()
if wardenclient:
wardenclient.close()
trap.finalize()
from warden_client import Client, Error, read_cfg, format_timestamp
import json
import string
import urllib
from time import time, gmtime, strftime, sleep
from math import trunc
from uuid import uuid4
from os import path
import base64
import sqlite3
import sys
import re
import warden_utils_flab as w3u
aconfig = read_cfg('warden_client_dio.cfg')
wconfig = read_cfg('warden_client.cfg')
aclient_name = aconfig['name']
wconfig['name'] = aclient_name
aanonymised = aconfig['anonymised']
aanonymised_net = aconfig['target_net']
aanonymised = aanonymised if (aanonymised_net != '0.0.0.0/0') or (aanonymised_net == 'omit') else '0.0.0.0/0'
awin = aconfig['awin'] * 60
abinpath = aconfig['binaries_path']
adbfile = aconfig['dbfile']
aconattempts = aconfig['con_attempts']
aretryinterval = aconfig['con_retry_interval']
areportbinaries = aconfig['report_binaries']
wconfig['secret'] = aconfig.get('secret', '')
#!/usr/bin/python
from twisted.protocols.policies import TimeoutMixin
from twisted.internet.protocol import Protocol, Factory
from twisted.internet import reactor
from warden_client import read_cfg, format_timestamp
import json, socket, re
import os
import sys
import logging
import warden_utils_flab as w3u
import time
import struct
import uuid
hconfig = read_cfg('jdwpd.cfg')
logger = w3u.getLogger(hconfig['logfile'])
DEBUG = False
DEFAULT_PORT = 8000
HANDSHAKE = "JDWP-Handshake"
AUTOTEST = "AUTOTEST\n"
REQUEST_PACKET_TYPE = 0x00
REPLY_PACKET_TYPE = 0x80
NO_ERROR = 0x00
EVENT_FIRE_INTERVAL = 2
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
from warden_client import Client, Error, read_cfg, format_timestamp
from time import time, gmtime, strftime
from math import trunc
from uuid import uuid4
import json
import string
import os
import sys
import warden_utils_flab as w3u
aconfig = read_cfg('warden_client_jdwpd.cfg')
wconfig = read_cfg('warden_client.cfg')
aclient_name = aconfig['name']
wconfig['name'] = aclient_name
aanonymised = aconfig['anonymised']
aanonymised_net = aconfig['target_net']
aanonymised = aanonymised if (aanonymised_net != '0.0.0.0/0') or (aanonymised_net == 'omit') else '0.0.0.0/0'
wclient = Client(**wconfig)
def gen_event_idea_jdwpd(detect_time, src_ip, src_port, dst_ip, dst_port, proto, category, method, cstring, data):
event = {
"Format": "IDEA0",
"ID": str(uuid4()),
"DetectTime": detect_time,
"Category": [category],
"Note": "jdwpd event",
"ConnCount": 1,
#!/usr/bin/python
from twisted.internet.protocol import Protocol, Factory
from twisted.internet import reactor
from warden_client import read_cfg, format_timestamp
import json, socket, re
import os
import sys
import logging
import warden_utils_flab as w3u
hconfig = read_cfg('uchotcp.cfg')
skipports = hconfig.get('port_skip', [])
logger = w3u.getLogger(hconfig['logfile'])
class UchoTCP(Protocol):
def connectionMade(self):
self._peer = self.transport.getPeer()
self._proto = self._peer.type.lower()
self._socket = self.transport.socket.getsockname()
self._dtime = format_timestamp()
self._data = []
def connectionLost(self, reason):
data = ''.join(self._data)
data2log = {
"detect_time" : self._dtime,
"proto" : [self._proto],
"src_ip" : self._peer.host,
"src_port" : self._peer.port,
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
from warden_client import Client, Error, read_cfg, format_timestamp
from time import time, gmtime, strftime
from math import trunc
from uuid import uuid4
import os
import sys
import warden_utils_flab as w3u
import dateutil.parser, calendar
import json
import string
aconfig = read_cfg('warden_client_elastichoney.cfg')
wconfig = read_cfg('warden_client.cfg')
aclient_name = aconfig['name']
wconfig['name'] = aclient_name
aanonymised = aconfig['anonymised']
aanonymised_net = aconfig['target_net']
aanonymised = aanonymised if (aanonymised_net != '0.0.0.0/0') or (aanonymised_net == 'omit') else '0.0.0.0/0'
wclient = Client(**wconfig)
def gen_event_idea_elastichoney(detect_time, src_ip, dst_ip, data):
event = {
"Format": "IDEA0",
"ID": str(uuid4()),
"DetectTime": detect_time,
"Category": ["Recon.Scanning", "Test"],
