def get_configs(): config = read_cfg(args.config) # Allow inline or external Warden config wconfig = config.get("warden", "warden_client.cfg") if isinstance(wconfig, basestring): wconfig = read_cfg(wconfig) fconfig = config.get(args.func, {}) return wconfig, fconfig
def main(): parser = argparse.ArgumentParser() parser.add_argument('-c', '--count') args = parser.parse_args() count = 100 if args.count: count = int(args.count) wclient = Client(**read_cfg("warden_client_tester.cfg")) # Also inline arguments are possible: # wclient = Client( # url = 'https://warden.example.com/warden3', # keyfile = '/opt/warden3/etc/key.pem', # certfile = '/opt/warden3/etc/cert.pem', # cafile = '/opt/warden3/etc/tcs-ca-bundle.pem', # timeout=10, # errlog={"level": "debug"}, # filelog={"level": "debug"}, # idstore="MyClient.id", # name="cz.example.warden.test") #info = wclient.getDebug() #wclient.logger.debug(info) #info = wclient.getInfo() #wclient.logger.info(info) #wclient.logger.debug("Sending %d event(s)" % count) start = time() ret = wclient.sendEvents([gen_random_idea(client_name=wclient.name) for i in range(count)]) ret['time'] = (time()-start) wclient.logger.info(ret)
def Run(module_name, module_desc, req_type, req_format, conv_func, arg_parser = None): """Run the main loop of the reporter module called `module_name` with `module_desc` (used in help). The module requires data format of `req_type` type and `req_format` specifier - these must be given by author of the module. `conv_func(rec, args)` is a callback function that must translate given incoming alert `rec` (typically in UniRec according to `req_type`) into IDEA message. `args` contains CLI arguments parsed by ArgumentParser. `conv_func` must return dict(). """ global trap # *** Parse command-line arguments *** if arg_parser is None: arg_parser = argparse.ArgumentParser() arg_parser.formatter_class = argparse.RawDescriptionHelpFormatter # Set description arg_parser.description = str.format(desc_template, name=module_name, type={pytrap.FMT_RAW:'raw', pytrap.FMT_UNIREC:'UniRec', pytrap.FMT_JSON:'JSON'}.get(req_type,'???'), fmt=req_format, original_desc = module_desc+"\n\n " if module_desc else "", ) # Add arguments defining outputs # TRAP output arg_parser.add_argument('-T', '--trap', action='store_true', help='Enable output via TRAP interface (JSON type with format id "IDEA"). Parameters are set using "-i" option as usual.') # Config file arg_parser.add_argument('-c', '--config', metavar="FILE", default="./config.yaml", type=str, help='Specify YAML config file path which to load.') arg_parser.add_argument('-d', '--dry', action='store_true', help="""Do not run, just print loaded config.""") # Warden3 output arg_parser.add_argument('-W', '--warden', metavar="CONFIG_FILE", help='Send IDEA messages to Warden server. Load configuration of Warden client from CONFIG_FILE.') # Other options arg_parser.add_argument('-n', '--name', metavar='NODE_NAME', help='Name of the node, filled into "Node.Name" element of the IDEA message. Required argument.') arg_parser.add_argument('-v', '--verbose', metavar='VERBOSE_LEVEL', default=3, type=int, help="""Enable verbose mode (may be used by some modules, common part doesn't print anything).\nLevel 1 logs everything, level 5 only critical errors. Level 0 doesn't log.""") # TRAP parameters trap_args = arg_parser.add_argument_group('Common TRAP parameters') trap_args.add_argument('-i', metavar="IFC_SPEC", required=True, help='See http://nemea.liberouter.org/trap-ifcspec/ for more information.') # Parse arguments args = arg_parser.parse_args() # Set log level logging.basicConfig(level=(args.verbose*10), format=FORMAT) # Check if node name is set if Warden output is enabled if args.name is None: #if args.warden: # sys.stderr.write(module_name+": Error: Node name must be specified if Warden output is used (set param --name).\n") # exit(1) logger.warning("Node name is not specified.") # *** Initialize TRAP *** logger.info("Trap arguments: %s", args.i) trap.init(["-i", args.i], 1, 1 if args.trap else 0) #trap.setVerboseLevel(3) signal.signal(signal.SIGINT, signal_h) # Set required input format trap.setRequiredFmt(0, req_type, req_format) if args.trap: trap.setDataFmt(0, pytrap.FMT_JSON, "IDEA") # *** Create output handles/clients/etc *** wardenclient = None if args.warden: try: import warden_client except: logger.error("There is no available warden_client python module. Install it or remove '--warden' from the module's arguments.") sys.exit(1) config = warden_client.read_cfg(args.warden) config['name'] = args.name wardenclient = warden_client.Client(**config) # Initialize configuration config = Config.Config(args.config, trap = trap, warden = wardenclient) if not args.dry: # *** Main loop *** URInputTmplt = None if req_type == pytrap.FMT_UNIREC and req_format != "": URInputTmplt = pytrap.UnirecTemplate(req_format) # TRAP expects us to have predefined template for required set of fields rec = URInputTmplt stop = False while not stop: logger.info("Starting receiving") # *** Read data from input interface *** try: data = trap.recv() except pytrap.FormatMismatch: logger.error("Input data format mismatch in receiving from TRAP interface") break except pytrap.FormatChanged as e: # Get negotiated input data format (fmttype, fmtspec) = trap.getDataFmt(0) # If data type is UniRec, create UniRec template if fmttype == pytrap.FMT_UNIREC: URInputTmplt = pytrap.UnirecTemplate(fmtspec) else: URInputTmplt = None rec = URInputTmplt data = e.data except pytrap.Terminated: break # Check for "end-of-stream" record if len(data) <= 1: if args.trap: # If we have output, send "end-of-stream" record and exit trap.send(0, b"0") break # Assert that if UniRec input is required, input template is set assert(req_type != pytrap.FMT_UNIREC or URInputTmplt is not None) # Convert raw input data to UniRec object (if UniRec input is expected) if req_type == pytrap.FMT_UNIREC: rec.setData(data) elif req_type == pytrap.FMT_JSON: rec = json.loads(str(data)) else: # TRAP_FMT_RAW rec = data # *** Convert input record to IDEA *** # Pass the input record to conversion function to create IDEA message idea = conv_func(rec, args) if idea is None: # Record can't be converted - skip it continue if args.name is not None: idea['Node'][0]['Name'] = args.name # *** Send IDEA to outputs *** # Perform rule matching and action running on the idea message try: config.match(idea) except pytrap.Terminated: logger.error("PyTrap was terminated") break except DropMsg: logger.info("Message was dropped by Drop action.") continue except Exception as e: logger.error(str(e)) break else: # DRY argument given, just print config and exit print(config) if wardenclient: wardenclient.close() trap.finalize()
def main(): wclient = Client(**read_cfg("warden_client.cfg")) # Also inline arguments are possible: # wclient = Client( # url = 'https://warden.example.com/warden3', # keyfile = '/opt/warden3/etc/key.pem', # certfile = '/opt/warden3/etc/cert.pem', # cafile = '/opt/warden3/etc/tcs-ca-bundle.pem', # timeout=10, # errlog={"level": "debug"}, # filelog={"level": "debug"}, # idstore="MyClient.id", # name="cz.example.warden.test") print("=== Debug ===") info = wclient.getDebug() pprint(info) # All methods return something. # If you want to catch possible errors (for example implement some # form of persistent retry, or save failed events for later, you may # check for Error instance and act based on contained info. # If you want just to be informed, this is not necessary, just # configure logging correctly and check logs. if isinstance(info, Error): print(info) print("=== Server info ===") info = wclient.getInfo() print("=== Sending 10 event(s) ===") start = time() ret = wclient.sendEvents( [gen_random_idea(client_name=wclient.name) for i in range(10)]) print(ret) print("Time: %f" % (time() - start)) print("=== Getting 10 events ===") start = time() # cat = ['Availability', 'Abusive.Spam','Attempt.Login'] # cat = ['Attempt', 'Information','Fraud.Scam','Malware.Virus'] # cat = ['Fraud', 'Abusive.Spam'] # nocat = ['Availability', 'Information', 'Fraud.Scam'] cat = [] nocat = [] #tag = ['Log', 'Data'] #notag = ['Flow', 'Datagram'] tag = [] notag = [] #group = ['cz.tul.ward.kippo','cz.vsb.buldog.kippo'] #nogroup = ['cz.zcu.civ.afrodita','cz.vutbr.net.bee.hpscan'] group = [] nogroup = [] ret = wclient.getEvents(count=10, cat=cat, nocat=nocat, tag=tag, notag=notag, group=group, nogroup=nogroup) print("Time: %f" % (time() - start)) print("Got %i events" % len(ret)) for e in ret: print(e.get("Category"), e.get("Node")[0].get("Type"), e.get("Node")[0].get("Name"))
#!/usr/bin/python # -*- coding: utf-8 -*- # from warden_client import read_cfg, format_timestamp from twisted.conch.telnet import TelnetTransport, StatefulTelnetProtocol from twisted.internet.protocol import ServerFactory from twisted.internet import reactor from time import sleep import json import warden_utils_flab as w3u import os hconfig = read_cfg('telnetd.cfg') logger = w3u.getLogger(hconfig['logfile']) class telnetd(StatefulTelnetProtocol, object): DEF_USER = "******" DEF_PASS = "******" DEF_AUTOTEST_USER = "******" DEF_AUTOTEST_PASS = "******" MAX_LOGIN_COUNT = 3 LOGIN_DELAY = 1 PROMPT = "/ # " NEWLINE = "\n" MSG_PRELOGIN = "******" MSG_BADPASS = "******" state = 'User' logcount = 0
#!/usr/bin/python # -*- coding: utf-8 -*- # # Copyright (C) 2011-2015 Cesnet z.s.p.o # Use of this source is governed by a 3-clause BSD-style license, see LICENSE file. from warden_client import Client, Error, read_cfg, format_timestamp import json from time import time, gmtime, strftime, sleep from uuid import uuid4 import sqlite3 import warden_utils_flab as w3u aconfig = read_cfg('warden_client_glastopf.cfg') wconfig = read_cfg('warden_client.cfg') aclient_name = aconfig['name'] wconfig['name'] = aclient_name aanonymised = aconfig['anonymised'] aanonymised_net = aconfig['target_net'] aanonymised = aanonymised if (aanonymised_net != '0.0.0.0/0') or (aanonymised_net == 'omit') else '0.0.0.0/0' awin = aconfig['awin'] * 60 aconattempts = aconfig['con_attempts'] aretryinterval = aconfig['con_retry_interval'] adbfile = aconfig['dbfile'] wclient = Client(**wconfig) def gen_event_idea_gl(detect_time, src_ip, src_port, request_url, request_raw, pattern, filename):
#!/usr/bin/python # -*- coding: utf-8 -*- # from warden_client import Client, Error, read_cfg, format_timestamp from time import time, gmtime, strftime from math import trunc from uuid import uuid4 import json import string import os import sys import warden_utils_flab as w3u aconfig = read_cfg('warden_client_uchotcp.cfg') wconfig = read_cfg('warden_client.cfg') aclient_name = aconfig['name'] wconfig['name'] = aclient_name aanonymised = aconfig['anonymised'] aanonymised_net = aconfig['target_net'] aanonymised = aanonymised if (aanonymised_net != '0.0.0.0/0') or (aanonymised_net == 'omit') else '0.0.0.0/0' wclient = Client(**wconfig) def gen_event_idea_uchotcp(detect_time, src_ip, src_port, dst_ip, dst_port, proto, decoded, smart, data): event = { "Format": "IDEA0", "ID": str(uuid4()), "DetectTime": detect_time, "Category": ["Recon.Scanning", "Test"], "Note": "Uchotcp event", "ConnCount": 1,
#!/usr/bin/python from warden_client import read_cfg, format_timestamp from jinja2 import Environment, FileSystemLoader import SimpleHTTPServer, SocketServer, logging, sys import json import os import sys import re import base64 import warden_utils_flab as w3u import mimetypes hconfig = read_cfg('uchoweb.cfg') content_base = os.path.join(os.getcwd(), "content/") templates_base = os.path.join(os.getcwd(), "templates/") port = hconfig.get('port', 8081) personality = hconfig.get('personality', 'Apache Tomcat/7.0.56') logger = w3u.getLogger(hconfig['logfile']) class ServerHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): server_version = personality sys_version = "" def doAbort(self): j2_env = Environment(loader=FileSystemLoader(templates_base), trim_blocks=True) output = j2_env.get_template('404').render(path=self.path) self.send_response(404) self.end_headers()
def main(): wclient = Client(**read_cfg("warden_client.cfg")) # Also inline arguments are possible: # wclient = Client( # url = 'https://warden.example.com/warden3', # keyfile = '/opt/warden3/etc/key.pem', # certfile = '/opt/warden3/etc/cert.pem', # cafile = '/opt/warden3/etc/tcs-ca-bundle.pem', # timeout=10, # errlog={"level": "debug"}, # filelog={"level": "debug"}, # idstore="MyClient.id", # name="cz.example.warden.test") print "=== Debug ===" info = wclient.getDebug() pprint(info) # All methods return something. # If you want to catch possible errors (for example implement some # form of persistent retry, or save failed events for later, you may # check for Error instance and act based on contained info. # If you want just to be informed, this is not necessary, just # configure logging correctly and check logs. if isinstance(info, Error): print info print "=== Server info ===" info = wclient.getInfo() print "=== Sending 10 event(s) ===" start = time() ret = wclient.sendEvents([gen_random_idea(client_name=wclient.name) for i in range(10)]) print ret print "Time: %f" % (time()-start) print "=== Getting 10 events ===" start = time() # cat = ['Availability', 'Abusive.Spam','Attempt.Login'] # cat = ['Attempt', 'Information','Fraud.Scam','Malware.Virus'] # cat = ['Fraud', 'Abusive.Spam'] # nocat = ['Availability', 'Information', 'Fraud.Scam'] cat = [] nocat = [] #tag = ['Log', 'Data'] #notag = ['Flow', 'Datagram'] tag = [] notag = [] #group = ['cz.tul.ward.kippo','cz.vsb.buldog.kippo'] #nogroup = ['cz.zcu.civ.afrodita','cz.vutbr.net.bee.hpscan'] group = [] nogroup = [] ret = wclient.getEvents(count=10, cat=cat, nocat=nocat, tag=tag, notag=notag, group=group, nogroup=nogroup) print "Time: %f" % (time()-start) print "Got %i events" % len(ret) for e in ret: print e.get("Category"), e.get("Node")[0].get("Type"), e.get("Node")[0].get("Name")
def Run(module_name, module_desc, req_type, req_format, conv_func, arg_parser=None): """ TODO doc """ # *** Parse command-line arguments *** if arg_parser is None: arg_parser = argparse.ArgumentParser() arg_parser.formatter_class = argparse.RawDescriptionHelpFormatter # Set description arg_parser.description = str.format( desc_template, name=module_name, type={ pytrap.FMT_RAW: 'raw', pytrap.FMT_UNIREC: 'UniRec', pytrap.FMT_JSON: 'JSON' }.get(req_type, '???'), fmt=req_format, original_desc=module_desc + "\n\n " if module_desc else "", ) # Add arguments defining outputs # TRAP output arg_parser.add_argument( '--trap', action='store_true', help= 'Enable output via TRAP interface (JSON type with format id "IDEA"). Parameters are set using "-i" option as usual.' ) # File output arg_parser.add_argument( '--file', metavar="FILE", type=str, help= 'Enable output to file (each IDEA message printed to new line in JSON format). Set to "-" to use standard output.' ) arg_parser.add_argument( '--file-indent', metavar="N", type=int, help='Pretty-format JSON in output file using N spaces for indentation.' ) arg_parser.add_argument('--file-append', action='store_true', help='Append to file instead of overwrite.') # MongoDB output arg_parser.add_argument( '--mongodb', metavar="DBNAME", help='Enable output to MongoDB. Connect to database named DBNAME.') arg_parser.add_argument( '--mongodb-coll', metavar="COLL", default='alerts', help='Put IDEA messages into collection named COLL (default: "alerts").' ) arg_parser.add_argument( '--mongodb-host', metavar="HOSTNAME", default='localhost', help='Connect to MongoDB running on HOSTNAME (default: "localhost").') arg_parser.add_argument( '--mongodb-port', metavar="PORT", type=int, default=27017, help='Connect to MongoDB running on port number PORT (default: 27017).' ) # Warden3 output arg_parser.add_argument( '--warden', metavar="CONFIG_FILE", help= 'Send IDEA messages to Warden server. Load configuration of Warden client from CONFIG_FILE.' ) # Other options arg_parser.add_argument( '-n', '--name', metavar='NODE_NAME', help= 'Name of the node, filled into "Node.Name" element of the IDEA message. Required if Warden output is used, recommended otherwise.' ) arg_parser.add_argument( '--test', action='store_true', help='Add "Test" to "Category" before sending a message to output(s).') arg_parser.add_argument( '-v', '--verbose', action='store_true', help= "Enable verbose mode (may be used by some modules, common part donesn't print anything" ) arg_parser.add_argument( '--srcwhitelist-file', metavar="FILE", type=str, help= "File with addresses/subnets in format: <ip address>/<mask>,<data>\\n \n where /<mask>,<data> is optional, <data> is a user-specific optional content. Whitelist is applied to SRC_IP field. If SRC_IP from the alert is on whitelist, the alert IS NOT reported." ) arg_parser.add_argument( '--dstwhitelist-file', metavar="FILE", type=str, help= "File with addresses/subnets, whitelist is applied on DST_IP, see --srcwhitelist-file help." ) # TRAP parameters trap_args = arg_parser.add_argument_group('Common TRAP parameters') trap_args.add_argument( '-i', metavar="IFC_SPEC", required=True, help='TODO (ideally this section should be added by TRAP') # Parse arguments args = arg_parser.parse_args() # Check if at least one output is enabled if not (args.file or args.trap or args.mongodb or args.warden): sys.stderr.write(module_name + ": Error: At least one output must be selected\n") exit(1) # Check if node name is set if Warden output is enabled if args.name is None: if args.warden: sys.stderr.write( module_name + ": Error: Node name must be specified if Warden output is used (set param --name).\n" ) exit(1) else: sys.stderr.write(module_name + ": Warning: Node name is not specified.\n") # *** Initialize TRAP *** trap = pytrap.TrapCtx() trap.init(["-i", args.i], 1, 1 if args.trap else 0) #trap.setVerboseLevel(3) #trap.registerDefaultSignalHandler() # Set required input format trap.setRequiredFmt(0, req_type, req_format) # If TRAP output is enabled, set output format (JSON, format id "IDEA") if args.trap: trap.setDataFmt(0, pytrap.FMT_JSON, "IDEA") # *** Create output handles/clients/etc *** filehandle = None mongoclient = None mongocoll = None wardenclient = None if args.file: if args.file == '-': filehandle = sys.stdout else: filehandle = open(args.file, "a" if args.file_append else "w") if args.mongodb: import pymongo mongoclient = pymongo.MongoClient(args.mongodb_host, args.mongodb_port) mongocoll = mongoclient[args.mongodb][args.mongodb_coll] if args.warden: import warden_client try: config = warden_client.read_cfg(args.warden) except ValueError as e: sys.stderr.write( "{0}: Failed to load Warden config file '{1}'\n{2}\n".format( module_name, args.warden, e)) exit(1) config['name'] = args.name wardenclient = warden_client.Client(**config) # Check if a whitelist is set, parse the file and prepare context for binary search import ip_prefix_search if args.srcwhitelist_file: if 'ipaddr SRC_IP' in req_format.split(","): srcwhitelist = ip_prefix_search.IPPSContext.fromFile( args.srcwhitelist_file) else: srcwhitelist = None if args.dstwhitelist_file: if 'ipaddr DST_IP' in req_format.split(","): dstwhitelist = ip_prefix_search.IPPSContext.fromFile( args.dstwhitelist_file) else: dstwhitelist = None # *** Main loop *** URInputTmplt = None if req_type == pytrap.FMT_UNIREC and req_format != "": URInputTmplt = pytrap.UnirecTemplate( req_format ) # TRAP expects us to have predefined template for required set of fields rec = URInputTmplt stop = False while not stop: # *** Read data from input interface *** try: data = trap.recv() except pytrap.FormatMismatch: sys.stderr.write( module_name + ": Error: input data format mismatch\n" ) #Required: "+str((req_type,req_format))+"\nReceived: "+str(trap.get_data_fmt(trap.IFC_INPUT, 0))+"\n") break except pytrap.FormatChanged as e: # TODO: This should be handled by trap.recv transparently # Get negotiated input data format (fmttype, fmtspec) = trap.getDataFmt(0) # If data type is UniRec, create UniRec template if fmttype == pytrap.FMT_UNIREC: URInputTmplt = pytrap.UnirecTemplate(fmtspec) else: URInputTmplt = None rec = URInputTmplt data = e.data except pytrap.Terminated: break # Check for "end-of-stream" record if len(data) <= 1: # If we have output, send "end-of-stream" record and exit if args.trap: trap.send(0, b"0") break # Assert that if UniRec input is required, input template is set assert (req_type != pytrap.FMT_UNIREC or URInputTmplt is not None) # Convert raw input data to UniRec object (if UniRec input is expected) if req_type == pytrap.FMT_UNIREC: rec.setData(data) elif req_type == pytrap.FMT_JSON: rec = json.loads(data) else: # TRAP_FMT_RAW rec = data # Check whitelists if srcwhitelist and srcwhitelist.ip_search(rec.SRC_IP): continue if dstwhitelist and dstwhitelist.ip_search(rec.DST_IP): continue # *** Convert input record to IDEA *** # Pass the input record to conversion function to create IDEA message idea = conv_func(rec, args) if idea is None: continue # Record can't be converted - skip it (notice should be printed by the conv function) if args.name is not None: idea['Node'][0]['Name'] = args.name if args.test: idea['Category'].append('Test') # *** Send IDEA to outputs *** # File output if filehandle: filehandle.write(json.dumps(idea, indent=args.file_indent) + '\n') # TRAP output if args.trap: try: trap.send(json.dumps(idea), 0) except pytrap.TimeoutError: # skip this message pass except pytrap.Terminated: # don't exit immediately, first finish sending to other outputs stop = True # MongoDB output if mongocoll: # We need to change IDEA message here, but we may need it unchanged # later -> copy it (shallow copy is sufficient) idea2 = idea.copy() # Convert timestamps from string to Date format idea2['DetectTime'] = datetime.strptime(idea2['DetectTime'], "%Y-%m-%dT%H:%M:%SZ") for i in ['CreateTime', 'EventTime', 'CeaseTime']: if idea2.has_key(i): idea2[i] = datetime.strptime(idea2[i], "%Y-%m-%dT%H:%M:%SZ") try: mongocoll.insert(idea2) except pymongo.errors.AutoReconnect: sys.stderr.write(module_name + ": Error: MongoDB connection failure.\n") stop = True # Warden output if wardenclient: wardenclient.sendEvents([idea]) # *** Cleanup *** if filehandle and filehandle != sys.stdout: filehandle.close() if mongoclient: mongoclient.close() if wardenclient: wardenclient.close() trap.finalize()
#!/usr/bin/python # -*- coding: utf-8 -*- # from warden_client import Client, Error, read_cfg, format_timestamp from time import time, gmtime, strftime from math import trunc from uuid import uuid4 import json import string import os import sys import warden_utils_flab as w3u aconfig = read_cfg('warden_client_telnetd.cfg') wconfig = read_cfg('warden_client.cfg') aclient_name = aconfig['name'] wconfig['name'] = aclient_name aanonymised = aconfig['anonymised'] aanonymised_net = aconfig['target_net'] aanonymised = aanonymised if (aanonymised_net != '0.0.0.0/0') or (aanonymised_net == 'omit') else '0.0.0.0/0' wclient = Client(**wconfig) def gen_event_idea_telnetd(detect_time, src_ip, src_port, dst_ip, dst_port, proto, category, data): event = { "Format": "IDEA0", "ID": str(uuid4()), "DetectTime": detect_time, "Category": [category, "Test"], "Note": "telnetd event", "ConnCount": 1,
finally: sock.shutdown(socket.SHUT_RDWR) sock.close() return len(ret) if __name__ == "__main__": signal.signal(signal.SIGTERM , handler) if sys.stdout.name == '<stdout>': sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) if sys.stderr.name == '<stderr>': sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0) aconfig = read_cfg(DEFAULT_ACONFIG) wconfig = read_cfg(aconfig.get('warden', DEFAULT_WCONFIG)) aname = aconfig.get('name', DEFAULT_NAME) wconfig['name'] = aname wclient = Client(**wconfig) while True: try: #fetch until queue drain and have a rest for while while (fetch_and_send(wclient) != 0): pass sleep(60) except KeyboardInterrupt as e: break except Exception as e: print e
Error(message="Error saving event", exc=sys.exc_info(), file=dest_file, event_ids=[event.get("ID")]).log(wclient.logger) count_err += 1 wclient.logger.info("warden_archiver: received %d, errors %d" % (count_ok, count_err)) if __name__ == "__main__": # Parse arguments args = argparser.parse_args() # Create Warden client wclient = Client(**read_cfg(args.warden_config)) wclient.logger.info("Warden archiver started") # Ensure the archive directory exists os.makedirs(args.archive_dir, exist_ok=True) # Setup signal handlers for (signum, handler) in signals.items(): signal.signal(signum, handler) # Run receiver receiver(config, wclient, args.archive_dir) wclient.logger.info("Warden archiver stopped")
#!/usr/bin/python from twisted.internet.error import CannotListenError from twisted.internet.protocol import DatagramProtocol from twisted.internet import reactor from warden_client import read_cfg, format_timestamp from time import time import re import json import string import logging import warden_utils_flab as w3u import scapy.all hconfig = read_cfg('uchoudp.cfg') skipports = hconfig.get('port_skip', []) logger = w3u.getLogger(hconfig['logfile']) class UchoUDP(DatagramProtocol): proto = 'udp' dst_ip = '0.0.0.0' iface = 'eth0' def __init__(self): self.dst_ip = w3u.get_ip_address(self.iface) def datagramReceived(self, data, (host, port)): if re.match("autotest.*", data): self.transport.write(data, (host, port))
def Run(module_name, module_desc, req_type, req_format, conv_func, arg_parser = None): """ TODO doc """ # *** Parse command-line arguments *** if arg_parser is None: arg_parser = argparse.ArgumentParser() arg_parser.formatter_class = argparse.RawDescriptionHelpFormatter # Set description arg_parser.description = str.format(desc_template, name=module_name, type={trap.TRAP_FMT_RAW:'raw', trap.TRAP_FMT_UNIREC:'UniRec', trap.TRAP_FMT_JSON:'JSON'}.get(req_type,'???'), fmt=req_format, original_desc = module_desc+"\n\n " if module_desc else "", ) # Add arguments defining outputs # TRAP output arg_parser.add_argument('--trap', action='store_true', help='Enable output via TRAP interface (JSON type with format id "IDEA"). Parameters are set using "-i" option as usual.') # File output arg_parser.add_argument('--file', metavar="FILE", type=str, help='Enable output to file (each IDEA message printed to new line in JSON format). Set to "-" to use standard output.') arg_parser.add_argument('--file-indent', metavar="N", type=int, help='Pretty-format JSON in output file using N spaces for indentation.') arg_parser.add_argument('--file-append', action='store_true', help='Append to file instead of overwrite.') # MongoDB output arg_parser.add_argument('--mongodb', metavar="DBNAME", help='Enable output to MongoDB. Connect to database named DBNAME.') arg_parser.add_argument('--mongodb-coll', metavar="COLL", default='alerts', help='Put IDEA messages into collection named COLL (default: "alerts").') arg_parser.add_argument('--mongodb-host', metavar="HOSTNAME", default='localhost', help='Connect to MongoDB running on HOSTNAME (default: "localhost").') arg_parser.add_argument('--mongodb-port', metavar="PORT", type=int, default=27017, help='Connect to MongoDB running on port number PORT (default: 27017).') # Warden3 output arg_parser.add_argument('--warden', metavar="CONFIG_FILE", help='Send IDEA messages to Warden server. Load configuration of Warden client from CONFIG_FILE.') # Other options arg_parser.add_argument('-n', '--name', metavar='NODE_NAME', help='Name of the node, filled into "Node.Name" element of the IDEA message. Required if Warden output is used, recommended otherwise.') arg_parser.add_argument('--test', action='store_true', help='Add "Test" to "Category" before sending a message to output(s).') arg_parser.add_argument('-v', '--verbose', action='store_true', help="Enable verbose mode (may be used by some modules, common part donesn't print anything") # TRAP parameters trap_args = arg_parser.add_argument_group('Common TRAP parameters') trap_args.add_argument('-i', metavar="IFC_SPEC", required=True, help='TODO (ideally this section should be added by TRAP') # Parse arguments args = arg_parser.parse_args() # Check if at least one output is enabled if not (args.file or args.trap or args.mongodb or args.warden): sys.stderr.write(module_name+": Error: At least one output must be selected\n") exit(1) # Check if node name is set if Warden output is enabled if args.name is None: if args.warden: sys.stderr.write(module_name+": Error: Node name must be specified if Warden output is used (set param --name).\n") exit(1) else: sys.stderr.write(module_name+": Warning: Node name is not specified.\n") # *** Initialize TRAP *** module_info = trap.CreateModuleInfo( module_name, # Module name "", # Description 1, # Number of input interfaces 1 if args.trap else 0, # Number of output interfaces None # optionParser ) ifc_spec = trap.parseParams(['-i', args.i], module_info) trap.init(module_info, ifc_spec) trap.registerDefaultSignalHandler() # Set required input format trap.set_required_fmt(0, req_type, req_format) # If TRAP output is enabled, set output format (JSON, format id "IDEA") if args.trap: trap.set_data_fmt(0, trap.TRAP_FMT_JSON, "IDEA") # *** Create output handles/clients/etc *** filehandle = None mongoclient = None mongocoll = None wardenclient = None if args.file: if args.file == '-': filehandle = sys.stdout else: filehandle = open(args.file, "a" if args.file_append else "w") if args.mongodb: import pymongo mongoclient = pymongo.MongoClient(args.mongodb_host, args.mongodb_port) mongocoll = mongoclient[args.mongodb][args.mongodb_coll] if args.warden: import warden_client wardenclient = warden_client.Client(**warden_client.read_cfg(args.warden)) # *** Main loop *** URInputTmplt = None if req_type == trap.TRAP_FMT_UNIREC and req_format != "": URInputTmplt = unirec.CreateTemplate("URInputTmplt", req_format) # TRAP expects us to have predefined template for required set of fields while not trap.stop: # *** Read data from input interface *** try: data = trap.recv(0) except trap.EFMTMismatch: sys.stderr.write(module_name+": Error: input data format mismatch\n")#Required: "+str((req_type,req_format))+"\nReceived: "+str(trap.get_data_fmt(trap.IFC_INPUT, 0))+"\n") break except trap.EFMTChanged as e: # TODO: This should be handled by trap.recv transparently # Get negotiated input data format (fmttype, fmtspec) = trap.get_data_fmt(trap.IFC_INPUT, 0) # If data type is UniRec, create UniRec template #print "Creating template", fmtspec if fmttype == trap.TRAP_FMT_UNIREC: URInputTmplt = unirec.CreateTemplate("URInputTmplt", fmtspec) else: URInputTmplt = None data = e.data except trap.ETerminated: break # Check for "end-of-stream" record if len(data) <= 1: # If we have output, send "end-of-stream" record and exit if args.trap: trap.send(0, "0") break # Assert that if UniRec input is required, input template is set assert(req_type != trap.TRAP_FMT_UNIREC or URInputTmplt is not None) # Convert raw input data to UniRec object (if UniRec input is expected) if req_type == trap.TRAP_FMT_UNIREC: rec = URInputTmplt(data) elif req_type == trap.TRAP_FMT_JSON: rec = json.loads(data) else: # TRAP_FMT_RAW rec = data # *** Convert input record to IDEA *** # Pass the input record to conversion function to create IDEA message idea = conv_func(rec, args) if idea is None: continue # Record can't be converted - skip it (notice should be printed by the conv function) if args.name is not None: idea['Node'][0]['Name'] = args.name if args.test: idea['Category'].append('Test') # *** Send IDEA to outputs *** # File output if filehandle: filehandle.write(json.dumps(idea, indent=args.file_indent)+'\n') # TRAP output if args.trap: try: trap.send(0, json.dumps(idea)) except trap.ETerminated: # don't exit immediately, first finish sending to other outputs trap.stop = 1 # MongoDB output if mongocoll: # Convert timestamps from string to Date format idea['DetectTime'] = datetime.strptime(idea['DetectTime'], "%Y-%m-%dT%H:%M:%SZ") for i in [ 'CreateTime', 'EventTime', 'CeaseTime' ]: if idea.has_key(i): idea[i] = datetime.strptime(idea[i], "%Y-%m-%dT%H:%M:%SZ") try: mongocoll.insert(idea) except pymongo.errors.AutoReconnect: sys.stderr.write(module_name+": Error: MongoDB connection failure.\n") trap.stop = 1 # Warden output if wardenclient: wardenclient.sendEvents([idea]) # *** Cleanup *** if filehandle and filehandle != sys.stdout: filehandle.close() if mongoclient: mongoclient.close() if wardenclient: wardenclient.close() trap.finalize()
def Run(module_name, module_desc, req_type, req_format, conv_func, arg_parser=None): """ TODO doc """ # *** Parse command-line arguments *** if arg_parser is None: arg_parser = argparse.ArgumentParser() arg_parser.formatter_class = argparse.RawDescriptionHelpFormatter # Set description arg_parser.description = str.format( desc_template, name=module_name, type={ trap.TRAP_FMT_RAW: 'raw', trap.TRAP_FMT_UNIREC: 'UniRec', trap.TRAP_FMT_JSON: 'JSON' }.get(req_type, '???'), fmt=req_format, original_desc=module_desc + "\n\n " if module_desc else "", ) # Add arguments defining outputs # TRAP output arg_parser.add_argument( '--trap', action='store_true', help= 'Enable output via TRAP interface (JSON type with format id "IDEA"). Parameters are set using "-i" option as usual.' ) # File output arg_parser.add_argument( '--file', metavar="FILE", type=str, help= 'Enable output to file (each IDEA message printed to new line in JSON format). Set to "-" to use standard output.' ) arg_parser.add_argument( '--file-indent', metavar="N", type=int, help='Pretty-format JSON in output file using N spaces for indentation.' ) arg_parser.add_argument('--file-append', action='store_true', help='Append to file instead of overwrite.') # MongoDB output arg_parser.add_argument( '--mongodb', metavar="DBNAME", help='Enable output to MongoDB. Connect to database named DBNAME.') arg_parser.add_argument( '--mongodb-coll', metavar="COLL", default='alerts', help='Put IDEA messages into collection named COLL (default: "alerts").' ) arg_parser.add_argument( '--mongodb-host', metavar="HOSTNAME", default='localhost', help='Connect to MongoDB running on HOSTNAME (default: "localhost").') arg_parser.add_argument( '--mongodb-port', metavar="PORT", type=int, default=27017, help='Connect to MongoDB running on port number PORT (default: 27017).' ) # Warden3 output arg_parser.add_argument( '--warden', metavar="CONFIG_FILE", help= 'Send IDEA messages to Warden server. Load configuration of Warden client from CONFIG_FILE.' ) # Other options arg_parser.add_argument( '-n', '--name', metavar='NODE_NAME', help= 'Name of the node, filled into "Node.Name" element of the IDEA message. Required if Warden output is used, recommended otherwise.' ) arg_parser.add_argument( '--test', action='store_true', help='Add "Test" to "Category" before sending a message to output(s).') arg_parser.add_argument( '-v', '--verbose', action='store_true', help= "Enable verbose mode (may be used by some modules, common part donesn't print anything" ) # TRAP parameters trap_args = arg_parser.add_argument_group('Common TRAP parameters') trap_args.add_argument( '-i', metavar="IFC_SPEC", required=True, help='TODO (ideally this section should be added by TRAP') # Parse arguments args = arg_parser.parse_args() # Check if at least one output is enabled if not (args.file or args.trap or args.mongodb or args.warden): sys.stderr.write(module_name + ": Error: At least one output must be selected\n") exit(1) # Check if node name is set if Warden output is enabled if args.name is None: if args.warden: sys.stderr.write( module_name + ": Error: Node name must be specified if Warden output is used (set param --name).\n" ) exit(1) else: sys.stderr.write(module_name + ": Warning: Node name is not specified.\n") # *** Initialize TRAP *** module_info = trap.CreateModuleInfo( module_name, # Module name "", # Description 1, # Number of input interfaces 1 if args.trap else 0, # Number of output interfaces None # optionParser ) ifc_spec = trap.parseParams(['-i', args.i], module_info) trap.init(module_info, ifc_spec) trap.registerDefaultSignalHandler() # Set required input format trap.set_required_fmt(0, req_type, req_format) # If TRAP output is enabled, set output format (JSON, format id "IDEA") if args.trap: trap.set_data_fmt(0, trap.TRAP_FMT_JSON, "IDEA") # *** Create output handles/clients/etc *** filehandle = None mongoclient = None mongocoll = None wardenclient = None if args.file: if args.file == '-': filehandle = sys.stdout else: filehandle = open(args.file, "a" if args.file_append else "w") if args.mongodb: import pymongo mongoclient = pymongo.MongoClient(args.mongodb_host, args.mongodb_port) mongocoll = mongoclient[args.mongodb][args.mongodb_coll] if args.warden: import warden_client wardenclient = warden_client.Client( **warden_client.read_cfg(args.warden)) # *** Main loop *** URInputTmplt = None if req_type == trap.TRAP_FMT_UNIREC and req_format != "": URInputTmplt = unirec.CreateTemplate( "URInputTmplt", req_format ) # TRAP expects us to have predefined template for required set of fields while not trap.stop: # *** Read data from input interface *** try: data = trap.recv(0) except trap.EFMTMismatch: sys.stderr.write( module_name + ": Error: input data format mismatch\n" ) #Required: "+str((req_type,req_format))+"\nReceived: "+str(trap.get_data_fmt(trap.IFC_INPUT, 0))+"\n") break except trap.EFMTChanged as e: # TODO: This should be handled by trap.recv transparently # Get negotiated input data format (fmttype, fmtspec) = trap.get_data_fmt(trap.IFC_INPUT, 0) # If data type is UniRec, create UniRec template #print "Creating template", fmtspec if fmttype == trap.TRAP_FMT_UNIREC: URInputTmplt = unirec.CreateTemplate("URInputTmplt", fmtspec) else: URInputTmplt = None data = e.data except trap.ETerminated: break # Check for "end-of-stream" record if len(data) <= 1: # If we have output, send "end-of-stream" record and exit if args.trap: trap.send(0, "0") break # Assert that if UniRec input is required, input template is set assert (req_type != trap.TRAP_FMT_UNIREC or URInputTmplt is not None) # Convert raw input data to UniRec object (if UniRec input is expected) if req_type == trap.TRAP_FMT_UNIREC: rec = URInputTmplt(data) elif req_type == trap.TRAP_FMT_JSON: rec = json.loads(data) else: # TRAP_FMT_RAW rec = data # *** Convert input record to IDEA *** # Pass the input record to conversion function to create IDEA message idea = conv_func(rec, args) if idea is None: continue # Record can't be converted - skip it (notice should be printed by the conv function) if args.name is not None: idea['Node'][0]['Name'] = args.name if args.test: idea['Category'].append('Test') # *** Send IDEA to outputs *** # File output if filehandle: filehandle.write(json.dumps(idea, indent=args.file_indent) + '\n') # TRAP output if args.trap: try: trap.send(0, json.dumps(idea)) except trap.ETerminated: # don't exit immediately, first finish sending to other outputs trap.stop = 1 # MongoDB output if mongocoll: # Convert timestamps from string to Date format idea['DetectTime'] = datetime.strptime(idea['DetectTime'], "%Y-%m-%dT%H:%M:%SZ") for i in ['CreateTime', 'EventTime', 'CeaseTime']: if idea.has_key(i): idea[i] = datetime.strptime(idea[i], "%Y-%m-%dT%H:%M:%SZ") try: mongocoll.insert(idea) except pymongo.errors.AutoReconnect: sys.stderr.write(module_name + ": Error: MongoDB connection failure.\n") trap.stop = 1 # Warden output if wardenclient: wardenclient.sendEvents([idea]) # *** Cleanup *** if filehandle and filehandle != sys.stdout: filehandle.close() if mongoclient: mongoclient.close() if wardenclient: wardenclient.close() trap.finalize()
from warden_client import Client, Error, read_cfg, format_timestamp import json import string import urllib from time import time, gmtime, strftime, sleep from math import trunc from uuid import uuid4 from os import path import base64 import sqlite3 import sys import re import warden_utils_flab as w3u aconfig = read_cfg('warden_client_dio.cfg') wconfig = read_cfg('warden_client.cfg') aclient_name = aconfig['name'] wconfig['name'] = aclient_name aanonymised = aconfig['anonymised'] aanonymised_net = aconfig['target_net'] aanonymised = aanonymised if (aanonymised_net != '0.0.0.0/0') or (aanonymised_net == 'omit') else '0.0.0.0/0' awin = aconfig['awin'] * 60 abinpath = aconfig['binaries_path'] adbfile = aconfig['dbfile'] aconattempts = aconfig['con_attempts'] aretryinterval = aconfig['con_retry_interval'] areportbinaries = aconfig['report_binaries'] wconfig['secret'] = aconfig.get('secret', '')
#!/usr/bin/python from twisted.protocols.policies import TimeoutMixin from twisted.internet.protocol import Protocol, Factory from twisted.internet import reactor from warden_client import read_cfg, format_timestamp import json, socket, re import os import sys import logging import warden_utils_flab as w3u import time import struct import uuid hconfig = read_cfg('jdwpd.cfg') logger = w3u.getLogger(hconfig['logfile']) DEBUG = False DEFAULT_PORT = 8000 HANDSHAKE = "JDWP-Handshake" AUTOTEST = "AUTOTEST\n" REQUEST_PACKET_TYPE = 0x00 REPLY_PACKET_TYPE = 0x80 NO_ERROR = 0x00 EVENT_FIRE_INTERVAL = 2
#!/usr/bin/python # -*- coding: utf-8 -*- # from warden_client import Client, Error, read_cfg, format_timestamp from time import time, gmtime, strftime from math import trunc from uuid import uuid4 import json import string import os import sys import warden_utils_flab as w3u aconfig = read_cfg('warden_client_jdwpd.cfg') wconfig = read_cfg('warden_client.cfg') aclient_name = aconfig['name'] wconfig['name'] = aclient_name aanonymised = aconfig['anonymised'] aanonymised_net = aconfig['target_net'] aanonymised = aanonymised if (aanonymised_net != '0.0.0.0/0') or (aanonymised_net == 'omit') else '0.0.0.0/0' wclient = Client(**wconfig) def gen_event_idea_jdwpd(detect_time, src_ip, src_port, dst_ip, dst_port, proto, category, method, cstring, data): event = { "Format": "IDEA0", "ID": str(uuid4()), "DetectTime": detect_time, "Category": [category], "Note": "jdwpd event", "ConnCount": 1,
#!/usr/bin/python from twisted.internet.protocol import Protocol, Factory from twisted.internet import reactor from warden_client import read_cfg, format_timestamp import json, socket, re import os import sys import logging import warden_utils_flab as w3u hconfig = read_cfg('uchotcp.cfg') skipports = hconfig.get('port_skip', []) logger = w3u.getLogger(hconfig['logfile']) class UchoTCP(Protocol): def connectionMade(self): self._peer = self.transport.getPeer() self._proto = self._peer.type.lower() self._socket = self.transport.socket.getsockname() self._dtime = format_timestamp() self._data = [] def connectionLost(self, reason): data = ''.join(self._data) data2log = { "detect_time" : self._dtime, "proto" : [self._proto], "src_ip" : self._peer.host, "src_port" : self._peer.port,
#!/usr/bin/python # -*- coding: utf-8 -*- # from warden_client import Client, Error, read_cfg, format_timestamp from time import time, gmtime, strftime from math import trunc from uuid import uuid4 import os import sys import warden_utils_flab as w3u import dateutil.parser, calendar import json import string aconfig = read_cfg('warden_client_elastichoney.cfg') wconfig = read_cfg('warden_client.cfg') aclient_name = aconfig['name'] wconfig['name'] = aclient_name aanonymised = aconfig['anonymised'] aanonymised_net = aconfig['target_net'] aanonymised = aanonymised if (aanonymised_net != '0.0.0.0/0') or (aanonymised_net == 'omit') else '0.0.0.0/0' wclient = Client(**wconfig) def gen_event_idea_elastichoney(detect_time, src_ip, dst_ip, data): event = { "Format": "IDEA0", "ID": str(uuid4()), "DetectTime": detect_time, "Category": ["Recon.Scanning", "Test"],