"--file",
dest="filename",
help="write report to FILE",
metavar="FILE")
parser.add_option("-q",
"--quiet",
action="store_false",
dest="verbose",
default=True,
help="don't print status messages to stdout")
# usage of OptionParser can be found in its help
module_info = trap.CreateModuleInfo(
"PythonExample", # Module name
"An example TRAP module written in Python", # Description
1, # Number of input interfaces
1, # Number of output interfaces
parser # use previously defined OptionParser
)
# Initialize module
ifc_spec = trap.parseParams(sys.argv, module_info)
trap.init(module_info, ifc_spec)
trap.registerDefaultSignalHandler(
) # This is needed to allow module termination using s SIGINT or SIGTERM signal
# Parse remaining command-line arguments
(options, args) = parser.parse_args()
import os.path
sys.path.append(
os.path.join(os.path.dirname(__file__), "..", "..", "nemea-framework",
"python"))
import trap
import unirec
from unirec import Timestamp, IPAddr
import readline
from re import match
module_info = trap.CreateModuleInfo(name="DebugSender",
description="""\
This module allows to manually send arbitrary UniRec records to a TRAP
interface. You have to specify UniRec format at startup, everything other is
done interactively by writing simple commands.
Usage:
python debug_sender.py -i IFC_SPEC UNIREC_FORMAT
""",
num_ifc_in=0,
num_ifc_out=1)
fieldOrder = []
# ----------------------------------------------------------------------------
def init_trap():
trap.init(module_info, ifc_spec)
trap.registerDefaultSignalHandler()
trap.ifcctl(trap.IFC_OUTPUT, 0, trap.CTL_BUFFERSWITCH,
def Run(module_name,
module_desc,
req_type,
req_format,
conv_func,
arg_parser=None):
""" TODO doc
"""
# *** Parse command-line arguments ***
if arg_parser is None:
arg_parser = argparse.ArgumentParser()
arg_parser.formatter_class = argparse.RawDescriptionHelpFormatter
# Set description
arg_parser.description = str.format(
desc_template,
name=module_name,
type={
trap.TRAP_FMT_RAW: 'raw',
trap.TRAP_FMT_UNIREC: 'UniRec',
trap.TRAP_FMT_JSON: 'JSON'
}.get(req_type, '???'),
fmt=req_format,
original_desc=module_desc + "\n\n " if module_desc else "",
)
# Add arguments defining outputs
# TRAP output
arg_parser.add_argument(
'--trap',
action='store_true',
help=
'Enable output via TRAP interface (JSON type with format id "IDEA"). Parameters are set using "-i" option as usual.'
)
# File output
arg_parser.add_argument(
'--file',
metavar="FILE",
type=str,
help=
'Enable output to file (each IDEA message printed to new line in JSON format). Set to "-" to use standard output.'
)
arg_parser.add_argument(
'--file-indent',
metavar="N",
type=int,
help='Pretty-format JSON in output file using N spaces for indentation.'
)
arg_parser.add_argument('--file-append',
action='store_true',
help='Append to file instead of overwrite.')
# MongoDB output
arg_parser.add_argument(
'--mongodb',
metavar="DBNAME",
help='Enable output to MongoDB. Connect to database named DBNAME.')
arg_parser.add_argument(
'--mongodb-coll',
metavar="COLL",
default='alerts',
help='Put IDEA messages into collection named COLL (default: "alerts").'
)
arg_parser.add_argument(
'--mongodb-host',
metavar="HOSTNAME",
default='localhost',
help='Connect to MongoDB running on HOSTNAME (default: "localhost").')
arg_parser.add_argument(
'--mongodb-port',
metavar="PORT",
type=int,
default=27017,
help='Connect to MongoDB running on port number PORT (default: 27017).'
)
# Warden3 output
arg_parser.add_argument(
'--warden',
metavar="CONFIG_FILE",
help=
'Send IDEA messages to Warden server. Load configuration of Warden client from CONFIG_FILE.'
)
# Other options
arg_parser.add_argument(
'-n',
'--name',
metavar='NODE_NAME',
help=
'Name of the node, filled into "Node.Name" element of the IDEA message. Required if Warden output is used, recommended otherwise.'
)
arg_parser.add_argument(
'--test',
action='store_true',
help='Add "Test" to "Category" before sending a message to output(s).')
arg_parser.add_argument(
'-v',
'--verbose',
action='store_true',
help=
"Enable verbose mode (may be used by some modules, common part donesn't print anything"
)
# TRAP parameters
trap_args = arg_parser.add_argument_group('Common TRAP parameters')
trap_args.add_argument(
'-i',
metavar="IFC_SPEC",
required=True,
help='TODO (ideally this section should be added by TRAP')
# Parse arguments
args = arg_parser.parse_args()
# Check if at least one output is enabled
if not (args.file or args.trap or args.mongodb or args.warden):
sys.stderr.write(module_name +
": Error: At least one output must be selected\n")
exit(1)
# Check if node name is set if Warden output is enabled
if args.name is None:
if args.warden:
sys.stderr.write(
module_name +
": Error: Node name must be specified if Warden output is used (set param --name).\n"
)
exit(1)
else:
sys.stderr.write(module_name +
": Warning: Node name is not specified.\n")
# *** Initialize TRAP ***
module_info = trap.CreateModuleInfo(
module_name, # Module name
"", # Description
1, # Number of input interfaces
1 if args.trap else 0, # Number of output interfaces
None # optionParser
)
ifc_spec = trap.parseParams(['-i', args.i], module_info)
trap.init(module_info, ifc_spec)
trap.registerDefaultSignalHandler()
# Set required input format
trap.set_required_fmt(0, req_type, req_format)
# If TRAP output is enabled, set output format (JSON, format id "IDEA")
if args.trap:
trap.set_data_fmt(0, trap.TRAP_FMT_JSON, "IDEA")
# *** Create output handles/clients/etc ***
filehandle = None
mongoclient = None
mongocoll = None
wardenclient = None
if args.file:
if args.file == '-':
filehandle = sys.stdout
else:
filehandle = open(args.file, "a" if args.file_append else "w")
if args.mongodb:
import pymongo
mongoclient = pymongo.MongoClient(args.mongodb_host, args.mongodb_port)
mongocoll = mongoclient[args.mongodb][args.mongodb_coll]
if args.warden:
import warden_client
wardenclient = warden_client.Client(
**warden_client.read_cfg(args.warden))
# *** Main loop ***
URInputTmplt = None
if req_type == trap.TRAP_FMT_UNIREC and req_format != "":
URInputTmplt = unirec.CreateTemplate(
"URInputTmplt", req_format
) # TRAP expects us to have predefined template for required set of fields
while not trap.stop:
# *** Read data from input interface ***
try:
data = trap.recv(0)
except trap.EFMTMismatch:
sys.stderr.write(
module_name + ": Error: input data format mismatch\n"
) #Required: "+str((req_type,req_format))+"\nReceived: "+str(trap.get_data_fmt(trap.IFC_INPUT, 0))+"\n")
break
except trap.EFMTChanged as e:
# TODO: This should be handled by trap.recv transparently
# Get negotiated input data format
(fmttype, fmtspec) = trap.get_data_fmt(trap.IFC_INPUT, 0)
# If data type is UniRec, create UniRec template
#print "Creating template", fmtspec
if fmttype == trap.TRAP_FMT_UNIREC:
URInputTmplt = unirec.CreateTemplate("URInputTmplt", fmtspec)
else:
URInputTmplt = None
data = e.data
except trap.ETerminated:
break
# Check for "end-of-stream" record
if len(data) <= 1:
# If we have output, send "end-of-stream" record and exit
if args.trap:
trap.send(0, "0")
break
# Assert that if UniRec input is required, input template is set
assert (req_type != trap.TRAP_FMT_UNIREC or URInputTmplt is not None)
# Convert raw input data to UniRec object (if UniRec input is expected)
if req_type == trap.TRAP_FMT_UNIREC:
rec = URInputTmplt(data)
elif req_type == trap.TRAP_FMT_JSON:
rec = json.loads(data)
else: # TRAP_FMT_RAW
rec = data
# *** Convert input record to IDEA ***
# Pass the input record to conversion function to create IDEA message
idea = conv_func(rec, args)
if idea is None:
continue # Record can't be converted - skip it (notice should be printed by the conv function)
if args.name is not None:
idea['Node'][0]['Name'] = args.name
if args.test:
idea['Category'].append('Test')
# *** Send IDEA to outputs ***
# File output
if filehandle:
filehandle.write(json.dumps(idea, indent=args.file_indent) + '\n')
# TRAP output
if args.trap:
try:
trap.send(0, json.dumps(idea))
except trap.ETerminated:
# don't exit immediately, first finish sending to other outputs
trap.stop = 1
# MongoDB output
if mongocoll:
# Convert timestamps from string to Date format
idea['DetectTime'] = datetime.strptime(idea['DetectTime'],
"%Y-%m-%dT%H:%M:%SZ")
for i in ['CreateTime', 'EventTime', 'CeaseTime']:
if idea.has_key(i):
idea[i] = datetime.strptime(idea[i], "%Y-%m-%dT%H:%M:%SZ")
try:
mongocoll.insert(idea)
except pymongo.errors.AutoReconnect:
sys.stderr.write(module_name +
": Error: MongoDB connection failure.\n")
trap.stop = 1
# Warden output
if wardenclient:
wardenclient.sendEvents([idea])
# *** Cleanup ***
if filehandle and filehandle != sys.stdout:
filehandle.close()
if mongoclient:
mongoclient.close()
if wardenclient:
wardenclient.close()
trap.finalize()
"-f",
"--format",
dest="format",
default="",
help=
"Set format identifier string (this must match the format ID required by receiving module)",
metavar="FMT_ID")
parser.add_option("-n",
"--no-eos",
action="store_true",
help="Don't send end-of-stream message at the end.")
module_info = trap.CreateModuleInfo(
"JSON replay", # Module name
"Replay JSON data read from input file or stdin to TRAP interface. "
"Each line of input must contain exactly one JSON record.",
0, # Number of input interfaces
1, # Number of output interfaces
parser)
# Initialize module
ifc_spec = trap.parseParams(sys.argv, module_info)
trap.init(module_info, ifc_spec)
trap.registerDefaultSignalHandler()
# Parse remaining command-line arguments
(options, args) = parser.parse_args()
if len(args) > 1:
sys.stderr.write("Error: Only one input file can be specified.")
elif len(args) == 1:
import trap
import json
import optparse # TODO change TRAP Python wrapper to use argparse
from optparse import OptionParser
parser = OptionParser(add_help_option=False)
parser.add_option("-w", dest="filename",
help="Write dump to FILE instead of stdout (overwrite file)", metavar="FILE")
parser.add_option("-a", dest="filename_append",
help="Write dump to FILE instead of stdout (append to file)", metavar="FILE")
parser.add_option("-I", "--indent", metavar="N", type=int,
help="Pretty-print JSON with indentation set to N spaces. Note that such format can't be read by json_replay module.")
module_info = trap.CreateModuleInfo(
"JSON dump", # Module name
"Dump JSON data from input TRAP interface to file or standard output. Each JSON record is written to separate line.", # Description
1, # Number of input interfaces
0 # Number of output interfaces
)
# Initialize module
ifc_spec = trap.parseParams(sys.argv, module_info)
trap.init(module_info, ifc_spec)
trap.registerDefaultSignalHandler()
# Parse remaining command-line arguments
(options, args) = parser.parse_args()
# Open output file
if options.filename and options.filename_append:
sys.stderr.write("Error: -w and -a are mutually exclusive.")
module_info = trap.CreateModuleInfo(
"Email reporter", # Module name
"""\
Each UniRec record received is transformed to an email of specified
template and send to specified address.
Usage:
python email_reporter.py -i "ifc_spec" [options] CONFIG_FILE
Parameters:
CONFIG_FILE File with configuration. It should contain information about
SMTP server to connect to and a template of the message.
Format of this file is given below.
-d, --dry-run Dry-run mode - nothing is sent, messages are printed to
stdout instead.
--skip-smtp-test By default, the module tries to connect to specified SMTP
server on startup to check that the connection (and login
credentials, if specified) works. You can skip the test
using this option.
"""
# --limit=N Set maximal number of emails sent per hour. If more records arrive on input,...
"""\
The config file has two sections. First, there is specification of required
UniRec fields, SMTP server to be used, optionally login credentials and so on.
Format of this section is given by example below. You can use comments (start
them with '#') in this section.
The second section starts after a blank line and it contains template of
an email message in RFC2822 format. That is: headers (e.g. From, To, Subject),
blank line and message body. The "Date:" header is added automatically.
The template may contain references to fields of UniRec record which will be
substituted by corresponding values in each message. References are made using
'$' followed by name of the field.
An example of config file:
unirec=ipaddr SRC_IP,ipaddr DST_IP,string FILENAME,time TIMESTAMP
server=smtp.example.com
port=25 # optional, default: 25
starttls=1 # optional, default: 0
login=username:password # optional, default: no login
From: NEMEA <*****@*****.**>
To: Main Recipient <*****@*****.**>
Cc: <*****@*****.**>
Bcc: <*****@*****.**>
Subject: [Nemea-alert] $SRC_IP sent a picture of cat
NEMEA system has detected a picture of cat being sent over the internet.
Details:
Source: $SRC_IP
Destination: $DST_IP
File name: $FILENAME
Time of detection: $TIMESTAMP
-----
This is automatically generated email, don't reply.
You can contact Nemea administrators at [email protected].
""",
1, # Number of input interfaces
0 # Number of output interfaces
)