def preModule(self):
'''parse the source and target lists'''
if self.target:
for tstr in self.target:
targets=util.strtok(tstr, as_list=True)[0]
for t in targets:
try:
parts=t.split(':')
if len(parts)==2: ip,port=parts #IP:port
else: ip,port=t,None #IPv6 addr
except: ip,port=t,None #IP
if ip=='': ip=None # :port
self.targets.append((ip,port))
if self.source:
for sstr in self.source:
sources=util.strtok(sstr, as_list=True)[0]
for ip in sources:
self.sources.append(ip)
dshell.TCPDecoder.preModule(self)
def preModule(self):
'''parse the source and target lists'''
if self.target:
for tstr in self.target:
targets = util.strtok(tstr, as_list=True)[0]
for t in targets:
try:
parts = t.split(':')
if len(parts) == 2: ip, port = parts #IP:port
else: ip, port = t, None #IPv6 addr
except:
ip, port = t, None #IP
if ip == '': ip = None # :port
self.targets.append((ip, port))
if self.source:
for sstr in self.source:
sources = util.strtok(sstr, as_list=True)[0]
for ip in sources:
self.sources.append(ip)
dshell.TCPDecoder.preModule(self)
def main(*largs, **kwargs):
global log
bin_path = os.environ['BINPATH']
sys.path.insert(0, bin_path)
# get map of name to module import path
decoder_map = getDecoders(setDecoderPath(os.environ['DECODERPATH']))
# The main argument parser. It will have every command line option
# available and should be used when actually parsing
parser = dshellOptionParser(
usage="usage: %prog [options] [decoder options] file1 file2 ... filen [-- [decoder args]+]",
version="%prog " + str(dshell.__version__), add_help_option=False)
# A short argument parser, meant to only hold the shorter list of
# arguments for when a decoder is called without a pcap file. DO
# NOT USE for any serious argument parsing.
parser_short = dshellOptionParser(
usage="usage: %prog [options] [decoder options] file1 file2 ... filen [-- [decoder args]+]",
version="%prog " + str(dshell.__version__), add_help_option=False)
parser.add_option('-h', '-?', '--help', dest='help',
help="Print common command-line flags and exit", action='store_true',
default=False)
parser_short.add_option('-h', '-?', '--help', dest='help',
help="Print common command-line flags and exit", action='store_true',
default=False)
parser.add_option('-d', '--decoder', dest="decoder",
action='append', help="Use a specific decoder module")
parser.add_option('-l', '--ls', '--list', action="store_true",
help='List all available decoders', dest='list')
parser.add_option(
'-C', '--config', dest='config', help='specify config.ini file')
parser.add_option('--tmpdir', dest='tmpdir', type='string', default=tempfile.gettempdir(),
help='alternate temp directory (for use when processing compressed pcap files)')
parser.add_option('-r', '--recursive', dest='recursive', action='store_true',
help='recursively process all PCAP files under input directory')
group = optparse.OptionGroup(parser, "Multiprocessing options")
group.add_option('-p', '--parallel', dest='parallel',
action='store_true', help='process multiple files in parallel')
group.add_option('-t', '--threaded', dest='threaded',
action='store_true', help='run multiple decoders in parallel')
group.add_option('-n', '--nprocs', dest='numprocs', type='int',
default=4, help='number of simultaneous processes')
parser.add_option_group(group)
# decode-pcap specific options
group = optparse.OptionGroup(parser, "Input options")
group.add_option('-i', '--interface', dest='interface',
default=None, help='listen live on INTERFACE')
group.add_option('-c', '--count', dest='count', type='int',
help='number of packets to process', default=0)
group.add_option('-f', '--bpf', dest='bpf',
help='replace default decoder filter (use carefully)')
group.add_option('--nofilterfn', dest='nofilterfn',
action="store_true", help='Set filterfn to pass-thru')
group.add_option('-F', dest='filefilter',
help='Use filefilter as input for the filter expression. An additional expression given on the command line is ignored.')
group.add_option(
'--ebpf', dest='ebpf', help='BPF filter to exclude traffic, extends other filters')
group.add_option('--no-vlan', dest='novlan', action="store_true",
help='do not examine traffic which has VLAN headers present')
group.add_option('--layer2', dest='layer2', default='ethernet.Ethernet',
help='select the layer-2 protocol module')
group.add_option('--strip', dest='striplayers', default=0,
help='extra data-link layers to strip')
parser.add_option_group(group)
group = optparse.OptionGroup(parser_short, "Input options")
group.add_option('-i', '--interface', dest='interface',
default=None, help='listen live on INTERFACE')
group.add_option('-c', '--count', dest='count', type='int',
help='number of packets to process', default=0)
group.add_option('-f', '--bpf', dest='bpf',
help='replace default decoder filter (use carefully)')
group.add_option('--nofilterfn', dest='nofilterfn',
action="store_true", help='Set filterfn to pass-thru')
group.add_option('-F', dest='filefilter',
help='Use filefilter as input for the filter expression. An additional expression given on the command line is ignored.')
group.add_option(
'--ebpf', dest='ebpf', help='BPF filter to exclude traffic, extends other filters')
group.add_option('--no-vlan', dest='novlan', action="store_true",
help='do not examine traffic which has VLAN headers present')
group.add_option('--layer2', dest='layer2', default='ethernet.Ethernet',
help='select the layer-2 protocol module')
group.add_option('--strip', dest='striplayers', default=0,
help='extra data-link layers to strip')
parser_short.add_option_group(group)
group = optparse.OptionGroup(parser, "Output options")
group.add_option('-o', '--outfile', dest='outfile', help='write output to the file OUTFILE. Additional output can be set with KEYWORD=VALUE,...\n' +
'\tmode=<w: write (default), a: append, noclobber: do not overwrite, use a a OUTFILE.1 (.2,.3) file if file(s) exists\n' +
'\tpcap=PCAPFILE to write packets to a PCAP file\n' +
'\tsession=SESSION to write session text\n' +
'\tdirection=data direction to write (c,s,both,split)')
group.add_option('-w', '--session', dest='session',
help='write session file, same as -o session=')
group.add_option('-W', '--pcap', dest='pcap', default=None,
help='output decoded packets to PCAP (same as -o pcap=....)')
group.add_option('--db', dest='db', default=None,
help='output to db. Supply "config=file" or "param=...,param=..." ')
group.add_option(
'--oformat', dest='oformat', help='define the output format')
group.add_option('-x', '--extra', dest='oextra',
action='store_true', help='output a lot of extra information')
group.add_option('-O', '--output', dest='output', default=None,
help='Use a custom output module. Supply "modulename,option=value,..."')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Logging options")
group.add_option('-L', '--logfile', dest="logfile", help="log to file")
group.add_option('--debug', action="store_true", dest="debug",
help="debug logging (debug may also affect decoding behavior)")
group.add_option('-v', '--verbose', action="store_true",
dest="verbose", help="verbose logging")
group.add_option('-q', '--quiet', action="store_true",
dest="quiet", help="practically zero logging")
parser.add_option_group(group)
group = optparse.OptionGroup(parser_short, "Logging options")
group.add_option('-L', '--logfile', dest="logfile", help="log to file")
group.add_option('--debug', action="store_true", dest="debug",
help="debug logging (debug may also affect decoding behavior)")
group.add_option(
'-v', '--verbose', action="store_true", dest="verbose", help="verbose logging")
group.add_option('-q', '--quiet', action="store_true",
dest="quiet", help="practically zero logging")
parser_short.add_option_group(group)
# [decoder][option]=value dict of decoder options, set by config file
decoder_options = {}
decoder_args = []
args = []
extra_args = False
for x in largs:
if x == '--':
extra_args = True
continue
if extra_args:
decoder_args.append(x)
else:
args.append(x)
# parse basic options and crdate the options object
options = parser.parse_args(args, **kwargs)[0]
if options == None:
print "\nError processing provided arguments"
return
# dump list
if options.list:
printDecoders(decoder_map, not options.debug)
return
# parse config file, updating the options and decoder_options dicts
if options.config:
try:
import ConfigParser
config = ConfigParser.ConfigParser()
config.read(options.config)
for s in config.sections():
# this is the main section, set the options
if s.lower() == 'dshell':
for k, v in config.items(s, raw=True):
if k in options.__dict__:
options.__dict__[k] = v
except:
raise # :-(
# are we a thread outputting to a queue?
if 'queue' in options.__dict__:
out = output.QueueOutput(options.queue)
# if not, parse output args
else:
outfile = None
outkw = {}
# set output file (and other args if -o filename,key=val...)
if options.outfile:
outfile, outkw = util.strtok(options.outfile)
# output extra?
if options.oextra:
outkw.update(extra=True)
# set session writer?
if options.session:
outkw.update(session=options.session)
# add default pcap writer?
if options.pcap:
outkw.update(pcap=options.pcap)
# use database?
if options.db:
a, kw = util.strtok(options.db, as_list=True)
# add output options
kw.update(outkw)
out = output.DBOutput(*a, **kw)
# if not db mode and no out module specd
# use default output lib to get default module
elif not options.output:
options.output = 'output'
# init output module
if options.output:
# parse output arglist (-O module,args..,k=val...)
a, kw = util.strtok(options.output, as_list=True)
kw.update(outkw) # set output options
if outfile:
# set filename arg if -o given (can also be first arg in module
# arglist)
kw.update(file=outfile)
outmod = import_module(name=os.path.basename(a[0])) # load module
if outmod:
# pass remaining args and keyword args to init object
out = outmod.obj(*a[1:], **kw)
# set the output format
if options.oformat != None:
out.setformat(options.oformat)
# set global log functions
out.logger = logging.getLogger('dshell')
log = out.log
# start up the logger
if options.debug:
level = logging.DEBUG
elif options.verbose:
level = logging.INFO
elif options.quiet:
level = logging.FATAL
else:
level = logging.WARNING
logging.basicConfig(filename=options.logfile, level=level)
decoders = {}
decoderNames = set()
# check for a decoder
if options.decoder != None:
# single decoder or came from config file
if type(options.decoder) == str:
options.decoder = util.strtok(
options.decoder, as_list=True)[0] # make it a list
# we have a list of decoders
for dnames in options.decoder:
chain = dnames.split('+')
# last module does not have a subdecoder
module = chain.pop()
try:
module, n = module.split(':', 1)
except:
n = None
m = import_module(module, search=decoder_map)
if m:
# create copy in case we import multiple times under different
# names
dObj = copy.copy(m.dObj)
if n:
dObj.name = n
else:
dObj = None
try:
decoderNames.add(dObj.name)
except AttributeError:
decoderNames.add(module)
# walk up the chain, setting sub-decoders
while chain:
subObj = dObj
module = chain.pop()
try:
module, n = module.split(':', 1)
except: # :-(
n = None
m = import_module(module, search=decoder_map)
if m:
dObj = copy.copy(m.dObj)
if n:
dObj.name = n
else:
dObj = None
try:
decoderNames.add(dObj.name)
except AttributeError:
decoderNames.add(module)
if dObj and dObj.chainable:
dObj.subDecoder = subObj
elif dObj:
sys.stderr.write("Error %s is not chainable\n" % module)
# insert the top decoder in the dict
if dObj:
decoders[dObj.name] = dObj
# save option dict
options_dict = options.__dict__.copy()
# add in options for loaded decoders and subdecoders
for d in decoders.itervalues():
parser.add_decoder_options(d)
for d in decoders.itervalues():
parser_short.add_decoder_options(d)
# reparse args to handle decoder options
optionerror = False
try:
options, args = parser.parse_args(args, **kwargs)
except:
optionerror = True
# replace base options
options.__dict__.update(options_dict)
# look for name_option keys and put them in decoder_options[name][option]
for k, v in options.__dict__.iteritems():
for decName in decoderNames:
try:
n = k.split(decName + '_', 1)[1]
decoder_options.setdefault(decName, {}).setdefault(n, v)
except IndexError:
continue
# reparse config file to handle decoder options
if options.config:
for s in config.sections():
# set the options for loaded decoders if they are present in the
# config file
if s.lower() in decoder_options:
for k, v in config.items(s, raw=True):
if k in decoder_options[s]: # if this is a valid option
if v.isdigit():
v = int(v) # try conversion to int/float
elif '.' in v:
try:
v = float(v)
except:
pass
decoder_options[s][k] = v
if any(x in ('-h', '-?', '--help') for x in sys.argv[1:]):
# Print the verbose help message
parser.print_help()
printDecoderBriefs(decoders)
return
if optionerror or (not args and not options.interface):
# Print the short help message
parser_short.print_help()
printDecoderBriefs(decoders)
return
#######################################################################
# listen live on the interface
# this will not process any files
if options.interface != None:
if len(decoders) != 1:
print 'Can only run one module live on an interface'
return
# handles options and arguments for dumping live on an interface
decode_live(out, options, dObj, decoder_args, decoder_options)
# close output
out.close()
return
#######################################################################
# take all other command line arguments as files to process
####################################################
# Works if directory (e.g. ~/data/) or * (e.g. ~/data/*
# used on command line. Does not work for partial
# wildcards (e.g. ~/data/*.dat) because bash
# expands wildcards before passing arguments into
# decode-pcap.py. Will work if no matches in root of
# path specified.
####################################################
inputs = []
for file_path in args:
# If this argument is a directory and RECURSIVE specified, then add
# entire directory tree to the list of input files
if os.path.isdir(file_path) and options.recursive:
addFilesFromDirectory(inputs, file_path)
# If a wildcard is specified, then handle accordingly
elif file_path.find('*') > -1:
(path, wildcard) = os.path.split(file_path)
# If just file is specified (no path)
if len(path) == 0:
inputs.extend(glob.glob(wildcard))
# If there is a path, but recursion not specified,
# then just add matching files from specified dir
elif not len(path) == 0 and not options.recursive:
inputs.extend(glob.glob(file_path))
# Otherwise, recursion specified and there is a directory.
# Recurse directory and add files
else:
addFilesFromDirectory(inputs, path, wildcard)
# Just a normal file, append to list of inputs
else:
inputs.append(file_path)
if options.parallel or options.threaded:
import multiprocessing
procs = []
q = multiprocessing.Queue()
kwargs = options.__dict__.copy() # put parsed base options in kwargs
kwargs.update(config=None, outfile=None, queue=q) # pass the q,
# do not pass the config file or outfile because we handled that here
for d in decoder_options: # put pre-parsed decoder options in kwargs
for k, v in decoder_options[d].items():
kwargs[d + '_' + k] = v
# check here to see if we are running in parallel-file mode
if options.parallel and len(inputs) > 1:
for f in inputs:
# create a child process for each input file
procs.append(
multiprocessing.Process(target=main, kwargs=kwargs, args=[f]))
runChildProcs(procs, q, out, numprocs=options.numprocs)
# check here to see if we are running decoders multithreaded
elif options.threaded and len(options.decoder) > 1:
for d in options.decoder:
# create a child for each decoder
kwargs.update(decoder=d)
procs.append(
multiprocessing.Process(target=main, kwargs=kwargs, args=inputs))
runChildProcs(procs, q, out, numprocs=options.numprocs)
# fall through to here (single threaded or child process)
else:
#
# Here is where we use the decoder(s) to process the pcap
#
temporaryFiles = [] # used when uncompressing files
for module in decoders.keys():
decoder = decoders[module]
initDecoderOptions(
decoder, out, options, decoder_args, decoder_options)
# If the decoder has a preModule function, will execute it now
decoder.preModule()
for input_file in inputs:
# Decoder-specific options may be seen as input files
# Skip anything starts with "--"
if input_file[:2] == '--':
continue
# Recursive directory processing is handled elsewhere,
# so we should only be dealing with files at this point.
if os.path.isdir(input_file):
continue
log('+Processing file %s' % input_file)
# assume the input_file is not compressed
# Allows the processing of .pcap files that are compressed with
# gzip, bzip2, or zip. Writes uncompressed file to a
# NamedTemporaryFile and unlinks the file once it is no longer
# needed. Might consider using mkstemp() since this implementation
# requires Python >= 2.6.
try:
exts = ['.gz', '.bz2', '.zip']
if os.path.splitext(input_file)[1] not in exts:
pcapfile = input_file
else:
# we have a compressed file
tmpfile = expandCompressedFile(
input_file, options.verbose, options.tmpdir)
temporaryFiles.append(tmpfile)
pcapfile = tmpfile
except:
if options.verbose:
sys.stderr.write(
'+Error processing file %s' % (input_file))
continue
# give the decoder access to the input filename
# motivation: run a decoder against a large number of pcap
# files and have the decoder print the filename
# so you can go straight to the pcap file for
# further analysis
decoder.input_file = input_file
# Check to see if the decoder has a preFile function
# This will be called before the decoder processes each
# input file
decoder.preFile()
try:
if not pcap:
raise NotImplementedError(
"pcap support not implemented")
decoder.capture = pcap.pcap(pcapfile)
if decoder.filter:
decoder.capture.setfilter(decoder.filter)
while not options.count or decoder.count < options.count:
try:
# read next packet and break on EOF
ts, pkt = decoder.capture.next()
except:
break # no data
decoder.decode(ts, pkt)
except KeyboardInterrupt:
raise
except:
traceback.print_exc()
if options.verbose:
log('+Done processing %s' % (input_file))
# call that decoder's processFile()
decoder.postFile()
# check to see if the decoder is using the Messages class
# if so, we need to clean up the connection store to
# purge any unfinished connections
if 'cleanConnectionStore' in dir(decoder):
decoder.cleanConnectionStore()
# Check to see if the decoder has a postModule function
# A postModule function will be called when the module
# has finished running against all of the input files
if 'postModule' in dir(decoder):
decoder.postModule()
# remove any temporary files that were created during execution
for tmpfile in temporaryFiles:
if options.verbose:
log('+Unlinking %s' % (tmpfile))
os.unlink(tmpfile)
# close output
out.close()
return
def main(*largs,**kwargs):
global log
bin_path = os.environ['BINPATH']
sys.path.insert(0,bin_path)
#get map of name to module import path
decoder_map=getDecoders(setDecoderPath(os.environ['DECODERPATH']))
# The main argument parser. It will have every command line option available and should be used when actually parsing
parser = dshellOptionParser(usage="usage: %prog [options] [decoder options] file1 file2 ... filen [-- [decoder args]+]", version="%prog "+str(dshell.__version__), add_help_option=False)
# A short argument parser, meant to only hold the shorter list of arguments for when a decoder is called without a pcap file. DO NOT USE for any serious argument parsing.
parser_short = dshellOptionParser(usage="usage: %prog [options] [decoder options] file1 file2 ... filen [-- [decoder args]+]", version="%prog "+str(dshell.__version__), add_help_option=False)
parser.add_option('-h', '-?', '--help', dest='help', help="Print common command-line flags and exit", action='store_true', default=False)
parser_short.add_option('-h', '-?', '--help', dest='help', help="Print common command-line flags and exit", action='store_true', default=False)
parser.add_option('-d','--decoder',dest="decoder",action='append',help="Use a specific decoder module")
parser.add_option('-l','--ls','--list',action="store_true",help='List all available decoders',dest='list')
parser.add_option('-C','--config',dest='config',help='specify config.ini file')
parser.add_option('--tmpdir', dest='tmpdir', type='string', default=tempfile.gettempdir(), help='alternate temp directory (for use when processing compressed pcap files)')
parser.add_option('-r','--recursive',dest='recursive', action='store_true',help='recursively process all PCAP files under input directory')
group = optparse.OptionGroup(parser,"Multiprocessing options")
group.add_option('-p','--parallel',dest='parallel',action='store_true',help='process multiple files in parallel')
group.add_option('-t','--threaded',dest='threaded',action='store_true',help='run multiple decoders in parallel')
group.add_option('-n','--nprocs',dest='numprocs',type='int',default=4,help='number of simultaneous processes')
parser.add_option_group(group)
# decode-pcap specific options
group = optparse.OptionGroup(parser,"Input options")
group.add_option('-i','--interface',dest='interface',default=None,help='listen live on INTERFACE')
group.add_option('-c','--count',dest='count',type='int',help='number of packets to process',default=0)
group.add_option('-f','--bpf',dest='bpf',help='replace default decoder filter (use carefully)')
group.add_option('--nofilterfn',dest='nofilterfn',action="store_true",help='Set filterfn to pass-thru')
group.add_option('-F',dest='filefilter',help='Use filefilter as input for the filter expression. An additional expression given on the command line is ignored.')
group.add_option('--ebpf',dest='ebpf',help='BPF filter to exclude traffic, extends other filters')
group.add_option('--no-vlan',dest='novlan',action="store_true",help='do not examine traffic which has VLAN headers present')
group.add_option('--layer2',dest='layer2',default='ethernet.Ethernet',help='select the layer-2 protocol module')
group.add_option('--strip',dest='striplayers',default=0,help='extra data-link layers to strip')
parser.add_option_group(group)
group = optparse.OptionGroup(parser_short,"Input options")
group.add_option('-i','--interface',dest='interface',default=None,help='listen live on INTERFACE')
group.add_option('-c','--count',dest='count',type='int',help='number of packets to process',default=0)
group.add_option('-f','--bpf',dest='bpf',help='replace default decoder filter (use carefully)')
group.add_option('--nofilterfn',dest='nofilterfn',action="store_true",help='Set filterfn to pass-thru')
group.add_option('-F',dest='filefilter',help='Use filefilter as input for the filter expression. An additional expression given on the command line is ignored.')
group.add_option('--ebpf',dest='ebpf',help='BPF filter to exclude traffic, extends other filters')
group.add_option('--no-vlan',dest='novlan',action="store_true",help='do not examine traffic which has VLAN headers present')
group.add_option('--layer2',dest='layer2',default='ethernet.Ethernet',help='select the layer-2 protocol module')
group.add_option('--strip',dest='striplayers',default=0,help='extra data-link layers to strip')
parser_short.add_option_group(group)
group = optparse.OptionGroup(parser,"Output options")
group.add_option('-o','--outfile',dest='outfile',help='write output to the file OUTFILE. Additional output can be set with KEYWORD=VALUE,...\n'+\
'\tmode=<w: write (default), a: append, noclobber: do not overwrite, use a a OUTFILE.1 (.2,.3) file if file(s) exists\n'+\
'\tpcap=PCAPFILE to write packets to a PCAP file\n'+\
'\tsession=SESSION to write session text\n'+\
'\tdirection=data direction to write (c,s,both,split)')
group.add_option('-w','--session',dest='session',help='write session file, same as -o session=')
group.add_option('-W','--pcap',dest='pcap',default=None,help='output decoded packets to PCAP (same as -o pcap=....)')
group.add_option('--db',dest='db',default=None,help='output to db. Supply "config=file" or "param=...,param=..." ')
group.add_option('--oformat',dest='oformat',help='define the output format')
group.add_option('-x','--extra',dest='oextra',action='store_true',help='output a lot of extra information')
group.add_option('-O','--output',dest='output',default=None,help='Use a custom output module. Supply "modulename,option=value,..."')
parser.add_option_group(group)
group = optparse.OptionGroup(parser,"Logging options")
group.add_option('-L','--logfile',dest="logfile",help="log to file")
group.add_option('--debug',action="store_true", dest="debug",help="debug logging (debug may also affect decoding behavior)")
group.add_option('-v','--verbose',action="store_true", dest="verbose",help="verbose logging")
group.add_option('-q','--quiet',action="store_true", dest="quiet",help="practically zero logging")
parser.add_option_group(group)
group = optparse.OptionGroup(parser_short,"Logging options")
group.add_option('-L','--logfile',dest="logfile",help="log to file")
group.add_option('--debug',action="store_true", dest="debug",help="debug logging (debug may also affect decoding behavior)")
group.add_option('-v','--verbose',action="store_true", dest="verbose",help="verbose logging")
group.add_option('-q','--quiet',action="store_true", dest="quiet",help="practically zero logging")
parser_short.add_option_group(group)
decoder_options={} #[decoder][option]=value dict of decoder options, set by config file
decoder_args = []
args = []
extra_args = False
for x in largs:
if x == '--':
extra_args = True
continue
if extra_args:
decoder_args.append(x)
else:
args.append(x)
#parse basic options and crdate the options object
options = parser.parse_args(args,**kwargs)[0]
if options == None:
print "\nError processing provided arguments"
return
#dump list
if options.list:
printDecoders(decoder_map,not options.debug)
return
#parse config file, updating the options and decoder_options dicts
if options.config:
try:
import ConfigParser
config = ConfigParser.ConfigParser()
config.read(options.config)
for s in config.sections():
if s.lower()=='dshell': #this is the main section, set the options
for k,v in config.items(s,raw=True):
if k in options.__dict__: options.__dict__[k]=v
except: raise
#are we a thread outputting to a queue?
if 'queue' in options.__dict__: out=output.QueueOutput(options.queue)
#if not, parse output args
else:
outfile=None
outkw={}
#set output file (and other args if -o filename,key=val...)
if options.outfile: outfile,outkw=util.strtok(options.outfile)
#output extra?
if options.oextra: outkw.update(extra=True)
#set session writer?
if options.session: outkw.update(session=options.session)
#add default pcap writer?
if options.pcap: outkw.update(pcap=options.pcap)
#use database?
if options.db:
a,kw=util.strtok(options.db,as_list=True)
#add output options
kw.update(outkw)
out=output.DBOutput(*a,**kw)
#if not db mode and no out module specd
# use default output lib to get default module
elif not options.output: options.output='output'
#init output module
if options.output:
a,kw=util.strtok(options.output,as_list=True) #parse output arglist (-O module,args..,k=val...)
kw.update(outkw) #set output options
if outfile: kw.update(file=outfile) #set filename arg if -o given (can also be first arg in module arglist)
outmod = import_module(name=os.path.basename(a[0])) #load module
if outmod: out=outmod.obj(*a[1:],**kw) #pass remaining args and keyword args to init object
#set the output format
if options.oformat != None: out.setformat(options.oformat)
#set global log functions
out.logger=logging.getLogger('dshell')
log=out.log
#start up the logger
if options.debug: level=logging.DEBUG
elif options.verbose: level=logging.INFO
elif options.quiet: level=logging.FATAL
else: level=logging.WARNING
logging.basicConfig(filename=options.logfile,level=level)
decoders = {}
decoderNames = set()
# check for a decoder
if options.decoder != None:
if type(options.decoder)==str: #single decoder or came from config file
options.decoder=util.strtok(options.decoder,as_list=True)[0] #make it a list
# we have a list of decoders
for dnames in options.decoder:
chain=dnames.split('+')
#last module does not have a subdecoder
module=chain.pop()
try: module,n=module.split(':',1)
except: n=None
m=import_module(module,search=decoder_map)
if m:
dObj=copy.copy(m.dObj) #create copy in case we import multiple times under different names
if n: dObj.name=n
else: dObj=None
try: decoderNames.add(dObj.name)
except AttributeError: decoderNames.add(module)
#walk up the chain, setting sub-decoders
while chain:
subObj=dObj
module=chain.pop()
try: module,n=module.split(':',1)
except: n=None
m=import_module(module,search=decoder_map)
if m:
dObj=copy.copy(m.dObj)
if n: dObj.name=n
else: dObj=None
try: decoderNames.add(dObj.name)
except AttributeError: decoderNames.add(module)
if dObj and dObj.chainable: dObj.subDecoder=subObj
elif dObj: sys.stderr.write( "Error %s is not chainable\n"%module)
#insert the top decoder in the dict
if dObj: decoders[dObj.name]=dObj
#save option dict
options_dict=options.__dict__.copy()
#add in options for loaded decoders and subdecoders
for d in decoders.itervalues(): parser.add_decoder_options(d)
for d in decoders.itervalues(): parser_short.add_decoder_options(d)
#reparse args to handle decoder options
optionerror=False
try: options,args=parser.parse_args(args,**kwargs)
except: optionerror=True
#replace base options
options.__dict__.update(options_dict)
#look for name_option keys and put them in decoder_options[name][option]
for k,v in options.__dict__.iteritems():
for decName in decoderNames:
try:
n=k.split(decName+'_', 1)[1]
decoder_options.setdefault(decName,{}).setdefault(n,v)
except IndexError: continue
#reparse config file to handle decoder options
if options.config:
for s in config.sections():
if s.lower() in decoder_options: #set the options for loaded decoders if they are present in the config file
for k,v in config.items(s,raw=True):
if k in decoder_options[s]: #if this is a valid option
if v.isdigit(): v=int(v) #try conversion to int/float
elif '.' in v:
try: v=float(v)
except: pass
decoder_options[s][k]=v
if any(x in ('-h', '-?', '--help') for x in sys.argv[1:]):
# Print the verbose help message
parser.print_help()
printDecoderBriefs(decoders)
return
if optionerror or (not args and not options.interface):
# Print the short help message
parser_short.print_help()
printDecoderBriefs(decoders)
return
#####################################################################################
# listen live on the interface
# this will not process any files
if options.interface != None:
if len(decoders)!=1:
print 'Can only run one module live on an interface'
return
# handles options and arguments for dumping live on an interface
decode_live(out,options,dObj, decoder_args,decoder_options)
#close output
out.close()
return
#####################################################################################
# take all other command line arguments as files to process
####################################################
# Works if directory (e.g. ~/data/) or * (e.g. ~/data/*
# used on command line. Does not work for partial
# wildcards (e.g. ~/data/*.dat) because bash
# expands wildcards before passing arguments into
# decode-pcap.py. Will work if no matches in root of
# path specified.
####################################################
inputs = []
for file_path in args:
# If this argument is a directory and RECURSIVE specified, then add
# entire directory tree to the list of input files
if os.path.isdir(file_path) and options.recursive:
addFilesFromDirectory(inputs,file_path)
# If a wildcard is specified, then handle accordingly
elif file_path.find('*') > -1:
(path,wildcard) = os.path.split(file_path)
# If just file is specified (no path)
if len(path) == 0:
inputs.extend(glob.glob(wildcard))
# If there is a path, but recursion not specified,
# then just add matching files from specified dir
elif not len(path) == 0 and not options.recursive:
inputs.extend(glob.glob(file_path))
# Otherwise, recursion specified and there is a directory.
# Recurse directory and add files
else:
addFilesFromDirectory(inputs,path,wildcard)
# Just a normal file, append to list of inputs
else:
inputs.append(file_path)
if options.parallel or options.threaded:
import multiprocessing
procs=[]
q=multiprocessing.Queue()
kwargs=options.__dict__.copy() #put parsed base options in kwargs
kwargs.update(config=None,outfile=None,queue=q) #pass the q,
#do not pass the config file or outfile because we handled that here
for d in decoder_options: #put pre-parsed decoder options in kwargs
for k,v in decoder_options[d].items(): kwargs[d+'_'+k]=v
#check here to see if we are running in parallel-file mode
if options.parallel and len(inputs)>1:
for f in inputs:
#create a child process for each input file
procs.append(multiprocessing.Process(target=main,kwargs=kwargs,args=[f]))
runChildProcs(procs,q,out,numprocs=options.numprocs)
#check here to see if we are running decoders multithreaded
elif options.threaded and len(options.decoder)>1:
for d in options.decoder:
#create a child for each decoder
kwargs.update(decoder=d)
procs.append(multiprocessing.Process(target=main,kwargs=kwargs,args=inputs))
runChildProcs(procs,q,out,numprocs=options.numprocs)
#fall through to here (single threaded or child process)
else:
#
# Here is where we use the decoder(s) to process the pcap
#
temporaryFiles = [] # used when uncompressing files
for module in decoders.keys():
decoder = decoders[module]
initDecoderOptions(decoder,out,options,decoder_args,decoder_options)
# If the decoder has a preModule function, will execute it now
decoder.preModule()
for input_file in inputs:
# Decoder-specific options may be seen as input files
# Skip anything starts with "--"
if input_file[:2] == '--':
continue
# Recursive directory processing is handled elsewhere,
# so we should only be dealing with files at this point.
if os.path.isdir(input_file): continue
log('+Processing file %s' % input_file)
# assume the input_file is not compressed
# Allows the processing of .pcap files that are compressed with
# gzip, bzip2, or zip. Writes uncompressed file to a
# NamedTemporaryFile and unlinks the file once it is no longer
# needed. Might consider using mkstemp() since this implementation
# requires Python >= 2.6.
try:
exts = ['.gz','.bz2','.zip']
if os.path.splitext(input_file)[1] not in exts:
pcapfile=input_file
else:
# we have a compressed file
tmpfile = expandCompressedFile(input_file,options.verbose,options.tmpdir)
temporaryFiles.append(tmpfile)
pcapfile=tmpfile
except:
if options.verbose: sys.stderr.write('+Error processing file %s' % (input_file))
continue
# give the decoder access to the input filename
# motivation: run a decoder against a large number of pcap
# files and have the decoder print the filename
# so you can go straight to the pcap file for
# further analysis
decoder.input_file = input_file
# Check to see if the decoder has a preFile function
# This will be called before the decoder processes each
# input file
decoder.preFile()
try:
if not pcap: raise NotImplementedError("pcap support not implemented")
decoder.capture=pcap.pcap(pcapfile)
if decoder.filter: decoder.capture.setfilter(decoder.filter)
while not options.count or decoder.count<options.count:
try: ts,pkt=decoder.capture.next() #read next packet and break on EOF
except: break #no data
decoder.decode(ts,pkt)
except KeyboardInterrupt: raise
except: traceback.print_exc()
if options.verbose:
log('+Done processing %s' % (input_file))
# call that decoder's processFile()
decoder.postFile()
# check to see if the decoder is using the Messages class
# if so, we need to clean up the connection store to
# purge any unfinished connections
if 'cleanConnectionStore' in dir(decoder): decoder.cleanConnectionStore()
# Check to see if the decoder has a postModule function
# A postModule function will be called when the module
# has finished running against all of the input files
if 'postModule' in dir(decoder): decoder.postModule()
# remove any temporary files that were created during execution
for tmpfile in temporaryFiles:
if options.verbose: log('+Unlinking %s'%(tmpfile))
os.unlink(tmpfile)
#close output
out.close()
return
def _parseCookies(self, data):
p, kwp = util.strtok(data, sep='; ')
return dict((urllib.unquote(k), urllib.unquote(kwp[k]))for k in kwp.keys())
def _parseCookies(self, data):
p,kwp=util.strtok(data,sep='; ')
return dict((urllib.unquote(k),urllib.unquote(kwp[k]))for k in kwp.keys())