def process_stream(self, stream, factories):
combined_inode = "I%s|S%s/%s" % (stream.fd.name, stream.inode_id, stream.reverse)
pyflaglog.log(pyflaglog.DEBUG,"Openning %s for SMTP" % combined_inode)
## We open the file and scan it for emails:
fd = self.fsfd.open(inode=combined_inode)
dbh=DB.DBO(self.case)
p=SMTP(fd,dbh,self.fsfd)
## Iterate over all the messages in this connection
for f in p.parse():
if not f: continue
## message number and its offset:
count, offset, length = f
## Create the VFS node:
path, combined_inode, inode_id =self.fsfd.lookup(inode=combined_inode)
path=posixpath.normpath(path+"/../../../../../")
new_inode="%s|o%s:%s" % (combined_inode,offset,length)
ds_timestamp = Time.convert(stream.ts_sec, case=self.case, evidence_tz="UTC")
date_str = ds_timestamp.split(" ")[0]
self.fsfd.VFSCreate(None, new_inode,
"%s/SMTP/%s/Message_%s" % (path,
date_str,
count),
mtime = stream.ts_sec, size=length
)
## Scan the new file using the scanner train. If
## the user chose the RFC2822 scanner, we will be
## able to understand this:
self.scan_as_file(new_inode, factories)
def external_process(self,fd):
""" This is run on the extracted file """
pyflaglog.log(pyflaglog.VERBOSE_DEBUG, "Decompressing Zip File %s" % fd.inode)
cache_key = "%s:%s" % (self.case , self.fd.inode)
## Try to read the fd as a zip file
z = zipfile.ZipFile(fd)
pathname, inode, inode_id = self.ddfs.lookup(inode = self.inode)
## retrieve evidence timezone, this is necessary because zip files
## store time in localtime
evidence_tz = Time.get_evidence_tz_name(self.case, self.fd)
## List all the files in the zip file:
dircount = 0
inodes = []
namelist = z.namelist()
for i in range(len(namelist)):
## Add the file into the VFS
try:
## Convert the time to case timezone
t = Time.convert(z.infolist()[i].date_time, case=self.case, evidence_tz=evidence_tz)
except:
t=0
## If the entry corresponds to just a directory we ignore it.
if not posixpath.basename(namelist[i]): continue
info = z.infolist()[i]
inode = "%s|Z%s:%s" % (self.inode,info.header_offset, info.compress_size)
inodes.append(inode)
inode_id = self.ddfs.VFSCreate(None,
inode,DB.expand("%s/%s",(pathname,namelist[i])),
size=info.file_size,
mtime=t, _fast=True)
for inode in inodes:
## Now call the scanners on this new file (FIXME limit
## the recursion level here)
fd = self.ddfs.open(inode_id = inode_id)
Scanner.scanfile(self.ddfs,fd,self.factories)
new_inode += "|G1"
except KeyError:
pass
try:
if 'deflate' in p.response['content-encoding']:
new_inode += "|d1"
except KeyError:
pass
## stream.ts_sec is already formatted in DB format
## need to convert back to utc/gmt as paths are UTC
timestamp = fd.get_packet_ts(offset)
ds_timestamp = Time.convert(timestamp, case=self.case, evidence_tz="UTC")
try:
date_str = ds_timestamp.split(" ")[0]
except:
date_str = stream.ts_sec.split(" ")[0]
path,inode,inode_id=self.fsfd.lookup(inode=combined_inode)
## Try to put the HTTP inodes at the mount point. FIXME:
## This should not be needed when a http stats viewer is
## written.
path=posixpath.normpath(path+"/../../../../../")
inode_id = self.fsfd.VFSCreate(None,new_inode,
"%s/HTTP/%s/%s" % (path,date_str,
escape(p.request['url'])),
def process_stream(self, stream, factories):
""" We look for HTTP requests to identify the stream. This
allows us to processes HTTP connections on unusual ports. This
situation might arise if HTTP proxies are used for example.
"""
if stream.reverse:
combined_inode = "I%s|S%s/%s" % (stream.fd.name, stream.inode_id, stream.reverse)
try:
fd = self.fsfd.open(inode=combined_inode)
## If we cant open the combined stream, we quit (This could
## happen if we are trying to operate on a combined stream
## already
except IOError: return
else:
fd = stream
p=HTTP(fd,self.fsfd)
## Check that this is really HTTP
if not p.identify():
return
pyflaglog.log(pyflaglog.DEBUG,"Openning %s for HTTP" % combined_inode)
## Iterate over all the messages in this connection
for f in p.parse():
if not f: continue
offset, size = f
## Create the VFS node:
new_inode="%s|H%s:%s" % (combined_inode,offset,size)
try:
if 'chunked' in p.response['transfer-encoding']:
new_inode += "|c0"
except KeyError:
pass
try:
if 'gzip' in p.response['content-encoding']:
new_inode += "|G1"
except KeyError:
pass
try:
if 'deflate' in p.response['content-encoding']:
new_inode += "|d1"
except KeyError:
pass
## stream.ts_sec is already formatted in DB format
## need to convert back to utc/gmt as paths are UTC
timestamp = fd.get_packet_ts(offset)
ds_timestamp = Time.convert(timestamp, case=self.case, evidence_tz="UTC")
try:
date_str = ds_timestamp.split(" ")[0]
except:
date_str = stream.ts_sec.split(" ")[0]
path,inode,inode_id=self.fsfd.lookup(inode=combined_inode)
## Try to put the HTTP inodes at the mount point. FIXME:
## This should not be needed when a http stats viewer is
## written.
path=posixpath.normpath(path+"/../../../../../")
inode_id = self.fsfd.VFSCreate(None,new_inode,
"%s/HTTP/%s/%s" % (path,date_str,
escape(p.request['url'])),
mtime=timestamp, size=size
)
## Update the inode again:
#new_inode = new_inode % inode_id
## This updates the inode table with the new inode
#self.fsfd.VFSCreate(None,new_inode,
# None, update_only = True,
# inode_id = inode_id
# )
## Store information about this request in the
## http table:
host = p.request.get("host",IP2str(stream.dest_ip))
url = HTML.url_unquote(p.request.get("url"))
try:
date = p.response["date"]
date = Time.parse(date, case=self.case, evidence_tz=None)
except (KeyError,ValueError):
date = 0
## Two forms for the referrer:
referer = p.request.get('referer', p.request.get('referrer',''))
if not url.startswith("http://") and not url.startswith("ftp://"):
url = "http://%s%s" % (host, url)
## Not sure if we really care about this?
## Find referred page:
## parent = 0
dbh = DB.DBO(self.case)
## if referer:
## dbh.execute("select inode_id from http where url=%r order by inode_id desc limit 1", referer)
## row = dbh.fetch()
## ## If there is no referrer we just make a psuedo entry
## if not row:
## ## Find out the host
## m=re.match("(http://|ftp://)([^/]+)([^\?\&\=]*)",
## "%s" % referer)
## if m:
## host = m.group(2)
## dbh.insert("http", url=referer, host=host)
## parent = dbh.autoincrement()
## else:
## parent = row['inode_id']
args = dict(inode_id = inode_id,
request_packet = p.request.get("packet_id",0),
method = p.request.get("method","-"),
url = url,
response_packet= p.response.get("packet_id"),
status = p.response.get("HTTP_code"),
content_type = p.response.get("content-type","text/html"),
referrer = referer[:500],
host = host,
tld = make_tld(host),
useragent = p.request.get('user-agent', '-'),
)
if date:
args['date'] = date
dbh.insert('http', **args)
# parent = parent)
## Replicate the information about the subobjects in the
## connection_details table - this makes it easier to do
## some queries:
dbh.insert("connection_details",
ts_sec = stream.ts_sec,
inode_id = inode_id,
src_ip = stream.src_ip,
src_port = stream.src_port,
dest_ip = stream.dest_ip,
dest_port = stream.dest_port,
)
## handle the request's parameters:
try:
self.handle_parameters(p.request, inode_id)
except (KeyError, TypeError):
pass
## Only scan the new file using the scanner train if its
## size of bigger than 0:
if size>0:
self.scan_as_file(new_inode, factories)
