def __init__(self, case, fd, inode):
File.__init__(self, case, fd, inode)
# strategy: must determine basepath from parent, get our path
# from db and then return the file:
## Note this _must_ work because we can only ever be called on
## a mounted iosource - it is an error otherwise:
basepath = fd.io.directory
self.case = case
dbh = DB.DBO(case)
dbh.check_index("file", "inode")
dbh.execute("select path,name from file where inode=%r limit 1", (inode))
row = dbh.fetch()
path = row["path"]
mount_point = fd.io.mount_point
## Prune the path down to the mount point:
if path[: len(mount_point)] != mount_point:
raise RuntimeError(DB.expand("Something went wrong - %s should be mounted on %s", (path, mount_point)))
path = path[len(mount_point) :]
path = basepath + "/" + path + "/" + row["name"]
if not path.startswith(posixpath.normpath(config.UPLOADDIR)):
path = FlagFramework.sane_join(config.UPLOADDIR, path)
if os.path.isdir(path):
self.fd = StringIO.StringIO("")
else:
self.fd = open(path, "r")
s = os.stat(path)
self.size = s.st_size
def __init__(self, case, fd, inode):
File.__init__(self, case, fd, inode)
## Make sure our parent is cached:
self.fd.cache()
## Parse out inode - if we got the compressed length provided,
## we use that, otherwise we calculate it from the zipfile
## header
parts = inode.split('|')
ourpart = parts[-1][1:]
try:
offset, size = ourpart.split(":")
self.compressed_length = int(size)
offset = int(offset)
except:
offset = int(ourpart)
self.offset = offset
## Ensure that we can read the file header:
b = Zip.Buffer(fd=fd)[offset:]
self.header = Zip.ZipFileHeader(b)
## This is sometimes invalid and set to zero - should we query
## the db?
self.size = int(self.header['uncompr_size'])
if not self.compressed_length:
self.compressed_length = int(self.header['compr_size'])
self.type = int(self.header['compression_method'])
## Where does the data start?
self.init()
def __init__(self, case, fd, inode):
File.__init__(self,case, fd, inode)
dbh = DB.DBO(self.case)
## Ensure we have an index on this column
dbh.check_index("connection","inode_id")
dbh.check_index("connection_details","inode_id")
## We use the inode column in the connection_details table to
## cache this so we only have to combine the streams once.
inode = inode.split("|")[-1]
self.inode_ids = [ int(x) for x in inode[1:].split("/")]
## Fill in some vital stats
dbh.execute("select inode.inode_id, reverse, src_ip, dest_ip, src_port, dest_port, ts_sec from `connection_details` join inode on inode.inode_id = connection_details.inode_id where inode.inode=%r limit 1", self.inode)
row=dbh.fetch()
if not row:
dbh.execute("select inode_id,reverse, src_ip, dest_ip, src_port, dest_port, ts_sec from `connection_details` where inode_id = %r", self.inode_ids[0])
row = dbh.fetch()
## This updates our properties from the db
if not row: return
self.__dict__.update(row)
## We allow the user to ask for a number of streams which will
## be combined at the same time. This allows us to create a
## VFS node for both forward and reverse streams, or even
## totally unrelated streams which happen at the same time.
self.look_for_cached()
self.read(0)
## This is a cache of packet lists that we keep so we do not
## have to hit the db all the time.
self.packet_list = None
def __init__(self, case, fd, inode):
self.urn = inode
fd = aff4.oracle.open(inode, 'r')
try:
if not fd: raise IOError("Unable to open %s" % inode)
finally:
aff4.oracle.cache_return(fd)
File.__init__(self, case, fd, inode)
def seek(self,offset,rel=None):
File.seek(self,offset,rel)
if self.cached_fd: return
## If we were asked to seek in a gzip file:
if self.readptr!=0:
pyflaglog.log(pyflaglog.VERBOSE_DEBUG,"Asked to seek to %s in gzip file %s. This is expensive, caching on disk." % (self.readptr, self.inode))
self.cache()
self.seek(offset,rel)
## Force a new decompressor when rereading:
self.gz = None
def read(self,len=None):
## Call our baseclass to see if we have cached data:
try:
return File.read(self,len)
except IOError:
pass
self.create_new_stream(self.inode_ids)
self.look_for_cached()
## For the reassembler its sometimes legitimate to have no
## cached file - this is because the file length is 0 bytes,
## and the reassembler does not bother to create it
try:
return File.read(self,len)
except IOError: return ''
def seek(self, offset, rel=None):
File.seek(self,offset,rel)
if self.cached_fd: return
if self.readptr == 0:
self.init()
## We want to reinitialise the file pointer:
elif self.readptr!=0 and self.type == Zip.ZIP_DEFLATED:
pyflaglog.log(pyflaglog.VERBOSE_DEBUG, "Required to seek to offset %s in Zip File %s (%s,%s). This is inefficient, forcing disk caching." % (self.readptr, self.inode, offset,rel))
self.init()
self.cache()
self.seek(offset, rel)
return
def read(self, length=None):
try:
return File.read(self,length)
except IOError:
pass
if self.readptr > 0:
return ''
self.fd.seek(0)
a=email.message_from_file(self.fd)
my_part = self.inode.split('|')[-1]
attachment_number = int(my_part[1:])
#print "attchement number %s" % attachment_number
count = 0
for part in a.walk():
if part.get_content_maintype() == 'multipart':
continue
if count==attachment_number:
self.message = part.get_payload(decode=1)
self.readptr += len(self.message)
#print "Returning %s" % part.get_payload()
return self.message
count+=1
return ''
def make_tabs(self):
names, cbs = File.make_tabs(self)
names.extend(["Show Packets", "Combined stream"])
cbs.extend([self.show_packets, self.combine_streams])
return names, cbs
def read(self, length=None):
try:
return File.read(self,length)
except IOError:
pass
if not self.gz:
self.fd.seek(0)
self.gz = gzip.GzipFile(fileobj=self.fd, mode='r')
count = 0
step = 1024
result = ''
## Copy ourself into the file - This is in case we have errors
## in the file, we try to read as much as possible:
while 1:
try:
data=self.gz.read(step)
except IOError,e:
step /= 2
if step<10:
pyflaglog.log(pyflaglog.DEBUG, "Error reading from %s(%s), could only get %s bytes (wanted %s/%s)" % (self.fd.inode, e, count, length,self.size));
break
else:
continue
except Exception, e:
#pyflaglog.log(pyflaglog.WARNING, "Unable to decompress inode (%s) %s" % (self.inode, e))
break
def read(self, length=None):
try:
return File.read(self, length)
except IOError:
pass
if self.readptr > 0:
return ''
self.fd.seek(0)
a = email.message_from_file(self.fd)
my_part = self.inode.split('|')[-1]
attachment_number = int(my_part[1:])
#print "attchement number %s" % attachment_number
count = 0
for part in a.walk():
if part.get_content_maintype() == 'multipart':
continue
if count == attachment_number:
self.message = part.get_payload(decode=1)
self.readptr += len(self.message)
#print "Returning %s" % part.get_payload()
return self.message
count += 1
return ''
def __init__(self, case, fd, inode):
File.__init__(self,case,fd,inode)
cache_key = "%s:%s" % (self.case, self.fd.inode)
try:
fs = SKCACHE.get(cache_key)
except KeyError:
fs = sk.skfs(self.fd)
SKCACHE.put(fs, key=cache_key)
inode = self.inode[self.inode.find('|K')+2:]
self.skfd = fs.open(inode=inode)
self.skfd.seek(0,2)
self.size = self.skfd.tell()
self.skfd.seek(0)
self.block_size = fs.block_size
def read(self, length=None):
try:
return File.read(self,length)
except IOError:
pass
if not self.gz:
self.fd.seek(0)
self.gz = gzip.zlib.decompressobj(-15)
count = 0
step = 1024
result = ''
## Copy ourself into the file - This is in case we have errors
## in the file, we try to read as much as possible:
while 1:
try:
data=self.gz.decompress(self.fd.read(step))
except IOError,e:
step /= 2
if step<10:
pyflaglog.log(pyflaglog.DEBUG, "Error reading from %s, could only get %s bytes" % (self.fd.inode, count));
break
else:
continue
except Exception, e:
pyflaglog.log(pyflaglog.WARNING, "Unable to decompress inode %s" % e)
break
def make_tabs(self):
names, cbs = File.make_tabs(self)
names.extend( [ "Show Packets", "Combined stream"])
cbs.extend([ self.show_packets, self.combine_streams ])
return names, cbs
def read(self, len=None):
## Call our baseclass to see if we have cached data:
try:
return File.read(self, len)
except IOError:
pass
self.create_new_stream(self.inode_ids)
self.look_for_cached()
## For the reassembler its sometimes legitimate to have no
## cached file - this is because the file length is 0 bytes,
## and the reassembler does not bother to create it
try:
return File.read(self, len)
except IOError:
return ''
def __init__(self, case, fd, inode):
""" This is a top level File driver for opening pcap files.
Note that pcap files are stored in their own filesystem. We expect the following initialisation:
@arg fd: is an io source for the pcap file
@arg inode: The inode of the pcap file in the pcap filesystem, currently ignored.
"""
File.__init__(self, case, fd, inode)
## Calculates the size of this file:
dbh = DB.DBO(self.case)
self.private_dbh = dbh.clone()
dbh.execute("select max(id) as max from pcap")
row=dbh.fetch()
if row['max']:
self.size = row['max']
else:
self.size = 0
self.private_dbh.execute("select id,offset,link_type,ts_sec,length from pcap where id>%r" % int(self.size))
self.iosource = fd
def make_tabs(self):
names, cbs = File.make_tabs(self)
for name, cb in zip(names, cbs):
if name == "Show Packets" or name == "Combined stream":
names.remove(name)
cbs.remove(cb)
names.extend( [ "Show Packets", "Combined stream", "IPID plot"])
cbs.extend([ self.show_packets, self.combine_streams, self.ipid_plot ])
return names, cbs
def make_tabs(self):
names, cbs = File.make_tabs(self)
for name, cb in zip(names, cbs):
if name == "Show Packets" or name == "Combined stream":
names.remove(name)
cbs.remove(cb)
names.extend(["Show Packets", "Combined stream", "IPID plot"])
cbs.extend([self.show_packets, self.combine_streams, self.ipid_plot])
return names, cbs
def __init__(self, case, fd, inode):
File.__init__(self, case, fd, inode)
## Tar file handling requires repeated access into the tar
## file. Caching our input fd really helps to speed things
## up...
fd.cache()
# strategy:
# inode is the index into the namelist of the tar file (i hope this is consistant!!)
# just read that file!
parts = inode.split('|')
try:
t = ZIPCACHE.get(self.fd.inode)
except (AttributeError, KeyError):
try:
t = tarfile.TarFile(name='/', fileobj=fd)
ZIPCACHE.put(t, key=self.fd.inode)
except tarfile.CompressionError,e:
raise IOError("Tar file: %s" % e)
def read(self,len=None):
## Call our baseclass to see if we have cached data:
try:
return File.read(self,len)
except IOError:
pass
if len:
temp=self.data[self.readptr:self.readptr+len]
self.readptr+=len
return temp
else: return self.data
def read(self, length=None):
## Call our baseclass to see if we have cached data:
try:
return File.read(self, length)
except IOError:
pass
## Read as much as possible
if length == None:
length = sys.maxint
## This is done in order to decompress the file in small
## chunks. We try to return as much data as was required
## and not much more
try:
## Consume the data left over from previous reads
result = self.left_over[:length]
self.left_over = self.left_over[length:]
## We keep reading compressed data until we can satify
## the desired length
while len(result) < length and self.clength > 0:
## Read up to 1k of the file:
available_clength = min(self.blocksize, self.clength)
cdata = self.fd.read(available_clength)
self.clength -= available_clength
if self.type == Zip.ZIP_DEFLATED:
## Now Decompress that:
try:
ddata = self.d.decompress(cdata)
except:
ddata = ''
elif self.type == Zip.ZIP_STORED:
ddata = cdata
else:
raise RuntimeError(
"Compression method %s is not supported" % self.type)
## How much data do we require?
required_length = length - len(result)
result += ddata[:required_length]
## This will be '' if we have not finished making
## up the result, and store the rest for next time
## if we have
self.left_over = ddata[required_length:]
except (IndexError, KeyError, zipfile.BadZipfile), e:
raise IOError("Zip_File: (%s)" % e)
def read(self,length=None):
## Call our baseclass to see if we have cached data:
try:
return File.read(self,length)
except IOError:
pass
## Read as much as possible
if length==None:
length = sys.maxint
## This is done in order to decompress the file in small
## chunks. We try to return as much data as was required
## and not much more
try:
## Consume the data left over from previous reads
result = self.left_over[:length]
self.left_over=self.left_over[length:]
## We keep reading compressed data until we can satify
## the desired length
while len(result)<length and self.clength>0:
## Read up to 1k of the file:
available_clength = min(self.blocksize,self.clength)
cdata = self.fd.read(available_clength)
self.clength -= available_clength
if self.type == Zip.ZIP_DEFLATED:
## Now Decompress that:
try:
ddata = self.d.decompress(cdata)
except:
ddata = ''
elif self.type == Zip.ZIP_STORED:
ddata = cdata
else:
raise RuntimeError("Compression method %s is not supported" % self.type)
## How much data do we require?
required_length = length - len(result)
result += ddata[:required_length]
## This will be '' if we have not finished making
## up the result, and store the rest for next time
## if we have
self.left_over = ddata[required_length:]
except (IndexError, KeyError, zipfile.BadZipfile),e:
raise IOError("Zip_File: (%s)" % e)
def read(self,length=None):
## Call our baseclass to see if we have cached data:
try:
return File.read(self,length)
except IOError:
pass
item = self.pst.get_item(self.item_id)
result =''
properties = item.properties()
if self.attach_number == None:
result = format_properties(properties)
else:
attachment = properties['_attachments'][self.attach_number]
result = attachment['body']
self.size = len(result)
return result
def read(self, length=None):
## Call our baseclass to see if we have cached data:
try:
return File.read(self, length)
except IOError:
pass
item = self.pst.get_item(self.item_id)
result = ''
properties = item.properties()
if self.attach_number == None:
result = format_properties(properties)
else:
attachment = properties['_attachments'][self.attach_number]
result = attachment['body']
self.size = len(result)
return result
def __init__(self, case, fd, inode):
File.__init__(self, case, fd, inode)
self.gz = None
def __init__(self, case, fd, inode):
File.__init__(self, case, fd, inode)
self.cache()
def seek(self, offset, rel=None):
File.seek(self,offset,rel)
if self.cached_fd: return
self.skfd.seek(self.readptr, slack=self.slack, overread=self.overread)
def read(self,length=None):
try:
return File.read(self, length)
except IOError,e:
pass