Пример #1
0
    def __init__(self, case, fd, inode):
        File.__init__(self, case, fd, inode)
        # strategy: must determine basepath from parent, get our path
        # from db and then return the file:

        ## Note this _must_ work because we can only ever be called on
        ## a mounted iosource - it is an error otherwise:
        basepath = fd.io.directory

        self.case = case
        dbh = DB.DBO(case)
        dbh.check_index("file", "inode")
        dbh.execute("select path,name from file where inode=%r limit 1", (inode))
        row = dbh.fetch()

        path = row["path"]
        mount_point = fd.io.mount_point
        ## Prune the path down to the mount point:
        if path[: len(mount_point)] != mount_point:
            raise RuntimeError(DB.expand("Something went wrong - %s should be mounted on %s", (path, mount_point)))

        path = path[len(mount_point) :]
        path = basepath + "/" + path + "/" + row["name"]
        if not path.startswith(posixpath.normpath(config.UPLOADDIR)):
            path = FlagFramework.sane_join(config.UPLOADDIR, path)

        if os.path.isdir(path):
            self.fd = StringIO.StringIO("")
        else:
            self.fd = open(path, "r")

        s = os.stat(path)
        self.size = s.st_size
Пример #2
0
    def __init__(self, case, fd, inode):
        File.__init__(self, case, fd, inode)

        ## Make sure our parent is cached:
        self.fd.cache()

        ## Parse out inode - if we got the compressed length provided,
        ## we use that, otherwise we calculate it from the zipfile
        ## header
        parts = inode.split('|')
        ourpart = parts[-1][1:]
        try:
            offset, size = ourpart.split(":")
            self.compressed_length = int(size)
            offset = int(offset)
        except:
            offset = int(ourpart)

        self.offset = offset
        ## Ensure that we can read the file header:
        b = Zip.Buffer(fd=fd)[offset:]
        self.header = Zip.ZipFileHeader(b)

        ## This is sometimes invalid and set to zero - should we query
        ## the db?
        self.size = int(self.header['uncompr_size'])
        
        if not self.compressed_length:
            self.compressed_length = int(self.header['compr_size'])
            
        self.type = int(self.header['compression_method'])

        ## Where does the data start?
        self.init()
Пример #3
0
    def __init__(self, case, fd, inode):
        File.__init__(self,case, fd, inode)
        dbh = DB.DBO(self.case)
        
        ## Ensure we have an index on this column
        dbh.check_index("connection","inode_id")
        dbh.check_index("connection_details","inode_id")
        
        ## We use the inode column in the connection_details table to
        ## cache this so we only have to combine the streams once.
        inode = inode.split("|")[-1]
        self.inode_ids = [ int(x) for x in inode[1:].split("/")]

        ## Fill in some vital stats
        dbh.execute("select inode.inode_id, reverse, src_ip, dest_ip, src_port, dest_port, ts_sec from `connection_details` join inode on inode.inode_id = connection_details.inode_id where inode.inode=%r limit 1", self.inode)
        row=dbh.fetch()
        if not row:
            dbh.execute("select inode_id,reverse, src_ip, dest_ip, src_port, dest_port, ts_sec from `connection_details` where inode_id = %r", self.inode_ids[0])
            row = dbh.fetch()

        ## This updates our properties from the db
        if not row: return
        self.__dict__.update(row)

        ## We allow the user to ask for a number of streams which will
        ## be combined at the same time. This allows us to create a
        ## VFS node for both forward and reverse streams, or even
        ## totally unrelated streams which happen at the same time.
        self.look_for_cached()
        self.read(0)
        
        ## This is a cache of packet lists that we keep so we do not
        ## have to hit the db all the time.
        self.packet_list = None
Пример #4
0
 def __init__(self, case, fd, inode):
     self.urn = inode
     fd = aff4.oracle.open(inode, 'r')
     try:
         if not fd: raise IOError("Unable to open %s" % inode)
     finally:
         aff4.oracle.cache_return(fd)
         
     File.__init__(self, case, fd, inode)
Пример #5
0
    def seek(self,offset,rel=None):
        File.seek(self,offset,rel)

        if self.cached_fd: return

        ## If we were asked to seek in a gzip file:
        if self.readptr!=0:
            pyflaglog.log(pyflaglog.VERBOSE_DEBUG,"Asked to seek to %s in gzip file %s. This is expensive, caching on disk." % (self.readptr, self.inode))
            self.cache()

            self.seek(offset,rel)

        ## Force a new decompressor when rereading:
        self.gz = None
Пример #6
0
    def read(self,len=None):
        ## Call our baseclass to see if we have cached data:
        try:
            return File.read(self,len)
        except IOError:
            pass

        self.create_new_stream(self.inode_ids)
        self.look_for_cached()
        ## For the reassembler its sometimes legitimate to have no
        ## cached file - this is because the file length is 0 bytes,
        ## and the reassembler does not bother to create it
        try:
            return File.read(self,len)
        except IOError: return ''
Пример #7
0
    def seek(self, offset, rel=None):
        File.seek(self,offset,rel)

        if self.cached_fd: return

        if self.readptr == 0:
            self.init()

        ## We want to reinitialise the file pointer:
        elif self.readptr!=0 and self.type == Zip.ZIP_DEFLATED:
            pyflaglog.log(pyflaglog.VERBOSE_DEBUG, "Required to seek to offset %s in Zip File %s (%s,%s). This is inefficient, forcing disk caching." % (self.readptr, self.inode, offset,rel))
            self.init()
            self.cache()

            self.seek(offset, rel)
            return
Пример #8
0
    def read(self, length=None):
        try:
            return File.read(self,length)
        except IOError:
           pass

        if self.readptr > 0:
            return ''
        
        self.fd.seek(0)
        a=email.message_from_file(self.fd)
        my_part = self.inode.split('|')[-1]
        attachment_number = int(my_part[1:])
        #print "attchement number %s" % attachment_number
        count = 0

        for part in a.walk():
            if part.get_content_maintype() == 'multipart':
                continue

            if count==attachment_number:
                self.message = part.get_payload(decode=1)
                self.readptr += len(self.message)
                #print "Returning %s" % part.get_payload()
                return self.message

            count+=1

        return ''
Пример #9
0
    def make_tabs(self):
        names, cbs = File.make_tabs(self)

        names.extend(["Show Packets", "Combined stream"])
        cbs.extend([self.show_packets, self.combine_streams])

        return names, cbs
Пример #10
0
    def read(self, length=None):
        try:
            return File.read(self,length)
        except IOError:
            pass

        if not self.gz:
            self.fd.seek(0)
            self.gz = gzip.GzipFile(fileobj=self.fd, mode='r')

        count = 0
        step = 1024

        result = ''
        
        ## Copy ourself into the file - This is in case we have errors
        ## in the file, we try to read as much as possible:
        while 1:
            try:
                data=self.gz.read(step)
            except IOError,e:
                step /= 2
                if step<10:
                    pyflaglog.log(pyflaglog.DEBUG, "Error reading from %s(%s), could only get %s bytes (wanted %s/%s)" % (self.fd.inode, e, count, length,self.size));
                    break
                
                else:
                    continue

            except Exception, e:
                #pyflaglog.log(pyflaglog.WARNING, "Unable to decompress inode (%s) %s" % (self.inode, e))
                break
Пример #11
0
    def read(self, length=None):
        try:
            return File.read(self, length)
        except IOError:
            pass

        if self.readptr > 0:
            return ''

        self.fd.seek(0)
        a = email.message_from_file(self.fd)
        my_part = self.inode.split('|')[-1]
        attachment_number = int(my_part[1:])
        #print "attchement number %s" % attachment_number
        count = 0

        for part in a.walk():
            if part.get_content_maintype() == 'multipart':
                continue

            if count == attachment_number:
                self.message = part.get_payload(decode=1)
                self.readptr += len(self.message)
                #print "Returning %s" % part.get_payload()
                return self.message

            count += 1

        return ''
Пример #12
0
    def __init__(self, case, fd, inode):
        File.__init__(self,case,fd,inode)
        
        cache_key = "%s:%s" % (self.case, self.fd.inode)
        try:
            fs = SKCACHE.get(cache_key)
        except KeyError:
            fs = sk.skfs(self.fd)
            SKCACHE.put(fs, key=cache_key)

        inode = self.inode[self.inode.find('|K')+2:]
        self.skfd = fs.open(inode=inode)
        self.skfd.seek(0,2)
        self.size = self.skfd.tell()
        self.skfd.seek(0)
        self.block_size = fs.block_size
Пример #13
0
    def read(self, length=None):
        try:
            return File.read(self,length)
        except IOError:
            pass

        if not self.gz:
            self.fd.seek(0)
            self.gz = gzip.zlib.decompressobj(-15)

        count = 0
        step = 1024

        result = ''
        
        ## Copy ourself into the file - This is in case we have errors
        ## in the file, we try to read as much as possible:
        while 1:
            try:
                data=self.gz.decompress(self.fd.read(step))
            except IOError,e:
                step /= 2
                if step<10:
                    pyflaglog.log(pyflaglog.DEBUG, "Error reading from %s, could only get %s bytes" % (self.fd.inode, count));
                    break
                
                else:
                    continue

            except Exception, e:
                pyflaglog.log(pyflaglog.WARNING, "Unable to decompress inode %s" % e)
                break
Пример #14
0
    def make_tabs(self):
        names, cbs = File.make_tabs(self)

        names.extend( [ "Show Packets", "Combined stream"])
        cbs.extend([ self.show_packets, self.combine_streams ])

        return names, cbs
Пример #15
0
    def read(self, len=None):
        ## Call our baseclass to see if we have cached data:
        try:
            return File.read(self, len)
        except IOError:
            pass

        self.create_new_stream(self.inode_ids)
        self.look_for_cached()
        ## For the reassembler its sometimes legitimate to have no
        ## cached file - this is because the file length is 0 bytes,
        ## and the reassembler does not bother to create it
        try:
            return File.read(self, len)
        except IOError:
            return ''
Пример #16
0
    def __init__(self, case, fd, inode):
        """ This is a top level File driver for opening pcap files.

        Note that pcap files are stored in their own filesystem. We expect the following initialisation:
        @arg fd: is an io source for the pcap file
        @arg inode: The inode of the pcap file in the pcap filesystem, currently ignored.
        """
        File.__init__(self, case, fd, inode)
        ## Calculates the size of this file:
        dbh = DB.DBO(self.case)    
        self.private_dbh = dbh.clone()
        dbh.execute("select max(id) as max from pcap")
        row=dbh.fetch()
        if row['max']:
            self.size = row['max']
        else:
            self.size = 0
        
        self.private_dbh.execute("select id,offset,link_type,ts_sec,length from pcap where id>%r" % int(self.size))
        self.iosource = fd
Пример #17
0
    def make_tabs(self):
        names, cbs = File.make_tabs(self)
        
        for name, cb in zip(names, cbs):
            if name == "Show Packets" or name == "Combined stream":
                names.remove(name)
                cbs.remove(cb)

        names.extend( [ "Show Packets", "Combined stream", "IPID plot"])
        cbs.extend([ self.show_packets, self.combine_streams, self.ipid_plot ])
        return names, cbs
Пример #18
0
    def make_tabs(self):
        names, cbs = File.make_tabs(self)

        for name, cb in zip(names, cbs):
            if name == "Show Packets" or name == "Combined stream":
                names.remove(name)
                cbs.remove(cb)

        names.extend(["Show Packets", "Combined stream", "IPID plot"])
        cbs.extend([self.show_packets, self.combine_streams, self.ipid_plot])
        return names, cbs
Пример #19
0
    def __init__(self, case, fd, inode):
        File.__init__(self, case, fd, inode)

        ## Tar file handling requires repeated access into the tar
        ## file. Caching our input fd really helps to speed things
        ## up...
        fd.cache()
        
        # strategy:
        # inode is the index into the namelist of the tar file (i hope this is consistant!!)
        # just read that file!
        parts = inode.split('|')

        try:
            t = ZIPCACHE.get(self.fd.inode)
        except (AttributeError, KeyError):
            try:
                t = tarfile.TarFile(name='/', fileobj=fd)
                ZIPCACHE.put(t, key=self.fd.inode)
            except tarfile.CompressionError,e:
                raise IOError("Tar file: %s" % e)
Пример #20
0
    def read(self,len=None):
        ## Call our baseclass to see if we have cached data:
        try:
            return File.read(self,len)
        except IOError:
            pass

        if len:
            temp=self.data[self.readptr:self.readptr+len]
            self.readptr+=len
            return temp
        else: return self.data
Пример #21
0
    def read(self, length=None):
        ## Call our baseclass to see if we have cached data:
        try:
            return File.read(self, length)
        except IOError:
            pass

        ## Read as much as possible
        if length == None:
            length = sys.maxint

        ## This is done in order to decompress the file in small
        ## chunks. We try to return as much data as was required
        ## and not much more
        try:
            ## Consume the data left over from previous reads
            result = self.left_over[:length]
            self.left_over = self.left_over[length:]

            ## We keep reading compressed data until we can satify
            ## the desired length
            while len(result) < length and self.clength > 0:
                ## Read up to 1k of the file:
                available_clength = min(self.blocksize, self.clength)
                cdata = self.fd.read(available_clength)
                self.clength -= available_clength

                if self.type == Zip.ZIP_DEFLATED:
                    ## Now Decompress that:
                    try:
                        ddata = self.d.decompress(cdata)
                    except:
                        ddata = ''
                elif self.type == Zip.ZIP_STORED:
                    ddata = cdata
                else:
                    raise RuntimeError(
                        "Compression method %s is not supported" % self.type)

                ## How much data do we require?
                required_length = length - len(result)
                result += ddata[:required_length]

                ## This will be '' if we have not finished making
                ## up the result, and store the rest for next time
                ## if we have
                self.left_over = ddata[required_length:]

        except (IndexError, KeyError, zipfile.BadZipfile), e:
            raise IOError("Zip_File: (%s)" % e)
Пример #22
0
    def read(self,length=None):
        ## Call our baseclass to see if we have cached data:
        try:
            return File.read(self,length)
        except IOError:
            pass

        ## Read as much as possible
        if length==None:
            length = sys.maxint
            
        ## This is done in order to decompress the file in small
        ## chunks. We try to return as much data as was required
        ## and not much more
        try:
            ## Consume the data left over from previous reads
            result = self.left_over[:length]
            self.left_over=self.left_over[length:]

            ## We keep reading compressed data until we can satify
            ## the desired length
            while len(result)<length and self.clength>0:
                ## Read up to 1k of the file:
                available_clength = min(self.blocksize,self.clength)
                cdata = self.fd.read(available_clength)
                self.clength -= available_clength

                if self.type == Zip.ZIP_DEFLATED:
                    ## Now Decompress that:
                    try:
                        ddata = self.d.decompress(cdata)
                    except:
                        ddata = ''
                elif self.type == Zip.ZIP_STORED:
                    ddata = cdata
                else:
                    raise RuntimeError("Compression method %s is not supported" % self.type)

                ## How much data do we require?
                required_length = length - len(result)
                result += ddata[:required_length]

                ## This will be '' if we have not finished making
                ## up the result, and store the rest for next time
                ## if we have
                self.left_over = ddata[required_length:]

        except (IndexError, KeyError, zipfile.BadZipfile),e:
            raise IOError("Zip_File: (%s)" % e)
Пример #23
0
    def read(self,length=None):
        ## Call our baseclass to see if we have cached data:
        try:
            return File.read(self,length)
        except IOError:
            pass

        item = self.pst.get_item(self.item_id)
        result =''
        properties = item.properties()
        
        if self.attach_number == None:
            result = format_properties(properties)
        else:
            attachment = properties['_attachments'][self.attach_number]
            result = attachment['body']
            
        self.size = len(result)

        return result
Пример #24
0
    def read(self, length=None):
        ## Call our baseclass to see if we have cached data:
        try:
            return File.read(self, length)
        except IOError:
            pass

        item = self.pst.get_item(self.item_id)
        result = ''
        properties = item.properties()

        if self.attach_number == None:
            result = format_properties(properties)
        else:
            attachment = properties['_attachments'][self.attach_number]
            result = attachment['body']

        self.size = len(result)

        return result
Пример #25
0
 def __init__(self, case, fd, inode):
     File.__init__(self, case, fd, inode)
     self.gz = None
Пример #26
0
 def __init__(self, case, fd, inode):
     File.__init__(self, case, fd, inode)
     self.cache()
Пример #27
0
    def seek(self, offset, rel=None):
        File.seek(self,offset,rel)

        if self.cached_fd: return
        self.skfd.seek(self.readptr, slack=self.slack, overread=self.overread)
Пример #28
0
 def read(self,length=None):
     try:
         return File.read(self, length)
     except IOError,e:
         pass