Beispiel #1
0
 def __init__(self, filenames, offset):
     if type(filenames) == str:
         filenames = [
             filenames,
         ]
     fds = [IO.open_URL(filename) for filename in filenames]
     OffsettedFDFile.__init__(self, fds, offset)
Beispiel #2
0
    def search_next_text_region(self, query, result):
        """ searches for the next text region and updates query['period_number'] """
        ## Open all the disks
        filenames = query.getarray('filename')
        fds = [IO.open_URL(f) for f in filenames]
        if query.get('ismissing', False):
            fds.append(ParityFD(copy.deepcopy(filenames)))
        period_number = int(query.get('period_number', 0)) + 1
        blocksize = FlagFramework.calculate_offset_suffix(query['blocksize'])
        period = FlagFramework.calculate_offset_suffix(query['period'])

        p = 0
        while 1:
            offset = blocksize * (p + period_number * period)
            for fd in fds:
                fd.seek(offset)
                ## We classify a text region as one with 20 chars at
                ## the start of the period
                data = fd.read(20)
                if not data:
                    result.heading("Error")
                    result.para("Unable to read data from %r" % fd)
                    return

                m = self.text_re.match(data)
                if m:
                    period_number = period_number + p / period
                    query.set('period_number', period_number)
                    result.refresh(0, query, 'parent')
                    return

                p += 1
Beispiel #3
0
    def load(self, mount_point, iosource_name, loading_scanners = None):
        """ Sets up the schema for loading the filesystem.

        Note that derived classes need to actually do the loading
        after they call the base class.

        loading_scanners are the scanners which need to be run on
        every new Inode.
        """
        self.mount_point = mount_point
        dbh=DB.DBO(self.case)
        
        ## Commented out to fix Bug0035. This should be (and is) done
        ## by VFSCreate, since we want to avoid duplicate mount
        ## points.  mic: This is here because skfs.load does not use
        ## VFSCreate for speed reasons and therefore does not
        ## necessarily create the mount points when needed.  Ensure
        ## the VFS contains the mount point:
        self.VFSCreate(None, "I%s" % iosource_name, mount_point, 
                       directory=True)

        dbh.insert("filesystems",
                   iosource = iosource_name,
                   property = 'mount point',
                   value = mount_point)
        
        ## Ensure that we have the IOSource available
        self.iosource = IO.open(self.case, iosource_name)
Beispiel #4
0
    def load(self, mount_point, iosource_name, scanners = None, directory=None):
        ## Ensure that mount point is normalised:
        self.iosource_name = iosource_name
        mount_point = os.path.normpath(mount_point)
        self.mount_point = mount_point
        
        FileSystem.DBFS.load(self, mount_point, iosource_name)
        
        # open the iosource
        self.iosrc = IO.open(self.case, iosource_name)

        ## Make a volatility object available FIXME allow options in
        ## here
        op = vutils.get_standard_parser("")

        ## Create an address space for the kernel
        self.kernel_VA_inode_id = self.VFSCreate(None, "I%s|A0"  % iosource_name,
                                                 "%s/mem" % self.mount_point)
        
        ## Build a fake command line
        self.filename = '%s/%s' % (self.case, iosource_name)
        self.args = ['-f', self.filename ]
        opts, args = op.parse_args(self.args)

        ## This identifies the image
        (self.addr_space, self.symtab, self.types) = vutils.load_and_identify_image(op, opts)

        for loader in Registry.FSLOADERS.classes:
            if loader.filesystem != "WindowsMemory": continue
            
            ## Instantiate them
            loader = loader()
            
            ## Ask them to load this memory image
            loader.load(self)
Beispiel #5
0
    def __init__(self, case, iosource_name, profile, map):
        ## Try to open the image
        self.iosource_name = iosource_name
        self.case = case
        self.map = map
        self.tasks = {}
        print "Opening memory image"
        path_to_profiles = "%s/%s" % (config.memory_profile_dir,
                                      profile)

        print path_to_profiles
        profile_file_name = [ m for m in os.listdir(path_to_profiles) \
                              if m.endswith(".py") ][0]

        print "Profile name %s" % profile_file_name
        self.profile = load_profile_from_file("%s/%s" % (path_to_profiles,
                                                                    profile_file_name))
        try:
            symdict = load_symboltable_from_file("%s/%s" % (path_to_profiles,
                                                                   map))
            self.symtable = SymbolTable(symdict)
        except:
            raise RuntimeError("Invalid or corrupt Symbol Table file %s/%s" % (path_to_profiles, profile_file_name))

        pgd = self.symtable.lookup('swapper_pg_dir')
        iosrc = IO.open(self.case, iosource_name)
        phyAS = IOSourceAddressSpace(iosrc)
        self.addr_space = IA32PagedMemory(phyAS, pgd - 0xc0000000)
        self.theProfile = Profile(abstract_types = self.profile)
Beispiel #6
0
    def search_next_text_region(self, query, result):
        """ searches for the next text region and updates query['period_number'] """
        ## Open all the disks
        filenames = query.getarray('filename') 
        fds = [ IO.open_URL(f) for f in filenames ]
        if query.get('ismissing',False):
            fds.append(ParityFD(copy.deepcopy(filenames)))
        period_number = int(query.get('period_number',0)) + 1
        blocksize = FlagFramework.calculate_offset_suffix(query['blocksize'])
        period = FlagFramework.calculate_offset_suffix(query['period'])

        p=0
        while 1:
            offset = blocksize  * (p + period_number * period)
            for fd in fds:
                fd.seek(offset)
                ## We classify a text region as one with 20 chars at
                ## the start of the period
                data = fd.read(20)
                if not data:
                    result.heading("Error")
                    result.para("Unable to read data from %r" % fd)
                    return
                
                m = self.text_re.match(data)
                if m:
                    period_number = period_number + p / period
                    query.set('period_number',period_number)
                    result.refresh(0, query, 'parent')
                    return

                p += 1
Beispiel #7
0
 def __init__(self, name, mode='rb', fast=False):
     self.case, self.iosource = name.split("/",1)
     fd = IO.open(self.case, self.iosource)
     self.fhandle = fd
     self.fsize = fd.size
     self.fast_fhandle = fd
     self.fname = name
     self.name = name
Beispiel #8
0
def process(case,subsys,extension=None):
    """ A generator to produce all the recoverable files within the io object identified by identifier

    @arg subsys: Either an IO object to use, or the string name of an io object that will be opened using IO.open().
    @arg extension: A list of extensions we would like to see
    """
    if type(subsys)==types.StringType:
        io=IO.open(case,subsys)
    else:
        io=subsys
        
    blocksize=1024*1024*10
    windowsize=100
    count=0
    bytes_read=0
    window=''
    while(1):
        ## This implements a sliding window of window bytes to ensure
        ## we do not miss a signature that was split across blocksize:
        try:
            data=io.read(blocksize)
            if not len(data): break
        except IOError:
            break
        
        f=window+data
        bytes_read+=len(data)
        pyflaglog.log(pyflaglog.INFO,"Processed %u Mb" % (bytes_read/1024/1024))
        for cut in definitions:
            if extension and cut['Extension'] not in extension: continue
            pos=0
            while pos<blocksize:
                match=cut['CStartRE'].search(f,pos)
                if match:
                    offset=match.start()+count-len(window)
                    length=cut['MaxLength']
                    ## If there is an end RE, we try to read the entire length in, and then look for the end to we can adjust the length acurately. This is essential for certain file types which do not tolerate garbage at the end of the file, e.g. pdfs.
                    if cut.has_key('CEndRE'):
                        tell=io.tell()
                        io.seek(offset)
                        file_data=io.read(length)
                        io.seek(tell)

                        end_match=cut['CEndRE'].search(file_data,0)
                        if end_match:
                            length=end_match.end()
                  
                    yield({'offset':offset,'length':length,'type':cut['Extension']})
                    pos=match.start()+1
                else:
                    pos=blocksize

        window=f[-windowsize:]
        count+=blocksize
        
    io.close()
def write_stream(con_id):
    dbh2= dbh.clone()
    dbh.execute("select pcap.id as id,iosource,offset,pcap.length from `connection`,pcap where pcap.id=`connection`.packet_id and con_id = %r order by packet_id" % con_id)
    for row in dbh:
        ## This should not be too slow as its cached in the IO module
        ## Store
        io = IO.open(options.case, row['iosource'])
        io.seek(row['offset'])
        data = io.read(row['length'])
        outfd.write(data)
Beispiel #10
0
 def tree_cb(path):
     fd = IO.open_URL(query['file'])
     b = Buffer(fd = fd)
     header = RegFile.RegF(b)
     key = header.get_key(path)
     for k in key.keys():
         try:
             name = k['key_name'].get_value()
         except:
             name = None
         yield (name,name,'branch')
Beispiel #11
0
    def create(self, name,case, query):
        offset = FlagFramework.calculate_offset_suffix(query.get('offset','0'))
        filenames = self.glob_filenames(query.getarray('filename'))

        ## Open the io sources here
        fds = [ IO.open_URL(f) for f in filenames ]
        if query.get('ismissing',False):
            fds += [ParityFD(copy.deepcopy(filenames))]
        blocksize = FlagFramework.calculate_offset_suffix(query.get('blocksize','32k'))
        period = int(query.get('period',3))
        return RAIDFD(fds, blocksize, query['map'], offset, period)
Beispiel #12
0
    def __init__(self, case, fd, inode):
        FileSystem.File.__init__(self, case, fd, inode)

        ## The format of the inode is Iname .Where name is the name of
        ## the IO source.
        self.name = inode[1:]
        self.io = IO.open(case, self.name)
        self.size = self.io.size

        ## This source should not be scanned directly.
        self.ignore = True
Beispiel #13
0
    def iosubsystem(self, field, query):
        """ Check to see that the io subsystem is adequitely filled in

        instantiate an IO object, and try to pass the query to it.
        If the query is ok, we then pass the test, else we fail.
        """
        import pyflag.IO as IO

        try:
            IO.IOFactory(query, subsys=field)
        except (IOError, KeyError, RuntimeError):
            return False
Beispiel #14
0
    def form(self, query, result):
        result.start_table()
        try:
            result.case_selector()
            result.ruler()
            result.selector(
                "Select IO Data Source",
                "iosource",
                "select name as `key`,name as `value` from iosources",
                case=query["case"],
            )

            # initialise/open the subsystem
            fd = IO.open(query["case"], query["iosource"])

            ## We need to ask all our filesystems to have a go at
            ## this:
            metadata = {}
            order = []
            for c in Registry.FILESYSTEMS.classes:
                c = c(query["case"], query)
                guess = c.guess(fd, result, metadata)
                if guess > 0:
                    order.append((guess, c.name))

            ## Sort according to the guess value
            order.sort(lambda x, y: y[0] - x[0])

            ## We only show those whose guess is within 100 off
            ## the top - this allows a strong guess to bump out
            ## weaker guesses.
            maximum = order[0][0]
            fs_types = [x[1] for x in order if x[0] > maximum - 100]

            ## If only one option is possible here,
            if len(fs_types) == 1:
                result.hidden("fstype", fs_types[0], exclusive=True)
            else:
                result.const_selector("Enter Filesystem type", "fstype", fs_types, fs_types)

            result.textfield("VFS Mount Point:", "mount_point")
            result.ruler()

            ## Allow the filesystem to draw a form:
            try:
                c = Registry.FILESYSTEMS.dispatch(query["fstype"])
                c = c(query["case"], query)
                c.form(query, result)
            except KeyError:
                pass

        except IOError, e:
            result.text("IOError %s" % e, style="red")
Beispiel #15
0
 def read(self,length):
     fds = [ IO.open_URL(f) for f in self.filenames ]
     for fd in fds:
         fd.seek(self.readptr)
     data = False
     for fd in fds:
         if data == False:
             data = map(lambda x: ord(x), fd.read(length))
         else:
             data = map(lambda (x,y): x^y,zip(data,map(lambda z: ord(z), fd.read(length))))
     self.readptr += length
     return ''.join(map(lambda x: chr(x), data))
Beispiel #16
0
 def read(self, length):
     fds = [IO.open_URL(f) for f in self.filenames]
     for fd in fds:
         fd.seek(self.readptr)
     data = False
     for fd in fds:
         if data == False:
             data = map(lambda x: ord(x), fd.read(length))
         else:
             data = map(lambda (x, y): x ^ y,
                        zip(data, map(lambda z: ord(z), fd.read(length))))
     self.readptr += length
     return ''.join(map(lambda x: chr(x), data))
Beispiel #17
0
 def details(query,result):
     fd = IO.open_URL(query['file'])
     b = Buffer(fd = fd)
     header = RegFile.RegF(b)
     key = header.get_key(path)
     result.heading("Key %s" % path)
     result.text("%s" % key, font='typewriter', wrap='full')
     
     for v in key.values():
         try:
             name = "%s"%  v['keyname']
             result.heading("%s" % name)
             result.text("%s" % v, font='typewriter', wrap='full')
         except: pass
Beispiel #18
0
    def form(self,query,result):
        result.start_table()
        try:
            result.case_selector()
            result.ruler()
            result.selector('Select IO Data Source', 'iosource',
                            "select name as `key`,name as `value` from iosources",
                            case=query['case'])
            
            # initialise/open the subsystem
            fd=IO.open(query['case'],query['iosource'])

            ## We need to ask all our filesystems to have a go at
            ## this:
            metadata = {}
            order = []
            for c in Registry.FILESYSTEMS.classes:
                c = c(query['case'], query)
                guess = c.guess(fd, result, metadata)
                if guess>0:
                    order.append((guess , c.name))

            ## Sort according to the guess value
            order.sort(lambda x,y: y[0] - x[0])
                    
            ## We only show those whose guess is within 100 off
            ## the top - this allows a strong guess to bump out
            ## weaker guesses.
            maximum = order[0][0]
            fs_types = [ x[1] for x in order if x[0] > maximum - 100 ]

            ## If only one option is possible here, 
            if len(fs_types)==1:
                result.hidden('fstype',fs_types[0], exclusive = True)
            else:
                result.const_selector("Enter Filesystem type",'fstype',fs_types,fs_types)

            result.textfield("VFS Mount Point:","mount_point")
            result.ruler()

            ## Allow the filesystem to draw a form:
            try:
                c = Registry.FILESYSTEMS.dispatch(query['fstype'])
                c = c(query['case'], query)
                c.form(query, result)
            except KeyError:
                pass

        except IOError,e:
            result.text("IOError %s" % e,style='red')
Beispiel #19
0
        def map_popup(query, result):
            result.decoration = 'naked'
            try:
                map = str(query['map']).split('.')
            except KeyError:
                map = []

            result.start_form(query)
            ## Open all the disks
            filenames = query.getarray('filename')
            fds = [IO.open_URL(f) for f in filenames]

            if query.get('ismissing', False):
                fds += [ParityFD(copy.deepcopy(filenames))]
                filenames += ('Missing Disk', )

            uis = [result.__class__(result) for f in filenames]

            period_number = int(query.get('period_number', 0))
            blocksize = FlagFramework.calculate_offset_suffix(
                query['blocksize'])
            period = FlagFramework.calculate_offset_suffix(query['period'])
            logical_period_size = period * (len(fds) - 1)
            ## Let the user know our disk offset
            result.para("Disk Offset is %s (%s periods)" %
                        (blocksize * (period_number * period), period_number))

            if query.has_key("__submit__"):
                ## Build the new map variable
                query.clear('map')
                map = []
                for x in range(period * len(fds)):
                    map.append(query['position_%s' % x])
                    query.clear('position_%s' % x)

                query.set('map', '.'.join(map))
                ## Now check that this map is valid by instantiating a
                ## RAIDFD (we will raise if anything is wrong:
                try:
                    RAIDFD(fds, blocksize, query['map'], 0, period)
                    result.refresh(0, query, pane='parent')
                except Exception, e:
                    result.heading("Error with map")
                    result.para("%s" % e)

                return result
Beispiel #20
0
        def map_popup(query,result):
            result.decoration = 'naked'
            try:
                map = str(query['map']).split('.')
            except KeyError:
                map = []

            result.start_form(query)
            ## Open all the disks
            filenames = query.getarray('filename') 
            fds = [ IO.open_URL(f) for f in filenames ]

            if query.get('ismissing',False):
                fds += [ParityFD(copy.deepcopy(filenames))]
                filenames += ('Missing Disk',)


            uis = [ result.__class__(result) for f in filenames ]

            period_number = int(query.get('period_number',0))
            blocksize = FlagFramework.calculate_offset_suffix(query['blocksize'])
            period = FlagFramework.calculate_offset_suffix(query['period'])
            logical_period_size = period * (len(fds)-1)
            ## Let the user know our disk offset
            result.para("Disk Offset is %s (%s periods)" %  (blocksize  * (period_number * period), period_number))

            if query.has_key("__submit__"):
                ## Build the new map variable
                query.clear('map')
                map = []
                for x in range(period * len(fds)):
                    map.append(query['position_%s' % x])
                    query.clear('position_%s' % x)
                    
                query.set('map', '.'.join(map))
                ## Now check that this map is valid by instantiating a
                ## RAIDFD (we will raise if anything is wrong:
                try:
                    RAIDFD(fds, blocksize, query['map'], 0, period)
                    result.refresh(0,query,pane='parent')
                except Exception,e:
                    result.heading("Error with map")
                    result.para("%s" % e)

                return result
Beispiel #21
0
        def pane_cb(path, result):
            fd = IO.open_URL(query['file'])
            b = Buffer(fd=fd)
            header = RegFile.RegF(b)
            key = header.get_key(path)
            result.text("Timestamp: %s" % key['WriteTS'], style='red')
            result.start_table(**{'class': 'GeneralTable'})

            ## We dont want to reference the keys because we
            ## will leak memeory while the callback remains stored.
            def details(query, result):
                fd = IO.open_URL(query['file'])
                b = Buffer(fd=fd)
                header = RegFile.RegF(b)
                key = header.get_key(path)
                result.heading("Key %s" % path)
                result.text("%s" % key, font='typewriter', wrap='full')

                for v in key.values():
                    try:
                        name = "%s" % v['keyname']
                        result.heading("%s" % name)
                        result.text("%s" % v, font='typewriter', wrap='full')
                    except:
                        pass

            result.toolbar(cb=details,
                           text="Examine Details",
                           icon="examine.png")

            result.row('Type', 'Length', 'Name', 'Value',
                       **{'class': 'hilight'})
            for v in key.values():
                try:
                    t = "%s" % v['data']['val_type']
                    length = "%s" % v['data']['len_data']
                    name = "%s" % v['keyname']
                    data = "%s" % v['data']
                    data = RAW(data[:100])
                    result.row(t, length, name, data)
                except Exception, e:
                    print e
                    pass
Beispiel #22
0
    def read_record(self, ignore_comment = True):
        """ Generates records.

        This can handle multiple files as provided in the constructor.
        """
        
        blank = re.compile("^\s*$")

        if self.datafile==None:
            raise IOError("Datafile is not set!!!")
        
        for file in self.datafile:
            ## open the file as a url:
            fd = IO.open_URL(file)
            buffer = ''
            while 1:
                if len(buffer) < 1024:
                    data = fd.read(1024)
                    buffer = buffer + data

                tmp = buffer.split("\n",1)
                if len(tmp) == 0: break

                line = tmp[0]
               
                try:
                    buffer = tmp[1]
                except:
                    data = fd.read(1024)
                    if len(data) == 0:
                        break
                    buffer = line + data
                    continue
                
                if blank.match(line) or not line:
                    continue
                if line.startswith('#') and ignore_comment:
                    continue
                else:
                    yield line
Beispiel #23
0
    def read_record(self, ignore_comment=True):
        """ Generates records.

        This can handle multiple files as provided in the constructor.
        """

        blank = re.compile("^\s*$")

        if self.datafile == None:
            raise IOError("Datafile is not set!!!")

        for file in self.datafile:
            ## open the file as a url:
            fd = IO.open_URL(file)
            buffer = ''
            while 1:
                if len(buffer) < 1024:
                    data = fd.read(1024)
                    buffer = buffer + data

                tmp = buffer.split("\n", 1)
                if len(tmp) == 0: break

                line = tmp[0]

                try:
                    buffer = tmp[1]
                except:
                    data = fd.read(1024)
                    if len(data) == 0:
                        break
                    buffer = line + data
                    continue

                if blank.match(line) or not line:
                    continue
                if line.startswith('#') and ignore_comment:
                    continue
                else:
                    yield line
Beispiel #24
0
        def pane_cb(path, result):
            fd = IO.open_URL(query['file'])
            b = Buffer(fd = fd)
            header = RegFile.RegF(b)
            key = header.get_key(path)
            result.text("Timestamp: %s" % key['WriteTS'], style='red')
            result.start_table(**{'class':'GeneralTable'})

            ## We dont want to reference the keys because we
            ## will leak memeory while the callback remains stored.
            def details(query,result):
                fd = IO.open_URL(query['file'])
                b = Buffer(fd = fd)
                header = RegFile.RegF(b)
                key = header.get_key(path)
                result.heading("Key %s" % path)
                result.text("%s" % key, font='typewriter', wrap='full')
                
                for v in key.values():
                    try:
                        name = "%s"%  v['keyname']
                        result.heading("%s" % name)
                        result.text("%s" % v, font='typewriter', wrap='full')
                    except: pass

            result.toolbar(cb = details, text = "Examine Details", icon = "examine.png")
            
            result.row('Type','Length','Name','Value', **{'class':'hilight'})
            for v in key.values():
                try:
                    t = "%s" % v['data']['val_type']
                    length = "%s" % v['data']['len_data']
                    name = "%s"%  v['keyname']
                    data = "%s" % v['data']
                    data = RAW(data[:100])
                    result.row(t,length,name,data)
                except Exception,e:
                    print e
                    pass
Beispiel #25
0
 def __init__(self, filenames, offset):
     if type(filenames)==str:
         filenames = [ filenames,]
     fds = [ IO.open_URL(filename) for filename in filenames ]
     OffsettedFDFile.__init__(self, fds, offset)
Beispiel #26
0
    def load(self, mount_point, iosource_name, scanners = None, directory=None):
        ## Ensure that mount point is normalised:
        self.iosource_name = iosource_name
        mount_point = os.path.normpath(mount_point)
        self.mount_point = mount_point
        
        DBFS.load(self, mount_point, iosource_name)
        
        # open the iosource
        iosrc = IO.open(self.case, iosource_name)

        ## Get a db handle
        dbh = DB.DBO(self.case)
        dbh.mass_insert_start('tasks')
        
        (addr_space, symtab, types) = load_and_identify_image(iosrc)
        self.load_open_files(dbh, addr_space, types, symtab)

        ## process_list should probably be a generator here (or not,
        ## the list is unlikely to be that big)
        for task in process_list(addr_space, types, symtab):
            ## Skip invalid tasks (This should probably be done in
            ## process_list itself so it doesnt yield rubbish)
            if not addr_space.is_valid_address(task): continue
            pid = process_pid(addr_space, types, task) or -1
            create_time = process_create_time(addr_space, types, task)

            task_info = {
                'iosource':        iosource_name,
                'image_file_name': process_imagename(addr_space, types, task) or "UNKNOWN",
                'pid':             pid,
                'offset':          task,
                'active_threads':  process_num_active_threads(addr_space, types, task) or -1,
                'inherited_from':  process_inherited_from(addr_space, types,task) or -1,
                'handle_count':    process_handle_count(addr_space, types, task) or -1,
                '_create_time':    "from_unixtime('%s')" % create_time
                }

            ## Put the data in the db
            dbh.mass_insert(**task_info)

            ## Create some VFS nodes:
            new_inode = "I%s|N%s" % (iosource_name, task)
            inode_id = self.VFSCreate(None, new_inode,
                                      "%s/%s/exe" % (mount_point, task_info['pid']),
                                      _mtime = create_time,
                                      link = task_info['image_file_name'],
                                      _fast = True)

            ## Try to read the PEB:
            peb = process_peb(addr_space, types, task)
            process_address_space = process_addr_space(addr_space, types, task, None)
            command_line = process_command_line(process_address_space, types, peb)
            if command_line:
                dbh.insert('xattr',
                           inode_id=inode_id,
                           property = "command_line",
                           value = command_line,
                           _fast = True)

            if peb:
                modules = process_ldrs(process_address_space, types, peb)
                for module in modules:
                    if not process_address_space.is_valid_address(module): continue
                    path = module_path(process_address_space, types, module)
                    base = module_base(process_address_space, types, module) or 0
                    size = module_size(process_address_space, types, module)

                    dbh.insert("modules", iosource = iosource_name,
                               pid = pid,
                               path = path,
                               base = base,
                               _fast = True
                               )

                    self.VFSCreate(None, None,
                                   "%s/%s/Modules/Base 0x%X" % ( mount_point,
                                                                 task_info['pid'],
                                                                 base),
                                   _mtime = create_time,
                                   link = path,
                                   size = size,
                                   _fast = True)
                    
        ## Now look for the connections:
        for connection in tcb_connections(addr_space, types, symtab):
            if not addr_space.is_valid_address(connection):
                continue

            dbh.insert("mconnections",
                       pid = connection_pid(addr_space, types, connection),
                       lport   = connection_lport(addr_space, types, connection),
                       laddr   = connection_laddr(addr_space, types, connection),
                       rport   = connection_rport(addr_space, types, connection),
                       raddr   = connection_raddr(addr_space, types, connection),
                       iosource = iosource_name,
                       _fast = True)

        ## Now do the sockets:
        for socket in open_sockets(addr_space, types, symtab):
            if not addr_space.is_valid_address(connection):
                continue

            dbh.insert("sockets",
                       pid   = socket_pid(addr_space, types, socket),
                       proto = socket_protocol(addr_space, types, socket),
                       port  = socket_local_port(addr_space, types, socket),
                       _create_time  = "from_unixtime('%s')" % socket_create_time(addr_space, types, socket),
                       iosource = iosource_name
                       )
parser.add_option("-f", "--file", default=None,
                  help = "A single file of connection ids, one per line")

(options, args) = parser.parse_args()

if not options.case or not options.write:
    print "Mandatory args missing - run me with --help for help"
    sys.exit(1)

## Create the PCAP header for the new file:
dbh=DB.DBO(options.case)
dbh.execute("select * from pcap limit 1")
row = dbh.fetch()

io = IO.open(options.case, row['iosource'])
fd = Buffer(fd=io)

## Read the header:
header = PCAP.FileHeader(fd)

io.seek(0)
data = io.read(header.start_of_data)

## Open file for writing:
outfd = open(options.write, 'w')
outfd.write(data)

## Now grab the data packets for all the relevant streams:
def write_stream(con_id):
    dbh2= dbh.clone()
Beispiel #28
0
                m = self.text_re.match(data)
                if m:
                    period_number = period_number + p / period
                    query.set('period_number', period_number)
                    result.refresh(0, query, 'parent')
                    return

                p += 1

    def create(self, name, case, query):
        offset = FlagFramework.calculate_offset_suffix(query.get(
            'offset', '0'))
        filenames = self.glob_filenames(query.getarray('filename'))

        ## Open the io sources here
        fds = [IO.open_URL(f) for f in filenames]
        if query.get('ismissing', False):
            fds += [ParityFD(copy.deepcopy(filenames))]
        blocksize = FlagFramework.calculate_offset_suffix(
            query.get('blocksize', '32k'))
        period = int(query.get('period', 3))
        return RAIDFD(fds, blocksize, query['map'], offset, period)


import pyflag.tests
import pyflag.pyflagsh as pyflagsh


class RaidTest(pyflag.tests.ScannerTest):
    """ Test the RAID IOSource loader """
    test_case = "PyFlagTestCase"
    arg = iter.next()

    try:
        while 1:
            if arg == '-i' or arg == '--subsystem':
                query['subsys'] = iter.next()
            elif arg.startswith('-'):
                try:
                    while 1:
                        opt = iter.next()
                        if opt.startswith('-'):
                            arg = opt
                            continue
                        query["io_%s" % arg[1:]] = opt
                except StopIteration:
                    break

            arg = iter.next()

    except StopIteration:
        pass

    ## We try to instantiate the IO object:
    io = IO.IOFactory(query)

    #Now we create a fuse object with that IO subsystem:
    server = Xmp(io=io)
    server.flags = 0
    server.multithreaded = 1
    server.main()
Beispiel #30
0

def clean_up_tmp_directory(tmp_dir):
    #Recursively remove the tmp directory
    for root, dirs, files in os.walk(tmp_dir, topdown=False):
        for name in files:
            os.remove(join(root, name))
        for name in dirs:
            os.rmdir(join(root, name))


# set up flag io subsystem
query_target = FlagFramework.query_type(())
query_target['subsys'] = 'standard'
query_target['io_filename'] = target
io_target = IO.IOFactory(query_target)
io_source = IO.IOFactory(query_target)

#iterate file copying
image_count = 0
total_image_count = 0
collection_size = 0
for image in Exgrep.process(None, io_target, ("jpg")):
    io_source.seek(image['offset'])
    tmp_image = open(tmp_image_filename, 'w')
    if extracted_length > 0:
        tmp_image.write(io_source.read(extracted_length * 1024))
    else:
        tmp_image.write(io_source.read(image['length']))

    tmp_image.close()
Beispiel #31
0
    def get_fields(self):
        if self.datafile == None:
            raise IOError("Datafile is not set!!!")

        print "Datafile %s" % (self.datafile, )

        for file in self.datafile:
            ## open the file as a url:
            fd = IO.open_URL(file)
            dbh = DB.DBO()
            buffer = Buffer(fd=fd)
            header = EVTLog.Header(buffer)
            buffer = buffer[header.size():]

            while 1:
                try:
                    event = EVTLog.Event(buffer)

                    source = event['Source'].get_value()
                    machine = event['Machine'].get_value()

                    ## Find the filename for this source:
                    dbh.execute(
                        "select filename from EventMessageSources where source=%r",
                        source)
                    row = dbh.fetch()
                    if row:
                        dbh.execute(
                            "select message from EventMessages where filename=%r and message_id=%r",
                            (row['filename'], event['EventID'].get_value()))
                        row = dbh.fetch()
                        if row:
                            message = EVTLog.format_message(
                                row['message'], event['Strings'])
                        ## Message not found
                        else:
                            message = "Unable to find message format string (Maybe file was not loaded with --mode=dll?). Parameters are: %s" % event[
                                'Strings']

                    ## Filename not found for this source:
                    else:
                        message = "Unable to locate file for source %s. Maybe you need to run EventLogTool with the --reg flag on the SYSTEM registry hive? Parameters are: %s " % (
                            source, event['Strings'])

                    buffer = buffer[event.size():]
                    result = dict(
                        _time="from_unixtime('%s')" %
                        event['TimeGenerated'].get_value(),
                        message=message,
                        event=event['EventID'].get_value(),
                        Source=event['Source'].get_value(),
                        record=event['RecordNumber'].get_value(),
                    )
                    try:
                        result['arg1'] = event['Strings'][0].get_value()
                    except:
                        pass

                    try:
                        result['arg2'] = event['Strings'][1].get_value()
                    except:
                        pass

                    try:
                        result['arg3'] = event['Strings'][2].get_value()
                    except:
                        pass

                    yield result

                except IOError:
                    break
Beispiel #32
0
    def display(self,query,result):
        dbh = DB.DBO(query['case'])
        dbh.execute("select * from pcap where id=%r limit 1", query['id'])
        row=dbh.fetch()
        
        io = IO.open(query['case'], row['iosource'])
        packet = pypcap.PyPCAP(io)
        packet.seek(row['offset'])
        dissected_packet = packet.dissect()
        
        id = int(query['id'])
        
        def get_node(branch):
            """ Locate the node specified by the branch.

            branch is a list of attribute names.
            """
            result = dissected_packet
            for b in branch:
                try:
                    result = getattr(result, b)
                except: pass

            return result
        
        def tree_cb(path):
            branch = FlagFramework.splitpath(path)
            
            node = get_node(branch)
            try:
                for field in node.list():
                    if field.startswith("_"): continue

                    child = getattr(node, field)
                    try:
                        yield  ( field, child.get_name(), 'branch')
                    except AttributeError:
                        yield  ( field, field, 'leaf')

            except AttributeError:
                pass
            
            return
        
        def pane_cb(path,result):
            branch = FlagFramework.splitpath(path)
            
            node = get_node(branch)

            result.heading("Packet %s" % id)
            data = dissected_packet.serialise()
            
            h=FlagFramework.HexDump(data, result)
            try:
                result.text("%s" % node.get_name(), font='bold')
                result.text('',style='black', font='normal')
                start,length = node.get_range()
                
            except AttributeError:
                result.text("%s\n" % node, style='red', wrap='full', font='typewriter', sanitise='full')
                result.text('',style='black', font='normal')
                node = get_node(branch[:-1])
                start,length = node.get_range(branch[-1])

            h.dump(highlight=[[start,length,'highlight'],])

            return

        result.tree(tree_cb=tree_cb, pane_cb=pane_cb, branch=[''])

        ## We add forward and back toolbar buttons to let people move
        ## to next or previous packet:
        dbh.execute("select min(id) as id from pcap")
        row = dbh.fetch()

        new_query=query.clone()
        if id>row['id']:
            del new_query['id']
            new_query['id']=id-1
            result.toolbar(text="Previous Packet",icon="stock_left.png",link=new_query)
        else:
            result.toolbar(text="Previous Packet",icon="stock_left_gray.png")
            
        dbh.execute("select max(id) as id from pcap")
        row = dbh.fetch()
        
        if id<row['id']:
            del new_query['id']
            new_query['id']=id+1
            result.toolbar(text="Next Packet",icon="stock_right.png",link=new_query)
        else:
            result.toolbar(text="Next Packet",icon="stock_right_gray.png")
Beispiel #33
0
    def load(self, mount_point, iosource_name,scanners = None):
        DBFS.load(self, mount_point, iosource_name)
        
        ## Open the file descriptor
        self.fd = IO.open(self.case, iosource_name)

        ## Use the C implementation to read the pcap files:
        pcap_file = pypcap.PyPCAP(self.fd)

        ## Build our streams:
        pyflaglog.log(pyflaglog.DEBUG, "Reassembling streams, this might take a while")
        pcap_dbh = DB.DBO(self.case)
        pcap_dbh.mass_insert_start("pcap")

        pcap_dbh.execute("select max(id) as m from pcap")
        max_id = pcap_dbh.fetch()['m'] or 0
        cookie, processor = self.make_processor(iosource_name, scanners)

        ## Process the file with it:
        while 1:
            try:
                packet = pcap_file.dissect()
                max_id += 1

                ## FIXME - this is a bottleneck. For now we use mass
                ## insert but this will break when we have multiple
                ## concurrent loaders.  Record the packet in the pcap
                ## table:
                args = dict(
                    iosource = iosource_name,
                    offset = packet.offset,
                    length = packet.caplen,
                    _ts_sec =  "from_unixtime('%s')" % packet.ts_sec,
                    ts_usec = packet.ts_usec,
                    )

                ## Try to insert the ipid field
                try:
                    args['ipid']= packet.root.eth.payload.id
                except: pass

                pcap_dbh.mass_insert(**args)
                #pcap_id = pcap_dbh.autoincrement()
                pcap_id = max_id
                pcap_file.set_id(pcap_id)

                
                ## Some progress reporting
                if pcap_id % 10000 == 0:
                    pyflaglog.log(pyflaglog.DEBUG, "processed %s packets (%s bytes)" % (pcap_id, packet.offset))

                processor.process(packet)
            except StopIteration:
                break

        processor.flush()

        pcap_dbh.check_index("connection_details",'src_ip')
        pcap_dbh.check_index("connection_details",'src_port')
        pcap_dbh.check_index("connection_details",'dest_ip')
        pcap_dbh.check_index("connection_details",'dest_port')
        pcap_dbh.check_index('connection_details','inode_id')
Beispiel #34
0
#!/usr/bin/env python
import pyflag.IO as IO
import sys
import sk
from stat import *

#img = open(sys.argv[1],'r')
#print "Will open %s" % sys.argv[1]

## This assumes that we have a case loaded
img = IO.open('winxp','test')
fs = sk.skfs(img)

def readfile(fs,inode):
    print "reading: %s" % inode
    fd = fs.open(inode=inode)
    while True:
        if not fd.read(4000000):
            break

# walk the directory tree
for root, dirs, files in fs.walk('/', unalloc=True, inodes=True):
    for f in files:
        try:
            print "processing: (%u) %s" % (f[0], f[1])
            s=fs.stat(inode=str(f[0]))
            print "length %s" % s[ST_SIZE]
            if int(f[0])==0: continue
            readfile(fs,str(f[0]))
        except IOError, e:
            print "Got error: %s" % e
Beispiel #35
0
    def get_fields(self):
        if self.datafile==None:
            raise IOError("Datafile is not set!!!")

        print "Datafile %s" % (self.datafile,)
        
        for file in self.datafile:
            ## open the file as a url:
            fd = IO.open_URL(file)
            dbh = DB.DBO()
            buffer = Buffer(fd=fd)
            header = EVTLog.Header(buffer)
            buffer = buffer[header.size():]
             
            while 1:
                try:
                    event = EVTLog.Event(buffer)

                    source = event['Source'].get_value()
                    machine = event['Machine'].get_value()
                    
                    ## Find the filename for this source:
                    dbh.execute("select filename from EventMessageSources where source=%r", source)
                    row=dbh.fetch()
                    if row:
                        dbh.execute("select message from EventMessages where filename=%r and message_id=%r", (row['filename'], event['EventID'].get_value()))
                        row = dbh.fetch()
                        if row:
                            message=EVTLog.format_message(row['message'],event['Strings'])
                        ## Message not found
                        else:
                            message="Unable to find message format string (Maybe file was not loaded with --mode=dll?). Parameters are: %s" % event['Strings']
                        
                    ## Filename not found for this source:
                    else: message="Unable to locate file for source %s. Maybe you need to run EventLogTool with the --reg flag on the SYSTEM registry hive? Parameters are: %s " % (source,event['Strings'])


                    buffer=buffer[event.size():]
                    result = dict(
                        _time= "from_unixtime('%s')" % event['TimeGenerated'].get_value(),
                        message= message,
                        event = event['EventID'].get_value(),
                        Source = event['Source'].get_value(),
                        record = event['RecordNumber'].get_value(),
                        )
                    try:
                        result['arg1'] = event['Strings'][0].get_value()
                    except: pass

                    try:
                        result['arg2'] = event['Strings'][1].get_value()
                    except: pass

                    try:
                        result['arg3'] = event['Strings'][2].get_value()
                    except: pass
                    
                    yield result
                    
                except IOError:
                    break