示例#1
0
    def run(self):
        """
            This function will read a raw disk capture and use a scanned disk 
            image to reconstruct the recorded SATA traffic and output the 
            semantic output.
        """

        # copy our disk image to a temporary working image
        self.working_disk_img = os.path.join(self.output_dir, "disk.img.tmp")
        print "* Creating temporary working image from disk scan. (%s)"%self.working_disk_img
        
        # Delete, copy, chmod new file
        try:
            os.unlink(self.working_disk_img)
        except:
            pass
        cmd = "cp --sparse=always %s %s" % (self.disk_img, self.working_disk_img)
        subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read()
        os.chmod(self.working_disk_img, 0755)

        # Set up our semantic bridge
        print "* Parsing disk image %s into our semantic engine... (This may take a while)" % self.working_disk_img
        semantic_engine = SemanticEngineDisk(self.working_disk_img)
  
        # Start processing our dcap
        print "* Processing dcap file %s..." % self.dcap_filename
        # SATA Interpreter
        sata = SATAInterpreter()  
        """
            @TODO Extract sector size from PyTSK
        """
        sata_reconstructor = SATAReconstructor(sector_size=G.SENSOR_DISK.DEFAULT_SECTOR_SIZE)
    
        # read from the cap file in real time
        reader = CaptureReader(self.dcap_filename)

        # Tailing or terminating?
        reader_iter = reader
        if self.tail_enable:
            reader_iter = reader.tail()

        # Loop over all of the dcap contents
        for (timestamp, data) in reader_iter:
            
            if self.sensor_type == G.MACHINE_TYPES.PHYSICAL:
                (header, data) = sata.extract_sata_data(data)

                # deal with SATA NCQ reordering
                disk_sensor_pkts = sata_reconstructor.process_packet(PhysicalPacket(header, data))
            else:
                disk_sensor_pkts = [DiskSensorPacket(data)]
        
            # Process all of our disk packets
            if disk_sensor_pkts:
                for dsp in disk_sensor_pkts:
                    # Skip empty packets
                    if not dsp:
                        continue
      
                    try:
                        fs_operations = semantic_engine.get_access(dsp.sector, 
                                                                   dsp.num_sectors, 
                                                                   dsp.disk_operation, 
                                                                   dsp.data)   
                        self.log_output(timestamp, fs_operations)                   
                               
                    except:
                        logging.exception("Encountered error while trying to bridge semantic gap for this disk access.")
示例#2
0
    def run(self):
        """
            Figure out which type of host we are examining and call the 
            appropriate function.
        """

        from lophi_semanticgap.disk.sata import SATAInterpreter
        from lophi_semanticgap.disk.sata_reconstructor import SATAReconstructor
        from lophi_semanticgap.disk.filesystem_reconstructor import SemanticEngineDisk

        logger.debug("DiskEngine Started.")

        # Scan in our starting point
        if self.machine.type == G.MACHINE_TYPES.PHYSICAL:
            if not self._check_disk_scan():
                logger.error(
                    "Analysis cannot continue without a valid scan file.")
                return

            disk_img = self.machine.config.disk_scan
        else:
            # Get our image name for the virtual HDD on this host
            image_name = self.machine.disk_get_filename()
            logger.debug("Scanning disk image (%s)..." % (image_name))

            if image_name is None:
                logger.error("No disk found for VM (%s)." %
                             self.machine.config.name)
                return

            if image_name.endswith("qcow2"):
                logger.warning(
                    "Got qcow2 image, scanning the base image, ensure that you reset the machine!"
                )
                disk_img = self.machine.config.disk_base
            else:
                disk_img = image_name

        # Setup our tmp file
        self.working_disk_img = os.path.join(
            G.DIR_ROOT, G.DIR_TMP, self.machine.config.name + "-disk.img.tmp")

        # Create a backup
        logger.debug("Copying %s to %s..." % (disk_img, self.working_disk_img))
        cmd = "cp --sparse=always %s %s" % (disk_img, self.working_disk_img)
        subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read()
        os.chmod(self.working_disk_img, 0755)

        # Set up our semantic bridge
        logger.info(
            "Parsing disk image %s into our semantic engine... (This may take a while)"
            % self.working_disk_img)
        semantic_engine = SemanticEngineDisk(self.working_disk_img)

        # SATA Interpreter
        sata = SATAInterpreter()

        # SATA Interpreter
        sata_reconstructor = SATAReconstructor(
            sector_size=G.SENSOR_DISK.DEFAULT_SECTOR_SIZE)

        # Get data forever and report it back
        self.RUNNING = True
        while self.RUNNING:

            # Accept commands
            try:
                cmd = self.command_queue.get(False).split(" ")
                logger.debug("Got cmd: %s" % cmd)
                if cmd[0] == G.CTRL_CMD_PAUSE:
                    logger.debug("Pausing analysis")
                    self.PAUSED = True
                    self.machine.disk._disconnect()
                if cmd[0] == G.CTRL_CMD_UNPAUSE:
                    logger.debug("Resuming Analysis")
                    self.PAUSED = False
                    self.machine.disk._connect()
                if cmd[0] == G.CTRL_CMD_KILL or cmd[0] == G.CTRL_CMD_STOP:
                    logger.debug("Got kill command")
                    self.RUNNING = False
                    self.machine.disk._disconnect()
                    break
            except:
                # Do nothing
                pass

            if self.PAUSED:
                time.sleep(1)
                continue

            # Get our packet
            try:
                data = self.machine.disk_get_packet()
                logger.debug("Got: %s" % data)
            except:
                G.print_traceback()
                logger.debug("Disk introspection socket closed.")
                break

            # Good data?
            if data is None:
                continue

            if self.machine.type == G.MACHINE_TYPES.PHYSICAL:
                lophi_packet = type('AnonClass', (object, ), {
                    "sata_header": None,
                    "sata_data": None
                })
                (lophi_packet.sata_header,
                 lophi_packet.sata_data) = sata.extract_sata_data( ` data `)

                # deal with SATA NCQ reordering
                disk_sensor_pkts = sata_reconstructor.process_packet(
                    lophi_packet)
            else:
                disk_sensor_pkts = [data]

            # Process all of our disk packets
            if disk_sensor_pkts:
                for dsp in disk_sensor_pkts:
                    # Skip empty packets
                    if not dsp:
                        continue

                    try:

                        fs_operations = semantic_engine.get_access(
                            dsp.sector, dsp.num_sectors, dsp.disk_operation,
                            dsp.data)
                        self.parse_actions(time.time(), fs_operations)

                    except:
                        logging.exception(
                            "Encountered error while trying to bridge semantic gap for this disk access."
                        )


#             logger.debug("got actions %s"%actions)
# Handle our output
#             self.parse_actions(actions)

        logger.debug("Disk analysis exiting...")
示例#3
0
    def replay(self):
        """
        Replays the raw log of SATA frames (captured as LOPHIPackets),
        bridges the semantic gap and returns human-readable text form
        """

        # copy our disk image to a temporary working image
        self.working_disk_img = os.path.join(self.output_dir, "disk.img.tmp")
        logger.debug("* Creating temporary working image from disk scan. (%s)" % self.working_disk_img)

        # Delete, copy, chmod new file

        try:
            os.unlink(self.working_disk_img)
        except OSError:
            pass
        
        cmd = "cp --sparse=always %s %s" % (self.disk_img, self.working_disk_img)
        subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read()
        os.chmod(self.working_disk_img, 0755)
        
        # Set up our semantic bridge
        logger.debug("* Parsing disk image %s into our semantic engine... (This may take a while)" % self.working_disk_img)
        semantic_engine = SemanticEngineDisk(self.working_disk_img)

        # Start processing our dcap
        logger.debug("* Processing dcap file %s..." % self.dcap_url)
        # SATA Interpreter
        sata = SATAInterpreter()
        """
            @TODO Extract sector size from PyTSK
        """
        sata_reconstructor = SATAReconstructor(sector_size=G.SENSOR_DISK.DEFAULT_SECTOR_SIZE)

        # read from the cap file
        reader = CaptureReader(self.dcap_url)

        # output
        output_log = []

        # Loop over all of the dcap contents
        for (timestamp, data) in reader:

            disk_sensor_pkts = None

            if self.machine_type == G.MACHINE_TYPES.PHYSICAL:
                lophi_packet = type('AnonClass', (object,), { "sata_header": None, "sata_data": None })
                (lophi_packet.sata_header, lophi_packet.sata_data) = sata.extract_sata_data(data)

                if (lophi_packet.sata_header == None or
                    lophi_packet.sata_data == None):
                    logger.warning("Skipping abnormal physical SATA capture packet -- either sata header and/or data is None.")
                    continue

                # deal with SATA NCQ reordering
                disk_sensor_pkts = sata_reconstructor.process_packet(lophi_packet)
            else:
                # logger.debug(DiskSensorPacket(data))
                disk_sensor_pkts = [DiskSensorPacket(data)]

            # Process all of our disk packets
            if disk_sensor_pkts:
                for dsp in disk_sensor_pkts:
                    # Skip empty packets
                    if not dsp:
                        continue

                    try:
                        fs_operations = semantic_engine.get_access(dsp.sector,
                                                                   dsp.num_sectors,
                                                                   dsp.disk_operation,
                                                                   dsp.data)
                        if fs_operations == None:
                            logger.error("Got an operation to sector %s that is outside our disk." % dsp.sector)
                            continue

                        for op in fs_operations:
                            output_log.append(op)

                    except:
                        logging.exception("Encountered error while trying to bridge semantic gap for this disk access.")

        # return value
        fs_dict = {}

        # Consolidate filesystem activity into a set
        # fields should be 'sector', 'op', 'op_type', 'inode', 'filename', 'raw_data', 'semantic_data'
        # try to insert (operation, filename, inode)
        # for duplicates, use a counter
        for fs_operation in output_log:

            sector = fs_operation['sector']
            operation = fs_operation['op']
            operation_type = fs_operation['op_type']
            inode = fs_operation['inode']
            filename = fs_operation['filename']
            raw_data = fs_operation['raw_data']

            ## TODO - look at filesystem stuff for now -- more low level data later?
            # key = (sector, op, raw_data)
            key = (operation_type, filename, inode)

            if key in fs_dict:
                fs_dict[key] += 1
            else:
                fs_dict[key] = 1

        return fs_dict
示例#4
0
    def run(self):
        """
            Figure out which type of host we are examining and call the 
            appropriate function.
        """

        from lophi_semanticgap.disk.sata import SATAInterpreter
        from lophi_semanticgap.disk.sata_reconstructor import SATAReconstructor
        from lophi_semanticgap.disk.filesystem_reconstructor import SemanticEngineDisk

        logger.debug("DiskEngine Started.")

        # Scan in our starting point
        if self.machine.type == G.MACHINE_TYPES.PHYSICAL:
            if not self._check_disk_scan():
                logger.error("Analysis cannot continue without a valid scan file.")
                return
            
            disk_img = self.machine.config.disk_scan
        else:
            # Get our image name for the virtual HDD on this host
            image_name = self.machine.disk_get_filename()
            logger.debug("Scanning disk image (%s)..." % (image_name))

            if image_name is None:
                logger.error("No disk found for VM (%s)."%self.machine.config.name)
                return
            
            if image_name.endswith("qcow2"):
                logger.warning("Got qcow2 image, scanning the base image, ensure that you reset the machine!")
                disk_img = self.machine.config.disk_base
            else:
                disk_img = image_name
                
        # Setup our tmp file
        self.working_disk_img = os.path.join(G.DIR_ROOT,G.DIR_TMP,self.machine.config.name+"-disk.img.tmp")
        
        # Create a backup
        logger.debug("Copying %s to %s..."%(disk_img, self.working_disk_img))
        cmd = "cp --sparse=always %s %s" % (disk_img, self.working_disk_img)
        subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read()
        os.chmod(self.working_disk_img, 0755)
        
        # Set up our semantic bridge
        logger.info("Parsing disk image %s into our semantic engine... (This may take a while)" % self.working_disk_img)
        semantic_engine = SemanticEngineDisk(self.working_disk_img)

        # SATA Interpreter
        sata = SATAInterpreter() 
        
        # SATA Interpreter
        sata_reconstructor = SATAReconstructor(sector_size=G.SENSOR_DISK.DEFAULT_SECTOR_SIZE)
        
        # Get data forever and report it back
        self.RUNNING  = True
        while self.RUNNING:
            
            # Accept commands
            try:
                cmd = self.command_queue.get(False).split(" ")
                logger.debug("Got cmd: %s" % cmd)
                if cmd[0] == G.CTRL_CMD_PAUSE:
                    logger.debug("Pausing analysis")
                    self.PAUSED = True
                    self.machine.disk._disconnect()
                if cmd[0] == G.CTRL_CMD_UNPAUSE:
                    logger.debug("Resuming Analysis")
                    self.PAUSED = False
                    self.machine.disk._connect()
                if cmd[0] == G.CTRL_CMD_KILL or cmd[0] == G.CTRL_CMD_STOP:
                    logger.debug("Got kill command")
                    self.RUNNING = False
                    self.machine.disk._disconnect()
                    break
            except:
                # Do nothing
                pass
            
            if self.PAUSED:
                time.sleep(1)
                continue
            
            # Get our packet
            try:
                data = self.machine.disk_get_packet()
                logger.debug("Got: %s"%data)
            except:
                G.print_traceback()
                logger.debug("Disk introspection socket closed.")
                break
            
            # Good data?
            if data is None:
                continue

            if self.machine.type == G.MACHINE_TYPES.PHYSICAL:
                lophi_packet = type('AnonClass', (object,), { "sata_header": None, "sata_data": None })                    
                (lophi_packet.sata_header, lophi_packet.sata_data) = sata.extract_sata_data(`data`)
                
                # deal with SATA NCQ reordering
                disk_sensor_pkts = sata_reconstructor.process_packet(lophi_packet)
            else:
                disk_sensor_pkts = [data]
                
            
            # Process all of our disk packets
            if disk_sensor_pkts:
                for dsp in disk_sensor_pkts:
                    # Skip empty packets
                    if not dsp:
                        continue
      
                    try:

                        fs_operations = semantic_engine.get_access(dsp.sector, 
                                                                   dsp.num_sectors, 
                                                                   dsp.disk_operation, 
                                                                   dsp.data)   
                        self.parse_actions(time.time(), fs_operations)                   
                               
                    except:
                        logging.exception("Encountered error while trying to bridge semantic gap for this disk access.")

            
#             logger.debug("got actions %s"%actions)
            # Handle our output
#             self.parse_actions(actions)

        logger.debug("Disk analysis exiting...")