def run(self):
        """
            Run forever grabbing data from the VM and forwarding it to listening
            clients 
        """
        # Get our meta data which is always the first packet sent
        meta = MetaHeader()
        meta_data = self._conn.recv(len(meta))
        if len(meta_data) == 0:
            logger.debug("VM Disconnected.")
            self._conn.close()
            return

        meta._unpack(meta_data)

        #logger.debug("Got meta data")
        #logger.debug(meta)

        # Store our filename for this VM
        # NOTE: We must strip the null chars off or comparisons will fail!
        filename = meta.filename.strip("\x00")

        # Create our sensor packet, and save its default size
        sensor_packet = DiskSensorPacket()
        sensor_header_size = len(sensor_packet)

        # Read packets forever
        while True:

            # Read and unpack our header
            header_data = self._conn.recv(sensor_header_size)
            if len(header_data) == 0:
                #logger.debug("VM Disconnected.")
                break

            sensor_packet._unpack(header_data)

            # Get the accompanying data
            try:
                sensor_packet.data = self._conn.recv(sensor_packet.size)
            except:
                print sensor_packet
                G.print_traceback()

            if len(sensor_packet.data) == 0:
                logger.debug("VM Disconnected.")
                self._conn.close()
                return

            if filename in disk_stream_dict:
                #logger.debug("Found %s in dict."%filename)
                for queue in disk_stream_dict[filename]:
                    queue.put( ` sensor_packet `)
 def run(self):
     """
         Run forever grabbing data from the VM and forwarding it to listening
         clients 
     """
     # Get our meta data which is always the first packet sent
     meta = MetaHeader()
     meta_data = self._conn.recv(len(meta))
     if len(meta_data) == 0:
         logger.debug("VM Disconnected.")
         self._conn.close()
         return
     
     meta._unpack(meta_data)
     
     #logger.debug("Got meta data")
     #logger.debug(meta)
     
     # Store our filename for this VM
     # NOTE: We must strip the null chars off or comparisons will fail!
     filename = meta.filename.strip("\x00")
     
     # Create our sensor packet, and save its default size
     sensor_packet = DiskSensorPacket()
     sensor_header_size = len(sensor_packet)
     
     # Read packets forever
     while True:
         
         # Read and unpack our header
         header_data = self._conn.recv(sensor_header_size)
         if len(header_data) == 0:
             #logger.debug("VM Disconnected.")
             break
         
         sensor_packet._unpack(header_data)
         
         # Get the accompanying data
         try:
             sensor_packet.data = self._conn.recv(sensor_packet.size)
         except:
             print sensor_packet
             G.print_traceback()
             
         if len(sensor_packet.data) == 0:
             logger.debug("VM Disconnected.")
             self._conn.close()
             return
         
         if filename in disk_stream_dict:
             #logger.debug("Found %s in dict."%filename)
             for queue in disk_stream_dict[filename]:
                 queue.put(`sensor_packet`)
Esempio n. 3
0
    def get_disk_packet(self):
        """
            Get the next packet header/data
        """

        if self.SOCK is None:
            self._connect()

        access_packet = DiskSensorPacket()

        # Try forever to get a packet
        while 1:
            # Receive our header
            header = self._read_raw_packet(len(access_packet))

            # Did our socket get closed?
            if len(header) < len(access_packet):
                logger.warn("Introspection server disconnected.")
                # reconnect and try again
                self._connect()
                continue

            access_packet = DiskSensorPacket(header)

            # Read for as long as the header tells us to to get the content
            data = ""
            recvd = 0
            while len(data) < access_packet.size:
                logger.debug("Read data.")
                tmp = self.SOCK.recv(access_packet.size - recvd)
                data += tmp
                recvd = len(data)
                if len(tmp) == 0:
                    logger.warn("Introspection server disconnected.")
                    break

            # Did our socket get closed?
            if recvd < access_packet.size:
                logger.warn("Introspection server disconnected.")
                # reconnect and try again
                self._connect()
                continue

            logger.debug("Read %d bytes of data." % len(data))

            # Return our data
            access_packet.data = data

            return access_packet
Esempio n. 4
0
    def get_disk_packet(self):
        """
            Get the next packet header/data
        """
        
        if self.SOCK is None:
            self._connect()
        
        access_packet = DiskSensorPacket()

        # Try forever to get a packet
        while 1:
            # Receive our header
            header = self._read_raw_packet(len(access_packet))

            # Did our socket get closed?
            if len(header) < len(access_packet):
                logger.warn("Introspection server disconnected.")
                # reconnect and try again
                self._connect()
                continue
            
            access_packet = DiskSensorPacket(header)

            # Read for as long as the header tells us to to get the content
            data = ""
            recvd = 0
            while len(data) < access_packet.size:
                logger.debug("Read data.")
                tmp = self.SOCK.recv(access_packet.size - recvd);
                data += tmp
                recvd = len(data)
                if len(tmp) == 0:
                    logger.warn("Introspection server disconnected.")
                    break

            # Did our socket get closed?
            if recvd < access_packet.size:
                logger.warn("Introspection server disconnected.")
                # reconnect and try again
                self._connect()
                continue

            logger.debug("Read %d bytes of data."%len(data))
            
            # Return our data
            access_packet.data = data
            
            return access_packet
def read_disk(options):

    # Setup our log files
    dcap_filename = options.dcap_file

    # read from the cap file in real time
    reader = CaptureReader(options.dcap_file)

    # Tailing or terminating?
    reader_iter = reader
    if options.tail_enable:
        reader_iter = reader.tail()

    # Loop over all of the dcap contents
    for (timestamp, data) in reader_iter:

        print timestamp
        if options.sensor_type == G.MACHINE_TYPES.PHYSICAL:
            sata_frame = SATAFrame(data)
            print sata_frame
        else:
            disk_sensor_pkt = [DiskSensorPacket(data)]
            print disk_sensor_pkt
    def handle_register_HTD(self, register_packet):
        """
            Handle a Register HTD

            Always returns None
        """

        sata_header = register_packet.sata_header

        # Check if C bit is set to 0 -- see 11.2 Dl1 Note 1 -- no FIS is sent
        # I think we can ignore if C bit is set to 0 and go back to IDLE state
        # software reset could still happen -- not sure which bit is the SRST bit in the control register?
        if sata_header['C'] == 0:
            logger.debug(
                "Got HTD Register Packet but C bit is set to 0.  Ignoring.")
            return None

        # check if this is an NCQ command
        if sata_header[
                'command'] == NCQCommandType.ReadFPDMAQueued or sata_header[
                    'command'] == NCQCommandType.WriteFPDMAQueued:

            cmd = "READ"
            direction = G.SATA_OP.DIRECTION.READ
            if sata_header['command'] == NCQCommandType.WriteFPDMAQueued:
                cmd = "WRITE"
                direction = G.SATA_OP.DIRECTION.WRITE

            # NCQ uses features field for sector count
            logger.debug(
                "Got HTD Register Packet for NCQ %s -- TAG %d expecting %d bytes."
                % (cmd, sata_header['tag'],
                   sata_header['features'] * self.sector_size))

            #            if len(self.ncq_register_stack) > 0:
            #                logger.error("Our NCQ register stack will be greater than 1, so we have at least one outstanding NCQ register packet with NO Register ACK.  Stack size is %d" % len(self.ncq_register_stack))

            #            # add to our stack
            #            self.ncq_register_stack.append(register_packet)

            #No ACKS
            #            # change state to waiting for Register DTH
            #            self.STATE = self.WAIT_FOR_REGISTER_ACK
            #
            #            # deal with error checking when we get the Register DTH ACK back
            #            return None

            # check if the spot belonging to the associated TAG is empty
            if (not self.ncq_transactions_outstanding[sata_header['tag']]):
                # This TAG is not taken, so we just add
                self.ncq_transactions_outstanding[
                    sata_header['tag']] = register_packet

            else:
                logger.error(
                    "Received HTD NCQ Register Packet for TAG %d but it is already in use.  Using it anyway."
                    % sata_header['tag'])
                # still put it in there for now
                self.ncq_transactions_outstanding[
                    sata_header['tag']] = register_packet

                #don't need this if we accept the register packet anyway
                #return None

            # Associate this DTH Register packet
            # with the tag and prepare the disk sensor packet for aggregating data
            lba = sata_header['lba']

            # NOTE: NCQ uses features field instead of count for sector count
            sector_count = sata_header['features']

            disk_sensor_packet = DiskSensorPacket()
            disk_sensor_packet.sector = lba
            disk_sensor_packet.num_sectors = sector_count
            disk_sensor_packet.disk_operation = direction
            disk_sensor_packet.size = sector_count * self.sector_size
            disk_sensor_packet.data = ""

            self.ncq_data_outstanding[sata_header['tag']] = disk_sensor_packet

            # DO NOT set state to IDLE
            # Because now it looks like Register packets can come during other times
            # so we want to maintain the current state
            #            # set state to IDLE
            #
            #            self.STATE = self.DEVICE_IDLE

            return None

        # NCQ Management stuff
        elif sata_header['command'] == NCQCommandType.NCQQueueManagement:
            logger.debug(
                "Got Register Packet for NCQ Queue Management.  Ignoring for now."
            )
            # Don't need to change state
            return None

        # NON NCQ Register Packet
        else:
            logger.debug(
                "Got Non-NCQ HTD Register Packet, expected %d bytes." %
                (sata_header['count'] * self.sector_size))
            logger.debug("Command field is %d" % sata_header['command'])

            #            if len(self.ncq_register_stack) > 0:
            #                logger.error("Have more than 0 outstanding NCQ HTD Register Packets.  Non NCQ Register packets are not supposed to be sent while NCQ Register packets are still outstanding.  Size of stack is %d" % len(self.regular_register_stack))

            #            if len(self.regular_register_stack) > 0:
            #                logger.error("Will have more than one outstanding regular (non NCQ) HTD Register Packets.  Size of stack is %d" % len(self.regular_register_stack))

            # push this normal Register HTD packet on the stack
            self.regular_register_stack.append(register_packet)

            # prepare the disk sensor packet for aggregating data
            lba = sata_header['lba']
            sector_count = sata_header['count']
            direction = sata_header['direction']

            #             self.regular_disk_sensor_packet = DiskSensorPacket(lba,
            #                                          sector_count,
            #                                          direction,
            #                                          sector_count*self.sector_size,
            #                                          "")
            self.regular_disk_sensor_packet = DiskSensorPacket()
            self.regular_disk_sensor_packet.sector = lba
            self.regular_disk_sensor_packet.num_sectors = sector_count
            self.regular_disk_sensor_packet.disk_operation = direction
            self.regular_disk_sensor_packet.size = sector_count * self.sector_size
            self.regular_disk_sensor_packet.data = ""

            # Transition state now that we are waiting for data
            self.STATE = self.WAIT_FOR_NON_NCQ_DATA

            return None
Esempio n. 7
0
    def handle_register_HTD(self, register_packet):
        """
            Handle a Register HTD

            Always returns None
        """

        sata_header = register_packet.sata_header

        # Check if C bit is set to 0 -- see 11.2 Dl1 Note 1 -- no FIS is sent
        # I think we can ignore if C bit is set to 0 and go back to IDLE state
        # software reset could still happen -- not sure which bit is the SRST bit in the control register?
        if sata_header['C'] == 0:
            logger.debug("Got HTD Register Packet but C bit is set to 0.  Ignoring.")
            return None

        # check if this is an NCQ command
        if sata_header['command'] == NCQCommandType.ReadFPDMAQueued or sata_header['command'] == NCQCommandType.WriteFPDMAQueued:

            cmd = "READ"
            direction = G.SATA_OP.DIRECTION.READ
            if sata_header['command'] == NCQCommandType.WriteFPDMAQueued:
                cmd = "WRITE"
                direction = G.SATA_OP.DIRECTION.WRITE

            # NCQ uses features field for sector count
            logger.debug("Got HTD Register Packet for NCQ %s -- TAG %d expecting %d bytes." % (cmd, sata_header['tag'], sata_header['features']*self.sector_size))

            #            if len(self.ncq_register_stack) > 0:
            #                logger.error("Our NCQ register stack will be greater than 1, so we have at least one outstanding NCQ register packet with NO Register ACK.  Stack size is %d" % len(self.ncq_register_stack))

            #            # add to our stack
            #            self.ncq_register_stack.append(register_packet)

            #No ACKS
            #            # change state to waiting for Register DTH
            #            self.STATE = self.WAIT_FOR_REGISTER_ACK
            #
            #            # deal with error checking when we get the Register DTH ACK back
            #            return None

            # check if the spot belonging to the associated TAG is empty
            if (not self.ncq_transactions_outstanding[sata_header['tag']]):
                # This TAG is not taken, so we just add
                self.ncq_transactions_outstanding[sata_header['tag']] = register_packet

            else:
                logger.error("Received HTD NCQ Register Packet for TAG %d but it is already in use.  Using it anyway." % sata_header['tag'])
                # still put it in there for now
                self.ncq_transactions_outstanding[sata_header['tag']] = register_packet

                #don't need this if we accept the register packet anyway
                #return None


            # Associate this DTH Register packet
            # with the tag and prepare the disk sensor packet for aggregating data
            lba = sata_header['lba']

            # NOTE: NCQ uses features field instead of count for sector count
            sector_count = sata_header['features']


            disk_sensor_packet = DiskSensorPacket()
            disk_sensor_packet.sector = lba
            disk_sensor_packet.num_sectors =  sector_count
            disk_sensor_packet.disk_operation = direction
            disk_sensor_packet.size = sector_count*self.sector_size
            disk_sensor_packet.data = ""

            self.ncq_data_outstanding[sata_header['tag']] = disk_sensor_packet

            # DO NOT set state to IDLE
            # Because now it looks like Register packets can come during other times
            # so we want to maintain the current state
            #            # set state to IDLE
            #
            #            self.STATE = self.DEVICE_IDLE

            return None



        # NCQ Management stuff
        elif sata_header['command'] == NCQCommandType.NCQQueueManagement:
            logger.debug("Got Register Packet for NCQ Queue Management.  Ignoring for now.")
            # Don't need to change state
            return None

        # NON NCQ Register Packet
        else:
            logger.debug("Got Non-NCQ HTD Register Packet, expected %d bytes." % (sata_header['count']*self.sector_size))
            logger.debug("Command field is %d" % sata_header['command'])

            #            if len(self.ncq_register_stack) > 0:
            #                logger.error("Have more than 0 outstanding NCQ HTD Register Packets.  Non NCQ Register packets are not supposed to be sent while NCQ Register packets are still outstanding.  Size of stack is %d" % len(self.regular_register_stack))


            #            if len(self.regular_register_stack) > 0:
            #                logger.error("Will have more than one outstanding regular (non NCQ) HTD Register Packets.  Size of stack is %d" % len(self.regular_register_stack))

            # push this normal Register HTD packet on the stack
            self.regular_register_stack.append(register_packet)


            # prepare the disk sensor packet for aggregating data
            lba = sata_header['lba']
            sector_count = sata_header['count']
            direction = sata_header['direction']

            #             self.regular_disk_sensor_packet = DiskSensorPacket(lba,
            #                                          sector_count,
            #                                          direction,
            #                                          sector_count*self.sector_size,
            #                                          "")
            self.regular_disk_sensor_packet = DiskSensorPacket()
            self.regular_disk_sensor_packet.sector = lba
            self.regular_disk_sensor_packet.num_sectors = sector_count
            self.regular_disk_sensor_packet.disk_operation = direction
            self.regular_disk_sensor_packet.size = sector_count*self.sector_size
            self.regular_disk_sensor_packet.data = ""

            # Transition state now that we are waiting for data
            self.STATE = self.WAIT_FOR_NON_NCQ_DATA

            return None
Esempio n. 8
0
    def run(self):
        """
            This function will read a raw disk capture and use a scanned disk 
            image to reconstruct the recorded SATA traffic and output the 
            semantic output.
        """

        # copy our disk image to a temporary working image
        self.working_disk_img = os.path.join(self.output_dir, "disk.img.tmp")
        print "* Creating temporary working image from disk scan. (%s)"%self.working_disk_img
        
        # Delete, copy, chmod new file
        try:
            os.unlink(self.working_disk_img)
        except:
            pass
        cmd = "cp --sparse=always %s %s" % (self.disk_img, self.working_disk_img)
        subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read()
        os.chmod(self.working_disk_img, 0755)

        # Set up our semantic bridge
        print "* Parsing disk image %s into our semantic engine... (This may take a while)" % self.working_disk_img
        semantic_engine = SemanticEngineDisk(self.working_disk_img)
  
        # Start processing our dcap
        print "* Processing dcap file %s..." % self.dcap_filename
        # SATA Interpreter
        sata = SATAInterpreter()  
        """
            @TODO Extract sector size from PyTSK
        """
        sata_reconstructor = SATAReconstructor(sector_size=G.SENSOR_DISK.DEFAULT_SECTOR_SIZE)
    
        # read from the cap file in real time
        reader = CaptureReader(self.dcap_filename)

        # Tailing or terminating?
        reader_iter = reader
        if self.tail_enable:
            reader_iter = reader.tail()

        # Loop over all of the dcap contents
        for (timestamp, data) in reader_iter:
            
            if self.sensor_type == G.MACHINE_TYPES.PHYSICAL:
                (header, data) = sata.extract_sata_data(data)

                # deal with SATA NCQ reordering
                disk_sensor_pkts = sata_reconstructor.process_packet(PhysicalPacket(header, data))
            else:
                disk_sensor_pkts = [DiskSensorPacket(data)]
        
            # Process all of our disk packets
            if disk_sensor_pkts:
                for dsp in disk_sensor_pkts:
                    # Skip empty packets
                    if not dsp:
                        continue
      
                    try:
                        fs_operations = semantic_engine.get_access(dsp.sector, 
                                                                   dsp.num_sectors, 
                                                                   dsp.disk_operation, 
                                                                   dsp.data)   
                        self.log_output(timestamp, fs_operations)                   
                               
                    except:
                        logging.exception("Encountered error while trying to bridge semantic gap for this disk access.")
Esempio n. 9
0
    def replay(self):
        """
        Replays the raw log of SATA frames (captured as LOPHIPackets),
        bridges the semantic gap and returns human-readable text form
        """

        # copy our disk image to a temporary working image
        self.working_disk_img = os.path.join(self.output_dir, "disk.img.tmp")
        logger.debug("* Creating temporary working image from disk scan. (%s)" % self.working_disk_img)

        # Delete, copy, chmod new file

        try:
            os.unlink(self.working_disk_img)
        except OSError:
            pass
        
        cmd = "cp --sparse=always %s %s" % (self.disk_img, self.working_disk_img)
        subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read()
        os.chmod(self.working_disk_img, 0755)
        
        # Set up our semantic bridge
        logger.debug("* Parsing disk image %s into our semantic engine... (This may take a while)" % self.working_disk_img)
        semantic_engine = SemanticEngineDisk(self.working_disk_img)

        # Start processing our dcap
        logger.debug("* Processing dcap file %s..." % self.dcap_url)
        # SATA Interpreter
        sata = SATAInterpreter()
        """
            @TODO Extract sector size from PyTSK
        """
        sata_reconstructor = SATAReconstructor(sector_size=G.SENSOR_DISK.DEFAULT_SECTOR_SIZE)

        # read from the cap file
        reader = CaptureReader(self.dcap_url)

        # output
        output_log = []

        # Loop over all of the dcap contents
        for (timestamp, data) in reader:

            disk_sensor_pkts = None

            if self.machine_type == G.MACHINE_TYPES.PHYSICAL:
                lophi_packet = type('AnonClass', (object,), { "sata_header": None, "sata_data": None })
                (lophi_packet.sata_header, lophi_packet.sata_data) = sata.extract_sata_data(data)

                if (lophi_packet.sata_header == None or
                    lophi_packet.sata_data == None):
                    logger.warning("Skipping abnormal physical SATA capture packet -- either sata header and/or data is None.")
                    continue

                # deal with SATA NCQ reordering
                disk_sensor_pkts = sata_reconstructor.process_packet(lophi_packet)
            else:
                # logger.debug(DiskSensorPacket(data))
                disk_sensor_pkts = [DiskSensorPacket(data)]

            # Process all of our disk packets
            if disk_sensor_pkts:
                for dsp in disk_sensor_pkts:
                    # Skip empty packets
                    if not dsp:
                        continue

                    try:
                        fs_operations = semantic_engine.get_access(dsp.sector,
                                                                   dsp.num_sectors,
                                                                   dsp.disk_operation,
                                                                   dsp.data)
                        if fs_operations == None:
                            logger.error("Got an operation to sector %s that is outside our disk." % dsp.sector)
                            continue

                        for op in fs_operations:
                            output_log.append(op)

                    except:
                        logging.exception("Encountered error while trying to bridge semantic gap for this disk access.")

        # return value
        fs_dict = {}

        # Consolidate filesystem activity into a set
        # fields should be 'sector', 'op', 'op_type', 'inode', 'filename', 'raw_data', 'semantic_data'
        # try to insert (operation, filename, inode)
        # for duplicates, use a counter
        for fs_operation in output_log:

            sector = fs_operation['sector']
            operation = fs_operation['op']
            operation_type = fs_operation['op_type']
            inode = fs_operation['inode']
            filename = fs_operation['filename']
            raw_data = fs_operation['raw_data']

            ## TODO - look at filesystem stuff for now -- more low level data later?
            # key = (sector, op, raw_data)
            key = (operation_type, filename, inode)

            if key in fs_dict:
                fs_dict[key] += 1
            else:
                fs_dict[key] = 1

        return fs_dict