def run(self):
"""
Run forever grabbing data from the VM and forwarding it to listening
clients
"""
# Get our meta data which is always the first packet sent
meta = MetaHeader()
meta_data = self._conn.recv(len(meta))
if len(meta_data) == 0:
logger.debug("VM Disconnected.")
self._conn.close()
return
meta._unpack(meta_data)
#logger.debug("Got meta data")
#logger.debug(meta)
# Store our filename for this VM
# NOTE: We must strip the null chars off or comparisons will fail!
filename = meta.filename.strip("\x00")
# Create our sensor packet, and save its default size
sensor_packet = DiskSensorPacket()
sensor_header_size = len(sensor_packet)
# Read packets forever
while True:
# Read and unpack our header
header_data = self._conn.recv(sensor_header_size)
if len(header_data) == 0:
#logger.debug("VM Disconnected.")
break
sensor_packet._unpack(header_data)
# Get the accompanying data
try:
sensor_packet.data = self._conn.recv(sensor_packet.size)
except:
print sensor_packet
G.print_traceback()
if len(sensor_packet.data) == 0:
logger.debug("VM Disconnected.")
self._conn.close()
return
if filename in disk_stream_dict:
#logger.debug("Found %s in dict."%filename)
for queue in disk_stream_dict[filename]:
queue.put( ` sensor_packet `)
def run(self):
"""
Run forever grabbing data from the VM and forwarding it to listening
clients
"""
# Get our meta data which is always the first packet sent
meta = MetaHeader()
meta_data = self._conn.recv(len(meta))
if len(meta_data) == 0:
logger.debug("VM Disconnected.")
self._conn.close()
return
meta._unpack(meta_data)
#logger.debug("Got meta data")
#logger.debug(meta)
# Store our filename for this VM
# NOTE: We must strip the null chars off or comparisons will fail!
filename = meta.filename.strip("\x00")
# Create our sensor packet, and save its default size
sensor_packet = DiskSensorPacket()
sensor_header_size = len(sensor_packet)
# Read packets forever
while True:
# Read and unpack our header
header_data = self._conn.recv(sensor_header_size)
if len(header_data) == 0:
#logger.debug("VM Disconnected.")
break
sensor_packet._unpack(header_data)
# Get the accompanying data
try:
sensor_packet.data = self._conn.recv(sensor_packet.size)
except:
print sensor_packet
G.print_traceback()
if len(sensor_packet.data) == 0:
logger.debug("VM Disconnected.")
self._conn.close()
return
if filename in disk_stream_dict:
#logger.debug("Found %s in dict."%filename)
for queue in disk_stream_dict[filename]:
queue.put(`sensor_packet`)
def get_disk_packet(self):
"""
Get the next packet header/data
"""
if self.SOCK is None:
self._connect()
access_packet = DiskSensorPacket()
# Try forever to get a packet
while 1:
# Receive our header
header = self._read_raw_packet(len(access_packet))
# Did our socket get closed?
if len(header) < len(access_packet):
logger.warn("Introspection server disconnected.")
# reconnect and try again
self._connect()
continue
access_packet = DiskSensorPacket(header)
# Read for as long as the header tells us to to get the content
data = ""
recvd = 0
while len(data) < access_packet.size:
logger.debug("Read data.")
tmp = self.SOCK.recv(access_packet.size - recvd)
data += tmp
recvd = len(data)
if len(tmp) == 0:
logger.warn("Introspection server disconnected.")
break
# Did our socket get closed?
if recvd < access_packet.size:
logger.warn("Introspection server disconnected.")
# reconnect and try again
self._connect()
continue
logger.debug("Read %d bytes of data." % len(data))
# Return our data
access_packet.data = data
return access_packet
def get_disk_packet(self):
"""
Get the next packet header/data
"""
if self.SOCK is None:
self._connect()
access_packet = DiskSensorPacket()
# Try forever to get a packet
while 1:
# Receive our header
header = self._read_raw_packet(len(access_packet))
# Did our socket get closed?
if len(header) < len(access_packet):
logger.warn("Introspection server disconnected.")
# reconnect and try again
self._connect()
continue
access_packet = DiskSensorPacket(header)
# Read for as long as the header tells us to to get the content
data = ""
recvd = 0
while len(data) < access_packet.size:
logger.debug("Read data.")
tmp = self.SOCK.recv(access_packet.size - recvd);
data += tmp
recvd = len(data)
if len(tmp) == 0:
logger.warn("Introspection server disconnected.")
break
# Did our socket get closed?
if recvd < access_packet.size:
logger.warn("Introspection server disconnected.")
# reconnect and try again
self._connect()
continue
logger.debug("Read %d bytes of data."%len(data))
# Return our data
access_packet.data = data
return access_packet
def read_disk(options):
# Setup our log files
dcap_filename = options.dcap_file
# read from the cap file in real time
reader = CaptureReader(options.dcap_file)
# Tailing or terminating?
reader_iter = reader
if options.tail_enable:
reader_iter = reader.tail()
# Loop over all of the dcap contents
for (timestamp, data) in reader_iter:
print timestamp
if options.sensor_type == G.MACHINE_TYPES.PHYSICAL:
sata_frame = SATAFrame(data)
print sata_frame
else:
disk_sensor_pkt = [DiskSensorPacket(data)]
print disk_sensor_pkt
def handle_register_HTD(self, register_packet):
"""
Handle a Register HTD
Always returns None
"""
sata_header = register_packet.sata_header
# Check if C bit is set to 0 -- see 11.2 Dl1 Note 1 -- no FIS is sent
# I think we can ignore if C bit is set to 0 and go back to IDLE state
# software reset could still happen -- not sure which bit is the SRST bit in the control register?
if sata_header['C'] == 0:
logger.debug(
"Got HTD Register Packet but C bit is set to 0. Ignoring.")
return None
# check if this is an NCQ command
if sata_header[
'command'] == NCQCommandType.ReadFPDMAQueued or sata_header[
'command'] == NCQCommandType.WriteFPDMAQueued:
cmd = "READ"
direction = G.SATA_OP.DIRECTION.READ
if sata_header['command'] == NCQCommandType.WriteFPDMAQueued:
cmd = "WRITE"
direction = G.SATA_OP.DIRECTION.WRITE
# NCQ uses features field for sector count
logger.debug(
"Got HTD Register Packet for NCQ %s -- TAG %d expecting %d bytes."
% (cmd, sata_header['tag'],
sata_header['features'] * self.sector_size))
# if len(self.ncq_register_stack) > 0:
# logger.error("Our NCQ register stack will be greater than 1, so we have at least one outstanding NCQ register packet with NO Register ACK. Stack size is %d" % len(self.ncq_register_stack))
# # add to our stack
# self.ncq_register_stack.append(register_packet)
#No ACKS
# # change state to waiting for Register DTH
# self.STATE = self.WAIT_FOR_REGISTER_ACK
#
# # deal with error checking when we get the Register DTH ACK back
# return None
# check if the spot belonging to the associated TAG is empty
if (not self.ncq_transactions_outstanding[sata_header['tag']]):
# This TAG is not taken, so we just add
self.ncq_transactions_outstanding[
sata_header['tag']] = register_packet
else:
logger.error(
"Received HTD NCQ Register Packet for TAG %d but it is already in use. Using it anyway."
% sata_header['tag'])
# still put it in there for now
self.ncq_transactions_outstanding[
sata_header['tag']] = register_packet
#don't need this if we accept the register packet anyway
#return None
# Associate this DTH Register packet
# with the tag and prepare the disk sensor packet for aggregating data
lba = sata_header['lba']
# NOTE: NCQ uses features field instead of count for sector count
sector_count = sata_header['features']
disk_sensor_packet = DiskSensorPacket()
disk_sensor_packet.sector = lba
disk_sensor_packet.num_sectors = sector_count
disk_sensor_packet.disk_operation = direction
disk_sensor_packet.size = sector_count * self.sector_size
disk_sensor_packet.data = ""
self.ncq_data_outstanding[sata_header['tag']] = disk_sensor_packet
# DO NOT set state to IDLE
# Because now it looks like Register packets can come during other times
# so we want to maintain the current state
# # set state to IDLE
#
# self.STATE = self.DEVICE_IDLE
return None
# NCQ Management stuff
elif sata_header['command'] == NCQCommandType.NCQQueueManagement:
logger.debug(
"Got Register Packet for NCQ Queue Management. Ignoring for now."
)
# Don't need to change state
return None
# NON NCQ Register Packet
else:
logger.debug(
"Got Non-NCQ HTD Register Packet, expected %d bytes." %
(sata_header['count'] * self.sector_size))
logger.debug("Command field is %d" % sata_header['command'])
# if len(self.ncq_register_stack) > 0:
# logger.error("Have more than 0 outstanding NCQ HTD Register Packets. Non NCQ Register packets are not supposed to be sent while NCQ Register packets are still outstanding. Size of stack is %d" % len(self.regular_register_stack))
# if len(self.regular_register_stack) > 0:
# logger.error("Will have more than one outstanding regular (non NCQ) HTD Register Packets. Size of stack is %d" % len(self.regular_register_stack))
# push this normal Register HTD packet on the stack
self.regular_register_stack.append(register_packet)
# prepare the disk sensor packet for aggregating data
lba = sata_header['lba']
sector_count = sata_header['count']
direction = sata_header['direction']
# self.regular_disk_sensor_packet = DiskSensorPacket(lba,
# sector_count,
# direction,
# sector_count*self.sector_size,
# "")
self.regular_disk_sensor_packet = DiskSensorPacket()
self.regular_disk_sensor_packet.sector = lba
self.regular_disk_sensor_packet.num_sectors = sector_count
self.regular_disk_sensor_packet.disk_operation = direction
self.regular_disk_sensor_packet.size = sector_count * self.sector_size
self.regular_disk_sensor_packet.data = ""
# Transition state now that we are waiting for data
self.STATE = self.WAIT_FOR_NON_NCQ_DATA
return None
def handle_register_HTD(self, register_packet):
"""
Handle a Register HTD
Always returns None
"""
sata_header = register_packet.sata_header
# Check if C bit is set to 0 -- see 11.2 Dl1 Note 1 -- no FIS is sent
# I think we can ignore if C bit is set to 0 and go back to IDLE state
# software reset could still happen -- not sure which bit is the SRST bit in the control register?
if sata_header['C'] == 0:
logger.debug("Got HTD Register Packet but C bit is set to 0. Ignoring.")
return None
# check if this is an NCQ command
if sata_header['command'] == NCQCommandType.ReadFPDMAQueued or sata_header['command'] == NCQCommandType.WriteFPDMAQueued:
cmd = "READ"
direction = G.SATA_OP.DIRECTION.READ
if sata_header['command'] == NCQCommandType.WriteFPDMAQueued:
cmd = "WRITE"
direction = G.SATA_OP.DIRECTION.WRITE
# NCQ uses features field for sector count
logger.debug("Got HTD Register Packet for NCQ %s -- TAG %d expecting %d bytes." % (cmd, sata_header['tag'], sata_header['features']*self.sector_size))
# if len(self.ncq_register_stack) > 0:
# logger.error("Our NCQ register stack will be greater than 1, so we have at least one outstanding NCQ register packet with NO Register ACK. Stack size is %d" % len(self.ncq_register_stack))
# # add to our stack
# self.ncq_register_stack.append(register_packet)
#No ACKS
# # change state to waiting for Register DTH
# self.STATE = self.WAIT_FOR_REGISTER_ACK
#
# # deal with error checking when we get the Register DTH ACK back
# return None
# check if the spot belonging to the associated TAG is empty
if (not self.ncq_transactions_outstanding[sata_header['tag']]):
# This TAG is not taken, so we just add
self.ncq_transactions_outstanding[sata_header['tag']] = register_packet
else:
logger.error("Received HTD NCQ Register Packet for TAG %d but it is already in use. Using it anyway." % sata_header['tag'])
# still put it in there for now
self.ncq_transactions_outstanding[sata_header['tag']] = register_packet
#don't need this if we accept the register packet anyway
#return None
# Associate this DTH Register packet
# with the tag and prepare the disk sensor packet for aggregating data
lba = sata_header['lba']
# NOTE: NCQ uses features field instead of count for sector count
sector_count = sata_header['features']
disk_sensor_packet = DiskSensorPacket()
disk_sensor_packet.sector = lba
disk_sensor_packet.num_sectors = sector_count
disk_sensor_packet.disk_operation = direction
disk_sensor_packet.size = sector_count*self.sector_size
disk_sensor_packet.data = ""
self.ncq_data_outstanding[sata_header['tag']] = disk_sensor_packet
# DO NOT set state to IDLE
# Because now it looks like Register packets can come during other times
# so we want to maintain the current state
# # set state to IDLE
#
# self.STATE = self.DEVICE_IDLE
return None
# NCQ Management stuff
elif sata_header['command'] == NCQCommandType.NCQQueueManagement:
logger.debug("Got Register Packet for NCQ Queue Management. Ignoring for now.")
# Don't need to change state
return None
# NON NCQ Register Packet
else:
logger.debug("Got Non-NCQ HTD Register Packet, expected %d bytes." % (sata_header['count']*self.sector_size))
logger.debug("Command field is %d" % sata_header['command'])
# if len(self.ncq_register_stack) > 0:
# logger.error("Have more than 0 outstanding NCQ HTD Register Packets. Non NCQ Register packets are not supposed to be sent while NCQ Register packets are still outstanding. Size of stack is %d" % len(self.regular_register_stack))
# if len(self.regular_register_stack) > 0:
# logger.error("Will have more than one outstanding regular (non NCQ) HTD Register Packets. Size of stack is %d" % len(self.regular_register_stack))
# push this normal Register HTD packet on the stack
self.regular_register_stack.append(register_packet)
# prepare the disk sensor packet for aggregating data
lba = sata_header['lba']
sector_count = sata_header['count']
direction = sata_header['direction']
# self.regular_disk_sensor_packet = DiskSensorPacket(lba,
# sector_count,
# direction,
# sector_count*self.sector_size,
# "")
self.regular_disk_sensor_packet = DiskSensorPacket()
self.regular_disk_sensor_packet.sector = lba
self.regular_disk_sensor_packet.num_sectors = sector_count
self.regular_disk_sensor_packet.disk_operation = direction
self.regular_disk_sensor_packet.size = sector_count*self.sector_size
self.regular_disk_sensor_packet.data = ""
# Transition state now that we are waiting for data
self.STATE = self.WAIT_FOR_NON_NCQ_DATA
return None
def run(self):
"""
This function will read a raw disk capture and use a scanned disk
image to reconstruct the recorded SATA traffic and output the
semantic output.
"""
# copy our disk image to a temporary working image
self.working_disk_img = os.path.join(self.output_dir, "disk.img.tmp")
print "* Creating temporary working image from disk scan. (%s)"%self.working_disk_img
# Delete, copy, chmod new file
try:
os.unlink(self.working_disk_img)
except:
pass
cmd = "cp --sparse=always %s %s" % (self.disk_img, self.working_disk_img)
subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read()
os.chmod(self.working_disk_img, 0755)
# Set up our semantic bridge
print "* Parsing disk image %s into our semantic engine... (This may take a while)" % self.working_disk_img
semantic_engine = SemanticEngineDisk(self.working_disk_img)
# Start processing our dcap
print "* Processing dcap file %s..." % self.dcap_filename
# SATA Interpreter
sata = SATAInterpreter()
"""
@TODO Extract sector size from PyTSK
"""
sata_reconstructor = SATAReconstructor(sector_size=G.SENSOR_DISK.DEFAULT_SECTOR_SIZE)
# read from the cap file in real time
reader = CaptureReader(self.dcap_filename)
# Tailing or terminating?
reader_iter = reader
if self.tail_enable:
reader_iter = reader.tail()
# Loop over all of the dcap contents
for (timestamp, data) in reader_iter:
if self.sensor_type == G.MACHINE_TYPES.PHYSICAL:
(header, data) = sata.extract_sata_data(data)
# deal with SATA NCQ reordering
disk_sensor_pkts = sata_reconstructor.process_packet(PhysicalPacket(header, data))
else:
disk_sensor_pkts = [DiskSensorPacket(data)]
# Process all of our disk packets
if disk_sensor_pkts:
for dsp in disk_sensor_pkts:
# Skip empty packets
if not dsp:
continue
try:
fs_operations = semantic_engine.get_access(dsp.sector,
dsp.num_sectors,
dsp.disk_operation,
dsp.data)
self.log_output(timestamp, fs_operations)
except:
logging.exception("Encountered error while trying to bridge semantic gap for this disk access.")
def replay(self):
"""
Replays the raw log of SATA frames (captured as LOPHIPackets),
bridges the semantic gap and returns human-readable text form
"""
# copy our disk image to a temporary working image
self.working_disk_img = os.path.join(self.output_dir, "disk.img.tmp")
logger.debug("* Creating temporary working image from disk scan. (%s)" % self.working_disk_img)
# Delete, copy, chmod new file
try:
os.unlink(self.working_disk_img)
except OSError:
pass
cmd = "cp --sparse=always %s %s" % (self.disk_img, self.working_disk_img)
subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read()
os.chmod(self.working_disk_img, 0755)
# Set up our semantic bridge
logger.debug("* Parsing disk image %s into our semantic engine... (This may take a while)" % self.working_disk_img)
semantic_engine = SemanticEngineDisk(self.working_disk_img)
# Start processing our dcap
logger.debug("* Processing dcap file %s..." % self.dcap_url)
# SATA Interpreter
sata = SATAInterpreter()
"""
@TODO Extract sector size from PyTSK
"""
sata_reconstructor = SATAReconstructor(sector_size=G.SENSOR_DISK.DEFAULT_SECTOR_SIZE)
# read from the cap file
reader = CaptureReader(self.dcap_url)
# output
output_log = []
# Loop over all of the dcap contents
for (timestamp, data) in reader:
disk_sensor_pkts = None
if self.machine_type == G.MACHINE_TYPES.PHYSICAL:
lophi_packet = type('AnonClass', (object,), { "sata_header": None, "sata_data": None })
(lophi_packet.sata_header, lophi_packet.sata_data) = sata.extract_sata_data(data)
if (lophi_packet.sata_header == None or
lophi_packet.sata_data == None):
logger.warning("Skipping abnormal physical SATA capture packet -- either sata header and/or data is None.")
continue
# deal with SATA NCQ reordering
disk_sensor_pkts = sata_reconstructor.process_packet(lophi_packet)
else:
# logger.debug(DiskSensorPacket(data))
disk_sensor_pkts = [DiskSensorPacket(data)]
# Process all of our disk packets
if disk_sensor_pkts:
for dsp in disk_sensor_pkts:
# Skip empty packets
if not dsp:
continue
try:
fs_operations = semantic_engine.get_access(dsp.sector,
dsp.num_sectors,
dsp.disk_operation,
dsp.data)
if fs_operations == None:
logger.error("Got an operation to sector %s that is outside our disk." % dsp.sector)
continue
for op in fs_operations:
output_log.append(op)
except:
logging.exception("Encountered error while trying to bridge semantic gap for this disk access.")
# return value
fs_dict = {}
# Consolidate filesystem activity into a set
# fields should be 'sector', 'op', 'op_type', 'inode', 'filename', 'raw_data', 'semantic_data'
# try to insert (operation, filename, inode)
# for duplicates, use a counter
for fs_operation in output_log:
sector = fs_operation['sector']
operation = fs_operation['op']
operation_type = fs_operation['op_type']
inode = fs_operation['inode']
filename = fs_operation['filename']
raw_data = fs_operation['raw_data']
## TODO - look at filesystem stuff for now -- more low level data later?
# key = (sector, op, raw_data)
key = (operation_type, filename, inode)
if key in fs_dict:
fs_dict[key] += 1
else:
fs_dict[key] = 1
return fs_dict