def run(self):
"""
This function will read a raw disk capture and use a scanned disk
image to reconstruct the recorded SATA traffic and output the
semantic output.
"""
# copy our disk image to a temporary working image
self.working_disk_img = os.path.join(self.output_dir, "disk.img.tmp")
print "* Creating temporary working image from disk scan. (%s)"%self.working_disk_img
# Delete, copy, chmod new file
try:
os.unlink(self.working_disk_img)
except:
pass
cmd = "cp --sparse=always %s %s" % (self.disk_img, self.working_disk_img)
subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read()
os.chmod(self.working_disk_img, 0755)
# Set up our semantic bridge
print "* Parsing disk image %s into our semantic engine... (This may take a while)" % self.working_disk_img
semantic_engine = SemanticEngineDisk(self.working_disk_img)
# Start processing our dcap
print "* Processing dcap file %s..." % self.dcap_filename
# SATA Interpreter
sata = SATAInterpreter()
"""
@TODO Extract sector size from PyTSK
"""
sata_reconstructor = SATAReconstructor(sector_size=G.SENSOR_DISK.DEFAULT_SECTOR_SIZE)
# read from the cap file in real time
reader = CaptureReader(self.dcap_filename)
# Tailing or terminating?
reader_iter = reader
if self.tail_enable:
reader_iter = reader.tail()
# Loop over all of the dcap contents
for (timestamp, data) in reader_iter:
if self.sensor_type == G.MACHINE_TYPES.PHYSICAL:
(header, data) = sata.extract_sata_data(data)
# deal with SATA NCQ reordering
disk_sensor_pkts = sata_reconstructor.process_packet(PhysicalPacket(header, data))
else:
disk_sensor_pkts = [DiskSensorPacket(data)]
# Process all of our disk packets
if disk_sensor_pkts:
for dsp in disk_sensor_pkts:
# Skip empty packets
if not dsp:
continue
try:
fs_operations = semantic_engine.get_access(dsp.sector,
dsp.num_sectors,
dsp.disk_operation,
dsp.data)
self.log_output(timestamp, fs_operations)
except:
logging.exception("Encountered error while trying to bridge semantic gap for this disk access.")
def run(self):
"""
Figure out which type of host we are examining and call the
appropriate function.
"""
from lophi_semanticgap.disk.sata import SATAInterpreter
from lophi_semanticgap.disk.sata_reconstructor import SATAReconstructor
from lophi_semanticgap.disk.filesystem_reconstructor import SemanticEngineDisk
logger.debug("DiskEngine Started.")
# Scan in our starting point
if self.machine.type == G.MACHINE_TYPES.PHYSICAL:
if not self._check_disk_scan():
logger.error(
"Analysis cannot continue without a valid scan file.")
return
disk_img = self.machine.config.disk_scan
else:
# Get our image name for the virtual HDD on this host
image_name = self.machine.disk_get_filename()
logger.debug("Scanning disk image (%s)..." % (image_name))
if image_name is None:
logger.error("No disk found for VM (%s)." %
self.machine.config.name)
return
if image_name.endswith("qcow2"):
logger.warning(
"Got qcow2 image, scanning the base image, ensure that you reset the machine!"
)
disk_img = self.machine.config.disk_base
else:
disk_img = image_name
# Setup our tmp file
self.working_disk_img = os.path.join(
G.DIR_ROOT, G.DIR_TMP, self.machine.config.name + "-disk.img.tmp")
# Create a backup
logger.debug("Copying %s to %s..." % (disk_img, self.working_disk_img))
cmd = "cp --sparse=always %s %s" % (disk_img, self.working_disk_img)
subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read()
os.chmod(self.working_disk_img, 0755)
# Set up our semantic bridge
logger.info(
"Parsing disk image %s into our semantic engine... (This may take a while)"
% self.working_disk_img)
semantic_engine = SemanticEngineDisk(self.working_disk_img)
# SATA Interpreter
sata = SATAInterpreter()
# SATA Interpreter
sata_reconstructor = SATAReconstructor(
sector_size=G.SENSOR_DISK.DEFAULT_SECTOR_SIZE)
# Get data forever and report it back
self.RUNNING = True
while self.RUNNING:
# Accept commands
try:
cmd = self.command_queue.get(False).split(" ")
logger.debug("Got cmd: %s" % cmd)
if cmd[0] == G.CTRL_CMD_PAUSE:
logger.debug("Pausing analysis")
self.PAUSED = True
self.machine.disk._disconnect()
if cmd[0] == G.CTRL_CMD_UNPAUSE:
logger.debug("Resuming Analysis")
self.PAUSED = False
self.machine.disk._connect()
if cmd[0] == G.CTRL_CMD_KILL or cmd[0] == G.CTRL_CMD_STOP:
logger.debug("Got kill command")
self.RUNNING = False
self.machine.disk._disconnect()
break
except:
# Do nothing
pass
if self.PAUSED:
time.sleep(1)
continue
# Get our packet
try:
data = self.machine.disk_get_packet()
logger.debug("Got: %s" % data)
except:
G.print_traceback()
logger.debug("Disk introspection socket closed.")
break
# Good data?
if data is None:
continue
if self.machine.type == G.MACHINE_TYPES.PHYSICAL:
lophi_packet = type('AnonClass', (object, ), {
"sata_header": None,
"sata_data": None
})
(lophi_packet.sata_header,
lophi_packet.sata_data) = sata.extract_sata_data( ` data `)
# deal with SATA NCQ reordering
disk_sensor_pkts = sata_reconstructor.process_packet(
lophi_packet)
else:
disk_sensor_pkts = [data]
# Process all of our disk packets
if disk_sensor_pkts:
for dsp in disk_sensor_pkts:
# Skip empty packets
if not dsp:
continue
try:
fs_operations = semantic_engine.get_access(
dsp.sector, dsp.num_sectors, dsp.disk_operation,
dsp.data)
self.parse_actions(time.time(), fs_operations)
except:
logging.exception(
"Encountered error while trying to bridge semantic gap for this disk access."
)
# logger.debug("got actions %s"%actions)
# Handle our output
# self.parse_actions(actions)
logger.debug("Disk analysis exiting...")
def replay(self):
"""
Replays the raw log of SATA frames (captured as LOPHIPackets),
bridges the semantic gap and returns human-readable text form
"""
# copy our disk image to a temporary working image
self.working_disk_img = os.path.join(self.output_dir, "disk.img.tmp")
logger.debug("* Creating temporary working image from disk scan. (%s)" % self.working_disk_img)
# Delete, copy, chmod new file
try:
os.unlink(self.working_disk_img)
except OSError:
pass
cmd = "cp --sparse=always %s %s" % (self.disk_img, self.working_disk_img)
subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read()
os.chmod(self.working_disk_img, 0755)
# Set up our semantic bridge
logger.debug("* Parsing disk image %s into our semantic engine... (This may take a while)" % self.working_disk_img)
semantic_engine = SemanticEngineDisk(self.working_disk_img)
# Start processing our dcap
logger.debug("* Processing dcap file %s..." % self.dcap_url)
# SATA Interpreter
sata = SATAInterpreter()
"""
@TODO Extract sector size from PyTSK
"""
sata_reconstructor = SATAReconstructor(sector_size=G.SENSOR_DISK.DEFAULT_SECTOR_SIZE)
# read from the cap file
reader = CaptureReader(self.dcap_url)
# output
output_log = []
# Loop over all of the dcap contents
for (timestamp, data) in reader:
disk_sensor_pkts = None
if self.machine_type == G.MACHINE_TYPES.PHYSICAL:
lophi_packet = type('AnonClass', (object,), { "sata_header": None, "sata_data": None })
(lophi_packet.sata_header, lophi_packet.sata_data) = sata.extract_sata_data(data)
if (lophi_packet.sata_header == None or
lophi_packet.sata_data == None):
logger.warning("Skipping abnormal physical SATA capture packet -- either sata header and/or data is None.")
continue
# deal with SATA NCQ reordering
disk_sensor_pkts = sata_reconstructor.process_packet(lophi_packet)
else:
# logger.debug(DiskSensorPacket(data))
disk_sensor_pkts = [DiskSensorPacket(data)]
# Process all of our disk packets
if disk_sensor_pkts:
for dsp in disk_sensor_pkts:
# Skip empty packets
if not dsp:
continue
try:
fs_operations = semantic_engine.get_access(dsp.sector,
dsp.num_sectors,
dsp.disk_operation,
dsp.data)
if fs_operations == None:
logger.error("Got an operation to sector %s that is outside our disk." % dsp.sector)
continue
for op in fs_operations:
output_log.append(op)
except:
logging.exception("Encountered error while trying to bridge semantic gap for this disk access.")
# return value
fs_dict = {}
# Consolidate filesystem activity into a set
# fields should be 'sector', 'op', 'op_type', 'inode', 'filename', 'raw_data', 'semantic_data'
# try to insert (operation, filename, inode)
# for duplicates, use a counter
for fs_operation in output_log:
sector = fs_operation['sector']
operation = fs_operation['op']
operation_type = fs_operation['op_type']
inode = fs_operation['inode']
filename = fs_operation['filename']
raw_data = fs_operation['raw_data']
## TODO - look at filesystem stuff for now -- more low level data later?
# key = (sector, op, raw_data)
key = (operation_type, filename, inode)
if key in fs_dict:
fs_dict[key] += 1
else:
fs_dict[key] = 1
return fs_dict
def run(self):
"""
Figure out which type of host we are examining and call the
appropriate function.
"""
from lophi_semanticgap.disk.sata import SATAInterpreter
from lophi_semanticgap.disk.sata_reconstructor import SATAReconstructor
from lophi_semanticgap.disk.filesystem_reconstructor import SemanticEngineDisk
logger.debug("DiskEngine Started.")
# Scan in our starting point
if self.machine.type == G.MACHINE_TYPES.PHYSICAL:
if not self._check_disk_scan():
logger.error("Analysis cannot continue without a valid scan file.")
return
disk_img = self.machine.config.disk_scan
else:
# Get our image name for the virtual HDD on this host
image_name = self.machine.disk_get_filename()
logger.debug("Scanning disk image (%s)..." % (image_name))
if image_name is None:
logger.error("No disk found for VM (%s)."%self.machine.config.name)
return
if image_name.endswith("qcow2"):
logger.warning("Got qcow2 image, scanning the base image, ensure that you reset the machine!")
disk_img = self.machine.config.disk_base
else:
disk_img = image_name
# Setup our tmp file
self.working_disk_img = os.path.join(G.DIR_ROOT,G.DIR_TMP,self.machine.config.name+"-disk.img.tmp")
# Create a backup
logger.debug("Copying %s to %s..."%(disk_img, self.working_disk_img))
cmd = "cp --sparse=always %s %s" % (disk_img, self.working_disk_img)
subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read()
os.chmod(self.working_disk_img, 0755)
# Set up our semantic bridge
logger.info("Parsing disk image %s into our semantic engine... (This may take a while)" % self.working_disk_img)
semantic_engine = SemanticEngineDisk(self.working_disk_img)
# SATA Interpreter
sata = SATAInterpreter()
# SATA Interpreter
sata_reconstructor = SATAReconstructor(sector_size=G.SENSOR_DISK.DEFAULT_SECTOR_SIZE)
# Get data forever and report it back
self.RUNNING = True
while self.RUNNING:
# Accept commands
try:
cmd = self.command_queue.get(False).split(" ")
logger.debug("Got cmd: %s" % cmd)
if cmd[0] == G.CTRL_CMD_PAUSE:
logger.debug("Pausing analysis")
self.PAUSED = True
self.machine.disk._disconnect()
if cmd[0] == G.CTRL_CMD_UNPAUSE:
logger.debug("Resuming Analysis")
self.PAUSED = False
self.machine.disk._connect()
if cmd[0] == G.CTRL_CMD_KILL or cmd[0] == G.CTRL_CMD_STOP:
logger.debug("Got kill command")
self.RUNNING = False
self.machine.disk._disconnect()
break
except:
# Do nothing
pass
if self.PAUSED:
time.sleep(1)
continue
# Get our packet
try:
data = self.machine.disk_get_packet()
logger.debug("Got: %s"%data)
except:
G.print_traceback()
logger.debug("Disk introspection socket closed.")
break
# Good data?
if data is None:
continue
if self.machine.type == G.MACHINE_TYPES.PHYSICAL:
lophi_packet = type('AnonClass', (object,), { "sata_header": None, "sata_data": None })
(lophi_packet.sata_header, lophi_packet.sata_data) = sata.extract_sata_data(`data`)
# deal with SATA NCQ reordering
disk_sensor_pkts = sata_reconstructor.process_packet(lophi_packet)
else:
disk_sensor_pkts = [data]
# Process all of our disk packets
if disk_sensor_pkts:
for dsp in disk_sensor_pkts:
# Skip empty packets
if not dsp:
continue
try:
fs_operations = semantic_engine.get_access(dsp.sector,
dsp.num_sectors,
dsp.disk_operation,
dsp.data)
self.parse_actions(time.time(), fs_operations)
except:
logging.exception("Encountered error while trying to bridge semantic gap for this disk access.")
# logger.debug("got actions %s"%actions)
# Handle our output
# self.parse_actions(actions)
logger.debug("Disk analysis exiting...")
