def read_disk(options):
# Setup our log files
dcap_filename = options.dcap_file
# read from the cap file in real time
reader = CaptureReader(options.dcap_file)
# Tailing or terminating?
reader_iter = reader
if options.tail_enable:
reader_iter = reader.tail()
# Loop over all of the dcap contents
for (timestamp, data) in reader_iter:
print timestamp
if options.sensor_type == G.MACHINE_TYPES.PHYSICAL:
sata_frame = SATAFrame(data)
print sata_frame
else:
disk_sensor_pkt = [DiskSensorPacket(data)]
print disk_sensor_pkt
def read_disk(options):
# Setup our log files
dcap_filename = options.dcap_file
# read from the cap file in real time
reader = CaptureReader(options.dcap_file)
# Tailing or terminating?
reader_iter = reader
if options.tail_enable:
reader_iter = reader.tail()
# Loop over all of the dcap contents
for (timestamp, data) in reader_iter:
print timestamp
if options.sensor_type == G.MACHINE_TYPES.PHYSICAL:
sata_frame = SATAFrame(data)
print sata_frame
else:
disk_sensor_pkt = [DiskSensorPacket(data)]
print disk_sensor_pkt
def parse_dcap(filename):
reader = CaptureReader(filename)
start = None
end = None
data_size = 0
window_data = deque([])
window_time = deque([])
window_sum = 0
max_throughput = 0
for (ts, data) in reader:
data_size += len(data)
# Keep track of timestamps
if start is None:
start = ts
end = ts
window_sum += len(data)
window_data.append(len(data))
window_time.append(ts)
# Only compute with a certain number of samples
if len(window_data) >= 10000 and window_time[-1] != window_time[0]:
throughput = window_sum / (window_time[-1] - window_time[0])
if throughput > max_throughput:
max_throughput = throughput
while len(window_data) >= 1000 and window_time[-1] != window_time[0]:
data_remove = window_data.popleft()
window_sum -= data_remove
window_time.popleft()
elapsed_time = end - start
print "Time elapsed: ", elapsed_time
print "Bytes Read: ", data_size
print "Avg Rate: ", (data_size / elapsed_time)
print "Max Rate: ", max_throughput
def main(options):
if options.replay_file is not None:
cap_reader = CaptureReader(options.replay_file)
for (ts, data) in cap_reader:
print "Time: ", ts, "s"
print MemorySensorPacket(data)
return
if options.sensor_type == G.MACHINE_TYPES.PHYSICAL:
client = MemorySensorPhysical(options.target,
cache_timeout=0,
use_threading=False)
else:
client = MemorySensorVirtual(options.target)
READ_SIZE = int(options.read_size)
start_addr = options.startaddr
# Create our output file
try:
os.makedirs(os.path.dirname(options.output))
except:
pass
try:
mcap_writer = None
if options.loop_forever == True:
logger.debug("Creating capture file.")
options.output += ".mcap"
mcap_writer = CaptureWriter(options.output)
mcap_writer.start()
else:
logger.debug("Creating dump file.")
options.output += ".mfd"
output_file = open(options.output, "w+")
except:
print "ERROR: Could not open output file."
sys.exit(0)
# Read memory
count = 0
start = time.time()
sensor_packet = MemorySensorPacket()
while True:
try:
# Get memory from remote system
# Read memory
data = client.read(start_addr, READ_SIZE)
# Write to file?
if not options.loop_forever:
output_file.write(data)
else:
sensor_packet.address = start_addr
sensor_packet.data = data
sensor_packet.length = READ_SIZE
mcap_writer.put(sensor_packet)
# Just read once?
if not options.loop_forever:
break
else:
print "Completed read #%d" % count
count += 1
except:
# Just finish up
break
end = time.time()
# Do we have an mcap file to close?
if mcap_writer is not None:
mcap_writer.stop()
else:
# Close output file
output_file.close()
print "Memory dump (%d bytes) written to %s. Took %s seconds." % (
len(data), options.output, end - start)
def run(self):
"""
This function will read a raw disk capture and use a scanned disk
image to reconstruct the recorded SATA traffic and output the
semantic output.
"""
# copy our disk image to a temporary working image
self.working_disk_img = os.path.join(self.output_dir, "disk.img.tmp")
print "* Creating temporary working image from disk scan. (%s)"%self.working_disk_img
# Delete, copy, chmod new file
try:
os.unlink(self.working_disk_img)
except:
pass
cmd = "cp --sparse=always %s %s" % (self.disk_img, self.working_disk_img)
subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read()
os.chmod(self.working_disk_img, 0755)
# Set up our semantic bridge
print "* Parsing disk image %s into our semantic engine... (This may take a while)" % self.working_disk_img
semantic_engine = SemanticEngineDisk(self.working_disk_img)
# Start processing our dcap
print "* Processing dcap file %s..." % self.dcap_filename
# SATA Interpreter
sata = SATAInterpreter()
"""
@TODO Extract sector size from PyTSK
"""
sata_reconstructor = SATAReconstructor(sector_size=G.SENSOR_DISK.DEFAULT_SECTOR_SIZE)
# read from the cap file in real time
reader = CaptureReader(self.dcap_filename)
# Tailing or terminating?
reader_iter = reader
if self.tail_enable:
reader_iter = reader.tail()
# Loop over all of the dcap contents
for (timestamp, data) in reader_iter:
if self.sensor_type == G.MACHINE_TYPES.PHYSICAL:
(header, data) = sata.extract_sata_data(data)
# deal with SATA NCQ reordering
disk_sensor_pkts = sata_reconstructor.process_packet(PhysicalPacket(header, data))
else:
disk_sensor_pkts = [DiskSensorPacket(data)]
# Process all of our disk packets
if disk_sensor_pkts:
for dsp in disk_sensor_pkts:
# Skip empty packets
if not dsp:
continue
try:
fs_operations = semantic_engine.get_access(dsp.sector,
dsp.num_sectors,
dsp.disk_operation,
dsp.data)
self.log_output(timestamp, fs_operations)
except:
logging.exception("Encountered error while trying to bridge semantic gap for this disk access.")
def replay(self):
"""
Replays the raw log of SATA frames (captured as LOPHIPackets),
bridges the semantic gap and returns human-readable text form
"""
# copy our disk image to a temporary working image
self.working_disk_img = os.path.join(self.output_dir, "disk.img.tmp")
logger.debug("* Creating temporary working image from disk scan. (%s)" % self.working_disk_img)
# Delete, copy, chmod new file
try:
os.unlink(self.working_disk_img)
except OSError:
pass
cmd = "cp --sparse=always %s %s" % (self.disk_img, self.working_disk_img)
subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read()
os.chmod(self.working_disk_img, 0755)
# Set up our semantic bridge
logger.debug("* Parsing disk image %s into our semantic engine... (This may take a while)" % self.working_disk_img)
semantic_engine = SemanticEngineDisk(self.working_disk_img)
# Start processing our dcap
logger.debug("* Processing dcap file %s..." % self.dcap_url)
# SATA Interpreter
sata = SATAInterpreter()
"""
@TODO Extract sector size from PyTSK
"""
sata_reconstructor = SATAReconstructor(sector_size=G.SENSOR_DISK.DEFAULT_SECTOR_SIZE)
# read from the cap file
reader = CaptureReader(self.dcap_url)
# output
output_log = []
# Loop over all of the dcap contents
for (timestamp, data) in reader:
disk_sensor_pkts = None
if self.machine_type == G.MACHINE_TYPES.PHYSICAL:
lophi_packet = type('AnonClass', (object,), { "sata_header": None, "sata_data": None })
(lophi_packet.sata_header, lophi_packet.sata_data) = sata.extract_sata_data(data)
if (lophi_packet.sata_header == None or
lophi_packet.sata_data == None):
logger.warning("Skipping abnormal physical SATA capture packet -- either sata header and/or data is None.")
continue
# deal with SATA NCQ reordering
disk_sensor_pkts = sata_reconstructor.process_packet(lophi_packet)
else:
# logger.debug(DiskSensorPacket(data))
disk_sensor_pkts = [DiskSensorPacket(data)]
# Process all of our disk packets
if disk_sensor_pkts:
for dsp in disk_sensor_pkts:
# Skip empty packets
if not dsp:
continue
try:
fs_operations = semantic_engine.get_access(dsp.sector,
dsp.num_sectors,
dsp.disk_operation,
dsp.data)
if fs_operations == None:
logger.error("Got an operation to sector %s that is outside our disk." % dsp.sector)
continue
for op in fs_operations:
output_log.append(op)
except:
logging.exception("Encountered error while trying to bridge semantic gap for this disk access.")
# return value
fs_dict = {}
# Consolidate filesystem activity into a set
# fields should be 'sector', 'op', 'op_type', 'inode', 'filename', 'raw_data', 'semantic_data'
# try to insert (operation, filename, inode)
# for duplicates, use a counter
for fs_operation in output_log:
sector = fs_operation['sector']
operation = fs_operation['op']
operation_type = fs_operation['op_type']
inode = fs_operation['inode']
filename = fs_operation['filename']
raw_data = fs_operation['raw_data']
## TODO - look at filesystem stuff for now -- more low level data later?
# key = (sector, op, raw_data)
key = (operation_type, filename, inode)
if key in fs_dict:
fs_dict[key] += 1
else:
fs_dict[key] = 1
return fs_dict