def __init__(self, case, iosource_name, profile, map):
## Try to open the image
self.iosource_name = iosource_name
self.case = case
self.map = map
self.tasks = {}
print "Opening memory image"
path_to_profiles = "%s/%s" % (config.memory_profile_dir,
profile)
print path_to_profiles
profile_file_name = [ m for m in os.listdir(path_to_profiles) \
if m.endswith(".py") ][0]
print "Profile name %s" % profile_file_name
self.profile = load_profile_from_file("%s/%s" % (path_to_profiles,
profile_file_name))
try:
symdict = load_symboltable_from_file("%s/%s" % (path_to_profiles,
map))
self.symtable = SymbolTable(symdict)
except:
raise RuntimeError("Invalid or corrupt Symbol Table file %s/%s" % (path_to_profiles, profile_file_name))
pgd = self.symtable.lookup('swapper_pg_dir')
iosrc = IO.open(self.case, iosource_name)
phyAS = IOSourceAddressSpace(iosrc)
self.addr_space = IA32PagedMemory(phyAS, pgd - 0xc0000000)
self.theProfile = Profile(abstract_types = self.profile)
def load(self, mount_point, iosource_name, loading_scanners = None):
""" Sets up the schema for loading the filesystem.
Note that derived classes need to actually do the loading
after they call the base class.
loading_scanners are the scanners which need to be run on
every new Inode.
"""
self.mount_point = mount_point
dbh=DB.DBO(self.case)
## Commented out to fix Bug0035. This should be (and is) done
## by VFSCreate, since we want to avoid duplicate mount
## points. mic: This is here because skfs.load does not use
## VFSCreate for speed reasons and therefore does not
## necessarily create the mount points when needed. Ensure
## the VFS contains the mount point:
self.VFSCreate(None, "I%s" % iosource_name, mount_point,
directory=True)
dbh.insert("filesystems",
iosource = iosource_name,
property = 'mount point',
value = mount_point)
## Ensure that we have the IOSource available
self.iosource = IO.open(self.case, iosource_name)
def load(self, mount_point, iosource_name, scanners = None, directory=None):
## Ensure that mount point is normalised:
self.iosource_name = iosource_name
mount_point = os.path.normpath(mount_point)
self.mount_point = mount_point
FileSystem.DBFS.load(self, mount_point, iosource_name)
# open the iosource
self.iosrc = IO.open(self.case, iosource_name)
## Make a volatility object available FIXME allow options in
## here
op = vutils.get_standard_parser("")
## Create an address space for the kernel
self.kernel_VA_inode_id = self.VFSCreate(None, "I%s|A0" % iosource_name,
"%s/mem" % self.mount_point)
## Build a fake command line
self.filename = '%s/%s' % (self.case, iosource_name)
self.args = ['-f', self.filename ]
opts, args = op.parse_args(self.args)
## This identifies the image
(self.addr_space, self.symtab, self.types) = vutils.load_and_identify_image(op, opts)
for loader in Registry.FSLOADERS.classes:
if loader.filesystem != "WindowsMemory": continue
## Instantiate them
loader = loader()
## Ask them to load this memory image
loader.load(self)
def __init__(self, name, mode='rb', fast=False):
self.case, self.iosource = name.split("/",1)
fd = IO.open(self.case, self.iosource)
self.fhandle = fd
self.fsize = fd.size
self.fast_fhandle = fd
self.fname = name
self.name = name
def process(case,subsys,extension=None):
""" A generator to produce all the recoverable files within the io object identified by identifier
@arg subsys: Either an IO object to use, or the string name of an io object that will be opened using IO.open().
@arg extension: A list of extensions we would like to see
"""
if type(subsys)==types.StringType:
io=IO.open(case,subsys)
else:
io=subsys
blocksize=1024*1024*10
windowsize=100
count=0
bytes_read=0
window=''
while(1):
## This implements a sliding window of window bytes to ensure
## we do not miss a signature that was split across blocksize:
try:
data=io.read(blocksize)
if not len(data): break
except IOError:
break
f=window+data
bytes_read+=len(data)
pyflaglog.log(pyflaglog.INFO,"Processed %u Mb" % (bytes_read/1024/1024))
for cut in definitions:
if extension and cut['Extension'] not in extension: continue
pos=0
while pos<blocksize:
match=cut['CStartRE'].search(f,pos)
if match:
offset=match.start()+count-len(window)
length=cut['MaxLength']
## If there is an end RE, we try to read the entire length in, and then look for the end to we can adjust the length acurately. This is essential for certain file types which do not tolerate garbage at the end of the file, e.g. pdfs.
if cut.has_key('CEndRE'):
tell=io.tell()
io.seek(offset)
file_data=io.read(length)
io.seek(tell)
end_match=cut['CEndRE'].search(file_data,0)
if end_match:
length=end_match.end()
yield({'offset':offset,'length':length,'type':cut['Extension']})
pos=match.start()+1
else:
pos=blocksize
window=f[-windowsize:]
count+=blocksize
io.close()
def write_stream(con_id):
dbh2= dbh.clone()
dbh.execute("select pcap.id as id,iosource,offset,pcap.length from `connection`,pcap where pcap.id=`connection`.packet_id and con_id = %r order by packet_id" % con_id)
for row in dbh:
## This should not be too slow as its cached in the IO module
## Store
io = IO.open(options.case, row['iosource'])
io.seek(row['offset'])
data = io.read(row['length'])
outfd.write(data)
def __init__(self, case, fd, inode):
FileSystem.File.__init__(self, case, fd, inode)
## The format of the inode is Iname .Where name is the name of
## the IO source.
self.name = inode[1:]
self.io = IO.open(case, self.name)
self.size = self.io.size
## This source should not be scanned directly.
self.ignore = True
def form(self, query, result):
result.start_table()
try:
result.case_selector()
result.ruler()
result.selector(
"Select IO Data Source",
"iosource",
"select name as `key`,name as `value` from iosources",
case=query["case"],
)
# initialise/open the subsystem
fd = IO.open(query["case"], query["iosource"])
## We need to ask all our filesystems to have a go at
## this:
metadata = {}
order = []
for c in Registry.FILESYSTEMS.classes:
c = c(query["case"], query)
guess = c.guess(fd, result, metadata)
if guess > 0:
order.append((guess, c.name))
## Sort according to the guess value
order.sort(lambda x, y: y[0] - x[0])
## We only show those whose guess is within 100 off
## the top - this allows a strong guess to bump out
## weaker guesses.
maximum = order[0][0]
fs_types = [x[1] for x in order if x[0] > maximum - 100]
## If only one option is possible here,
if len(fs_types) == 1:
result.hidden("fstype", fs_types[0], exclusive=True)
else:
result.const_selector("Enter Filesystem type", "fstype", fs_types, fs_types)
result.textfield("VFS Mount Point:", "mount_point")
result.ruler()
## Allow the filesystem to draw a form:
try:
c = Registry.FILESYSTEMS.dispatch(query["fstype"])
c = c(query["case"], query)
c.form(query, result)
except KeyError:
pass
except IOError, e:
result.text("IOError %s" % e, style="red")
def form(self,query,result):
result.start_table()
try:
result.case_selector()
result.ruler()
result.selector('Select IO Data Source', 'iosource',
"select name as `key`,name as `value` from iosources",
case=query['case'])
# initialise/open the subsystem
fd=IO.open(query['case'],query['iosource'])
## We need to ask all our filesystems to have a go at
## this:
metadata = {}
order = []
for c in Registry.FILESYSTEMS.classes:
c = c(query['case'], query)
guess = c.guess(fd, result, metadata)
if guess>0:
order.append((guess , c.name))
## Sort according to the guess value
order.sort(lambda x,y: y[0] - x[0])
## We only show those whose guess is within 100 off
## the top - this allows a strong guess to bump out
## weaker guesses.
maximum = order[0][0]
fs_types = [ x[1] for x in order if x[0] > maximum - 100 ]
## If only one option is possible here,
if len(fs_types)==1:
result.hidden('fstype',fs_types[0], exclusive = True)
else:
result.const_selector("Enter Filesystem type",'fstype',fs_types,fs_types)
result.textfield("VFS Mount Point:","mount_point")
result.ruler()
## Allow the filesystem to draw a form:
try:
c = Registry.FILESYSTEMS.dispatch(query['fstype'])
c = c(query['case'], query)
c.form(query, result)
except KeyError:
pass
except IOError,e:
result.text("IOError %s" % e,style='red')
parser.add_option("-f", "--file", default=None,
help = "A single file of connection ids, one per line")
(options, args) = parser.parse_args()
if not options.case or not options.write:
print "Mandatory args missing - run me with --help for help"
sys.exit(1)
## Create the PCAP header for the new file:
dbh=DB.DBO(options.case)
dbh.execute("select * from pcap limit 1")
row = dbh.fetch()
io = IO.open(options.case, row['iosource'])
fd = Buffer(fd=io)
## Read the header:
header = PCAP.FileHeader(fd)
io.seek(0)
data = io.read(header.start_of_data)
## Open file for writing:
outfd = open(options.write, 'w')
outfd.write(data)
## Now grab the data packets for all the relevant streams:
def write_stream(con_id):
dbh2= dbh.clone()
def display(self,query,result):
dbh = DB.DBO(query['case'])
dbh.execute("select * from pcap where id=%r limit 1", query['id'])
row=dbh.fetch()
io = IO.open(query['case'], row['iosource'])
packet = pypcap.PyPCAP(io)
packet.seek(row['offset'])
dissected_packet = packet.dissect()
id = int(query['id'])
def get_node(branch):
""" Locate the node specified by the branch.
branch is a list of attribute names.
"""
result = dissected_packet
for b in branch:
try:
result = getattr(result, b)
except: pass
return result
def tree_cb(path):
branch = FlagFramework.splitpath(path)
node = get_node(branch)
try:
for field in node.list():
if field.startswith("_"): continue
child = getattr(node, field)
try:
yield ( field, child.get_name(), 'branch')
except AttributeError:
yield ( field, field, 'leaf')
except AttributeError:
pass
return
def pane_cb(path,result):
branch = FlagFramework.splitpath(path)
node = get_node(branch)
result.heading("Packet %s" % id)
data = dissected_packet.serialise()
h=FlagFramework.HexDump(data, result)
try:
result.text("%s" % node.get_name(), font='bold')
result.text('',style='black', font='normal')
start,length = node.get_range()
except AttributeError:
result.text("%s\n" % node, style='red', wrap='full', font='typewriter', sanitise='full')
result.text('',style='black', font='normal')
node = get_node(branch[:-1])
start,length = node.get_range(branch[-1])
h.dump(highlight=[[start,length,'highlight'],])
return
result.tree(tree_cb=tree_cb, pane_cb=pane_cb, branch=[''])
## We add forward and back toolbar buttons to let people move
## to next or previous packet:
dbh.execute("select min(id) as id from pcap")
row = dbh.fetch()
new_query=query.clone()
if id>row['id']:
del new_query['id']
new_query['id']=id-1
result.toolbar(text="Previous Packet",icon="stock_left.png",link=new_query)
else:
result.toolbar(text="Previous Packet",icon="stock_left_gray.png")
dbh.execute("select max(id) as id from pcap")
row = dbh.fetch()
if id<row['id']:
del new_query['id']
new_query['id']=id+1
result.toolbar(text="Next Packet",icon="stock_right.png",link=new_query)
else:
result.toolbar(text="Next Packet",icon="stock_right_gray.png")
def load(self, mount_point, iosource_name,scanners = None):
DBFS.load(self, mount_point, iosource_name)
## Open the file descriptor
self.fd = IO.open(self.case, iosource_name)
## Use the C implementation to read the pcap files:
pcap_file = pypcap.PyPCAP(self.fd)
## Build our streams:
pyflaglog.log(pyflaglog.DEBUG, "Reassembling streams, this might take a while")
pcap_dbh = DB.DBO(self.case)
pcap_dbh.mass_insert_start("pcap")
pcap_dbh.execute("select max(id) as m from pcap")
max_id = pcap_dbh.fetch()['m'] or 0
cookie, processor = self.make_processor(iosource_name, scanners)
## Process the file with it:
while 1:
try:
packet = pcap_file.dissect()
max_id += 1
## FIXME - this is a bottleneck. For now we use mass
## insert but this will break when we have multiple
## concurrent loaders. Record the packet in the pcap
## table:
args = dict(
iosource = iosource_name,
offset = packet.offset,
length = packet.caplen,
_ts_sec = "from_unixtime('%s')" % packet.ts_sec,
ts_usec = packet.ts_usec,
)
## Try to insert the ipid field
try:
args['ipid']= packet.root.eth.payload.id
except: pass
pcap_dbh.mass_insert(**args)
#pcap_id = pcap_dbh.autoincrement()
pcap_id = max_id
pcap_file.set_id(pcap_id)
## Some progress reporting
if pcap_id % 10000 == 0:
pyflaglog.log(pyflaglog.DEBUG, "processed %s packets (%s bytes)" % (pcap_id, packet.offset))
processor.process(packet)
except StopIteration:
break
processor.flush()
pcap_dbh.check_index("connection_details",'src_ip')
pcap_dbh.check_index("connection_details",'src_port')
pcap_dbh.check_index("connection_details",'dest_ip')
pcap_dbh.check_index("connection_details",'dest_port')
pcap_dbh.check_index('connection_details','inode_id')
#!/usr/bin/env python
import pyflag.IO as IO
import sys
import sk
from stat import *
#img = open(sys.argv[1],'r')
#print "Will open %s" % sys.argv[1]
## This assumes that we have a case loaded
img = IO.open('winxp','test')
fs = sk.skfs(img)
def readfile(fs,inode):
print "reading: %s" % inode
fd = fs.open(inode=inode)
while True:
if not fd.read(4000000):
break
# walk the directory tree
for root, dirs, files in fs.walk('/', unalloc=True, inodes=True):
for f in files:
try:
print "processing: (%u) %s" % (f[0], f[1])
s=fs.stat(inode=str(f[0]))
print "length %s" % s[ST_SIZE]
if int(f[0])==0: continue
readfile(fs,str(f[0]))
except IOError, e:
print "Got error: %s" % e
def load(self, mount_point, iosource_name, scanners = None, directory=None):
## Ensure that mount point is normalised:
self.iosource_name = iosource_name
mount_point = os.path.normpath(mount_point)
self.mount_point = mount_point
DBFS.load(self, mount_point, iosource_name)
# open the iosource
iosrc = IO.open(self.case, iosource_name)
## Get a db handle
dbh = DB.DBO(self.case)
dbh.mass_insert_start('tasks')
(addr_space, symtab, types) = load_and_identify_image(iosrc)
self.load_open_files(dbh, addr_space, types, symtab)
## process_list should probably be a generator here (or not,
## the list is unlikely to be that big)
for task in process_list(addr_space, types, symtab):
## Skip invalid tasks (This should probably be done in
## process_list itself so it doesnt yield rubbish)
if not addr_space.is_valid_address(task): continue
pid = process_pid(addr_space, types, task) or -1
create_time = process_create_time(addr_space, types, task)
task_info = {
'iosource': iosource_name,
'image_file_name': process_imagename(addr_space, types, task) or "UNKNOWN",
'pid': pid,
'offset': task,
'active_threads': process_num_active_threads(addr_space, types, task) or -1,
'inherited_from': process_inherited_from(addr_space, types,task) or -1,
'handle_count': process_handle_count(addr_space, types, task) or -1,
'_create_time': "from_unixtime('%s')" % create_time
}
## Put the data in the db
dbh.mass_insert(**task_info)
## Create some VFS nodes:
new_inode = "I%s|N%s" % (iosource_name, task)
inode_id = self.VFSCreate(None, new_inode,
"%s/%s/exe" % (mount_point, task_info['pid']),
_mtime = create_time,
link = task_info['image_file_name'],
_fast = True)
## Try to read the PEB:
peb = process_peb(addr_space, types, task)
process_address_space = process_addr_space(addr_space, types, task, None)
command_line = process_command_line(process_address_space, types, peb)
if command_line:
dbh.insert('xattr',
inode_id=inode_id,
property = "command_line",
value = command_line,
_fast = True)
if peb:
modules = process_ldrs(process_address_space, types, peb)
for module in modules:
if not process_address_space.is_valid_address(module): continue
path = module_path(process_address_space, types, module)
base = module_base(process_address_space, types, module) or 0
size = module_size(process_address_space, types, module)
dbh.insert("modules", iosource = iosource_name,
pid = pid,
path = path,
base = base,
_fast = True
)
self.VFSCreate(None, None,
"%s/%s/Modules/Base 0x%X" % ( mount_point,
task_info['pid'],
base),
_mtime = create_time,
link = path,
size = size,
_fast = True)
## Now look for the connections:
for connection in tcb_connections(addr_space, types, symtab):
if not addr_space.is_valid_address(connection):
continue
dbh.insert("mconnections",
pid = connection_pid(addr_space, types, connection),
lport = connection_lport(addr_space, types, connection),
laddr = connection_laddr(addr_space, types, connection),
rport = connection_rport(addr_space, types, connection),
raddr = connection_raddr(addr_space, types, connection),
iosource = iosource_name,
_fast = True)
## Now do the sockets:
for socket in open_sockets(addr_space, types, symtab):
if not addr_space.is_valid_address(connection):
continue
dbh.insert("sockets",
pid = socket_pid(addr_space, types, socket),
proto = socket_protocol(addr_space, types, socket),
port = socket_local_port(addr_space, types, socket),
_create_time = "from_unixtime('%s')" % socket_create_time(addr_space, types, socket),
iosource = iosource_name
)