def __init__(self, filenames, offset):
if type(filenames) == str:
filenames = [
filenames,
]
fds = [IO.open_URL(filename) for filename in filenames]
OffsettedFDFile.__init__(self, fds, offset)
def search_next_text_region(self, query, result):
""" searches for the next text region and updates query['period_number'] """
## Open all the disks
filenames = query.getarray('filename')
fds = [IO.open_URL(f) for f in filenames]
if query.get('ismissing', False):
fds.append(ParityFD(copy.deepcopy(filenames)))
period_number = int(query.get('period_number', 0)) + 1
blocksize = FlagFramework.calculate_offset_suffix(query['blocksize'])
period = FlagFramework.calculate_offset_suffix(query['period'])
p = 0
while 1:
offset = blocksize * (p + period_number * period)
for fd in fds:
fd.seek(offset)
## We classify a text region as one with 20 chars at
## the start of the period
data = fd.read(20)
if not data:
result.heading("Error")
result.para("Unable to read data from %r" % fd)
return
m = self.text_re.match(data)
if m:
period_number = period_number + p / period
query.set('period_number', period_number)
result.refresh(0, query, 'parent')
return
p += 1
def load(self, mount_point, iosource_name, loading_scanners = None):
""" Sets up the schema for loading the filesystem.
Note that derived classes need to actually do the loading
after they call the base class.
loading_scanners are the scanners which need to be run on
every new Inode.
"""
self.mount_point = mount_point
dbh=DB.DBO(self.case)
## Commented out to fix Bug0035. This should be (and is) done
## by VFSCreate, since we want to avoid duplicate mount
## points. mic: This is here because skfs.load does not use
## VFSCreate for speed reasons and therefore does not
## necessarily create the mount points when needed. Ensure
## the VFS contains the mount point:
self.VFSCreate(None, "I%s" % iosource_name, mount_point,
directory=True)
dbh.insert("filesystems",
iosource = iosource_name,
property = 'mount point',
value = mount_point)
## Ensure that we have the IOSource available
self.iosource = IO.open(self.case, iosource_name)
def load(self, mount_point, iosource_name, scanners = None, directory=None):
## Ensure that mount point is normalised:
self.iosource_name = iosource_name
mount_point = os.path.normpath(mount_point)
self.mount_point = mount_point
FileSystem.DBFS.load(self, mount_point, iosource_name)
# open the iosource
self.iosrc = IO.open(self.case, iosource_name)
## Make a volatility object available FIXME allow options in
## here
op = vutils.get_standard_parser("")
## Create an address space for the kernel
self.kernel_VA_inode_id = self.VFSCreate(None, "I%s|A0" % iosource_name,
"%s/mem" % self.mount_point)
## Build a fake command line
self.filename = '%s/%s' % (self.case, iosource_name)
self.args = ['-f', self.filename ]
opts, args = op.parse_args(self.args)
## This identifies the image
(self.addr_space, self.symtab, self.types) = vutils.load_and_identify_image(op, opts)
for loader in Registry.FSLOADERS.classes:
if loader.filesystem != "WindowsMemory": continue
## Instantiate them
loader = loader()
## Ask them to load this memory image
loader.load(self)
def __init__(self, case, iosource_name, profile, map):
## Try to open the image
self.iosource_name = iosource_name
self.case = case
self.map = map
self.tasks = {}
print "Opening memory image"
path_to_profiles = "%s/%s" % (config.memory_profile_dir,
profile)
print path_to_profiles
profile_file_name = [ m for m in os.listdir(path_to_profiles) \
if m.endswith(".py") ][0]
print "Profile name %s" % profile_file_name
self.profile = load_profile_from_file("%s/%s" % (path_to_profiles,
profile_file_name))
try:
symdict = load_symboltable_from_file("%s/%s" % (path_to_profiles,
map))
self.symtable = SymbolTable(symdict)
except:
raise RuntimeError("Invalid or corrupt Symbol Table file %s/%s" % (path_to_profiles, profile_file_name))
pgd = self.symtable.lookup('swapper_pg_dir')
iosrc = IO.open(self.case, iosource_name)
phyAS = IOSourceAddressSpace(iosrc)
self.addr_space = IA32PagedMemory(phyAS, pgd - 0xc0000000)
self.theProfile = Profile(abstract_types = self.profile)
def search_next_text_region(self, query, result):
""" searches for the next text region and updates query['period_number'] """
## Open all the disks
filenames = query.getarray('filename')
fds = [ IO.open_URL(f) for f in filenames ]
if query.get('ismissing',False):
fds.append(ParityFD(copy.deepcopy(filenames)))
period_number = int(query.get('period_number',0)) + 1
blocksize = FlagFramework.calculate_offset_suffix(query['blocksize'])
period = FlagFramework.calculate_offset_suffix(query['period'])
p=0
while 1:
offset = blocksize * (p + period_number * period)
for fd in fds:
fd.seek(offset)
## We classify a text region as one with 20 chars at
## the start of the period
data = fd.read(20)
if not data:
result.heading("Error")
result.para("Unable to read data from %r" % fd)
return
m = self.text_re.match(data)
if m:
period_number = period_number + p / period
query.set('period_number',period_number)
result.refresh(0, query, 'parent')
return
p += 1
def __init__(self, name, mode='rb', fast=False):
self.case, self.iosource = name.split("/",1)
fd = IO.open(self.case, self.iosource)
self.fhandle = fd
self.fsize = fd.size
self.fast_fhandle = fd
self.fname = name
self.name = name
def process(case,subsys,extension=None):
""" A generator to produce all the recoverable files within the io object identified by identifier
@arg subsys: Either an IO object to use, or the string name of an io object that will be opened using IO.open().
@arg extension: A list of extensions we would like to see
"""
if type(subsys)==types.StringType:
io=IO.open(case,subsys)
else:
io=subsys
blocksize=1024*1024*10
windowsize=100
count=0
bytes_read=0
window=''
while(1):
## This implements a sliding window of window bytes to ensure
## we do not miss a signature that was split across blocksize:
try:
data=io.read(blocksize)
if not len(data): break
except IOError:
break
f=window+data
bytes_read+=len(data)
pyflaglog.log(pyflaglog.INFO,"Processed %u Mb" % (bytes_read/1024/1024))
for cut in definitions:
if extension and cut['Extension'] not in extension: continue
pos=0
while pos<blocksize:
match=cut['CStartRE'].search(f,pos)
if match:
offset=match.start()+count-len(window)
length=cut['MaxLength']
## If there is an end RE, we try to read the entire length in, and then look for the end to we can adjust the length acurately. This is essential for certain file types which do not tolerate garbage at the end of the file, e.g. pdfs.
if cut.has_key('CEndRE'):
tell=io.tell()
io.seek(offset)
file_data=io.read(length)
io.seek(tell)
end_match=cut['CEndRE'].search(file_data,0)
if end_match:
length=end_match.end()
yield({'offset':offset,'length':length,'type':cut['Extension']})
pos=match.start()+1
else:
pos=blocksize
window=f[-windowsize:]
count+=blocksize
io.close()
def write_stream(con_id):
dbh2= dbh.clone()
dbh.execute("select pcap.id as id,iosource,offset,pcap.length from `connection`,pcap where pcap.id=`connection`.packet_id and con_id = %r order by packet_id" % con_id)
for row in dbh:
## This should not be too slow as its cached in the IO module
## Store
io = IO.open(options.case, row['iosource'])
io.seek(row['offset'])
data = io.read(row['length'])
outfd.write(data)
def tree_cb(path):
fd = IO.open_URL(query['file'])
b = Buffer(fd = fd)
header = RegFile.RegF(b)
key = header.get_key(path)
for k in key.keys():
try:
name = k['key_name'].get_value()
except:
name = None
yield (name,name,'branch')
def create(self, name,case, query):
offset = FlagFramework.calculate_offset_suffix(query.get('offset','0'))
filenames = self.glob_filenames(query.getarray('filename'))
## Open the io sources here
fds = [ IO.open_URL(f) for f in filenames ]
if query.get('ismissing',False):
fds += [ParityFD(copy.deepcopy(filenames))]
blocksize = FlagFramework.calculate_offset_suffix(query.get('blocksize','32k'))
period = int(query.get('period',3))
return RAIDFD(fds, blocksize, query['map'], offset, period)
def __init__(self, case, fd, inode):
FileSystem.File.__init__(self, case, fd, inode)
## The format of the inode is Iname .Where name is the name of
## the IO source.
self.name = inode[1:]
self.io = IO.open(case, self.name)
self.size = self.io.size
## This source should not be scanned directly.
self.ignore = True
def iosubsystem(self, field, query):
""" Check to see that the io subsystem is adequitely filled in
instantiate an IO object, and try to pass the query to it.
If the query is ok, we then pass the test, else we fail.
"""
import pyflag.IO as IO
try:
IO.IOFactory(query, subsys=field)
except (IOError, KeyError, RuntimeError):
return False
def form(self, query, result):
result.start_table()
try:
result.case_selector()
result.ruler()
result.selector(
"Select IO Data Source",
"iosource",
"select name as `key`,name as `value` from iosources",
case=query["case"],
)
# initialise/open the subsystem
fd = IO.open(query["case"], query["iosource"])
## We need to ask all our filesystems to have a go at
## this:
metadata = {}
order = []
for c in Registry.FILESYSTEMS.classes:
c = c(query["case"], query)
guess = c.guess(fd, result, metadata)
if guess > 0:
order.append((guess, c.name))
## Sort according to the guess value
order.sort(lambda x, y: y[0] - x[0])
## We only show those whose guess is within 100 off
## the top - this allows a strong guess to bump out
## weaker guesses.
maximum = order[0][0]
fs_types = [x[1] for x in order if x[0] > maximum - 100]
## If only one option is possible here,
if len(fs_types) == 1:
result.hidden("fstype", fs_types[0], exclusive=True)
else:
result.const_selector("Enter Filesystem type", "fstype", fs_types, fs_types)
result.textfield("VFS Mount Point:", "mount_point")
result.ruler()
## Allow the filesystem to draw a form:
try:
c = Registry.FILESYSTEMS.dispatch(query["fstype"])
c = c(query["case"], query)
c.form(query, result)
except KeyError:
pass
except IOError, e:
result.text("IOError %s" % e, style="red")
def read(self,length):
fds = [ IO.open_URL(f) for f in self.filenames ]
for fd in fds:
fd.seek(self.readptr)
data = False
for fd in fds:
if data == False:
data = map(lambda x: ord(x), fd.read(length))
else:
data = map(lambda (x,y): x^y,zip(data,map(lambda z: ord(z), fd.read(length))))
self.readptr += length
return ''.join(map(lambda x: chr(x), data))
def read(self, length):
fds = [IO.open_URL(f) for f in self.filenames]
for fd in fds:
fd.seek(self.readptr)
data = False
for fd in fds:
if data == False:
data = map(lambda x: ord(x), fd.read(length))
else:
data = map(lambda (x, y): x ^ y,
zip(data, map(lambda z: ord(z), fd.read(length))))
self.readptr += length
return ''.join(map(lambda x: chr(x), data))
def details(query,result):
fd = IO.open_URL(query['file'])
b = Buffer(fd = fd)
header = RegFile.RegF(b)
key = header.get_key(path)
result.heading("Key %s" % path)
result.text("%s" % key, font='typewriter', wrap='full')
for v in key.values():
try:
name = "%s"% v['keyname']
result.heading("%s" % name)
result.text("%s" % v, font='typewriter', wrap='full')
except: pass
def form(self,query,result):
result.start_table()
try:
result.case_selector()
result.ruler()
result.selector('Select IO Data Source', 'iosource',
"select name as `key`,name as `value` from iosources",
case=query['case'])
# initialise/open the subsystem
fd=IO.open(query['case'],query['iosource'])
## We need to ask all our filesystems to have a go at
## this:
metadata = {}
order = []
for c in Registry.FILESYSTEMS.classes:
c = c(query['case'], query)
guess = c.guess(fd, result, metadata)
if guess>0:
order.append((guess , c.name))
## Sort according to the guess value
order.sort(lambda x,y: y[0] - x[0])
## We only show those whose guess is within 100 off
## the top - this allows a strong guess to bump out
## weaker guesses.
maximum = order[0][0]
fs_types = [ x[1] for x in order if x[0] > maximum - 100 ]
## If only one option is possible here,
if len(fs_types)==1:
result.hidden('fstype',fs_types[0], exclusive = True)
else:
result.const_selector("Enter Filesystem type",'fstype',fs_types,fs_types)
result.textfield("VFS Mount Point:","mount_point")
result.ruler()
## Allow the filesystem to draw a form:
try:
c = Registry.FILESYSTEMS.dispatch(query['fstype'])
c = c(query['case'], query)
c.form(query, result)
except KeyError:
pass
except IOError,e:
result.text("IOError %s" % e,style='red')
def map_popup(query, result):
result.decoration = 'naked'
try:
map = str(query['map']).split('.')
except KeyError:
map = []
result.start_form(query)
## Open all the disks
filenames = query.getarray('filename')
fds = [IO.open_URL(f) for f in filenames]
if query.get('ismissing', False):
fds += [ParityFD(copy.deepcopy(filenames))]
filenames += ('Missing Disk', )
uis = [result.__class__(result) for f in filenames]
period_number = int(query.get('period_number', 0))
blocksize = FlagFramework.calculate_offset_suffix(
query['blocksize'])
period = FlagFramework.calculate_offset_suffix(query['period'])
logical_period_size = period * (len(fds) - 1)
## Let the user know our disk offset
result.para("Disk Offset is %s (%s periods)" %
(blocksize * (period_number * period), period_number))
if query.has_key("__submit__"):
## Build the new map variable
query.clear('map')
map = []
for x in range(period * len(fds)):
map.append(query['position_%s' % x])
query.clear('position_%s' % x)
query.set('map', '.'.join(map))
## Now check that this map is valid by instantiating a
## RAIDFD (we will raise if anything is wrong:
try:
RAIDFD(fds, blocksize, query['map'], 0, period)
result.refresh(0, query, pane='parent')
except Exception, e:
result.heading("Error with map")
result.para("%s" % e)
return result
def map_popup(query,result):
result.decoration = 'naked'
try:
map = str(query['map']).split('.')
except KeyError:
map = []
result.start_form(query)
## Open all the disks
filenames = query.getarray('filename')
fds = [ IO.open_URL(f) for f in filenames ]
if query.get('ismissing',False):
fds += [ParityFD(copy.deepcopy(filenames))]
filenames += ('Missing Disk',)
uis = [ result.__class__(result) for f in filenames ]
period_number = int(query.get('period_number',0))
blocksize = FlagFramework.calculate_offset_suffix(query['blocksize'])
period = FlagFramework.calculate_offset_suffix(query['period'])
logical_period_size = period * (len(fds)-1)
## Let the user know our disk offset
result.para("Disk Offset is %s (%s periods)" % (blocksize * (period_number * period), period_number))
if query.has_key("__submit__"):
## Build the new map variable
query.clear('map')
map = []
for x in range(period * len(fds)):
map.append(query['position_%s' % x])
query.clear('position_%s' % x)
query.set('map', '.'.join(map))
## Now check that this map is valid by instantiating a
## RAIDFD (we will raise if anything is wrong:
try:
RAIDFD(fds, blocksize, query['map'], 0, period)
result.refresh(0,query,pane='parent')
except Exception,e:
result.heading("Error with map")
result.para("%s" % e)
return result
def pane_cb(path, result):
fd = IO.open_URL(query['file'])
b = Buffer(fd=fd)
header = RegFile.RegF(b)
key = header.get_key(path)
result.text("Timestamp: %s" % key['WriteTS'], style='red')
result.start_table(**{'class': 'GeneralTable'})
## We dont want to reference the keys because we
## will leak memeory while the callback remains stored.
def details(query, result):
fd = IO.open_URL(query['file'])
b = Buffer(fd=fd)
header = RegFile.RegF(b)
key = header.get_key(path)
result.heading("Key %s" % path)
result.text("%s" % key, font='typewriter', wrap='full')
for v in key.values():
try:
name = "%s" % v['keyname']
result.heading("%s" % name)
result.text("%s" % v, font='typewriter', wrap='full')
except:
pass
result.toolbar(cb=details,
text="Examine Details",
icon="examine.png")
result.row('Type', 'Length', 'Name', 'Value',
**{'class': 'hilight'})
for v in key.values():
try:
t = "%s" % v['data']['val_type']
length = "%s" % v['data']['len_data']
name = "%s" % v['keyname']
data = "%s" % v['data']
data = RAW(data[:100])
result.row(t, length, name, data)
except Exception, e:
print e
pass
def read_record(self, ignore_comment = True):
""" Generates records.
This can handle multiple files as provided in the constructor.
"""
blank = re.compile("^\s*$")
if self.datafile==None:
raise IOError("Datafile is not set!!!")
for file in self.datafile:
## open the file as a url:
fd = IO.open_URL(file)
buffer = ''
while 1:
if len(buffer) < 1024:
data = fd.read(1024)
buffer = buffer + data
tmp = buffer.split("\n",1)
if len(tmp) == 0: break
line = tmp[0]
try:
buffer = tmp[1]
except:
data = fd.read(1024)
if len(data) == 0:
break
buffer = line + data
continue
if blank.match(line) or not line:
continue
if line.startswith('#') and ignore_comment:
continue
else:
yield line
def read_record(self, ignore_comment=True):
""" Generates records.
This can handle multiple files as provided in the constructor.
"""
blank = re.compile("^\s*$")
if self.datafile == None:
raise IOError("Datafile is not set!!!")
for file in self.datafile:
## open the file as a url:
fd = IO.open_URL(file)
buffer = ''
while 1:
if len(buffer) < 1024:
data = fd.read(1024)
buffer = buffer + data
tmp = buffer.split("\n", 1)
if len(tmp) == 0: break
line = tmp[0]
try:
buffer = tmp[1]
except:
data = fd.read(1024)
if len(data) == 0:
break
buffer = line + data
continue
if blank.match(line) or not line:
continue
if line.startswith('#') and ignore_comment:
continue
else:
yield line
def pane_cb(path, result):
fd = IO.open_URL(query['file'])
b = Buffer(fd = fd)
header = RegFile.RegF(b)
key = header.get_key(path)
result.text("Timestamp: %s" % key['WriteTS'], style='red')
result.start_table(**{'class':'GeneralTable'})
## We dont want to reference the keys because we
## will leak memeory while the callback remains stored.
def details(query,result):
fd = IO.open_URL(query['file'])
b = Buffer(fd = fd)
header = RegFile.RegF(b)
key = header.get_key(path)
result.heading("Key %s" % path)
result.text("%s" % key, font='typewriter', wrap='full')
for v in key.values():
try:
name = "%s"% v['keyname']
result.heading("%s" % name)
result.text("%s" % v, font='typewriter', wrap='full')
except: pass
result.toolbar(cb = details, text = "Examine Details", icon = "examine.png")
result.row('Type','Length','Name','Value', **{'class':'hilight'})
for v in key.values():
try:
t = "%s" % v['data']['val_type']
length = "%s" % v['data']['len_data']
name = "%s"% v['keyname']
data = "%s" % v['data']
data = RAW(data[:100])
result.row(t,length,name,data)
except Exception,e:
print e
pass
def __init__(self, filenames, offset):
if type(filenames)==str:
filenames = [ filenames,]
fds = [ IO.open_URL(filename) for filename in filenames ]
OffsettedFDFile.__init__(self, fds, offset)
def load(self, mount_point, iosource_name, scanners = None, directory=None):
## Ensure that mount point is normalised:
self.iosource_name = iosource_name
mount_point = os.path.normpath(mount_point)
self.mount_point = mount_point
DBFS.load(self, mount_point, iosource_name)
# open the iosource
iosrc = IO.open(self.case, iosource_name)
## Get a db handle
dbh = DB.DBO(self.case)
dbh.mass_insert_start('tasks')
(addr_space, symtab, types) = load_and_identify_image(iosrc)
self.load_open_files(dbh, addr_space, types, symtab)
## process_list should probably be a generator here (or not,
## the list is unlikely to be that big)
for task in process_list(addr_space, types, symtab):
## Skip invalid tasks (This should probably be done in
## process_list itself so it doesnt yield rubbish)
if not addr_space.is_valid_address(task): continue
pid = process_pid(addr_space, types, task) or -1
create_time = process_create_time(addr_space, types, task)
task_info = {
'iosource': iosource_name,
'image_file_name': process_imagename(addr_space, types, task) or "UNKNOWN",
'pid': pid,
'offset': task,
'active_threads': process_num_active_threads(addr_space, types, task) or -1,
'inherited_from': process_inherited_from(addr_space, types,task) or -1,
'handle_count': process_handle_count(addr_space, types, task) or -1,
'_create_time': "from_unixtime('%s')" % create_time
}
## Put the data in the db
dbh.mass_insert(**task_info)
## Create some VFS nodes:
new_inode = "I%s|N%s" % (iosource_name, task)
inode_id = self.VFSCreate(None, new_inode,
"%s/%s/exe" % (mount_point, task_info['pid']),
_mtime = create_time,
link = task_info['image_file_name'],
_fast = True)
## Try to read the PEB:
peb = process_peb(addr_space, types, task)
process_address_space = process_addr_space(addr_space, types, task, None)
command_line = process_command_line(process_address_space, types, peb)
if command_line:
dbh.insert('xattr',
inode_id=inode_id,
property = "command_line",
value = command_line,
_fast = True)
if peb:
modules = process_ldrs(process_address_space, types, peb)
for module in modules:
if not process_address_space.is_valid_address(module): continue
path = module_path(process_address_space, types, module)
base = module_base(process_address_space, types, module) or 0
size = module_size(process_address_space, types, module)
dbh.insert("modules", iosource = iosource_name,
pid = pid,
path = path,
base = base,
_fast = True
)
self.VFSCreate(None, None,
"%s/%s/Modules/Base 0x%X" % ( mount_point,
task_info['pid'],
base),
_mtime = create_time,
link = path,
size = size,
_fast = True)
## Now look for the connections:
for connection in tcb_connections(addr_space, types, symtab):
if not addr_space.is_valid_address(connection):
continue
dbh.insert("mconnections",
pid = connection_pid(addr_space, types, connection),
lport = connection_lport(addr_space, types, connection),
laddr = connection_laddr(addr_space, types, connection),
rport = connection_rport(addr_space, types, connection),
raddr = connection_raddr(addr_space, types, connection),
iosource = iosource_name,
_fast = True)
## Now do the sockets:
for socket in open_sockets(addr_space, types, symtab):
if not addr_space.is_valid_address(connection):
continue
dbh.insert("sockets",
pid = socket_pid(addr_space, types, socket),
proto = socket_protocol(addr_space, types, socket),
port = socket_local_port(addr_space, types, socket),
_create_time = "from_unixtime('%s')" % socket_create_time(addr_space, types, socket),
iosource = iosource_name
)
parser.add_option("-f", "--file", default=None,
help = "A single file of connection ids, one per line")
(options, args) = parser.parse_args()
if not options.case or not options.write:
print "Mandatory args missing - run me with --help for help"
sys.exit(1)
## Create the PCAP header for the new file:
dbh=DB.DBO(options.case)
dbh.execute("select * from pcap limit 1")
row = dbh.fetch()
io = IO.open(options.case, row['iosource'])
fd = Buffer(fd=io)
## Read the header:
header = PCAP.FileHeader(fd)
io.seek(0)
data = io.read(header.start_of_data)
## Open file for writing:
outfd = open(options.write, 'w')
outfd.write(data)
## Now grab the data packets for all the relevant streams:
def write_stream(con_id):
dbh2= dbh.clone()
m = self.text_re.match(data)
if m:
period_number = period_number + p / period
query.set('period_number', period_number)
result.refresh(0, query, 'parent')
return
p += 1
def create(self, name, case, query):
offset = FlagFramework.calculate_offset_suffix(query.get(
'offset', '0'))
filenames = self.glob_filenames(query.getarray('filename'))
## Open the io sources here
fds = [IO.open_URL(f) for f in filenames]
if query.get('ismissing', False):
fds += [ParityFD(copy.deepcopy(filenames))]
blocksize = FlagFramework.calculate_offset_suffix(
query.get('blocksize', '32k'))
period = int(query.get('period', 3))
return RAIDFD(fds, blocksize, query['map'], offset, period)
import pyflag.tests
import pyflag.pyflagsh as pyflagsh
class RaidTest(pyflag.tests.ScannerTest):
""" Test the RAID IOSource loader """
test_case = "PyFlagTestCase"
arg = iter.next()
try:
while 1:
if arg == '-i' or arg == '--subsystem':
query['subsys'] = iter.next()
elif arg.startswith('-'):
try:
while 1:
opt = iter.next()
if opt.startswith('-'):
arg = opt
continue
query["io_%s" % arg[1:]] = opt
except StopIteration:
break
arg = iter.next()
except StopIteration:
pass
## We try to instantiate the IO object:
io = IO.IOFactory(query)
#Now we create a fuse object with that IO subsystem:
server = Xmp(io=io)
server.flags = 0
server.multithreaded = 1
server.main()
def clean_up_tmp_directory(tmp_dir):
#Recursively remove the tmp directory
for root, dirs, files in os.walk(tmp_dir, topdown=False):
for name in files:
os.remove(join(root, name))
for name in dirs:
os.rmdir(join(root, name))
# set up flag io subsystem
query_target = FlagFramework.query_type(())
query_target['subsys'] = 'standard'
query_target['io_filename'] = target
io_target = IO.IOFactory(query_target)
io_source = IO.IOFactory(query_target)
#iterate file copying
image_count = 0
total_image_count = 0
collection_size = 0
for image in Exgrep.process(None, io_target, ("jpg")):
io_source.seek(image['offset'])
tmp_image = open(tmp_image_filename, 'w')
if extracted_length > 0:
tmp_image.write(io_source.read(extracted_length * 1024))
else:
tmp_image.write(io_source.read(image['length']))
tmp_image.close()
def get_fields(self):
if self.datafile == None:
raise IOError("Datafile is not set!!!")
print "Datafile %s" % (self.datafile, )
for file in self.datafile:
## open the file as a url:
fd = IO.open_URL(file)
dbh = DB.DBO()
buffer = Buffer(fd=fd)
header = EVTLog.Header(buffer)
buffer = buffer[header.size():]
while 1:
try:
event = EVTLog.Event(buffer)
source = event['Source'].get_value()
machine = event['Machine'].get_value()
## Find the filename for this source:
dbh.execute(
"select filename from EventMessageSources where source=%r",
source)
row = dbh.fetch()
if row:
dbh.execute(
"select message from EventMessages where filename=%r and message_id=%r",
(row['filename'], event['EventID'].get_value()))
row = dbh.fetch()
if row:
message = EVTLog.format_message(
row['message'], event['Strings'])
## Message not found
else:
message = "Unable to find message format string (Maybe file was not loaded with --mode=dll?). Parameters are: %s" % event[
'Strings']
## Filename not found for this source:
else:
message = "Unable to locate file for source %s. Maybe you need to run EventLogTool with the --reg flag on the SYSTEM registry hive? Parameters are: %s " % (
source, event['Strings'])
buffer = buffer[event.size():]
result = dict(
_time="from_unixtime('%s')" %
event['TimeGenerated'].get_value(),
message=message,
event=event['EventID'].get_value(),
Source=event['Source'].get_value(),
record=event['RecordNumber'].get_value(),
)
try:
result['arg1'] = event['Strings'][0].get_value()
except:
pass
try:
result['arg2'] = event['Strings'][1].get_value()
except:
pass
try:
result['arg3'] = event['Strings'][2].get_value()
except:
pass
yield result
except IOError:
break
def display(self,query,result):
dbh = DB.DBO(query['case'])
dbh.execute("select * from pcap where id=%r limit 1", query['id'])
row=dbh.fetch()
io = IO.open(query['case'], row['iosource'])
packet = pypcap.PyPCAP(io)
packet.seek(row['offset'])
dissected_packet = packet.dissect()
id = int(query['id'])
def get_node(branch):
""" Locate the node specified by the branch.
branch is a list of attribute names.
"""
result = dissected_packet
for b in branch:
try:
result = getattr(result, b)
except: pass
return result
def tree_cb(path):
branch = FlagFramework.splitpath(path)
node = get_node(branch)
try:
for field in node.list():
if field.startswith("_"): continue
child = getattr(node, field)
try:
yield ( field, child.get_name(), 'branch')
except AttributeError:
yield ( field, field, 'leaf')
except AttributeError:
pass
return
def pane_cb(path,result):
branch = FlagFramework.splitpath(path)
node = get_node(branch)
result.heading("Packet %s" % id)
data = dissected_packet.serialise()
h=FlagFramework.HexDump(data, result)
try:
result.text("%s" % node.get_name(), font='bold')
result.text('',style='black', font='normal')
start,length = node.get_range()
except AttributeError:
result.text("%s\n" % node, style='red', wrap='full', font='typewriter', sanitise='full')
result.text('',style='black', font='normal')
node = get_node(branch[:-1])
start,length = node.get_range(branch[-1])
h.dump(highlight=[[start,length,'highlight'],])
return
result.tree(tree_cb=tree_cb, pane_cb=pane_cb, branch=[''])
## We add forward and back toolbar buttons to let people move
## to next or previous packet:
dbh.execute("select min(id) as id from pcap")
row = dbh.fetch()
new_query=query.clone()
if id>row['id']:
del new_query['id']
new_query['id']=id-1
result.toolbar(text="Previous Packet",icon="stock_left.png",link=new_query)
else:
result.toolbar(text="Previous Packet",icon="stock_left_gray.png")
dbh.execute("select max(id) as id from pcap")
row = dbh.fetch()
if id<row['id']:
del new_query['id']
new_query['id']=id+1
result.toolbar(text="Next Packet",icon="stock_right.png",link=new_query)
else:
result.toolbar(text="Next Packet",icon="stock_right_gray.png")
def load(self, mount_point, iosource_name,scanners = None):
DBFS.load(self, mount_point, iosource_name)
## Open the file descriptor
self.fd = IO.open(self.case, iosource_name)
## Use the C implementation to read the pcap files:
pcap_file = pypcap.PyPCAP(self.fd)
## Build our streams:
pyflaglog.log(pyflaglog.DEBUG, "Reassembling streams, this might take a while")
pcap_dbh = DB.DBO(self.case)
pcap_dbh.mass_insert_start("pcap")
pcap_dbh.execute("select max(id) as m from pcap")
max_id = pcap_dbh.fetch()['m'] or 0
cookie, processor = self.make_processor(iosource_name, scanners)
## Process the file with it:
while 1:
try:
packet = pcap_file.dissect()
max_id += 1
## FIXME - this is a bottleneck. For now we use mass
## insert but this will break when we have multiple
## concurrent loaders. Record the packet in the pcap
## table:
args = dict(
iosource = iosource_name,
offset = packet.offset,
length = packet.caplen,
_ts_sec = "from_unixtime('%s')" % packet.ts_sec,
ts_usec = packet.ts_usec,
)
## Try to insert the ipid field
try:
args['ipid']= packet.root.eth.payload.id
except: pass
pcap_dbh.mass_insert(**args)
#pcap_id = pcap_dbh.autoincrement()
pcap_id = max_id
pcap_file.set_id(pcap_id)
## Some progress reporting
if pcap_id % 10000 == 0:
pyflaglog.log(pyflaglog.DEBUG, "processed %s packets (%s bytes)" % (pcap_id, packet.offset))
processor.process(packet)
except StopIteration:
break
processor.flush()
pcap_dbh.check_index("connection_details",'src_ip')
pcap_dbh.check_index("connection_details",'src_port')
pcap_dbh.check_index("connection_details",'dest_ip')
pcap_dbh.check_index("connection_details",'dest_port')
pcap_dbh.check_index('connection_details','inode_id')
#!/usr/bin/env python
import pyflag.IO as IO
import sys
import sk
from stat import *
#img = open(sys.argv[1],'r')
#print "Will open %s" % sys.argv[1]
## This assumes that we have a case loaded
img = IO.open('winxp','test')
fs = sk.skfs(img)
def readfile(fs,inode):
print "reading: %s" % inode
fd = fs.open(inode=inode)
while True:
if not fd.read(4000000):
break
# walk the directory tree
for root, dirs, files in fs.walk('/', unalloc=True, inodes=True):
for f in files:
try:
print "processing: (%u) %s" % (f[0], f[1])
s=fs.stat(inode=str(f[0]))
print "length %s" % s[ST_SIZE]
if int(f[0])==0: continue
readfile(fs,str(f[0]))
except IOError, e:
print "Got error: %s" % e
def get_fields(self):
if self.datafile==None:
raise IOError("Datafile is not set!!!")
print "Datafile %s" % (self.datafile,)
for file in self.datafile:
## open the file as a url:
fd = IO.open_URL(file)
dbh = DB.DBO()
buffer = Buffer(fd=fd)
header = EVTLog.Header(buffer)
buffer = buffer[header.size():]
while 1:
try:
event = EVTLog.Event(buffer)
source = event['Source'].get_value()
machine = event['Machine'].get_value()
## Find the filename for this source:
dbh.execute("select filename from EventMessageSources where source=%r", source)
row=dbh.fetch()
if row:
dbh.execute("select message from EventMessages where filename=%r and message_id=%r", (row['filename'], event['EventID'].get_value()))
row = dbh.fetch()
if row:
message=EVTLog.format_message(row['message'],event['Strings'])
## Message not found
else:
message="Unable to find message format string (Maybe file was not loaded with --mode=dll?). Parameters are: %s" % event['Strings']
## Filename not found for this source:
else: message="Unable to locate file for source %s. Maybe you need to run EventLogTool with the --reg flag on the SYSTEM registry hive? Parameters are: %s " % (source,event['Strings'])
buffer=buffer[event.size():]
result = dict(
_time= "from_unixtime('%s')" % event['TimeGenerated'].get_value(),
message= message,
event = event['EventID'].get_value(),
Source = event['Source'].get_value(),
record = event['RecordNumber'].get_value(),
)
try:
result['arg1'] = event['Strings'][0].get_value()
except: pass
try:
result['arg2'] = event['Strings'][1].get_value()
except: pass
try:
result['arg3'] = event['Strings'][2].get_value()
except: pass
yield result
except IOError:
break