def __init__(self, volume_location):
self.location_url = pyaff4.RDFURN(volume_location)
self.volume_url = pyaff4.RDFURN(volume_location)
fd = oracle.open(self.location_url, 'r')
if not fd:
self.create_new_volume()
elif not oracle.load(self.volume_url):
self.create_new_volume()
def read_request(self, request, forward_fd):
""" Checks if line looks like a URL request. If it is, we
continue reading the fd until we finish consuming the request
(headers including post content if its there).
We should be positioned at the start of the response after this.
"""
line = forward_fd.readline()
m = self.request_re.search(line)
if not m: return False
request['method'] = m.group(1)
self.read_headers(request, forward_fd)
try:
host = request['host']
except KeyError:
dest_ip = forward_fd.get_metadata(DEST_IP)
host = dest_ip.serialise(None)
url = pyaff4.RDFURN()
url.set("http://%s/%s" % (host, m.group(2)))
request['url'] = url
request['host'] = host
return True
def dissect_packet(stream_fd, stream_pkt_fd):
""" Return a dissected packet in stream fd. Based on the current readptr.
"""
## What is the current range?
offset = stream_fd.readptr
stream_pkt_fd.seek(offset)
urn = pyaff4.RDFURN()
(urn, target_offset_at_point,
available_to_read, urn_id) = stream_pkt_fd.map.get_range(offset)
## Get the file from cache
try:
pcap_file = PCAP_FILE_CACHE[urn.value]
except KeyError:
pcap_fd = oracle.open(urn, 'r')
pcap_file = pypcap.PyPCAP(pcap_fd)
PCAP_FILE_CACHE[urn.value] = pcap_file
if available_to_read:
## Go to the packet
pcap_file.seek(target_offset_at_point)
## Dissect it
try:
return pcap_file.dissect()
except: pass
def dump_blocks(self):
## Read all the data into a single string
data_blocks = []
## Open another read handle to the file private to our thread
fd = oracle.open(self.in_urn, 'r')
try:
for block, length in self.blocks:
fd.seek(block)
data_blocks.append(fd.read(length))
finally:
fd.cache_return()
data = ''.join(data_blocks)
## If there is not enough data here, we just write it to the
## miscelaneous stream
if len(data) < self.MIN_SIZE:
return self.dump_to_misc(data)
## Calculate the hash
m = hashlib.sha1()
m.update(data)
hashed_urn = pyaff4.RDFURN("aff4://" + m.hexdigest())
## Check if the hashed file already exists
if self.check_corpus(m.digest()):
log("Skipping.... %s" % hashed_urn.value)
else:
compression = pyaff4.XSDInteger(self.compression)
## Save a copy of the data
out_fd = oracle.create(pyaff4.AFF4_IMAGE)
out_fd.urn.set(hashed_urn.value)
out_fd.set(pyaff4.AFF4_STORED, self.volume_urn)
out_fd.set(pyaff4.AFF4_COMPRESSION, compression)
out_fd = out_fd.finish()
out_fd.write(data)
self.add_to_corpus(m.digest(),
self.attributes.get(FILENAME_URN, '1'))
## Attach the attributes for this new object
#pdb.set_trace()
if self.attributes:
for k, v in self.attributes.items():
try:
out_fd.set(k, v)
except:
out_fd.set(k, pyaff4.XSDString(str(v)))
out_fd.close()
if self.inode:
string = pyaff4.XSDString(self.inode)
log("Setting %s filename %s" % (hashed_urn.value, self.inode))
oracle.add_value(hashed_urn,
pyaff4.PREDICATE_NAMESPACE + "filename", string)
self.add_block_run(hashed_urn)
def __init__(self, urn=None, mode='w', *args):
pyaff4.AFFObject.__init__(self, urn, mode)
if urn:
self.columns = []
if mode == 'w':
print "Creating new table %s" % urn.value
Framework.set_metadata(urn, RDF_TYPE, self.dataType, graph='schema',
rdftype=pyaff4.RDFURN)
name = self.make_table_filename(urn)
Framework.set_metadata(urn, pyaff4.AFF4_TARGET, name,
graph = 'schema')
if mode == 'r':
print "Loading table %s" % urn.value
column_urn = pyaff4.RDFURN()
## Read all the columns
iter = oracle.get_iter(urn, PYFLAG_COLUMN)
while oracle.iter_next(iter, column_urn):
## We are holding the column open so we dont need
## to recreate it all the time. This is ok because
## we can have mulitiple objects opened for
## reading simultaneously.
column = oracle.open(column_urn, 'r')
self.columns.append(column)
## Get the target file
name = oracle.resolve_alloc(urn, pyaff4.AFF4_TARGET).value
self.filename = os.path.join(config.RESULTDIR, "%s.sqlite" % name)
self.con = sqlite3.connect(self.filename)
def dissect_packet(stream_fd):
""" Return a dissected packet in stream fd. Based on the current readptr.
"""
dbfs = FileSystem.DBFS(stream_fd.case)
urn = pyaff4.RDFURN()
urn.set(stream_fd.urn.value + ".pkt")
fd = dbfs.open(urn=urn)
if not fd or \
not oracle.resolve_value(stream_fd.urn, pyaff4.AFF4_TARGET, urn):
raise RuntimeError("%s is not a stream" % stream_fd.urn)
## Get the file from cache
try:
pcap_file = PCAP_FILE_CACHE.get(urn.value)
except KeyError:
pcap_fd = dbfs.open(urn=urn)
pcap_file = pypcap.PyPCAP(pcap_fd)
PCAP_FILE_CACHE.add(urn.value, pcap_file)
offset = stream_fd.tell()
## What is the current range?
(target_offset_at_point, available_to_read) = fd.get_range(offset, None)
if available_to_read:
## Go to the packet
pcap_file.seek(target_offset_at_point)
## Dissect it
try:
return pcap_file.dissect()
except:
pass
def scan(self, buffer, fd, scanners):
if self.request_re.search(buffer):
self.forward_fd = fd
## Find the reverse fd
base = os.path.dirname(fd.urn.value)
urn = pyaff4.RDFURN()
urn.set(base)
urn.add("forward.pkt")
self.forward_pkt_fd = oracle.open(urn, 'r')
urn.set(base)
urn.add("reverse")
self.reverse_fd = oracle.open(urn, 'r')
urn.set(base)
urn.add("reverse.pkt")
self.reverse_pkt_fd = oracle.open(urn, 'r')
try:
self.process(scanners)
finally:
self.forward_pkt_fd.cache_return()
self.reverse_fd.cache_return()
self.reverse_pkt_fd.cache_return()
def display(self, query, result):
filenames = query.getarray('filename')
print "Openning AFF4 volumes %s" % (filenames, )
result.heading("Loading AFF4 Volumes")
fsfd = DBFS(query['case'])
for filename in filenames:
## Filenames are always specified relative to the upload
## directory
urn = pyaff4.RDFURN()
urn.set(config.UPLOADDIR)
urn.add(filename)
## We try to load the volume contained in the URI given,
## but if that fails we just load the URI as a raw file:
if not oracle.load(urn):
fsfd.VFSCreate(urn, urn.parser.query, _fast=True, mode=-1)
return
stream_urn = pyaff4.RDFURN()
iter = oracle.get_iter(urn, pyaff4.AFF4_CONTAINS)
while oracle.iter_next(iter, stream_urn):
result.row("Adding %s" % stream_urn.value)
## FIXME - what kind of objects do we import?
## Segments might be too much
fsfd.VFSCreate(stream_urn,
stream_urn.parser.query,
_fast=True,
mode=-1)
return
## FIXME - record the fact that these volumes are loaded
## already into this case...
## Load all the objects inside the volumes
for v in loaded_volumes:
for urn in aff4.oracle.resolve_list(v, AFF4_CONTAINS):
type = aff4.oracle.resolve(urn, AFF4_TYPE)
if 1 or type in SUPPORTED_STREAMS:
if "/" in urn:
path = "%s/%s" % (base_dir, urn[urn.index("/"):])
else:
path = base_dir
fsfd.VFSCreate(urn, path, _fast=True, mode=-1)
def pane_cb(path, result):
urn = pyaff4.RDFURN()
urn.set(pyaff4.AFF4_NAVIGATION_ROOT)
urn.add(path)
if oracle.resolve_value(urn, pyaff4.AFF4_NAVIGATION_LINK, urn):
InformationNotebook(urn, result)
return
def tree_cb(path):
child = pyaff4.XSDString()
urn = pyaff4.RDFURN()
urn.set(pyaff4.AFF4_NAVIGATION_ROOT)
urn.add(path)
iter = oracle.get_iter(urn, pyaff4.AFF4_NAVIGATION_CHILD)
while oracle.iter_next(iter, child):
yield (child.value, child.value, 'branch')
def make_new_volume(self):
self.filename = pyaff4.RDFURN()
self.filename.set(self.make_zip_filename())
volume = oracle.create(pyaff4.AFF4_ZIP_VOLUME)
volume.set(pyaff4.AFF4_STORED, self.filename)
volume = volume.finish()
self.volume_urn = volume.urn
volume.cache_return()
def scan(self, fd, scanners, type, mime, cookie, **args):
if "HTTP Request stream" in type:
forward_fd = fd
reverse_urn = pyaff4.RDFURN()
oracle.resolve_value(fd.urn, PYFLAG_REVERSE_STREAM, reverse_urn)
self.cookie = cookie
dbfs = FileSystem.DBFS(fd.case)
reverse_fd = dbfs.open(urn=reverse_urn)
pyflaglog.log(pyflaglog.DEBUG,
"Openning %s for HTTP" % fd.inode_id)
self.parse(forward_fd, reverse_fd, scanners)
def close(self):
if self.mode == 'w':
for column in self.columns:
if type(column)==str:
urn = pyaff4.RDFURN()
urn.set(Framework.RESULT_VOLUME.volume_urn.value)
urn.add(column)
else:
urn = column
Framework.set_metadata(self.urn, PYFLAG_COLUMN, urn, graph='schema')
## Create the sqlite file
self.create_file()
def do_resolve(self, line):
globs = shlex.split(line)
attribute = pyaff4.XSDString()
subject = pyaff4.RDFURN()
iter = pyaff4.RESOLVER_ITER()
subject.set(globs[0])
try:
attribute.set(globs[1])
print attribute.value
self._display_attribute(iter)
except IndexError:
## Just display all the attributes
while oracle.attributes_iter(subject, attribute, iter):
print attribute.value
self._display_attribute(iter)
def record(self):
filename = pyaff4.RDFURN()
filename.set(self.make_zip_filename())
volume = oracle.create(pyaff4.AFF4_ZIP_VOLUME)
volume.set(pyaff4.AFF4_STORED, filename)
volume = volume.finish()
for name, data, compression in self.Files:
volume.writestr(name, data, compression)
volume.close()
fd = oracle.open(filename, 'r')
data = fd.read(fd.size.value)
fd.close()
self.store(AFF4ZipComparator(data))
def run(self):
volume = Volume(self.results['output_volume'])
inurn = pyaff4.RDFURN(self.results['input'])
infd = oracle.open(inurn)
outfd = volume.make_new_image(self.results['stream'],
threads=int(self.results['threads']))
total_size = infd.size.value
count = 0
while not self.stop:
data = infd.read(10 * 1024 * 1024)
if not data:
self.stop = True
self.footer_text.set_text("Closing volume, please wait")
self.loop.draw_screen()
outfd.close()
volume.close()
self.footer_text.set_text(
"Imaging complete, press any key to exit ... ")
self.loop.draw_screen()
break
outfd.write(data)
count += len(data)
## Work out how much time is left
try:
time_so_far = time.time() - self.starttime
rate = count / time_so_far
time_left = (total_size - count) / rate
except:
time_left = 'Unknown'
self.textline.set_text(
"Acquiring %s (%s / %s done)\nSpeed %.02d Mb/s, Estimated time left %02d s"
% (inurn.value, count, total_size, rate / 1024 / 1024,
time_left))
self.progress.set_completion(float(count) * 100 / total_size)
self.loop.draw_screen()
count += 1
def create_volume(self, case):
""" Create a new case AFF4 Result file """
urn = pyaff4.RDFURN()
urn.set(config.RESULTDIR)
urn.add("%s.aff4" % case)
## Try to open an existing volume
if not oracle.load(urn):
volume = oracle.create(pyaff4.AFF4_ZIP_VOLUME, 'w')
oracle.set_value(volume.urn, pyaff4.AFF4_STORED, urn)
volume = volume.finish()
urn.set(volume.urn.value)
volume.cache_return()
## Keep the volume urn associated with this case (NOTE this is
## not the same as the file URI for the volume itself.
self.volume_urns[case] = urn
return urn
def populate_tree(self, node):
## Make sure to clear previous children
try:
children = self.tree.get_children(node)
for child in children:
self.tree.delete(child)
except: pass
child = pyaff4.XSDString()
urn = pyaff4.RDFURN()
urn.set(self.tree.set(node, 'url'))
path = self.tree.set(node, "path")
iter = oracle.get_iter(urn, pyaff4.AFF4_NAVIGATION_CHILD)
while oracle.iter_next(iter, child):
new_path = path + "/" + child.value
self.tree.insert(node, "end", text=child.value,
values=[new_path, urn.value + "/" + child.value])
def do_cp(self, line):
""" Copy a stream from a source to a destination. """
globs = shlex.split(line)
src = globs[0]
dest = globs[1]
if (len(globs) > 2):
print "usage: cp src dest"
return
if not src.startswith("/"):
src = self.CWD + src
src = os.path.normpath(src)
src_urn = pyaff4.RDFURN()
src_urn.set("aff4:/" + src)
## Try to open the stream
try:
fd = oracle.open(src_urn, 'r')
except IOError, e:
raise
def __init__(self):
""" Given an output volume URN or a path we create this for writing.
"""
## A Cache of RDFValue types that aviods us having to recreate
## them all the time
self.rdftypes = {}
## A cache of all schema objects keyed by name
self.schema = {}
## A list of all active tables.
self.tables = []
self.volume_urn = pyaff4.RDFURN()
self.volume_urn.set(config.RESULTDIR)
self.volume_urn.add("Output.aff4")
## Try to append to an existing volume
if not oracle.load(self.volume_urn):
## Nope just make it then
volume = oracle.create(pyaff4.AFF4_ZIP_VOLUME)
volume.set(pyaff4.AFF4_STORED, self.volume_urn)
volume = volume.finish()
self.volume_urn = volume.urn
volume.cache_return()
## Now make the navigation graph
graph = pyaff4.Graph(mode='w')
graph.urn.set(self.volume_urn.value)
graph.urn.add("pyflag/navigation")
graph.set(pyaff4.AFF4_STORED, self.volume_urn)
self.navigation_graph_urn = graph.urn
self.path_cache = PathManager(self.navigation_graph_urn)
graph = graph.finish()
graph.cache_return()
## Make the metadata graph
self.make_graph("metadata")
self.make_graph("schema")
import pyaff4
import time
time.sleep(1)
oracle = pyaff4.Resolver()
URN = "aff4://navigation/root"
URN = "aff4://69581d02-36ae-47a4-b0ba-888368dc2e11/192.168.1.34-192.168.1.1/38099-25/reverse"
urn = pyaff4.RDFURN()
urn.set(URN)
attribute = pyaff4.XSDString()
iter = pyaff4.RESOLVER_ITER()
next = 0
while oracle.attributes_iter(urn, attribute, iter):
print
print attribute.value
while 1:
obj = oracle.alloc_from_iter(iter)
if not obj: break
print " -> type (%s) " % (obj.dataType)
print " -> data (%s) " % (obj.serialise(urn))
def __init__(self, url):
self.fd = oracle.open(pyaff4.RDFURN(url), 'r')
if not self.fd:
raise IOError("Unable to open %s" % url)
pytsk3.Img_Info.__init__(self, '')
import pyaff4
import pdb
import time
time.sleep(1)
urn_base = 'http://www.google.com/'
cache = pyaff4.Cache(max_cache_size=10)
for x in range(0, 15):
urn = urn_base + "%s.html" % x
r = pyaff4.RDFURN()
r.set(urn)
cache.put("%s" % x, r)
del r
print cache.cache_size, cache.max_cache_size
## Get an item
try:
a = cache.get("1")
except KeyError:
a = cache.get("10")
print a.value
print cache.cache_size, cache.max_cache_size
print "Testing multiple values with the same key"
## Now test setting multiple values for a single key
def image(output, options, fds):
""" Copy the file like objects specified in the fds list into an
output volume specified by its URI.
"""
#pdb.set_trace()
base, ext = os.path.splitext(output)
volume_count = 0
## These are all the volumes we have created - we only close them
## at the end in case some thread is still writing to them:
volume_urns = []
output_URI = pyaff4.RDFURN()
output_URI.set(output)
size = pyaff4.XSDInteger()
out_fd = oracle.open(output_URI, 'w')
## A file urn was specified
if isinstance(out_fd, pyaff4.FileLikeObject):
out_fd.cache_return()
## Make a volume object
volume_fd = oracle.create(pyaff4.AFF4_ZIP_VOLUME, 'w')
oracle.set_value(volume_fd.urn, pyaff4.AFF4_STORED, output_URI)
volume_fd = volume_fd.finish()
else:
volume_fd = out_fd
volume_urns.append(volume_fd.urn)
volume_fd.cache_return()
for fd in fds:
image_fd = oracle.create(pyaff4.AFF4_IMAGE, 'w')
image_fd.urn.set(volume_urns[-1].value)
image_fd.urn.add(fd.urn.parser.query)
warn("New image %s%s%s stored on volume %s%s" %
(colors['cyan'], image_fd.urn.value, colors['end'],
colors['yellow'], output_URI.value))
## We want to make the image URI the same as the volume URI
## with the query stem of the source appended to it:
image_fd.set_workers(options.threads)
oracle.set_value(image_fd.urn, pyaff4.AFF4_STORED, volume_urns[-1])
image_fd = image_fd.finish()
while 1:
data = fd.read(1024 * 1024)
if not data: break
image_fd.write(data)
## Check if we need to change volumes
if options.max_size > 0 and oracle.resolve_value(
volume_urns[-1], pyaff4.AFF4_DIRECTORY_OFFSET, size) and \
size.value > options.max_size:
## Make a new volume:
volume_count += 1
new_filename = "%s.%02d%s" % (base, volume_count, ext)
output_URI.set(new_filename)
volume = oracle.create(pyaff4.AFF4_ZIP_VOLUME, 'w')
volume.set(pyaff4.AFF4_STORED, output_URI)
volume = volume.finish()
volume_urns.append(volume.urn)
volume.cache_return()
## The image is now stored in the new volume
image_fd.set(pyaff4.AFF4_STORED, volume_urns[-1])
warn("New volume created on %s%s" %
(colors['cyan'], output_URI.value))
## This will block until all threads have finished.
image_fd.close()
## Now close all the volumes off
for volume_urn in volume_urns:
print "Closing %s" % volume_urn.value
volume_fd = oracle.open(volume_urn, 'w')
volume_fd.close()
import threading
import pytsk3
import time
#time.sleep(1)
## This is a list of files we should not bother compressing
## because they will not be very compressible. For now we use
## filename extension to make this call
DO_NOT_COMPRESS = set(
"zip cab jpg avi mpg jar tgz msi swf mp3 mp4 wma".split())
## This attribute is used to specify the filename of the hash blocks
FILENAME_URN = 'http://afflib.org/2009/aff4#hash_filename'
out_urn = pyaff4.RDFURN("/tmp/test3.aff4")
## Offset into the image where the filesystem can be found
OFFSET = 0
THREADS = 0
IN_FILENAME = "/var/tmp/uploads/testimages/winxp.E01"
IN_FILENAME = "/var/tmp/uploads/testimages/winxp.dd"
IN_FILENAME = "/var/tmp/uploads/testimages/ntfs_image.dd"
#IN_FILENAME = "/tmp/image.dd"
CORPUS_FILENAME = "/tmp/corpus.aff4"
CORPUS_TDB = pyaff4.TDB(CORPUS_FILENAME)
in_urn = pyaff4.RDFURN(IN_FILENAME)
ZERO_URN = pyaff4.RDFURN("aff4://zero")
import pyaff4
import pdb, os
import time, sys
time.sleep(1)
DD_DIR = "/var/tmp/uploads/testimages/raid/linux/"
d1 = DD_DIR + "d1.dd"
d2 = DD_DIR + "d2.dd"
d3 = DD_DIR + "d3.dd"
oracle = pyaff4.Resolver()
target_urn = pyaff4.RDFURN()
target_urn.set("/bin/ls")
volume_urn = pyaff4.RDFURN()
volume_urn.set("/tmp/foobar.zip")
try:
os.unlink(volume_urn.parser.query)
except:
pass
## New volume
volume = oracle.create(pyaff4.AFF4_ZIP_VOLUME, 'w')
oracle.set_value(volume.urn, pyaff4.AFF4_STORED, volume_urn)
volume = volume.finish()
volume_urn = volume.urn
def __init__(self):
self.flush()
self.target_id = 0
self.targets = []
self.image_urn = pyaff4.RDFURN()
print "Setting value"
self.value = value
## Register the class for handling RDF objects
oracle.register_rdf_value_class(pyaff4.ProxiedRDFValue(RDFSpecial))
## We can obtain a new value of this type by nameing the dataType:
value = oracle.new_rdfvalue(RDFSpecial.dataType)
## Note that this method is called in the proxy class, but ends up
## calling the RDFSpecial class instance:
value.set("foobar")
## Make up some URI to attach to:
urn = pyaff4.RDFURN()
urn.set("hello")
## We make up an attribute within the aff4 namespace
attr = pyaff4.PREDICATE_NAMESPACE + "sample_attribute"
## Now we can use it in the resolver as normal
oracle.set_value(urn, attr, value)
## Now set the same value using an alternative dataType:
value2 = pyaff4.XSDString()
value2.set("foobar")
## note that the same attribute can have multiple values encoded using
## different dataTypes:
oracle.add_value(urn, attr, value2)
"""
def passphrase(self, cipher, subject):
print "Setting passphrase for subject %s" % subject.value
return "Hello"
def x509_private_key(self, cert_name, subject):
""" Returns the private key (in pem format) for the certificate name provided. """
print "Certificate for %s" % cert_name
return open(CERT_LOCATION).read()
## This registers the security provider
oracle.register_security_provider(
pyaff4.ProxiedSecurityProvider(SecurityProvider()))
url = pyaff4.RDFURN()
url.set("/tmp/test.zip")
try:
url.set(sys.argv[1])
fd = oracle.open(url, 'r')
while 1:
data = fd.read(1024 * 1024)
if not data: break
sys.stdout.write(data)
sys.exit(0)
except IndexError:
pass
import pypcap, reassembler
import time
import pyaff4, os, pdb
time.sleep(1)
import gc
gc.set_debug(gc.DEBUG_LEAK)
oracle = pyaff4.Resolver(pyaff4.RESOLVER_MODE_DEBUG_MEMORY)
image_urn = pyaff4.RDFURN()
image_urn.set("/var/tmp/uploads/testimages/stdcapture_0.4.pcap")
image_urn.set("/var/tmp/uploads/a5912_01_03.pcap")
image = oracle.open(image_urn, 'r')
pcap_file = pypcap.PyPCAP(image)
## Create a new volume on the output file
outfile = pyaff4.RDFURN()
outfile.set("/tmp/output.aff4")
try:
os.unlink(outfile.parser.query)
except:
pass
volume = oracle.create(pyaff4.AFF4_ZIP_VOLUME)
volume.set(pyaff4.AFF4_STORED, outfile)
volume = volume.finish()
volume_urn = volume.urn
oracle.cache_return(volume)
