def __init__(self, volume_location): self.location_url = pyaff4.RDFURN(volume_location) self.volume_url = pyaff4.RDFURN(volume_location) fd = oracle.open(self.location_url, 'r') if not fd: self.create_new_volume() elif not oracle.load(self.volume_url): self.create_new_volume()
def read_request(self, request, forward_fd): """ Checks if line looks like a URL request. If it is, we continue reading the fd until we finish consuming the request (headers including post content if its there). We should be positioned at the start of the response after this. """ line = forward_fd.readline() m = self.request_re.search(line) if not m: return False request['method'] = m.group(1) self.read_headers(request, forward_fd) try: host = request['host'] except KeyError: dest_ip = forward_fd.get_metadata(DEST_IP) host = dest_ip.serialise(None) url = pyaff4.RDFURN() url.set("http://%s/%s" % (host, m.group(2))) request['url'] = url request['host'] = host return True
def dissect_packet(stream_fd, stream_pkt_fd): """ Return a dissected packet in stream fd. Based on the current readptr. """ ## What is the current range? offset = stream_fd.readptr stream_pkt_fd.seek(offset) urn = pyaff4.RDFURN() (urn, target_offset_at_point, available_to_read, urn_id) = stream_pkt_fd.map.get_range(offset) ## Get the file from cache try: pcap_file = PCAP_FILE_CACHE[urn.value] except KeyError: pcap_fd = oracle.open(urn, 'r') pcap_file = pypcap.PyPCAP(pcap_fd) PCAP_FILE_CACHE[urn.value] = pcap_file if available_to_read: ## Go to the packet pcap_file.seek(target_offset_at_point) ## Dissect it try: return pcap_file.dissect() except: pass
def dump_blocks(self): ## Read all the data into a single string data_blocks = [] ## Open another read handle to the file private to our thread fd = oracle.open(self.in_urn, 'r') try: for block, length in self.blocks: fd.seek(block) data_blocks.append(fd.read(length)) finally: fd.cache_return() data = ''.join(data_blocks) ## If there is not enough data here, we just write it to the ## miscelaneous stream if len(data) < self.MIN_SIZE: return self.dump_to_misc(data) ## Calculate the hash m = hashlib.sha1() m.update(data) hashed_urn = pyaff4.RDFURN("aff4://" + m.hexdigest()) ## Check if the hashed file already exists if self.check_corpus(m.digest()): log("Skipping.... %s" % hashed_urn.value) else: compression = pyaff4.XSDInteger(self.compression) ## Save a copy of the data out_fd = oracle.create(pyaff4.AFF4_IMAGE) out_fd.urn.set(hashed_urn.value) out_fd.set(pyaff4.AFF4_STORED, self.volume_urn) out_fd.set(pyaff4.AFF4_COMPRESSION, compression) out_fd = out_fd.finish() out_fd.write(data) self.add_to_corpus(m.digest(), self.attributes.get(FILENAME_URN, '1')) ## Attach the attributes for this new object #pdb.set_trace() if self.attributes: for k, v in self.attributes.items(): try: out_fd.set(k, v) except: out_fd.set(k, pyaff4.XSDString(str(v))) out_fd.close() if self.inode: string = pyaff4.XSDString(self.inode) log("Setting %s filename %s" % (hashed_urn.value, self.inode)) oracle.add_value(hashed_urn, pyaff4.PREDICATE_NAMESPACE + "filename", string) self.add_block_run(hashed_urn)
def __init__(self, urn=None, mode='w', *args): pyaff4.AFFObject.__init__(self, urn, mode) if urn: self.columns = [] if mode == 'w': print "Creating new table %s" % urn.value Framework.set_metadata(urn, RDF_TYPE, self.dataType, graph='schema', rdftype=pyaff4.RDFURN) name = self.make_table_filename(urn) Framework.set_metadata(urn, pyaff4.AFF4_TARGET, name, graph = 'schema') if mode == 'r': print "Loading table %s" % urn.value column_urn = pyaff4.RDFURN() ## Read all the columns iter = oracle.get_iter(urn, PYFLAG_COLUMN) while oracle.iter_next(iter, column_urn): ## We are holding the column open so we dont need ## to recreate it all the time. This is ok because ## we can have mulitiple objects opened for ## reading simultaneously. column = oracle.open(column_urn, 'r') self.columns.append(column) ## Get the target file name = oracle.resolve_alloc(urn, pyaff4.AFF4_TARGET).value self.filename = os.path.join(config.RESULTDIR, "%s.sqlite" % name) self.con = sqlite3.connect(self.filename)
def dissect_packet(stream_fd): """ Return a dissected packet in stream fd. Based on the current readptr. """ dbfs = FileSystem.DBFS(stream_fd.case) urn = pyaff4.RDFURN() urn.set(stream_fd.urn.value + ".pkt") fd = dbfs.open(urn=urn) if not fd or \ not oracle.resolve_value(stream_fd.urn, pyaff4.AFF4_TARGET, urn): raise RuntimeError("%s is not a stream" % stream_fd.urn) ## Get the file from cache try: pcap_file = PCAP_FILE_CACHE.get(urn.value) except KeyError: pcap_fd = dbfs.open(urn=urn) pcap_file = pypcap.PyPCAP(pcap_fd) PCAP_FILE_CACHE.add(urn.value, pcap_file) offset = stream_fd.tell() ## What is the current range? (target_offset_at_point, available_to_read) = fd.get_range(offset, None) if available_to_read: ## Go to the packet pcap_file.seek(target_offset_at_point) ## Dissect it try: return pcap_file.dissect() except: pass
def scan(self, buffer, fd, scanners): if self.request_re.search(buffer): self.forward_fd = fd ## Find the reverse fd base = os.path.dirname(fd.urn.value) urn = pyaff4.RDFURN() urn.set(base) urn.add("forward.pkt") self.forward_pkt_fd = oracle.open(urn, 'r') urn.set(base) urn.add("reverse") self.reverse_fd = oracle.open(urn, 'r') urn.set(base) urn.add("reverse.pkt") self.reverse_pkt_fd = oracle.open(urn, 'r') try: self.process(scanners) finally: self.forward_pkt_fd.cache_return() self.reverse_fd.cache_return() self.reverse_pkt_fd.cache_return()
def display(self, query, result): filenames = query.getarray('filename') print "Openning AFF4 volumes %s" % (filenames, ) result.heading("Loading AFF4 Volumes") fsfd = DBFS(query['case']) for filename in filenames: ## Filenames are always specified relative to the upload ## directory urn = pyaff4.RDFURN() urn.set(config.UPLOADDIR) urn.add(filename) ## We try to load the volume contained in the URI given, ## but if that fails we just load the URI as a raw file: if not oracle.load(urn): fsfd.VFSCreate(urn, urn.parser.query, _fast=True, mode=-1) return stream_urn = pyaff4.RDFURN() iter = oracle.get_iter(urn, pyaff4.AFF4_CONTAINS) while oracle.iter_next(iter, stream_urn): result.row("Adding %s" % stream_urn.value) ## FIXME - what kind of objects do we import? ## Segments might be too much fsfd.VFSCreate(stream_urn, stream_urn.parser.query, _fast=True, mode=-1) return ## FIXME - record the fact that these volumes are loaded ## already into this case... ## Load all the objects inside the volumes for v in loaded_volumes: for urn in aff4.oracle.resolve_list(v, AFF4_CONTAINS): type = aff4.oracle.resolve(urn, AFF4_TYPE) if 1 or type in SUPPORTED_STREAMS: if "/" in urn: path = "%s/%s" % (base_dir, urn[urn.index("/"):]) else: path = base_dir fsfd.VFSCreate(urn, path, _fast=True, mode=-1)
def pane_cb(path, result): urn = pyaff4.RDFURN() urn.set(pyaff4.AFF4_NAVIGATION_ROOT) urn.add(path) if oracle.resolve_value(urn, pyaff4.AFF4_NAVIGATION_LINK, urn): InformationNotebook(urn, result) return
def tree_cb(path): child = pyaff4.XSDString() urn = pyaff4.RDFURN() urn.set(pyaff4.AFF4_NAVIGATION_ROOT) urn.add(path) iter = oracle.get_iter(urn, pyaff4.AFF4_NAVIGATION_CHILD) while oracle.iter_next(iter, child): yield (child.value, child.value, 'branch')
def make_new_volume(self): self.filename = pyaff4.RDFURN() self.filename.set(self.make_zip_filename()) volume = oracle.create(pyaff4.AFF4_ZIP_VOLUME) volume.set(pyaff4.AFF4_STORED, self.filename) volume = volume.finish() self.volume_urn = volume.urn volume.cache_return()
def scan(self, fd, scanners, type, mime, cookie, **args): if "HTTP Request stream" in type: forward_fd = fd reverse_urn = pyaff4.RDFURN() oracle.resolve_value(fd.urn, PYFLAG_REVERSE_STREAM, reverse_urn) self.cookie = cookie dbfs = FileSystem.DBFS(fd.case) reverse_fd = dbfs.open(urn=reverse_urn) pyflaglog.log(pyflaglog.DEBUG, "Openning %s for HTTP" % fd.inode_id) self.parse(forward_fd, reverse_fd, scanners)
def close(self): if self.mode == 'w': for column in self.columns: if type(column)==str: urn = pyaff4.RDFURN() urn.set(Framework.RESULT_VOLUME.volume_urn.value) urn.add(column) else: urn = column Framework.set_metadata(self.urn, PYFLAG_COLUMN, urn, graph='schema') ## Create the sqlite file self.create_file()
def do_resolve(self, line): globs = shlex.split(line) attribute = pyaff4.XSDString() subject = pyaff4.RDFURN() iter = pyaff4.RESOLVER_ITER() subject.set(globs[0]) try: attribute.set(globs[1]) print attribute.value self._display_attribute(iter) except IndexError: ## Just display all the attributes while oracle.attributes_iter(subject, attribute, iter): print attribute.value self._display_attribute(iter)
def record(self): filename = pyaff4.RDFURN() filename.set(self.make_zip_filename()) volume = oracle.create(pyaff4.AFF4_ZIP_VOLUME) volume.set(pyaff4.AFF4_STORED, filename) volume = volume.finish() for name, data, compression in self.Files: volume.writestr(name, data, compression) volume.close() fd = oracle.open(filename, 'r') data = fd.read(fd.size.value) fd.close() self.store(AFF4ZipComparator(data))
def run(self): volume = Volume(self.results['output_volume']) inurn = pyaff4.RDFURN(self.results['input']) infd = oracle.open(inurn) outfd = volume.make_new_image(self.results['stream'], threads=int(self.results['threads'])) total_size = infd.size.value count = 0 while not self.stop: data = infd.read(10 * 1024 * 1024) if not data: self.stop = True self.footer_text.set_text("Closing volume, please wait") self.loop.draw_screen() outfd.close() volume.close() self.footer_text.set_text( "Imaging complete, press any key to exit ... ") self.loop.draw_screen() break outfd.write(data) count += len(data) ## Work out how much time is left try: time_so_far = time.time() - self.starttime rate = count / time_so_far time_left = (total_size - count) / rate except: time_left = 'Unknown' self.textline.set_text( "Acquiring %s (%s / %s done)\nSpeed %.02d Mb/s, Estimated time left %02d s" % (inurn.value, count, total_size, rate / 1024 / 1024, time_left)) self.progress.set_completion(float(count) * 100 / total_size) self.loop.draw_screen() count += 1
def create_volume(self, case): """ Create a new case AFF4 Result file """ urn = pyaff4.RDFURN() urn.set(config.RESULTDIR) urn.add("%s.aff4" % case) ## Try to open an existing volume if not oracle.load(urn): volume = oracle.create(pyaff4.AFF4_ZIP_VOLUME, 'w') oracle.set_value(volume.urn, pyaff4.AFF4_STORED, urn) volume = volume.finish() urn.set(volume.urn.value) volume.cache_return() ## Keep the volume urn associated with this case (NOTE this is ## not the same as the file URI for the volume itself. self.volume_urns[case] = urn return urn
def populate_tree(self, node): ## Make sure to clear previous children try: children = self.tree.get_children(node) for child in children: self.tree.delete(child) except: pass child = pyaff4.XSDString() urn = pyaff4.RDFURN() urn.set(self.tree.set(node, 'url')) path = self.tree.set(node, "path") iter = oracle.get_iter(urn, pyaff4.AFF4_NAVIGATION_CHILD) while oracle.iter_next(iter, child): new_path = path + "/" + child.value self.tree.insert(node, "end", text=child.value, values=[new_path, urn.value + "/" + child.value])
def do_cp(self, line): """ Copy a stream from a source to a destination. """ globs = shlex.split(line) src = globs[0] dest = globs[1] if (len(globs) > 2): print "usage: cp src dest" return if not src.startswith("/"): src = self.CWD + src src = os.path.normpath(src) src_urn = pyaff4.RDFURN() src_urn.set("aff4:/" + src) ## Try to open the stream try: fd = oracle.open(src_urn, 'r') except IOError, e: raise
def __init__(self): """ Given an output volume URN or a path we create this for writing. """ ## A Cache of RDFValue types that aviods us having to recreate ## them all the time self.rdftypes = {} ## A cache of all schema objects keyed by name self.schema = {} ## A list of all active tables. self.tables = [] self.volume_urn = pyaff4.RDFURN() self.volume_urn.set(config.RESULTDIR) self.volume_urn.add("Output.aff4") ## Try to append to an existing volume if not oracle.load(self.volume_urn): ## Nope just make it then volume = oracle.create(pyaff4.AFF4_ZIP_VOLUME) volume.set(pyaff4.AFF4_STORED, self.volume_urn) volume = volume.finish() self.volume_urn = volume.urn volume.cache_return() ## Now make the navigation graph graph = pyaff4.Graph(mode='w') graph.urn.set(self.volume_urn.value) graph.urn.add("pyflag/navigation") graph.set(pyaff4.AFF4_STORED, self.volume_urn) self.navigation_graph_urn = graph.urn self.path_cache = PathManager(self.navigation_graph_urn) graph = graph.finish() graph.cache_return() ## Make the metadata graph self.make_graph("metadata") self.make_graph("schema")
import pyaff4 import time time.sleep(1) oracle = pyaff4.Resolver() URN = "aff4://navigation/root" URN = "aff4://69581d02-36ae-47a4-b0ba-888368dc2e11/192.168.1.34-192.168.1.1/38099-25/reverse" urn = pyaff4.RDFURN() urn.set(URN) attribute = pyaff4.XSDString() iter = pyaff4.RESOLVER_ITER() next = 0 while oracle.attributes_iter(urn, attribute, iter): print print attribute.value while 1: obj = oracle.alloc_from_iter(iter) if not obj: break print " -> type (%s) " % (obj.dataType) print " -> data (%s) " % (obj.serialise(urn))
def __init__(self, url): self.fd = oracle.open(pyaff4.RDFURN(url), 'r') if not self.fd: raise IOError("Unable to open %s" % url) pytsk3.Img_Info.__init__(self, '')
import pyaff4 import pdb import time time.sleep(1) urn_base = 'http://www.google.com/' cache = pyaff4.Cache(max_cache_size=10) for x in range(0, 15): urn = urn_base + "%s.html" % x r = pyaff4.RDFURN() r.set(urn) cache.put("%s" % x, r) del r print cache.cache_size, cache.max_cache_size ## Get an item try: a = cache.get("1") except KeyError: a = cache.get("10") print a.value print cache.cache_size, cache.max_cache_size print "Testing multiple values with the same key" ## Now test setting multiple values for a single key
def image(output, options, fds): """ Copy the file like objects specified in the fds list into an output volume specified by its URI. """ #pdb.set_trace() base, ext = os.path.splitext(output) volume_count = 0 ## These are all the volumes we have created - we only close them ## at the end in case some thread is still writing to them: volume_urns = [] output_URI = pyaff4.RDFURN() output_URI.set(output) size = pyaff4.XSDInteger() out_fd = oracle.open(output_URI, 'w') ## A file urn was specified if isinstance(out_fd, pyaff4.FileLikeObject): out_fd.cache_return() ## Make a volume object volume_fd = oracle.create(pyaff4.AFF4_ZIP_VOLUME, 'w') oracle.set_value(volume_fd.urn, pyaff4.AFF4_STORED, output_URI) volume_fd = volume_fd.finish() else: volume_fd = out_fd volume_urns.append(volume_fd.urn) volume_fd.cache_return() for fd in fds: image_fd = oracle.create(pyaff4.AFF4_IMAGE, 'w') image_fd.urn.set(volume_urns[-1].value) image_fd.urn.add(fd.urn.parser.query) warn("New image %s%s%s stored on volume %s%s" % (colors['cyan'], image_fd.urn.value, colors['end'], colors['yellow'], output_URI.value)) ## We want to make the image URI the same as the volume URI ## with the query stem of the source appended to it: image_fd.set_workers(options.threads) oracle.set_value(image_fd.urn, pyaff4.AFF4_STORED, volume_urns[-1]) image_fd = image_fd.finish() while 1: data = fd.read(1024 * 1024) if not data: break image_fd.write(data) ## Check if we need to change volumes if options.max_size > 0 and oracle.resolve_value( volume_urns[-1], pyaff4.AFF4_DIRECTORY_OFFSET, size) and \ size.value > options.max_size: ## Make a new volume: volume_count += 1 new_filename = "%s.%02d%s" % (base, volume_count, ext) output_URI.set(new_filename) volume = oracle.create(pyaff4.AFF4_ZIP_VOLUME, 'w') volume.set(pyaff4.AFF4_STORED, output_URI) volume = volume.finish() volume_urns.append(volume.urn) volume.cache_return() ## The image is now stored in the new volume image_fd.set(pyaff4.AFF4_STORED, volume_urns[-1]) warn("New volume created on %s%s" % (colors['cyan'], output_URI.value)) ## This will block until all threads have finished. image_fd.close() ## Now close all the volumes off for volume_urn in volume_urns: print "Closing %s" % volume_urn.value volume_fd = oracle.open(volume_urn, 'w') volume_fd.close()
import threading import pytsk3 import time #time.sleep(1) ## This is a list of files we should not bother compressing ## because they will not be very compressible. For now we use ## filename extension to make this call DO_NOT_COMPRESS = set( "zip cab jpg avi mpg jar tgz msi swf mp3 mp4 wma".split()) ## This attribute is used to specify the filename of the hash blocks FILENAME_URN = 'http://afflib.org/2009/aff4#hash_filename' out_urn = pyaff4.RDFURN("/tmp/test3.aff4") ## Offset into the image where the filesystem can be found OFFSET = 0 THREADS = 0 IN_FILENAME = "/var/tmp/uploads/testimages/winxp.E01" IN_FILENAME = "/var/tmp/uploads/testimages/winxp.dd" IN_FILENAME = "/var/tmp/uploads/testimages/ntfs_image.dd" #IN_FILENAME = "/tmp/image.dd" CORPUS_FILENAME = "/tmp/corpus.aff4" CORPUS_TDB = pyaff4.TDB(CORPUS_FILENAME) in_urn = pyaff4.RDFURN(IN_FILENAME) ZERO_URN = pyaff4.RDFURN("aff4://zero")
import pyaff4 import pdb, os import time, sys time.sleep(1) DD_DIR = "/var/tmp/uploads/testimages/raid/linux/" d1 = DD_DIR + "d1.dd" d2 = DD_DIR + "d2.dd" d3 = DD_DIR + "d3.dd" oracle = pyaff4.Resolver() target_urn = pyaff4.RDFURN() target_urn.set("/bin/ls") volume_urn = pyaff4.RDFURN() volume_urn.set("/tmp/foobar.zip") try: os.unlink(volume_urn.parser.query) except: pass ## New volume volume = oracle.create(pyaff4.AFF4_ZIP_VOLUME, 'w') oracle.set_value(volume.urn, pyaff4.AFF4_STORED, volume_urn) volume = volume.finish() volume_urn = volume.urn
def __init__(self): self.flush() self.target_id = 0 self.targets = [] self.image_urn = pyaff4.RDFURN()
print "Setting value" self.value = value ## Register the class for handling RDF objects oracle.register_rdf_value_class(pyaff4.ProxiedRDFValue(RDFSpecial)) ## We can obtain a new value of this type by nameing the dataType: value = oracle.new_rdfvalue(RDFSpecial.dataType) ## Note that this method is called in the proxy class, but ends up ## calling the RDFSpecial class instance: value.set("foobar") ## Make up some URI to attach to: urn = pyaff4.RDFURN() urn.set("hello") ## We make up an attribute within the aff4 namespace attr = pyaff4.PREDICATE_NAMESPACE + "sample_attribute" ## Now we can use it in the resolver as normal oracle.set_value(urn, attr, value) ## Now set the same value using an alternative dataType: value2 = pyaff4.XSDString() value2.set("foobar") ## note that the same attribute can have multiple values encoded using ## different dataTypes: oracle.add_value(urn, attr, value2)
""" def passphrase(self, cipher, subject): print "Setting passphrase for subject %s" % subject.value return "Hello" def x509_private_key(self, cert_name, subject): """ Returns the private key (in pem format) for the certificate name provided. """ print "Certificate for %s" % cert_name return open(CERT_LOCATION).read() ## This registers the security provider oracle.register_security_provider( pyaff4.ProxiedSecurityProvider(SecurityProvider())) url = pyaff4.RDFURN() url.set("/tmp/test.zip") try: url.set(sys.argv[1]) fd = oracle.open(url, 'r') while 1: data = fd.read(1024 * 1024) if not data: break sys.stdout.write(data) sys.exit(0) except IndexError: pass
import pypcap, reassembler import time import pyaff4, os, pdb time.sleep(1) import gc gc.set_debug(gc.DEBUG_LEAK) oracle = pyaff4.Resolver(pyaff4.RESOLVER_MODE_DEBUG_MEMORY) image_urn = pyaff4.RDFURN() image_urn.set("/var/tmp/uploads/testimages/stdcapture_0.4.pcap") image_urn.set("/var/tmp/uploads/a5912_01_03.pcap") image = oracle.open(image_urn, 'r') pcap_file = pypcap.PyPCAP(image) ## Create a new volume on the output file outfile = pyaff4.RDFURN() outfile.set("/tmp/output.aff4") try: os.unlink(outfile.parser.query) except: pass volume = oracle.create(pyaff4.AFF4_ZIP_VOLUME) volume.set(pyaff4.AFF4_STORED, outfile) volume = volume.finish() volume_urn = volume.urn oracle.cache_return(volume)