def Callback(mode, packet, connection):
if mode == 'est':
if 'map' not in connection:
## Lookup the urn this packet came from
urn = urn_dispatcher[packet.pcap_file_id]
ip = packet.find_type("IP")
## We can only get tcp or udp packets here
try:
tcp = packet.find_type("TCP")
except AttributeError:
tcp = packet.find_type("UDP")
base_urn = "/%s-%s/%s-%s/" % (ip.source_addr, ip.dest_addr,
tcp.source, tcp.dest)
timestamp = pyaff4.XSDDatetime()
timestamp.set(packet.ts_sec)
map_stream = CacheManager.AFF4_MANAGER.create_cache_map(
case,
base_urn + "forward",
timestamp=timestamp,
target=urn)
connection['map'] = map_stream
## These streams are used to point at the start of
## each packet header - this helps us get back to
## the packet information for each bit of data
map_stream_pkt = CacheManager.AFF4_MANAGER.create_cache_map(
case,
base_urn + "forward.pkt",
timestamp=timestamp,
target=urn,
inherited=map_stream.urn)
connection['map.pkt'] = map_stream_pkt
r_map_stream = CacheManager.AFF4_MANAGER.create_cache_map(
case,
base_urn + "reverse",
timestamp=timestamp,
target=urn,
inherited=map_stream.urn)
connection['reverse']['map'] = r_map_stream
## These streams are used to point at the start of
## each packet header - this helps us get back to
## the packet information for each bit of data
r_map_stream_pkt = CacheManager.AFF4_MANAGER.create_cache_map(
case,
base_urn + "reverse.pkt",
timestamp=timestamp,
target=urn,
inherited=r_map_stream.urn)
connection['reverse']['map.pkt'] = r_map_stream_pkt
## Add to connection table
map_stream.insert_to_table(
"connection_details",
dict(
reverse=r_map_stream.inode_id,
src_ip=ip.src,
src_port=tcp.source,
dest_ip=ip.dest,
dest_port=tcp.dest,
_ts_sec="from_unixtime(%s)" % packet.ts_sec,
))
elif mode == 'data':
try:
tcp = packet.find_type("TCP")
except AttributeError:
tcp = packet.find_type("UDP")
try:
length = len(tcp.data)
except:
return
urn = urn_dispatcher[packet.pcap_file_id]
if packet.offset == 0: pdb.set_trace()
connection['map'].write_from(urn, packet.offset + tcp.data_offset,
length)
connection['map.pkt'].write_from(urn, packet.offset, length)
elif mode == 'destroy':
if connection['map'].size > 0 or connection['reverse'][
'map'].size > 0:
map_stream = connection['map']
r_map_stream = connection['reverse']['map']
map_stream_pkt = connection['map.pkt']
Magic.set_magic(case, map_stream_pkt.inode_id, "Packet Map")
r_map_stream_pkt = connection['reverse']['map.pkt']
Magic.set_magic(case, r_map_stream_pkt.inode_id, "Packet Map")
r_map_stream.set_attribute(PYFLAG_REVERSE_STREAM,
map_stream.urn)
map_stream.set_attribute(PYFLAG_REVERSE_STREAM,
r_map_stream.urn)
## Close all the streams
r_map_stream_pkt.close()
map_stream_pkt.close()
r_map_stream.close()
map_stream.close()
## FIXME - this needs to be done out of process using
## the distributed architecture!!!
## Open read only versions of these streams for
## scanning
dbfs = FileSystem.DBFS(case)
map_stream = dbfs.open(inode_id=map_stream.inode_id)
r_map_stream = dbfs.open(inode_id=r_map_stream.inode_id)
Scanner.scan_inode_distributed(case, map_stream.inode_id,
scanners, cookie)
Scanner.scan_inode_distributed(case, r_map_stream.inode_id,
scanners, cookie)
class AFF4Manager:
""" A Special Cache manager which maintains the main AFF4 Cache
"""
def __init__(self):
self.volume_urns = {}
def expire(self, case):
try:
volume_urn = self.volume_urns[case]
except KeyError:
volume_urn = self.create_volume(case)
pdb.set_trace()
try:
t = time.time()
oracle.expire(volume_urn)
print "Expired %s in %s sec" % (volume_urn.value, time.time() -t)
del self.volume_urns[case]
self.create_volume(case)
except KeyError:
pass
def create_volume(self, case):
""" Create a new case AFF4 Result file """
urn = pyaff4.RDFURN()
urn.set(config.RESULTDIR)
urn.add("%s.aff4" % case)
## Try to open an existing volume
if not oracle.load(urn):
volume = oracle.create(pyaff4.AFF4_ZIP_VOLUME, 'w')
oracle.set_value(volume.urn, pyaff4.AFF4_STORED, urn)
volume = volume.finish()
urn.set(volume.urn.value)
volume.cache_return()
## Keep the volume urn associated with this case (NOTE this is
## not the same as the file URI for the volume itself.
self.volume_urns[case] = urn
return urn
def close(self, case):
try:
volume_urn = self.volume_urns[case]
except KeyError:
volume_urn = self.create_volume(case)
volume = oracle.open(volume_urn, 'w')
volume.close()
def create_cache_data(self, case, path, data='', include_in_VFS=True,
timestamp = None, size=0,
inherited = None,**kwargs):
""" Creates a new AFF4 segment. A segment is useful for
storing small amounts of data in a single compressed file.
We return the URN of the created object so callers can use
this to set properties on it.
"""
try:
volume_urn = self.volume_urns[case]
except KeyError:
volume_urn = self.create_volume(case)
fd = oracle.create(pyaff4.AFF4_IMAGE, 'w')
fd.urn.set(self.volume_urns[case].value)
fd.urn.add_query(path)
fd.size.set(size)
if timestamp:
if not isinstance(timestamp, pyaff4.XSDDatetime):
t = pyaff4.XSDDatetime()
t.set(timestamp)
else:
t = timestamp
oracle.set_value(fd.urn, pyaff4.AFF4_TIMESTAMP, t)
oracle.set_value(fd.urn, pyaff4.AFF4_STORED, volume_urn)
fd = fd.finish()
fd = PyFlagImage(fd)
kwargs['path'] = path
fd.case = kwargs['case'] = case
if include_in_VFS:
fd.include_in_VFS = kwargs
return fd
try:
volume_urn = self.volume_urns[case]
except KeyError:
volume_urn = self.create_volume(case)
fd = oracle.create(pyaff4.AFF4_IMAGE, 'w')
fd.urn.set(self.volume_urns[case].value)
fd.urn.add_query(path)
fd.size.set(size)
oracle.set_value(fd.urn, pyaff4.AFF4_STORED, volume_urn)
## If a timestamp was specified we set it - otherwise we just
## take it from our inherited URN
if timestamp:
if not isinstance(timestamp, pyaff4.XSDDatetime):
t = pyaff4.XSDDatetime()
t.set(timestamp)
else:
t = timestamp
oracle.set_value(fd.urn, pyaff4.AFF4_TIMESTAMP, t)
fd = fd.finish()
fd = PyFlagImage(fd)
kwargs['path'] = path
fd.case = kwargs['case'] = case
if include_in_VFS:
fd.include_in_VFS = kwargs
return fd
import pyaff4, time, sys
import pdb
#pdb.set_trace()
#time.sleep(1)
date = pyaff4.XSDDatetime()
date.set(time.time())
for i in range(1, 100):
date.serialise()
print date.serialise()
def Callback(mode, packet, connection):
if packet:
try:
ip = packet.find_type("IP")
except AttributeError:
ip = None
try:
tcp = packet.find_type("TCP")
except AttributeError:
try:
tcp = packet.find_type("UDP")
except AttributeError:
tcp = None
if mode == 'est':
## Now do the reverse connection
if 'map' not in connection:
base = "%s-%s/%s-%s/" % (ip.source_addr, ip.dest_addr, tcp.source,
tcp.dest)
## Create the forward stream
forward_stream = oracle.create(pyaff4.AFF4_MAP)
forward_stream.urn.set(volume_urn.value)
forward_stream.urn.add(base + "forward")
forward_stream.set(pyaff4.AFF4_STORED, volume_urn)
timestamp = pyaff4.XSDDatetime()
timestamp.set(packet.ts_sec)
forward_stream.set(pyaff4.AFF4_TIMESTAMP, timestamp)
forward_stream = forward_stream.finish()
## Note that we hold the map locked while its in the
## reassembler - this prevents it from getting freed
connection['map'] = forward_stream
## Make the reverse map
reverse_stream = oracle.create(pyaff4.AFF4_MAP)
reverse_stream.urn.set(volume_urn.value)
reverse_stream.urn.add(base + "reverse")
reverse_stream.set(pyaff4.AFF4_STORED, volume_urn)
timestamp = pyaff4.XSDDatetime()
timestamp.set(packet.ts_sec)
reverse_stream.set(pyaff4.AFF4_TIMESTAMP, timestamp)
reverse_stream = reverse_stream.finish()
## Note that we hold the map locked while its in the
## reassembler - this prevents it from getting freed
connection['reverse']['map'] = reverse_stream
elif mode == 'data':
if tcp.data:
length = len(tcp.data)
connection['map'].write_from(image_urn,
packet.offset + tcp.data_offset,
length)
elif mode == 'destroy':
if connection['map'].size > 0 or connection['reverse']['map'].size > 0:
map_stream = connection['map']
map_stream.close()
r_map_stream = connection['reverse']['map']
r_map_stream.close()
def _Callback(self, mode, packet, connection):
if packet:
try:
ip = packet.find_type("IP")
except AttributeError:
ip = None
try:
tcp = packet.find_type("TCP")
except AttributeError:
try:
tcp = packet.find_type("UDP")
except AttributeError:
tcp = None
if mode == 'est':
## Now do the reverse connection
if 'map' not in connection:
base = "%s-%s/%s-%s/" % (
ip.source_addr, ip.dest_addr,
tcp.source, tcp.dest)
## Note that we hold the map locked while its in the
## reassembler - this prevents it from getting freed
connection['map'] = forward_stream = self.make_stream("forward", base)
connection['map.pkt'] = self.make_stream("forward.pkt", base, navigatable=False)
timestamp = pyaff4.XSDDatetime()
timestamp.set(packet.ts_sec)
forward_stream.set(pyaff4.AFF4_TIMESTAMP, timestamp)
## Make the reverse map
connection['reverse']['map'] = reverse_stream = self.make_stream("reverse", base)
connection['reverse']['map.pkt'] = self.make_stream(
"reverse.pkt", base, navigatable=False)
timestamp = pyaff4.XSDDatetime()
timestamp.set(packet.ts_sec)
reverse_stream.set(pyaff4.AFF4_TIMESTAMP, timestamp)
elif mode == 'data':
if tcp.data:
if oracle.get_urn_by_id(packet.pcap_file_id, self.image_urn):
length = len(tcp.data)
connection['map'].write_from(self.image_urn, packet.offset + tcp.data_offset, length)
connection['map.pkt'].write_from(self.image_urn, packet.offset, length)
elif mode == 'destroy':
if connection['map'].size > 0 or connection['reverse']['map'].size > 0:
map_stream = connection['map']
map_stream_urn = map_stream.urn
map_stream.close()
r_map_stream = connection['reverse']['map']
r_map_stream_urn = r_map_stream.urn
r_map_stream.close()
connection['map.pkt'].close()
connection['reverse']['map.pkt'].close()
## Now scan the resulting with the active scanners
Scanner.scan_urn(map_stream_urn, self.scanners)
Scanner.scan_urn(r_map_stream_urn, self.scanners)