def scan(self, fd, scanners, type, mime, cookie, **args):
if "MSN" in type and fd.urn.endswith("forward"):
pyflaglog.log(pyflaglog.DEBUG, "Openning %s for MSN" % fd.inode_id)
dbfs = FileSystem.DBFS(fd.case)
self.forward_fd = fd
self.reverse_fd = dbfs.open(urn="%s/reverse" %
os.path.dirname(fd.urn))
## Make back references to each other
self.forward_fd.reverse = self.reverse_fd
self.reverse_fd.reverse = self.forward_fd
## Install defaults
self.forward_fd.client_id = self.reverse_fd.client_id = ''
self.forward_fd.dest_id = self.reverse_fd.dest_id = ''
self.session_id = -1
for fd in NetworkScanner.generate_streams_in_time_order(
self.forward_fd, self.reverse_fd):
try:
line = fd.readline()
items = line.split()
command = items[0]
except IndexError:
continue
## Try to process the command
try:
handler = getattr(self, command)
except:
#print "Command %s not handled" % command
continue
handler(items, fd, scanners)
CacheManager.update_table_from_urn(fd.case, self.forward_fd.urn)
CacheManager.update_table_from_urn(fd.case, self.reverse_fd.urn)
def scan(self, fd, scanners, type, mime, cookie, **args):
if "MSN" in type and fd.urn.endswith("forward"):
pyflaglog.log(pyflaglog.DEBUG,"Openning %s for MSN" % fd.inode_id)
dbfs = FileSystem.DBFS(fd.case)
self.forward_fd = fd
self.reverse_fd = dbfs.open(urn = "%s/reverse" % os.path.dirname(fd.urn))
## Make back references to each other
self.forward_fd.reverse = self.reverse_fd
self.reverse_fd.reverse = self.forward_fd
## Install defaults
self.forward_fd.client_id = self.reverse_fd.client_id = ''
self.forward_fd.dest_id = self.reverse_fd.dest_id = ''
self.session_id = -1
for fd in NetworkScanner.generate_streams_in_time_order(
self.forward_fd, self.reverse_fd):
try:
line = fd.readline()
items = line.split()
command = items[0]
except IndexError: continue
## Try to process the command
try:
handler = getattr(self, command)
except:
#print "Command %s not handled" % command
continue
handler(items, fd, scanners)
CacheManager.update_table_from_urn(fd.case, self.forward_fd.urn)
CacheManager.update_table_from_urn(fd.case, self.reverse_fd.urn)
def parse(self, forward_fd, reverse_fd, scanners):
while True:
request = { 'url':'/unknown_request_%s' % forward_fd.inode_id,
'method': 'GET' }
response = {}
parse = False
request_body = response_body = None
## First parse both request and response
## Get the current timestamp of the request
packet = NetworkScanner.dissect_packet(forward_fd)
if self.read_request(request, forward_fd):
try:
request['timestamp'] = packet.ts_sec
except AttributeError:
request['timestamp'] = 0
parse = True
request_body = self.skip_body(request, forward_fd)
request_body.dirty = 0
packet = NetworkScanner.dissect_packet(reverse_fd)
if self.read_response(response, reverse_fd):
try:
response['timestamp'] = packet.ts_sec
except AttributeError:
response['timestamp'] = 0
parse = True
response_body = self.skip_body(response, reverse_fd)
## We hang all the parameters on the response object
## (i.e. file attachment, post parameters, cookies)
if response_body and request_body:
self.process_cookies(request, response_body)
self.process_post_body(request, request_body, response_body)
if request_body.size > 0:
request_body.close()
if response_body and response_body.size > 0:
## Store information about the object in the http table:
url = request.get('url','/')
## We try to store the url in a normalized form so we
## can find it regardless of the various permutations
## it can go though
response_body.insert_to_table("http",
dict(method = request.get('method'),
url = url,
status = response.get('HTTP_code'),
content_type = response.get('content-type'),
useragent = request.get('user-agent'),
host = request.get('host'),
tld = make_tld(request.get('host',''))
)
)
response_body.close()
Scanner.scan_inode_distributed(forward_fd.case, response_body.inode_id,
scanners, self.cookie)
if not parse: break
def parse(self, forward_fd, reverse_fd, scanners):
while True:
request = {
'url': '/unknown_request_%s' % forward_fd.inode_id,
'method': 'GET'
}
response = {}
parse = False
request_body = response_body = None
## First parse both request and response
## Get the current timestamp of the request
packet = NetworkScanner.dissect_packet(forward_fd)
if self.read_request(request, forward_fd):
try:
request['timestamp'] = packet.ts_sec
except AttributeError:
request['timestamp'] = 0
parse = True
request_body = self.skip_body(request, forward_fd)
request_body.dirty = 0
packet = NetworkScanner.dissect_packet(reverse_fd)
if self.read_response(response, reverse_fd):
try:
response['timestamp'] = packet.ts_sec
except AttributeError:
response['timestamp'] = 0
parse = True
response_body = self.skip_body(response, reverse_fd)
## We hang all the parameters on the response object
## (i.e. file attachment, post parameters, cookies)
if response_body and request_body:
self.process_cookies(request, response_body)
self.process_post_body(request, request_body, response_body)
if request_body.size > 0:
request_body.close()
if response_body and response_body.size > 0:
## Store information about the object in the http table:
url = request.get('url', '/')
## We try to store the url in a normalized form so we
## can find it regardless of the various permutations
## it can go though
response_body.insert_to_table(
"http",
dict(method=request.get('method'),
url=url,
status=response.get('HTTP_code'),
content_type=response.get('content-type'),
useragent=request.get('user-agent'),
host=request.get('host'),
tld=make_tld(request.get('host', ''))))
response_body.close()
Scanner.scan_inode_distributed(forward_fd.case,
response_body.inode_id,
scanners, self.cookie)
if not parse: break
## recreate case (only needed for testing)
env = pyflagsh.environment(case=config.case)
pyflagsh.shell_execv(command="delete_case", env=env,
argv=[config.case])
pyflagsh.shell_execv(command="create_case", env=env,
argv=[config.case])
files = os.listdir(directory)
files.sort()
#files = files[-70:]
baseurn = None
pcap_dispatch = {}
cookie = time.time()
processor = NetworkScanner.make_processor(config.case, scanners, pcap_dispatch, cookie)
now = time.time()
volume_urn = CacheManager.AFF4_MANAGER.make_volume_urn(config.case)
for f in files:
file_urn = "%s/%s" % (volume_urn, f)
if not aff4.oracle.resolve(file_urn, AFF4_TYPE):
## We need to add it in there
fd = open(os.path.join(directory, f))
outfd = CacheManager.AFF4_MANAGER.create_cache_fd(config.case, f,
include_in_VFS = False,
compression = False,
inherited = baseurn)
if not baseurn: baseurn = outfd.urn