def scan(self, fd, scanners, type, mime, cookie, **args): if "MSN" in type and fd.urn.endswith("forward"): pyflaglog.log(pyflaglog.DEBUG, "Openning %s for MSN" % fd.inode_id) dbfs = FileSystem.DBFS(fd.case) self.forward_fd = fd self.reverse_fd = dbfs.open(urn="%s/reverse" % os.path.dirname(fd.urn)) ## Make back references to each other self.forward_fd.reverse = self.reverse_fd self.reverse_fd.reverse = self.forward_fd ## Install defaults self.forward_fd.client_id = self.reverse_fd.client_id = '' self.forward_fd.dest_id = self.reverse_fd.dest_id = '' self.session_id = -1 for fd in NetworkScanner.generate_streams_in_time_order( self.forward_fd, self.reverse_fd): try: line = fd.readline() items = line.split() command = items[0] except IndexError: continue ## Try to process the command try: handler = getattr(self, command) except: #print "Command %s not handled" % command continue handler(items, fd, scanners) CacheManager.update_table_from_urn(fd.case, self.forward_fd.urn) CacheManager.update_table_from_urn(fd.case, self.reverse_fd.urn)
def scan(self, fd, scanners, type, mime, cookie, **args): if "MSN" in type and fd.urn.endswith("forward"): pyflaglog.log(pyflaglog.DEBUG,"Openning %s for MSN" % fd.inode_id) dbfs = FileSystem.DBFS(fd.case) self.forward_fd = fd self.reverse_fd = dbfs.open(urn = "%s/reverse" % os.path.dirname(fd.urn)) ## Make back references to each other self.forward_fd.reverse = self.reverse_fd self.reverse_fd.reverse = self.forward_fd ## Install defaults self.forward_fd.client_id = self.reverse_fd.client_id = '' self.forward_fd.dest_id = self.reverse_fd.dest_id = '' self.session_id = -1 for fd in NetworkScanner.generate_streams_in_time_order( self.forward_fd, self.reverse_fd): try: line = fd.readline() items = line.split() command = items[0] except IndexError: continue ## Try to process the command try: handler = getattr(self, command) except: #print "Command %s not handled" % command continue handler(items, fd, scanners) CacheManager.update_table_from_urn(fd.case, self.forward_fd.urn) CacheManager.update_table_from_urn(fd.case, self.reverse_fd.urn)
def parse(self, forward_fd, reverse_fd, scanners): while True: request = { 'url':'/unknown_request_%s' % forward_fd.inode_id, 'method': 'GET' } response = {} parse = False request_body = response_body = None ## First parse both request and response ## Get the current timestamp of the request packet = NetworkScanner.dissect_packet(forward_fd) if self.read_request(request, forward_fd): try: request['timestamp'] = packet.ts_sec except AttributeError: request['timestamp'] = 0 parse = True request_body = self.skip_body(request, forward_fd) request_body.dirty = 0 packet = NetworkScanner.dissect_packet(reverse_fd) if self.read_response(response, reverse_fd): try: response['timestamp'] = packet.ts_sec except AttributeError: response['timestamp'] = 0 parse = True response_body = self.skip_body(response, reverse_fd) ## We hang all the parameters on the response object ## (i.e. file attachment, post parameters, cookies) if response_body and request_body: self.process_cookies(request, response_body) self.process_post_body(request, request_body, response_body) if request_body.size > 0: request_body.close() if response_body and response_body.size > 0: ## Store information about the object in the http table: url = request.get('url','/') ## We try to store the url in a normalized form so we ## can find it regardless of the various permutations ## it can go though response_body.insert_to_table("http", dict(method = request.get('method'), url = url, status = response.get('HTTP_code'), content_type = response.get('content-type'), useragent = request.get('user-agent'), host = request.get('host'), tld = make_tld(request.get('host','')) ) ) response_body.close() Scanner.scan_inode_distributed(forward_fd.case, response_body.inode_id, scanners, self.cookie) if not parse: break
def parse(self, forward_fd, reverse_fd, scanners): while True: request = { 'url': '/unknown_request_%s' % forward_fd.inode_id, 'method': 'GET' } response = {} parse = False request_body = response_body = None ## First parse both request and response ## Get the current timestamp of the request packet = NetworkScanner.dissect_packet(forward_fd) if self.read_request(request, forward_fd): try: request['timestamp'] = packet.ts_sec except AttributeError: request['timestamp'] = 0 parse = True request_body = self.skip_body(request, forward_fd) request_body.dirty = 0 packet = NetworkScanner.dissect_packet(reverse_fd) if self.read_response(response, reverse_fd): try: response['timestamp'] = packet.ts_sec except AttributeError: response['timestamp'] = 0 parse = True response_body = self.skip_body(response, reverse_fd) ## We hang all the parameters on the response object ## (i.e. file attachment, post parameters, cookies) if response_body and request_body: self.process_cookies(request, response_body) self.process_post_body(request, request_body, response_body) if request_body.size > 0: request_body.close() if response_body and response_body.size > 0: ## Store information about the object in the http table: url = request.get('url', '/') ## We try to store the url in a normalized form so we ## can find it regardless of the various permutations ## it can go though response_body.insert_to_table( "http", dict(method=request.get('method'), url=url, status=response.get('HTTP_code'), content_type=response.get('content-type'), useragent=request.get('user-agent'), host=request.get('host'), tld=make_tld(request.get('host', '')))) response_body.close() Scanner.scan_inode_distributed(forward_fd.case, response_body.inode_id, scanners, self.cookie) if not parse: break
## recreate case (only needed for testing) env = pyflagsh.environment(case=config.case) pyflagsh.shell_execv(command="delete_case", env=env, argv=[config.case]) pyflagsh.shell_execv(command="create_case", env=env, argv=[config.case]) files = os.listdir(directory) files.sort() #files = files[-70:] baseurn = None pcap_dispatch = {} cookie = time.time() processor = NetworkScanner.make_processor(config.case, scanners, pcap_dispatch, cookie) now = time.time() volume_urn = CacheManager.AFF4_MANAGER.make_volume_urn(config.case) for f in files: file_urn = "%s/%s" % (volume_urn, f) if not aff4.oracle.resolve(file_urn, AFF4_TYPE): ## We need to add it in there fd = open(os.path.join(directory, f)) outfd = CacheManager.AFF4_MANAGER.create_cache_fd(config.case, f, include_in_VFS = False, compression = False, inherited = baseurn) if not baseurn: baseurn = outfd.urn