Beispiel #1
0
    def scan(self, fd, scanners, type, mime, cookie, **args):
        if "MSN" in type and fd.urn.endswith("forward"):
            pyflaglog.log(pyflaglog.DEBUG, "Openning %s for MSN" % fd.inode_id)
            dbfs = FileSystem.DBFS(fd.case)
            self.forward_fd = fd
            self.reverse_fd = dbfs.open(urn="%s/reverse" %
                                        os.path.dirname(fd.urn))

            ## Make back references to each other
            self.forward_fd.reverse = self.reverse_fd
            self.reverse_fd.reverse = self.forward_fd

            ## Install defaults
            self.forward_fd.client_id = self.reverse_fd.client_id = ''
            self.forward_fd.dest_id = self.reverse_fd.dest_id = ''

            self.session_id = -1

            for fd in NetworkScanner.generate_streams_in_time_order(
                    self.forward_fd, self.reverse_fd):
                try:
                    line = fd.readline()
                    items = line.split()
                    command = items[0]
                except IndexError:
                    continue

                ## Try to process the command
                try:
                    handler = getattr(self, command)
                except:
                    #print "Command %s not handled" % command
                    continue

                handler(items, fd, scanners)

            CacheManager.update_table_from_urn(fd.case, self.forward_fd.urn)
            CacheManager.update_table_from_urn(fd.case, self.reverse_fd.urn)
Beispiel #2
0
    def scan(self, fd, scanners, type, mime, cookie, **args):
        if "MSN" in type and fd.urn.endswith("forward"):
            pyflaglog.log(pyflaglog.DEBUG,"Openning %s for MSN" % fd.inode_id)
            dbfs = FileSystem.DBFS(fd.case)
            self.forward_fd = fd
            self.reverse_fd = dbfs.open(urn = "%s/reverse" % os.path.dirname(fd.urn))

            ## Make back references to each other
            self.forward_fd.reverse = self.reverse_fd
            self.reverse_fd.reverse = self.forward_fd

            ## Install defaults
            self.forward_fd.client_id =  self.reverse_fd.client_id = ''
            self.forward_fd.dest_id = self.reverse_fd.dest_id = ''
            
            self.session_id = -1

            for fd in NetworkScanner.generate_streams_in_time_order(
                self.forward_fd, self.reverse_fd):
                try:
                    line = fd.readline()
                    items = line.split()
                    command = items[0]
                except IndexError: continue

                ## Try to process the command
                try:
                    handler = getattr(self, command)
                except:
                    #print "Command %s not handled" % command
                    continue

                handler(items, fd, scanners)

            CacheManager.update_table_from_urn(fd.case, self.forward_fd.urn)
            CacheManager.update_table_from_urn(fd.case, self.reverse_fd.urn)
Beispiel #3
0
    def parse(self, forward_fd, reverse_fd, scanners):
        while True:
            request = { 'url':'/unknown_request_%s' % forward_fd.inode_id,
                        'method': 'GET' }
            response = {}
            parse = False
            request_body = response_body = None

            ## First parse both request and response
            ## Get the current timestamp of the request
            packet = NetworkScanner.dissect_packet(forward_fd)
            if self.read_request(request, forward_fd):
                try:
                    request['timestamp'] = packet.ts_sec
                except AttributeError:
                    request['timestamp'] = 0

                parse = True
                request_body = self.skip_body(request, forward_fd)
                request_body.dirty = 0

            packet = NetworkScanner.dissect_packet(reverse_fd)
            if self.read_response(response, reverse_fd):
                try:
                    response['timestamp'] = packet.ts_sec
                except AttributeError:
                    response['timestamp'] = 0

                parse = True
                response_body = self.skip_body(response, reverse_fd)

            ## We hang all the parameters on the response object
            ## (i.e. file attachment, post parameters, cookies)
            if response_body and request_body:
                self.process_cookies(request, response_body)
                self.process_post_body(request, request_body, response_body)
                if request_body.size > 0:
                    request_body.close()

            if response_body and response_body.size > 0:
                ## Store information about the object in the http table:
                url = request.get('url','/')

                ## We try to store the url in a normalized form so we
                ## can find it regardless of the various permutations
                ## it can go though
                response_body.insert_to_table("http",
                                              dict(method = request.get('method'),
                                                   url = url,
                                                   status = response.get('HTTP_code'),
                                                   content_type = response.get('content-type'),
                                                   useragent = request.get('user-agent'),
                                                   host = request.get('host'),
                                                   tld = make_tld(request.get('host',''))
                                                   )
                                              )
                response_body.close()
                Scanner.scan_inode_distributed(forward_fd.case, response_body.inode_id,
                                               scanners, self.cookie)

            if not parse: break
Beispiel #4
0
    def parse(self, forward_fd, reverse_fd, scanners):
        while True:
            request = {
                'url': '/unknown_request_%s' % forward_fd.inode_id,
                'method': 'GET'
            }
            response = {}
            parse = False
            request_body = response_body = None

            ## First parse both request and response
            ## Get the current timestamp of the request
            packet = NetworkScanner.dissect_packet(forward_fd)
            if self.read_request(request, forward_fd):
                try:
                    request['timestamp'] = packet.ts_sec
                except AttributeError:
                    request['timestamp'] = 0

                parse = True
                request_body = self.skip_body(request, forward_fd)
                request_body.dirty = 0

            packet = NetworkScanner.dissect_packet(reverse_fd)
            if self.read_response(response, reverse_fd):
                try:
                    response['timestamp'] = packet.ts_sec
                except AttributeError:
                    response['timestamp'] = 0

                parse = True
                response_body = self.skip_body(response, reverse_fd)

            ## We hang all the parameters on the response object
            ## (i.e. file attachment, post parameters, cookies)
            if response_body and request_body:
                self.process_cookies(request, response_body)
                self.process_post_body(request, request_body, response_body)
                if request_body.size > 0:
                    request_body.close()

            if response_body and response_body.size > 0:
                ## Store information about the object in the http table:
                url = request.get('url', '/')

                ## We try to store the url in a normalized form so we
                ## can find it regardless of the various permutations
                ## it can go though
                response_body.insert_to_table(
                    "http",
                    dict(method=request.get('method'),
                         url=url,
                         status=response.get('HTTP_code'),
                         content_type=response.get('content-type'),
                         useragent=request.get('user-agent'),
                         host=request.get('host'),
                         tld=make_tld(request.get('host', ''))))
                response_body.close()
                Scanner.scan_inode_distributed(forward_fd.case,
                                               response_body.inode_id,
                                               scanners, self.cookie)

            if not parse: break
## recreate case (only needed for testing)
env = pyflagsh.environment(case=config.case)
pyflagsh.shell_execv(command="delete_case", env=env,
                     argv=[config.case])
pyflagsh.shell_execv(command="create_case", env=env,
                     argv=[config.case])

files = os.listdir(directory)
files.sort()
#files = files[-70:]

baseurn = None
pcap_dispatch = {}
cookie = time.time()
processor = NetworkScanner.make_processor(config.case, scanners, pcap_dispatch, cookie)


now = time.time()
volume_urn = CacheManager.AFF4_MANAGER.make_volume_urn(config.case)

for f in files:
    file_urn = "%s/%s" % (volume_urn, f)
    if not aff4.oracle.resolve(file_urn, AFF4_TYPE):
        ## We need to add it in there
        fd = open(os.path.join(directory, f))
        outfd = CacheManager.AFF4_MANAGER.create_cache_fd(config.case, f,
                                                          include_in_VFS = False,
                                                          compression = False,
                                                          inherited = baseurn)
        if not baseurn: baseurn = outfd.urn