示例#1
0
    def import_one_file(self, filename, func, funcname):
        """ Import one file using specified parser function"""
        adapter = ParseAdapter()
        sys.stderr.write('\nImporting [%s]\n' % (filename))
        self.reset_script_begin_end()
        filters = []
        for script_env in self.capture_filter_scripts:
            self.call_script_method_with_filename(script_env, 'begin', filename)
            capture_filter = script_env.functions.get('capture_filter')
            if capture_filter:
                filters.append(capture_filter)

        count = 0
        commit_threshold = 100
        Data = self.Data
        cursor = Data.allocate_thread_cursor()
        try:
            Data.set_insert_pragmas(cursor)
            for result in func(filename):
                capture = adapter.adapt(result)
                skip = False
                for capture_filter in filters:
                    result = capture_filter(capture)
                    if not result:
                        skip = True
                        break
                if not skip:
                    insertlist = [None, capture.url, capture.request_headers, capture.request_body, capture.response_headers, capture.response_body,
                                  capture.status, capture.content_length, capture.elapsed, capture.datetime, capture.notes, None, capture.confirmed, 
                                  capture.method, capture.hostip, capture.content_type, '%s-%s' % (funcname, capture.origin), capture.host]
                    Data.insert_responses(cursor, insertlist, False)
                    count += 1
                    if (0 == (count % commit_threshold)):
                        Data.commit()

            if not (0 == (count % commit_threshold)):
                Data.commit()

            sys.stderr.write('\nInserted [%d] records\n' % (count))

        except Exception as error:
            Data.rollback()
            print(error)
            # TODO: should continue(?)
            raise error
        finally:
            Data.reset_pragmas(cursor)
            cursor.close()
            Data.release_thread_cursor(cursor)
            Data, cursor = None, None

        for script_env in self.capture_filter_scripts:
            self.call_script_method_with_filename(script_env, 'end', filename)
示例#2
0
    def parse_one_file(self, filename, func, funcname):
        """ Parse one file using specified parser function"""
        adapter = ParseAdapter()
        sys.stderr.write('\nProcessing [%s]\n' % (filename))
        self.reset_script_begin_end()
        filters = []
        processors = []
        for script_env in self.capture_filter_scripts:
            self.call_script_method_with_filename(script_env, 'begin',
                                                  filename)
            capture_filter = script_env.functions.get('capture_filter')
            if capture_filter:
                filters.append(capture_filter)
        for script_env in self.process_capture_scripts:
            self.call_script_method_with_filename(script_env, 'begin',
                                                  filename)
            process_capture = script_env.functions.get('process_capture')
            if process_capture:
                processors.append(process_capture)
        try:
            for result in func(filename):
                capture = adapter.adapt(result)
                skip = False
                for capture_filter in filters:
                    result = capture_filter(capture)
                    if not result:
                        skip = True
                        break
                if not skip:
                    for processor in processors:
                        result = processor(capture)

        except Exception as error:
            print(error)
            # TODO: should continue(?)
            raise error

        for script_env in self.capture_filter_scripts:
            self.call_script_method_with_filename(script_env, 'end', filename)
        for script_env in self.process_capture_scripts:
            self.call_script_method_with_filename(script_env, 'end', filename)
示例#3
0
    def parse_one_file(self, filename, func, funcname):
        """ Parse one file using specified parser function"""
        adapter = ParseAdapter()
        sys.stderr.write('\nProcessing [%s]\n' % (filename))
        self.reset_script_begin_end()
        filters = []
        processors = []
        for script_env in self.capture_filter_scripts:
            self.call_script_method_with_filename(script_env, 'begin', filename)
            capture_filter = script_env.functions.get('capture_filter')
            if capture_filter:
                filters.append(capture_filter)
        for script_env in self.process_capture_scripts:
            self.call_script_method_with_filename(script_env, 'begin', filename)
            process_capture = script_env.functions.get('process_capture')
            if process_capture:
                processors.append(process_capture)
        try:
            for result in func(filename):
                capture = adapter.adapt(result)
                skip = False
                for capture_filter in filters:
                    result = capture_filter(capture)
                    if not result:
                        skip = True
                        break
                if not skip:
                    for processor in processors:
                        result = processor(capture)

        except Exception as error:
            print(error)
            # TODO: should continue(?)
            raise error

        for script_env in self.capture_filter_scripts:
            self.call_script_method_with_filename(script_env, 'end', filename)
        for script_env in self.process_capture_scripts:
            self.call_script_method_with_filename(script_env, 'end', filename)
示例#4
0
    def import_one_file(self, filename, func, funcname):
        """ Import one file using specified parser function"""
        adapter = ParseAdapter()
        sys.stderr.write('\nImporting [%s]\n' % (filename))
        self.reset_script_begin_end()
        filters = []
        for script_env in self.capture_filter_scripts:
            self.call_script_method_with_filename(script_env, 'begin',
                                                  filename)
            capture_filter = script_env.functions.get('capture_filter')
            if capture_filter:
                filters.append(capture_filter)

        count = 0
        commit_threshold = 100
        Data = self.Data
        cursor = Data.allocate_thread_cursor()
        try:
            Data.set_insert_pragmas(cursor)
            for result in func(filename):
                capture = adapter.adapt(result)
                skip = False
                for capture_filter in filters:
                    result = capture_filter(capture)
                    if not result:
                        skip = True
                        break
                if not skip:
                    insertlist = [
                        None, capture.url, capture.request_headers,
                        capture.request_body, capture.response_headers,
                        capture.response_body, capture.status,
                        capture.content_length, capture.elapsed,
                        capture.datetime, capture.notes, None,
                        capture.confirmed, capture.method, capture.hostip,
                        capture.content_type,
                        '%s-%s' % (funcname, capture.origin), capture.host
                    ]
                    Data.insert_responses(cursor, insertlist, False)
                    count += 1
                    if (0 == (count % commit_threshold)):
                        Data.commit()

            if not (0 == (count % commit_threshold)):
                Data.commit()

            sys.stderr.write('\nInserted [%d] records\n' % (count))

        except Exception as error:
            Data.rollback()
            print(error)
            # TODO: should continue(?)
            raise error
        finally:
            Data.reset_pragmas(cursor)
            cursor.close()
            Data.release_thread_cursor(cursor)
            Data, cursor = None, None

        for script_env in self.capture_filter_scripts:
            self.call_script_method_with_filename(script_env, 'end', filename)