def export_to_raft_capture(self, filename, fhandle): """ Export to RAFT capture format """ sys.stderr.write('\nExporting to [%s]\n' % (filename)) self.reset_script_begin_end() self.setup_script_initializers() filters = [] for script_env in self.capture_filter_scripts: self.call_script_method_with_filename(script_env, 'begin', filename) capture_filter = script_env.functions.get('capture_filter') if capture_filter: filters.append(capture_filter) adapter = ParseAdapter() count = 0 Data = self.Data cursor = Data.allocate_thread_cursor() try: fhandle.write(b'<raft version="1.0">\n') for row in Data.read_all_responses(cursor): capture = RaftDbCapture() capture.populate_by_dbrow(row) skip = False for capture_filter in filters: result = capture_filter(capture) if not result: skip = True break if not skip: fhandle.write( adapter.format_as_xml(capture).encode('utf-8')) count += 1 fhandle.write(b'</raft>') fhandle.close() sys.stderr.write('\nExported [%d] records\n' % (count)) finally: cursor.close() Data.release_thread_cursor(cursor) Data, cursor = None, None for script_env in self.capture_filter_scripts: self.call_script_method_with_filename(script_env, 'end', filename) self.setup_script_finalizers()
def parse_one_file(self, filename, func, funcname): """ Parse one file using specified parser function""" adapter = ParseAdapter() sys.stderr.write('\nProcessing [%s]\n' % (filename)) self.reset_script_begin_end() filters = [] processors = [] for script_env in self.capture_filter_scripts: self.call_script_method_with_filename(script_env, 'begin', filename) capture_filter = script_env.functions.get('capture_filter') if capture_filter: filters.append(capture_filter) for script_env in self.process_capture_scripts: self.call_script_method_with_filename(script_env, 'begin', filename) process_capture = script_env.functions.get('process_capture') if process_capture: processors.append(process_capture) try: for result in func(filename): capture = adapter.adapt(result) skip = False for capture_filter in filters: result = capture_filter(capture) if not result: skip = True break if not skip: for processor in processors: result = processor(capture) except Exception as error: print(error) # TODO: should continue(?) raise error for script_env in self.capture_filter_scripts: self.call_script_method_with_filename(script_env, 'end', filename) for script_env in self.process_capture_scripts: self.call_script_method_with_filename(script_env, 'end', filename)
def export_to_raft_capture(self): # TODO: consider implementing a tasklet if this is the entire DB being exported filename = 'RaftExport-%s' % int(time.time()) file = QFileDialog.getSaveFileName( None, "Save to file", filename, "XML File (*.xml);;XZ XML File (*.xml.xz)") if file: adapter = ParseAdapter() # TODO: refactor filename = str(file) while '.xml.xml' in filename: filename = filename.replace('.xml.xml', '.xml') if filename.endswith('.xml.xz'): while '.xml.xz.xml.xz' in filename: filename = filename.replace('.xml.xz.xml.xz', '.xml.xz') fh = lzma.LZMAFile(filename, 'w') elif filename.endswith('.xml'): filename = filename.replace('.xml.xml', '.xml') fh = open(filename, 'wb') else: raise Exception('unhandled file type: %s' % (filename)) Data = self.framework.getDB() cursor = Data.allocate_thread_cursor() try: fh.write(b'<raft version="1.0">\n') for index in self.treeViewSelectionModel.selectedRows(): Id = interface.index_to_id(self.dataModel, index) if Id: capture = RaftDbCapture() capture.populate_by_id(self.framework, Data, cursor, Id) fh.write( adapter.format_as_xml(capture).encode('utf-8')) fh.write(b'</raft>') fh.close() finally: cursor.close() Data.release_thread_cursor(cursor) Data, cursor = None, None
def import_one_file(self, filename, func, funcname): """ Import one file using specified parser function""" adapter = ParseAdapter() sys.stderr.write('\nImporting [%s]\n' % (filename)) self.reset_script_begin_end() filters = [] for script_env in self.capture_filter_scripts: self.call_script_method_with_filename(script_env, 'begin', filename) capture_filter = script_env.functions.get('capture_filter') if capture_filter: filters.append(capture_filter) count = 0 commit_threshold = 100 Data = self.Data cursor = Data.allocate_thread_cursor() try: Data.set_insert_pragmas(cursor) for result in func(filename): capture = adapter.adapt(result) skip = False for capture_filter in filters: result = capture_filter(capture) if not result: skip = True break if not skip: insertlist = [ None, capture.url, capture.request_headers, capture.request_body, capture.response_headers, capture.response_body, capture.status, capture.content_length, capture.elapsed, capture.datetime, capture.notes, None, capture.confirmed, capture.method, capture.hostip, capture.content_type, '%s-%s' % (funcname, capture.origin), capture.host ] Data.insert_responses(cursor, insertlist, False) count += 1 if (0 == (count % commit_threshold)): Data.commit() if not (0 == (count % commit_threshold)): Data.commit() sys.stderr.write('\nInserted [%d] records\n' % (count)) except Exception as error: Data.rollback() print(error) # TODO: should continue(?) raise error finally: Data.reset_pragmas(cursor) cursor.close() Data.release_thread_cursor(cursor) Data, cursor = None, None for script_env in self.capture_filter_scripts: self.call_script_method_with_filename(script_env, 'end', filename)