def export_to_raft_capture(self):
# TODO: consider implementing a tasklet if this is the entire DB being exported
filename = 'RaftExport-%s' % int(time.time())
file = QFileDialog.getSaveFileName(None, "Save to file", filename, "XML File (*.xml);;XZ XML File (*.xml.xz)")
if file:
adapter = ParseAdapter()
# TODO: refactor
filename = str(file)
while '.xml.xml' in filename:
filename = filename.replace('.xml.xml', '.xml')
if filename.endswith('.xml.xz'):
while '.xml.xz.xml.xz' in filename:
filename = filename.replace('.xml.xz.xml.xz', '.xml.xz')
fh = lzma.LZMAFile(filename, 'w')
elif filename.endswith('.xml'):
filename = filename.replace('.xml.xml', '.xml')
fh = open(filename, 'wb')
else:
raise Exception('unhandled file type: %s' % (filename))
Data = self.framework.getDB()
cursor = Data.allocate_thread_cursor()
try:
fh.write(b'<raft version="1.0">\n')
for index in self.treeViewSelectionModel.selectedRows():
Id = interface.index_to_id(self.dataModel, index)
if Id:
capture = RaftDbCapture()
capture.populate_by_id(self.framework, Data, cursor, Id)
fh.write(adapter.format_as_xml(capture).encode('utf-8'))
fh.write(b'</raft>')
fh.close()
finally:
cursor.close()
Data.release_thread_cursor(cursor)
Data, cursor = None, None
def import_one_file(self, filename, func, funcname):
""" Import one file using specified parser function"""
adapter = ParseAdapter()
sys.stderr.write('\nImporting [%s]\n' % (filename))
self.reset_script_begin_end()
filters = []
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'begin', filename)
capture_filter = script_env.functions.get('capture_filter')
if capture_filter:
filters.append(capture_filter)
count = 0
commit_threshold = 100
Data = self.Data
cursor = Data.allocate_thread_cursor()
try:
Data.set_insert_pragmas(cursor)
for result in func(filename):
capture = adapter.adapt(result)
skip = False
for capture_filter in filters:
result = capture_filter(capture)
if not result:
skip = True
break
if not skip:
insertlist = [None, capture.url, capture.request_headers, capture.request_body, capture.response_headers, capture.response_body,
capture.status, capture.content_length, capture.elapsed, capture.datetime, capture.notes, None, capture.confirmed,
capture.method, capture.hostip, capture.content_type, '%s-%s' % (funcname, capture.origin), capture.host]
Data.insert_responses(cursor, insertlist, False)
count += 1
if (0 == (count % commit_threshold)):
Data.commit()
if not (0 == (count % commit_threshold)):
Data.commit()
sys.stderr.write('\nInserted [%d] records\n' % (count))
except Exception as error:
Data.rollback()
print(error)
# TODO: should continue(?)
raise error
finally:
Data.reset_pragmas(cursor)
cursor.close()
Data.release_thread_cursor(cursor)
Data, cursor = None, None
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'end', filename)
def export_to_raft_capture(self, filename, fhandle):
""" Export to RAFT capture format """
sys.stderr.write('\nExporting to [%s]\n' % (filename))
self.reset_script_begin_end()
self.setup_script_initializers()
filters = []
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'begin',
filename)
capture_filter = script_env.functions.get('capture_filter')
if capture_filter:
filters.append(capture_filter)
adapter = ParseAdapter()
count = 0
Data = self.Data
cursor = Data.allocate_thread_cursor()
try:
fhandle.write(b'<raft version="1.0">\n')
for row in Data.read_all_responses(cursor):
capture = RaftDbCapture()
capture.populate_by_dbrow(row)
skip = False
for capture_filter in filters:
result = capture_filter(capture)
if not result:
skip = True
break
if not skip:
fhandle.write(
adapter.format_as_xml(capture).encode('utf-8'))
count += 1
fhandle.write(b'</raft>')
fhandle.close()
sys.stderr.write('\nExported [%d] records\n' % (count))
finally:
cursor.close()
Data.release_thread_cursor(cursor)
Data, cursor = None, None
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'end', filename)
self.setup_script_finalizers()
def export_to_raft_capture(self, filename, fhandle):
""" Export to RAFT capture format """
sys.stderr.write('\nExporting to [%s]\n' % (filename))
self.reset_script_begin_end()
self.setup_script_initializers()
filters = []
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'begin', filename)
capture_filter = script_env.functions.get('capture_filter')
if capture_filter:
filters.append(capture_filter)
adapter = ParseAdapter()
count = 0
Data = self.Data
cursor = Data.allocate_thread_cursor()
try:
fhandle.write(b'<raft version="1.0">\n')
for row in Data.read_all_responses(cursor):
capture = RaftDbCapture()
capture.populate_by_dbrow(row)
skip = False
for capture_filter in filters:
result = capture_filter(capture)
if not result:
skip = True
break
if not skip:
fhandle.write(adapter.format_as_xml(capture).encode('utf-8'))
count += 1
fhandle.write(b'</raft>')
fhandle.close()
sys.stderr.write('\nExported [%d] records\n' % (count))
finally:
cursor.close()
Data.release_thread_cursor(cursor)
Data, cursor = None, None
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'end', filename)
self.setup_script_finalizers()
def parse_one_file(self, filename, func, funcname):
""" Parse one file using specified parser function"""
adapter = ParseAdapter()
sys.stderr.write('\nProcessing [%s]\n' % (filename))
self.reset_script_begin_end()
filters = []
processors = []
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'begin',
filename)
capture_filter = script_env.functions.get('capture_filter')
if capture_filter:
filters.append(capture_filter)
for script_env in self.process_capture_scripts:
self.call_script_method_with_filename(script_env, 'begin',
filename)
process_capture = script_env.functions.get('process_capture')
if process_capture:
processors.append(process_capture)
try:
for result in func(filename):
capture = adapter.adapt(result)
skip = False
for capture_filter in filters:
result = capture_filter(capture)
if not result:
skip = True
break
if not skip:
for processor in processors:
result = processor(capture)
except Exception as error:
print(error)
# TODO: should continue(?)
raise error
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'end', filename)
for script_env in self.process_capture_scripts:
self.call_script_method_with_filename(script_env, 'end', filename)
def export_to_raft_capture(self):
# TODO: consider implementing a tasklet if this is the entire DB being exported
filename = 'RaftExport-%s' % int(time.time())
file = QFileDialog.getSaveFileName(
None, "Save to file", filename,
"XML File (*.xml);;XZ XML File (*.xml.xz)")
if file:
adapter = ParseAdapter()
# TODO: refactor
filename = str(file)
while '.xml.xml' in filename:
filename = filename.replace('.xml.xml', '.xml')
if filename.endswith('.xml.xz'):
while '.xml.xz.xml.xz' in filename:
filename = filename.replace('.xml.xz.xml.xz', '.xml.xz')
fh = lzma.LZMAFile(filename, 'w')
elif filename.endswith('.xml'):
filename = filename.replace('.xml.xml', '.xml')
fh = open(filename, 'wb')
else:
raise Exception('unhandled file type: %s' % (filename))
Data = self.framework.getDB()
cursor = Data.allocate_thread_cursor()
try:
fh.write(b'<raft version="1.0">\n')
for index in self.treeViewSelectionModel.selectedRows():
Id = interface.index_to_id(self.dataModel, index)
if Id:
capture = RaftDbCapture()
capture.populate_by_id(self.framework, Data, cursor,
Id)
fh.write(
adapter.format_as_xml(capture).encode('utf-8'))
fh.write(b'</raft>')
fh.close()
finally:
cursor.close()
Data.release_thread_cursor(cursor)
Data, cursor = None, None
def parse_one_file(self, filename, func, funcname):
""" Parse one file using specified parser function"""
adapter = ParseAdapter()
sys.stderr.write('\nProcessing [%s]\n' % (filename))
self.reset_script_begin_end()
filters = []
processors = []
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'begin', filename)
capture_filter = script_env.functions.get('capture_filter')
if capture_filter:
filters.append(capture_filter)
for script_env in self.process_capture_scripts:
self.call_script_method_with_filename(script_env, 'begin', filename)
process_capture = script_env.functions.get('process_capture')
if process_capture:
processors.append(process_capture)
try:
for result in func(filename):
capture = adapter.adapt(result)
skip = False
for capture_filter in filters:
result = capture_filter(capture)
if not result:
skip = True
break
if not skip:
for processor in processors:
result = processor(capture)
except Exception as error:
print(error)
# TODO: should continue(?)
raise error
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'end', filename)
for script_env in self.process_capture_scripts:
self.call_script_method_with_filename(script_env, 'end', filename)
def import_one_file(self, filename, func, funcname):
""" Import one file using specified parser function"""
adapter = ParseAdapter()
sys.stderr.write('\nImporting [%s]\n' % (filename))
self.reset_script_begin_end()
filters = []
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'begin',
filename)
capture_filter = script_env.functions.get('capture_filter')
if capture_filter:
filters.append(capture_filter)
count = 0
commit_threshold = 100
Data = self.Data
cursor = Data.allocate_thread_cursor()
try:
Data.set_insert_pragmas(cursor)
for result in func(filename):
capture = adapter.adapt(result)
skip = False
for capture_filter in filters:
result = capture_filter(capture)
if not result:
skip = True
break
if not skip:
insertlist = [
None, capture.url, capture.request_headers,
capture.request_body, capture.response_headers,
capture.response_body, capture.status,
capture.content_length, capture.elapsed,
capture.datetime, capture.notes, None,
capture.confirmed, capture.method, capture.hostip,
capture.content_type,
'%s-%s' % (funcname, capture.origin), capture.host
]
Data.insert_responses(cursor, insertlist, False)
count += 1
if (0 == (count % commit_threshold)):
Data.commit()
if not (0 == (count % commit_threshold)):
Data.commit()
sys.stderr.write('\nInserted [%d] records\n' % (count))
except Exception as error:
Data.rollback()
print(error)
# TODO: should continue(?)
raise error
finally:
Data.reset_pragmas(cursor)
cursor.close()
Data.release_thread_cursor(cursor)
Data, cursor = None, None
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'end', filename)
