def import_one_file(self, filename, func, funcname):
""" Import one file using specified parser function"""
adapter = ParseAdapter()
sys.stderr.write('\nImporting [%s]\n' % (filename))
self.reset_script_begin_end()
filters = []
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'begin', filename)
capture_filter = script_env.functions.get('capture_filter')
if capture_filter:
filters.append(capture_filter)
count = 0
commit_threshold = 100
Data = self.Data
cursor = Data.allocate_thread_cursor()
try:
Data.set_insert_pragmas(cursor)
for result in func(filename):
capture = adapter.adapt(result)
skip = False
for capture_filter in filters:
result = capture_filter(capture)
if not result:
skip = True
break
if not skip:
insertlist = [None, capture.url, capture.request_headers, capture.request_body, capture.response_headers, capture.response_body,
capture.status, capture.content_length, capture.elapsed, capture.datetime, capture.notes, None, capture.confirmed,
capture.method, capture.hostip, capture.content_type, '%s-%s' % (funcname, capture.origin), capture.host]
Data.insert_responses(cursor, insertlist, False)
count += 1
if (0 == (count % commit_threshold)):
Data.commit()
if not (0 == (count % commit_threshold)):
Data.commit()
sys.stderr.write('\nInserted [%d] records\n' % (count))
except Exception as error:
Data.rollback()
print(error)
# TODO: should continue(?)
raise error
finally:
Data.reset_pragmas(cursor)
cursor.close()
Data.release_thread_cursor(cursor)
Data, cursor = None, None
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'end', filename)
def parse_one_file(self, filename, func, funcname):
""" Parse one file using specified parser function"""
adapter = ParseAdapter()
sys.stderr.write('\nProcessing [%s]\n' % (filename))
self.reset_script_begin_end()
filters = []
processors = []
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'begin',
filename)
capture_filter = script_env.functions.get('capture_filter')
if capture_filter:
filters.append(capture_filter)
for script_env in self.process_capture_scripts:
self.call_script_method_with_filename(script_env, 'begin',
filename)
process_capture = script_env.functions.get('process_capture')
if process_capture:
processors.append(process_capture)
try:
for result in func(filename):
capture = adapter.adapt(result)
skip = False
for capture_filter in filters:
result = capture_filter(capture)
if not result:
skip = True
break
if not skip:
for processor in processors:
result = processor(capture)
except Exception as error:
print(error)
# TODO: should continue(?)
raise error
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'end', filename)
for script_env in self.process_capture_scripts:
self.call_script_method_with_filename(script_env, 'end', filename)
def parse_one_file(self, filename, func, funcname):
""" Parse one file using specified parser function"""
adapter = ParseAdapter()
sys.stderr.write('\nProcessing [%s]\n' % (filename))
self.reset_script_begin_end()
filters = []
processors = []
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'begin', filename)
capture_filter = script_env.functions.get('capture_filter')
if capture_filter:
filters.append(capture_filter)
for script_env in self.process_capture_scripts:
self.call_script_method_with_filename(script_env, 'begin', filename)
process_capture = script_env.functions.get('process_capture')
if process_capture:
processors.append(process_capture)
try:
for result in func(filename):
capture = adapter.adapt(result)
skip = False
for capture_filter in filters:
result = capture_filter(capture)
if not result:
skip = True
break
if not skip:
for processor in processors:
result = processor(capture)
except Exception as error:
print(error)
# TODO: should continue(?)
raise error
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'end', filename)
for script_env in self.process_capture_scripts:
self.call_script_method_with_filename(script_env, 'end', filename)
def import_one_file(self, filename, func, funcname):
""" Import one file using specified parser function"""
adapter = ParseAdapter()
sys.stderr.write('\nImporting [%s]\n' % (filename))
self.reset_script_begin_end()
filters = []
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'begin',
filename)
capture_filter = script_env.functions.get('capture_filter')
if capture_filter:
filters.append(capture_filter)
count = 0
commit_threshold = 100
Data = self.Data
cursor = Data.allocate_thread_cursor()
try:
Data.set_insert_pragmas(cursor)
for result in func(filename):
capture = adapter.adapt(result)
skip = False
for capture_filter in filters:
result = capture_filter(capture)
if not result:
skip = True
break
if not skip:
insertlist = [
None, capture.url, capture.request_headers,
capture.request_body, capture.response_headers,
capture.response_body, capture.status,
capture.content_length, capture.elapsed,
capture.datetime, capture.notes, None,
capture.confirmed, capture.method, capture.hostip,
capture.content_type,
'%s-%s' % (funcname, capture.origin), capture.host
]
Data.insert_responses(cursor, insertlist, False)
count += 1
if (0 == (count % commit_threshold)):
Data.commit()
if not (0 == (count % commit_threshold)):
Data.commit()
sys.stderr.write('\nInserted [%d] records\n' % (count))
except Exception as error:
Data.rollback()
print(error)
# TODO: should continue(?)
raise error
finally:
Data.reset_pragmas(cursor)
cursor.close()
Data.release_thread_cursor(cursor)
Data, cursor = None, None
for script_env in self.capture_filter_scripts:
self.call_script_method_with_filename(script_env, 'end', filename)
