def test02LoadFile(self): """ Test that IIS Log files can be loaded """ dbh = DB.DBO(self.test_case) log = LogFile.load_preset(self.test_case, self.log_preset, [self.test_file]) t = time.time() ## Load the data: for a in log.load(self.test_table): pass print "Took %s seconds to load log" % (time.time() - t) ## Check that all rows were uploaded: dbh.execute("select count(*) as c from `%s_log`", self.test_table) row = dbh.fetch() self.assertEqual(row['c'], 8334) ## More specific tests dbh.execute( "select count(*) as c from `%s_log` where `IP Address` = 2921232307", self.test_table) row = dbh.fetch() self.assertEqual(row['c'], 129) dbh.execute( "select count(*) as c from `%s_log` where cs_username = '******'", self.test_table) row = dbh.fetch() self.assertEqual(row['c'], 7898)
def test03FilteredLoad(self): """ Test that filter expressions work when loading """ filter = ' "rhost" maxmind_country AUS and "status" = 404 ' ## See if we can load the preset again: log = LogFile.load_preset(self.test_case, self.log_preset, [self.datafile]) t = time.time() table_name = self.test_table + "filtered" ## Load the data: for a in log.load(table_name, filter=filter): print a print "Took %s seconds to load log" % (time.time()-t) ## How many rows were inserted dbh = DB.DBO(self.test_case) dbh.execute("select count(*) as c from `%s_log`", table_name) row = dbh.fetch() ## Now try to enforce the same filter on the original table ## (which has all the rows in it): for f in log.fields: f.table = self.test_table+"_log" filter_sql = parser.parse_to_sql(filter, log.fields, None) dbh.execute("select count(*) as c from `%s_log` where %s", (self.test_table, filter_sql)) row2 = dbh.fetch() self.assertEqual(row['c'], row2['c'])
def test04LoadFileTwo(self): """ Test that Simple Log files can be loaded when they contain weird spaces""" dbh = DB.DBO(self.test_case) log = LogFile.load_preset(self.test_case, self.log_preset_two, [self.test_file_two]) t = time.time() ## Load the data: for a in log.load(self.test_table_two): pass print "Took %s seconds to load log" % (time.time()-t) ## Check that all rows were uploaded: dbh.execute("select count(*) as c from `%s_log`", self.test_table_two) row = dbh.fetch() self.assertEqual(row['c'], 12)
def test02LoadFile(self): """ Test that Syslog Log files can be loaded """ dbh = DB.DBO(self.test_case) log = LogFile.load_preset(self.test_case, self.log_preset, [self.test_file]) t = time.time() ## Load the data: for a in log.load(self.test_table): pass print "Took %s seconds to load log" % (time.time()-t) ## Check that all rows were uploaded: dbh.execute("select count(*) as c from %s_log", self.test_table) row = dbh.fetch() self.assertEqual(row['c'], 2433)
def test02LoadFile(self): """ Test that Advanced Log files can be loaded """ dbh = DB.DBO(self.test_case) log = LogFile.load_preset(self.test_case, self.log_preset, [self.test_file]) t = time.time() ## Load the data: for a in log.load(self.test_table): pass print "Took %s seconds to load log" % (time.time()-t) ## Check that all rows were uploaded: dbh.execute("select count(*) as c from %s_log", self.test_table) row = dbh.fetch() self.assertEqual(row['c'], 3424)
def form(self, query, result): try: result.case_selector() result.selector( "Select Log Preset", "preset", "select name as `key`, name as `value` from log_presets", autosubmit=True ) result.textfield("Table name:", "table") result.fileselector("log files to use", "datafile") try: if query.getarray("datafile"): log = LogFile.load_preset(query["case"], query["preset"], query.getarray("datafile")) else: return result except Exception, e: result.end_table() result.text("\nError: %s\n\n" % e, style="red") # raise return result ## Draw the filter GUI: result.add_filter( query, query["case"], code_parser.parse_eval, log.fields, search_text="Pre-Filter Expression" ) result.ruler() result.end_table() result.para( "Please check the test table before continuing. Note that the filter is not applied to the test table." ) # show preview result.start_table() dbh = DB.DBO(query["case"]) try: # retrieve and display the temp table log.display_test_log(result, filter=None) except Exception, e: result.text( "Error: Unable to load a test set - maybe this log file is incompatible with this log preset?\nError was %s" % e, style="red", font="bold", ) pyflaglog.log(pyflaglog.DEBUG, "Unable to load test set - error returned was %s" % e) return
def test04LoadFileTwo(self): """ Test that Simple Log files can be loaded when they contain weird spaces""" dbh = DB.DBO(self.test_case) log = LogFile.load_preset(self.test_case, self.log_preset_two, [self.test_file_two]) t = time.time() ## Load the data: for a in log.load(self.test_table_two): pass print "Took %s seconds to load log" % (time.time() - t) ## Check that all rows were uploaded: dbh.execute("select count(*) as c from `%s_log`", self.test_table_two) row = dbh.fetch() self.assertEqual(row['c'], 12)
def test02LoadFile(self): """ Test that Apache Log files can be loaded """ ## See if we can load the preset again: log = LogFile.load_preset(self.test_case, self.log_preset, [self.datafile]) t = time.time() ## Load the data: for a in log.load(self.test_table, filter=None): print a print "Took %s seconds to load log" % (time.time()-t) ## Check that all rows were uploaded: dbh = DB.DBO(self.test_case) dbh.execute("select count(*) as c from `%s_log`", self.test_table) row = dbh.fetch() self.assertEqual(row['c'], 10000)
def display(self,query,result): if (query.has_key('limit')): result.heading("Log File in Table %s" % query['logtable']) else: result.heading("Log File in Table %s" % query['logtable']) dbh = DB.DBO(query['case']) ## Fetch the driver to use: dbh.execute("select * from log_tables where table_name='%s' limit 1",(query['logtable'])) row=dbh.fetch() if not row: raise Reports.ReportError("Log Table %s not found" % query['logtable']) try: ## Instantiate the driver on this case: log = LogFile.load_preset(query['case'], row['preset']) except KeyError,e: raise Reports.ReportError("Unable to load the preset %s for table %s " % (row['preset'],query['logtable']))
def form(self, query, result): try: result.case_selector() result.selector('Select Log Preset','preset', "select name as `key`, name as `value` from log_presets", autosubmit=True) result.textfield("Table name:","table") result.fileselector("log files to use", 'datafile') try: if query.getarray('datafile'): log = LogFile.load_preset(query['case'], query['preset'], query.getarray('datafile')) else: return result except Exception,e: result.end_table() result.text("\nError: %s\n\n" % e, style='red') # raise return result ## Draw the filter GUI: result.add_filter(query, query['case'], code_parser.parse_eval, log.fields, search_text = "Pre-Filter Expression") result.ruler() result.end_table() result.para("Please check the test table before continuing. Note that the filter is not applied to the test table.") # show preview result.start_table() dbh = DB.DBO(query['case']) try: # retrieve and display the temp table log.display_test_log(result, filter=None) except Exception,e: result.text("Error: Unable to load a test set - maybe this log file is incompatible with this log preset?\nError was %s" % e,style='red',font='bold') pyflaglog.log(pyflaglog.DEBUG,"Unable to load test set - error returned was %s" % e) return
def display(self, query, result): if (query.has_key('limit')): result.heading("Log File in Table %s" % query['logtable']) else: result.heading("Log File in Table %s" % query['logtable']) dbh = DB.DBO(query['case']) ## Fetch the driver to use: dbh.execute("select * from log_tables where table_name='%s' limit 1", (query['logtable'])) row = dbh.fetch() if not row: raise Reports.ReportError("Log Table %s not found" % query['logtable']) try: ## Instantiate the driver on this case: log = LogFile.load_preset(query['case'], row['preset']) except KeyError, e: raise Reports.ReportError( "Unable to load the preset %s for table %s " % (row['preset'], query['logtable']))
def test02LoadFile(self): """ Test that IIS Log files can be loaded """ dbh = DB.DBO(self.test_case) log = LogFile.load_preset(self.test_case, self.log_preset, [self.test_file]) t = time.time() ## Load the data: for a in log.load(self.test_table): pass print "Took %s seconds to load log" % (time.time() - t) ## Check that all rows were uploaded: dbh.execute("select count(*) as c from `%s_log`", self.test_table) row = dbh.fetch() self.assertEqual(row["c"], 8334) ## More specific tests dbh.execute("select count(*) as c from `%s_log` where `IP Address` = 2921232307", self.test_table) row = dbh.fetch() self.assertEqual(row["c"], 129) dbh.execute("select count(*) as c from `%s_log` where cs_username = '******'", self.test_table) row = dbh.fetch() self.assertEqual(row["c"], 7898)
def analyse(self, query): """ Load the log file into the table """ log = LogFile.load_preset(query['case'], query['preset'], query.getarray('datafile')) for progress in log.load(query['table'], filter=query.get('filter')): self.progress_str = progress
def analyse(self, query): """ Load the log file into the table """ log = LogFile.load_preset(query["case"], query["preset"], query.getarray("datafile")) for progress in log.load(query["table"], filter=query.get("filter")): self.progress_str = progress