def test02LoadFile(self):
""" Test that IIS Log files can be loaded """
dbh = DB.DBO(self.test_case)
log = LogFile.load_preset(self.test_case, self.log_preset,
[self.test_file])
t = time.time()
## Load the data:
for a in log.load(self.test_table):
pass
print "Took %s seconds to load log" % (time.time() - t)
## Check that all rows were uploaded:
dbh.execute("select count(*) as c from `%s_log`", self.test_table)
row = dbh.fetch()
self.assertEqual(row['c'], 8334)
## More specific tests
dbh.execute(
"select count(*) as c from `%s_log` where `IP Address` = 2921232307",
self.test_table)
row = dbh.fetch()
self.assertEqual(row['c'], 129)
dbh.execute(
"select count(*) as c from `%s_log` where cs_username = '******'",
self.test_table)
row = dbh.fetch()
self.assertEqual(row['c'], 7898)
def test03FilteredLoad(self):
""" Test that filter expressions work when loading """
filter = ' "rhost" maxmind_country AUS and "status" = 404 '
## See if we can load the preset again:
log = LogFile.load_preset(self.test_case, self.log_preset, [self.datafile])
t = time.time()
table_name = self.test_table + "filtered"
## Load the data:
for a in log.load(table_name, filter=filter):
print a
print "Took %s seconds to load log" % (time.time()-t)
## How many rows were inserted
dbh = DB.DBO(self.test_case)
dbh.execute("select count(*) as c from `%s_log`", table_name)
row = dbh.fetch()
## Now try to enforce the same filter on the original table
## (which has all the rows in it):
for f in log.fields: f.table = self.test_table+"_log"
filter_sql = parser.parse_to_sql(filter, log.fields, None)
dbh.execute("select count(*) as c from `%s_log` where %s", (self.test_table, filter_sql))
row2 = dbh.fetch()
self.assertEqual(row['c'], row2['c'])
def test04LoadFileTwo(self):
""" Test that Simple Log files can be loaded when they contain weird spaces"""
dbh = DB.DBO(self.test_case)
log = LogFile.load_preset(self.test_case, self.log_preset_two, [self.test_file_two])
t = time.time()
## Load the data:
for a in log.load(self.test_table_two):
pass
print "Took %s seconds to load log" % (time.time()-t)
## Check that all rows were uploaded:
dbh.execute("select count(*) as c from `%s_log`", self.test_table_two)
row = dbh.fetch()
self.assertEqual(row['c'], 12)
def test02LoadFile(self):
""" Test that Syslog Log files can be loaded """
dbh = DB.DBO(self.test_case)
log = LogFile.load_preset(self.test_case, self.log_preset, [self.test_file])
t = time.time()
## Load the data:
for a in log.load(self.test_table):
pass
print "Took %s seconds to load log" % (time.time()-t)
## Check that all rows were uploaded:
dbh.execute("select count(*) as c from %s_log", self.test_table)
row = dbh.fetch()
self.assertEqual(row['c'], 2433)
def test02LoadFile(self):
""" Test that Advanced Log files can be loaded """
dbh = DB.DBO(self.test_case)
log = LogFile.load_preset(self.test_case, self.log_preset, [self.test_file])
t = time.time()
## Load the data:
for a in log.load(self.test_table):
pass
print "Took %s seconds to load log" % (time.time()-t)
## Check that all rows were uploaded:
dbh.execute("select count(*) as c from %s_log", self.test_table)
row = dbh.fetch()
self.assertEqual(row['c'], 3424)
def form(self, query, result):
try:
result.case_selector()
result.selector(
"Select Log Preset", "preset", "select name as `key`, name as `value` from log_presets", autosubmit=True
)
result.textfield("Table name:", "table")
result.fileselector("log files to use", "datafile")
try:
if query.getarray("datafile"):
log = LogFile.load_preset(query["case"], query["preset"], query.getarray("datafile"))
else:
return result
except Exception, e:
result.end_table()
result.text("\nError: %s\n\n" % e, style="red")
# raise
return result
## Draw the filter GUI:
result.add_filter(
query, query["case"], code_parser.parse_eval, log.fields, search_text="Pre-Filter Expression"
)
result.ruler()
result.end_table()
result.para(
"Please check the test table before continuing. Note that the filter is not applied to the test table."
)
# show preview
result.start_table()
dbh = DB.DBO(query["case"])
try:
# retrieve and display the temp table
log.display_test_log(result, filter=None)
except Exception, e:
result.text(
"Error: Unable to load a test set - maybe this log file is incompatible with this log preset?\nError was %s"
% e,
style="red",
font="bold",
)
pyflaglog.log(pyflaglog.DEBUG, "Unable to load test set - error returned was %s" % e)
return
def test04LoadFileTwo(self):
""" Test that Simple Log files can be loaded when they contain weird spaces"""
dbh = DB.DBO(self.test_case)
log = LogFile.load_preset(self.test_case, self.log_preset_two,
[self.test_file_two])
t = time.time()
## Load the data:
for a in log.load(self.test_table_two):
pass
print "Took %s seconds to load log" % (time.time() - t)
## Check that all rows were uploaded:
dbh.execute("select count(*) as c from `%s_log`", self.test_table_two)
row = dbh.fetch()
self.assertEqual(row['c'], 12)
def test02LoadFile(self):
""" Test that Apache Log files can be loaded """
## See if we can load the preset again:
log = LogFile.load_preset(self.test_case, self.log_preset, [self.datafile])
t = time.time()
## Load the data:
for a in log.load(self.test_table, filter=None):
print a
print "Took %s seconds to load log" % (time.time()-t)
## Check that all rows were uploaded:
dbh = DB.DBO(self.test_case)
dbh.execute("select count(*) as c from `%s_log`", self.test_table)
row = dbh.fetch()
self.assertEqual(row['c'], 10000)
def display(self,query,result):
if (query.has_key('limit')):
result.heading("Log File in Table %s" % query['logtable'])
else:
result.heading("Log File in Table %s" % query['logtable'])
dbh = DB.DBO(query['case'])
## Fetch the driver to use:
dbh.execute("select * from log_tables where table_name='%s' limit 1",(query['logtable']))
row=dbh.fetch()
if not row:
raise Reports.ReportError("Log Table %s not found" % query['logtable'])
try:
## Instantiate the driver on this case:
log = LogFile.load_preset(query['case'],
row['preset'])
except KeyError,e:
raise Reports.ReportError("Unable to load the preset %s for table %s " % (row['preset'],query['logtable']))
def form(self, query, result):
try:
result.case_selector()
result.selector('Select Log Preset','preset',
"select name as `key`, name as `value` from log_presets",
autosubmit=True)
result.textfield("Table name:","table")
result.fileselector("log files to use", 'datafile')
try:
if query.getarray('datafile'):
log = LogFile.load_preset(query['case'], query['preset'], query.getarray('datafile'))
else:
return result
except Exception,e:
result.end_table()
result.text("\nError: %s\n\n" % e, style='red')
# raise
return result
## Draw the filter GUI:
result.add_filter(query, query['case'], code_parser.parse_eval,
log.fields, search_text = "Pre-Filter Expression")
result.ruler()
result.end_table()
result.para("Please check the test table before continuing. Note that the filter is not applied to the test table.")
# show preview
result.start_table()
dbh = DB.DBO(query['case'])
try:
# retrieve and display the temp table
log.display_test_log(result, filter=None)
except Exception,e:
result.text("Error: Unable to load a test set - maybe this log file is incompatible with this log preset?\nError was %s" % e,style='red',font='bold')
pyflaglog.log(pyflaglog.DEBUG,"Unable to load test set - error returned was %s" % e)
return
def display(self, query, result):
if (query.has_key('limit')):
result.heading("Log File in Table %s" % query['logtable'])
else:
result.heading("Log File in Table %s" % query['logtable'])
dbh = DB.DBO(query['case'])
## Fetch the driver to use:
dbh.execute("select * from log_tables where table_name='%s' limit 1",
(query['logtable']))
row = dbh.fetch()
if not row:
raise Reports.ReportError("Log Table %s not found" %
query['logtable'])
try:
## Instantiate the driver on this case:
log = LogFile.load_preset(query['case'], row['preset'])
except KeyError, e:
raise Reports.ReportError(
"Unable to load the preset %s for table %s " %
(row['preset'], query['logtable']))
def test02LoadFile(self):
""" Test that IIS Log files can be loaded """
dbh = DB.DBO(self.test_case)
log = LogFile.load_preset(self.test_case, self.log_preset, [self.test_file])
t = time.time()
## Load the data:
for a in log.load(self.test_table):
pass
print "Took %s seconds to load log" % (time.time() - t)
## Check that all rows were uploaded:
dbh.execute("select count(*) as c from `%s_log`", self.test_table)
row = dbh.fetch()
self.assertEqual(row["c"], 8334)
## More specific tests
dbh.execute("select count(*) as c from `%s_log` where `IP Address` = 2921232307", self.test_table)
row = dbh.fetch()
self.assertEqual(row["c"], 129)
dbh.execute("select count(*) as c from `%s_log` where cs_username = '******'", self.test_table)
row = dbh.fetch()
self.assertEqual(row["c"], 7898)
def analyse(self, query):
""" Load the log file into the table """
log = LogFile.load_preset(query['case'], query['preset'], query.getarray('datafile'))
for progress in log.load(query['table'], filter=query.get('filter')):
self.progress_str = progress
def analyse(self, query):
""" Load the log file into the table """
log = LogFile.load_preset(query["case"], query["preset"], query.getarray("datafile"))
for progress in log.load(query["table"], filter=query.get("filter")):
self.progress_str = progress
