def display(self, query, result):
if not query.has_key('table'):
result.heading("Delete a table from this case")
def DeleteIcon(value):
tmp = result.__class__(result)
target = query.clone()
target.set('table', value)
tmp.link("Delete", icon="no.png", target=target)
return tmp
result.table(elements=[
ColumnType("Delete?", 'table_name', callback=DeleteIcon),
StringType("Table Name", 'table_name'),
StringType("Type", "preset"),
],
table="log_tables",
case=query['case'])
elif query.has_key('confirm'):
LogFile.drop_table(query['case'], query['table'])
result.refresh(
0,
query_type(family=query['family'],
case=query['case'],
report=query['report']))
else:
result.heading("About to remove %s" % query['table'])
query['confirm'] = 1
result.link(
"Are you sure you want to drop table %s. Click here to confirm"
% query['table'], query)
def display(self,query, result):
if not query.has_key('table'):
result.heading("Delete a table from this case")
def DeleteIcon(value):
tmp=result.__class__(result)
target = query.clone()
target.set('table',value)
tmp.link("Delete", icon="no.png",
target=target)
return tmp
result.table(
elements = [ ColumnType("Delete?",'table_name',
callback = DeleteIcon),
StringType("Table Name",'table_name'),
StringType("Type", "preset"),
],
table="log_tables",
case=query['case']
)
elif query.has_key('confirm'):
LogFile.drop_table(query['case'] , query['table'])
result.refresh(0, query_type(family=query['family'], case=query['case'],
report=query['report']))
else:
result.heading("About to remove %s" % query['table'])
query['confirm']=1
result.link("Are you sure you want to drop table %s. Click here to confirm"% query['table'], query)
def display(self, query, result):
LogFile.drop_preset(query['log_preset'])
result.heading("Deleted preset %s from the database" %
query['log_preset'])
result.link(
"Manage more Presets",
query_type(family=query['family'], report="Manage Log Presets"))
def form(self,query,result):
try:
result.selector("Select Preset to delete",'log_preset',
"select name as `key`, name as `value` from log_presets",
case = None)
found=0
result.row("The following will also be dropped:",'')
tmp=result.__class__(result)
tmp.start_table(**{'class':'GeneralTable'})
left=result.__class__(result)
right=result.__class__(result)
left.text("Case",font='bold',style='red')
right.text("Table",font='bold',style='red')
tmp.row(left,right)
for case, tablename in LogFile.find_tables(query['log_preset']):
tmp.row(case, tablename)
result.row(tmp)
result.checkbox("Are you sure you want to do this?","confirm",'yes')
except KeyError,e:
print e
pass
def test02LoadFile(self):
""" Test that IIS Log files can be loaded """
dbh = DB.DBO(self.test_case)
log = LogFile.load_preset(self.test_case, self.log_preset,
[self.test_file])
t = time.time()
## Load the data:
for a in log.load(self.test_table):
pass
print "Took %s seconds to load log" % (time.time() - t)
## Check that all rows were uploaded:
dbh.execute("select count(*) as c from `%s_log`", self.test_table)
row = dbh.fetch()
self.assertEqual(row['c'], 8334)
## More specific tests
dbh.execute(
"select count(*) as c from `%s_log` where `IP Address` = 2921232307",
self.test_table)
row = dbh.fetch()
self.assertEqual(row['c'], 129)
dbh.execute(
"select count(*) as c from `%s_log` where cs_username = '******'",
self.test_table)
row = dbh.fetch()
self.assertEqual(row['c'], 7898)
def test03FilteredLoad(self):
""" Test that filter expressions work when loading """
filter = ' "rhost" maxmind_country AUS and "status" = 404 '
## See if we can load the preset again:
log = LogFile.load_preset(self.test_case, self.log_preset, [self.datafile])
t = time.time()
table_name = self.test_table + "filtered"
## Load the data:
for a in log.load(table_name, filter=filter):
print a
print "Took %s seconds to load log" % (time.time()-t)
## How many rows were inserted
dbh = DB.DBO(self.test_case)
dbh.execute("select count(*) as c from `%s_log`", table_name)
row = dbh.fetch()
## Now try to enforce the same filter on the original table
## (which has all the rows in it):
for f in log.fields: f.table = self.test_table+"_log"
filter_sql = parser.parse_to_sql(filter, log.fields, None)
dbh.execute("select count(*) as c from `%s_log` where %s", (self.test_table, filter_sql))
row2 = dbh.fetch()
self.assertEqual(row['c'], row2['c'])
def form(self, query, result):
try:
result.selector(
"Select Preset to delete",
'log_preset',
"select name as `key`, name as `value` from log_presets",
case=None)
found = 0
result.row("The following will also be dropped:", '')
tmp = result.__class__(result)
tmp.start_table(**{'class': 'GeneralTable'})
left = result.__class__(result)
right = result.__class__(result)
left.text("Case", font='bold', style='red')
right.text("Table", font='bold', style='red')
tmp.row(left, right)
for case, tablename in LogFile.find_tables(query['log_preset']):
tmp.row(case, tablename)
result.row(tmp)
result.checkbox("Are you sure you want to do this?", "confirm",
'yes')
except KeyError, e:
print e
pass
def test02LoadFile(self):
""" Test that Syslog Log files can be loaded """
dbh = DB.DBO(self.test_case)
log = LogFile.load_preset(self.test_case, self.log_preset, [self.test_file])
t = time.time()
## Load the data:
for a in log.load(self.test_table):
pass
print "Took %s seconds to load log" % (time.time()-t)
## Check that all rows were uploaded:
dbh.execute("select count(*) as c from %s_log", self.test_table)
row = dbh.fetch()
self.assertEqual(row['c'], 2433)
def test04LoadFileTwo(self):
""" Test that Simple Log files can be loaded when they contain weird spaces"""
dbh = DB.DBO(self.test_case)
log = LogFile.load_preset(self.test_case, self.log_preset_two, [self.test_file_two])
t = time.time()
## Load the data:
for a in log.load(self.test_table_two):
pass
print "Took %s seconds to load log" % (time.time()-t)
## Check that all rows were uploaded:
dbh.execute("select count(*) as c from `%s_log`", self.test_table_two)
row = dbh.fetch()
self.assertEqual(row['c'], 12)
def test02LoadFile(self):
""" Test that Advanced Log files can be loaded """
dbh = DB.DBO(self.test_case)
log = LogFile.load_preset(self.test_case, self.log_preset, [self.test_file])
t = time.time()
## Load the data:
for a in log.load(self.test_table):
pass
print "Took %s seconds to load log" % (time.time()-t)
## Check that all rows were uploaded:
dbh.execute("select count(*) as c from %s_log", self.test_table)
row = dbh.fetch()
self.assertEqual(row['c'], 3424)
def form(self, query, result):
try:
result.case_selector()
result.selector(
"Select Log Preset", "preset", "select name as `key`, name as `value` from log_presets", autosubmit=True
)
result.textfield("Table name:", "table")
result.fileselector("log files to use", "datafile")
try:
if query.getarray("datafile"):
log = LogFile.load_preset(query["case"], query["preset"], query.getarray("datafile"))
else:
return result
except Exception, e:
result.end_table()
result.text("\nError: %s\n\n" % e, style="red")
# raise
return result
## Draw the filter GUI:
result.add_filter(
query, query["case"], code_parser.parse_eval, log.fields, search_text="Pre-Filter Expression"
)
result.ruler()
result.end_table()
result.para(
"Please check the test table before continuing. Note that the filter is not applied to the test table."
)
# show preview
result.start_table()
dbh = DB.DBO(query["case"])
try:
# retrieve and display the temp table
log.display_test_log(result, filter=None)
except Exception, e:
result.text(
"Error: Unable to load a test set - maybe this log file is incompatible with this log preset?\nError was %s"
% e,
style="red",
font="bold",
)
pyflaglog.log(pyflaglog.DEBUG, "Unable to load test set - error returned was %s" % e)
return
def test04LoadFileTwo(self):
""" Test that Simple Log files can be loaded when they contain weird spaces"""
dbh = DB.DBO(self.test_case)
log = LogFile.load_preset(self.test_case, self.log_preset_two,
[self.test_file_two])
t = time.time()
## Load the data:
for a in log.load(self.test_table_two):
pass
print "Took %s seconds to load log" % (time.time() - t)
## Check that all rows were uploaded:
dbh.execute("select count(*) as c from `%s_log`", self.test_table_two)
row = dbh.fetch()
self.assertEqual(row['c'], 12)
def test02LoadFile(self):
""" Test that Apache Log files can be loaded """
## See if we can load the preset again:
log = LogFile.load_preset(self.test_case, self.log_preset, [self.datafile])
t = time.time()
## Load the data:
for a in log.load(self.test_table, filter=None):
print a
print "Took %s seconds to load log" % (time.time()-t)
## Check that all rows were uploaded:
dbh = DB.DBO(self.test_case)
dbh.execute("select count(*) as c from `%s_log`", self.test_table)
row = dbh.fetch()
self.assertEqual(row['c'], 10000)
def display(self,query,result):
if (query.has_key('limit')):
result.heading("Log File in Table %s" % query['logtable'])
else:
result.heading("Log File in Table %s" % query['logtable'])
dbh = DB.DBO(query['case'])
## Fetch the driver to use:
dbh.execute("select * from log_tables where table_name='%s' limit 1",(query['logtable']))
row=dbh.fetch()
if not row:
raise Reports.ReportError("Log Table %s not found" % query['logtable'])
try:
## Instantiate the driver on this case:
log = LogFile.load_preset(query['case'],
row['preset'])
except KeyError,e:
raise Reports.ReportError("Unable to load the preset %s for table %s " % (row['preset'],query['logtable']))
def form(self, query, result):
try:
result.case_selector()
result.selector('Select Log Preset','preset',
"select name as `key`, name as `value` from log_presets",
autosubmit=True)
result.textfield("Table name:","table")
result.fileselector("log files to use", 'datafile')
try:
if query.getarray('datafile'):
log = LogFile.load_preset(query['case'], query['preset'], query.getarray('datafile'))
else:
return result
except Exception,e:
result.end_table()
result.text("\nError: %s\n\n" % e, style='red')
# raise
return result
## Draw the filter GUI:
result.add_filter(query, query['case'], code_parser.parse_eval,
log.fields, search_text = "Pre-Filter Expression")
result.ruler()
result.end_table()
result.para("Please check the test table before continuing. Note that the filter is not applied to the test table.")
# show preview
result.start_table()
dbh = DB.DBO(query['case'])
try:
# retrieve and display the temp table
log.display_test_log(result, filter=None)
except Exception,e:
result.text("Error: Unable to load a test set - maybe this log file is incompatible with this log preset?\nError was %s" % e,style='red',font='bold')
pyflaglog.log(pyflaglog.DEBUG,"Unable to load test set - error returned was %s" % e)
return
def display(self, query, result):
if (query.has_key('limit')):
result.heading("Log File in Table %s" % query['logtable'])
else:
result.heading("Log File in Table %s" % query['logtable'])
dbh = DB.DBO(query['case'])
## Fetch the driver to use:
dbh.execute("select * from log_tables where table_name='%s' limit 1",
(query['logtable']))
row = dbh.fetch()
if not row:
raise Reports.ReportError("Log Table %s not found" %
query['logtable'])
try:
## Instantiate the driver on this case:
log = LogFile.load_preset(query['case'], row['preset'])
except KeyError, e:
raise Reports.ReportError(
"Unable to load the preset %s for table %s " %
(row['preset'], query['logtable']))
def test02LoadFile(self):
""" Test that IIS Log files can be loaded """
dbh = DB.DBO(self.test_case)
log = LogFile.load_preset(self.test_case, self.log_preset, [self.test_file])
t = time.time()
## Load the data:
for a in log.load(self.test_table):
pass
print "Took %s seconds to load log" % (time.time() - t)
## Check that all rows were uploaded:
dbh.execute("select count(*) as c from `%s_log`", self.test_table)
row = dbh.fetch()
self.assertEqual(row["c"], 8334)
## More specific tests
dbh.execute("select count(*) as c from `%s_log` where `IP Address` = 2921232307", self.test_table)
row = dbh.fetch()
self.assertEqual(row["c"], 129)
dbh.execute("select count(*) as c from `%s_log` where cs_username = '******'", self.test_table)
row = dbh.fetch()
self.assertEqual(row["c"], 7898)
def form(self, query, result):
if LogFile.save_preset(query,result, self):
query['finished']='yes'
result.refresh(0,query)
def form(self, query, result):
if LogFile.save_preset(query, result, self):
query['finished'] = 'yes'
result.refresh(0, query)
def execute(self):
for table in self.args:
yield "Deleting table %s" % table
LogFile.drop_table(self.environment._CASE, table)
def execute(self):
for preset in self.args:
yield "Deleting preset %s" % preset
LogFile.drop_preset(preset)
def analyse(self, query):
""" Load the log file into the table """
log = LogFile.load_preset(query['case'], query['preset'], query.getarray('datafile'))
for progress in log.load(query['table'], filter=query.get('filter')):
self.progress_str = progress
def analyse(self, query):
""" Load the log file into the table """
log = LogFile.load_preset(query["case"], query["preset"], query.getarray("datafile"))
for progress in log.load(query["table"], filter=query.get("filter")):
self.progress_str = progress
def display(self,query,result):
LogFile.drop_preset(query['log_preset'])
result.heading("Deleted preset %s from the database" % query['log_preset'])
result.link("Manage more Presets", query_type(family=query['family'],
report="Manage Log Presets"))
