def print_columns(self, paginate=None):
""" Print out data in a nice formatted table
`paginate` option will insert a new header after that many
rows have been printed. Defaults to None (only single header
output).
"""
if not self.data:
print "No data found."
return
if self.options.typelist:
# two columns only
headers = ['type_id', 'type']
data = self.data
elif self.options.ipaddr:
# single dict
headers = self.data.keys()
data = [self.data.values()]
else:
# assume objects are uniform and take keys from first one
headers = self.data[0].keys()
data = [d.values() for d in self.data]
Formatter.print_table(data, headers, paginate=30)
def main(self):
if self.options.list_groupbys:
header = ["GroupBy", "Id"]
data = [(k, v) for k,v in self.profiler.groupbys.iteritems()]
data.sort()
Formatter.print_table(data, header)
else:
if self.options.ids:
columns = self.profiler.get_columns_by_ids(self.options.ids)
else:
o = self.options
# find groupby looking in keys and values
if o.groupby in self.profiler.groupbys:
groupby = self.profiler.groupbys[o.groupby]
elif o.groupby in self.profiler.groupbys.values():
groupby = o.groupby
else:
groupby = None
args = {
'realms': [o.realm] if o.realm else None,
'centricities': [o.centricity] if o.centricity else None,
'groupbys': [groupby] if groupby else None,
}
columns = self.profiler.search_columns(**args)
columns.sort(key=lambda x: x.key)
self.print_columns(columns)
def handle(self, *args, **options):
""" Main command handler. """
if options['job_list']:
# print out the id's instead of processing anything
columns = ['ID', 'Table', 'Created', 'Touched', 'Sts', 'Refs',
'Progress', 'Data file']
data = []
for j in Job.objects.all():
datafile = j.datafile()
if not os.path.exists(datafile):
datafile += " (missing)"
data.append([j.id, j.table.name, j.created, j.touched,
j.status, j.refcount, j.progress, datafile])
Formatter.print_table(data, columns)
elif options['job_data']:
job = Job.objects.get(id=options['job_data'])
columns = [c.name for c in job.table.get_columns()]
if job.status == job.COMPLETE:
Formatter.print_table(job.values(), columns)
elif options['job_age']:
Job.age_jobs(force=True)
elif options['job_flush']:
Job.objects.all().delete()
def main(self):
""" Setup query and run report with default column set
"""
if self.options.timerange:
timefilter = TimeFilter.parse_range(self.options.timerange)
else:
timefilter = TimeFilter(self.options.time0, self.options.time1)
trafficexpr = TrafficFilter(self.options.trafficexpr)
columns = [self.profiler.columns.key.srv_host_ip,
self.profiler.columns.key.app_info,
self.profiler.columns.key.start_time,
self.profiler.columns.key.end_time,
self.profiler.columns.value.s2c_total_bytes,
self.profiler.columns.value.s2c_total_pkts,
self.profiler.columns.value.response_time,
self.profiler.columns.value.server_delay]
report = TrafficFlowListReport(self.profiler)
report.run(columns, timefilter=timefilter, trafficexpr=trafficexpr)
data = report.get_data()
report.delete()
headers = [c.key for c in columns]
Formatter.print_table(data, headers)
def handle(self, *args, **options):
""" Main command handler
"""
self.options = options
if options['job_list']:
# print out the id's instead of processing anything
columns = ['ID', 'Table', 'Created', 'Touched', 'Sts', 'Refs', 'Progress', 'Data file']
data = []
for j in Job.objects.all():
datafile = j.datafile()
if not os.path.exists(datafile):
datafile += " (missing)"
data.append ([j.id, j.table.name, j.created, j.touched,
j.status, j.refcount, j.progress, datafile])
Formatter.print_table(data, columns)
elif options['job_data']:
job = Job.objects.get(id=options['job_data'])
columns = [c.name for c in job.table.get_columns()]
if job.status == job.COMPLETE:
Formatter.print_table(job.values(), columns)
elif options['job_age']:
Job.age_jobs(force=True)
elif options['job_flush']:
Job.objects.all().delete()
def main(self):
headers = ['ID', 'Description', 'Type']
data = [(f.id, f.description, f.type) for f in self.shark.get_extractor_fields()]
if self.options.sort_id:
data.sort()
Formatter.print_table(data,
headers,
padding=2,
max_width=int(self.options.table_width),
long_column=1,
wrap_columns=(not self.options.truncate))
def print_columns(self, columns):
keys = []
values = []
for c in columns:
if (self.options.filter and
self.options.filter.lower() not in c.label.lower()):
continue
item = (c.key, c.label, c.id)
if c.iskey:
keys.append(item)
else:
values.append(item)
Formatter.print_table(keys, ['Key Columns', 'Label', 'ID'])
print ''
Formatter.print_table(values, ['Value Columns', 'Label', 'ID'])
def print_columns(self, columns):
keys = []
values = []
for c in columns:
if (self.options.filter
and self.options.filter.lower() not in c.label.lower()):
continue
item = (c.key, c.label, c.id)
if c.iskey:
keys.append(item)
else:
values.append(item)
Formatter.print_table(keys, ['Key Columns', 'Label', 'ID'])
print ''
Formatter.print_table(values, ['Value Columns', 'Label', 'ID'])
def main(self):
if self.options.list_groupbys:
header = ["GroupBy", "Id"]
data = [(k, v) for k,v in self.profiler.groupbys.iteritems()]
Formatter.print_table(data, header)
if self.options.list_columns:
if self.options.ids:
columns = self.profiler.get_columns_by_ids(self.options.ids)
else:
o = self.options
args = {
'realms': [o.realm] if o.realm else None,
'centricities': [o.centricity] if o.centricity else None,
'groupbys': [self.profiler.groupbys[o.groupby]] if o.groupby else None,
}
columns = self.profiler.search_columns(**args)
columns.sort(key=lambda x: x.key)
self.print_columns(columns)
def main(self):
if self.options.list_groupbys:
header = ["GroupBy", "Id"]
data = [(k, v) for k, v in self.profiler.groupbys.iteritems()]
Formatter.print_table(data, header)
if self.options.list_columns:
if self.options.ids:
columns = self.profiler.get_columns_by_ids(self.options.ids)
else:
o = self.options
args = {
'realms': [o.realm] if o.realm else None,
'centricities': [o.centricity] if o.centricity else None,
'groupbys':
[self.profiler.groupbys[o.groupby]] if o.groupby else None,
}
columns = self.profiler.search_columns(**args)
columns.sort(key=lambda x: x.key)
self.print_columns(columns)
def main(self):
""" Setup query and run report with default column set
"""
if self.options.timerange:
timefilter = TimeFilter.parse_range(self.options.timerange)
else:
timefilter = TimeFilter(self.options.time0, self.options.time1)
if self.options.trafficexpr:
trafficexpr = TrafficFilter(self.options.trafficexpr)
else:
trafficexpr = None
legend_columns, all_data = self.identity_report(timefilter=timefilter,
trafficexpr=trafficexpr,
testfile=self.options.testfile)
legend, activity = self.analyze_login_data(all_data, legend_columns)
if self.options.timeseries_report:
headers, tbl_data = self.generate_traffic(activity, legend, 'timeseries')
elif self.options.summary_report:
headers, tbl_data = self.generate_traffic(activity, legend, 'summary')
else:
headers = ('Host IP', 'Login Time', 'Logout Time', 'Duration')
tbl_data = [(x[0], format_time(x[1]), format_time(x[2]), x[3])
for x in activity]
if self.options.csv:
Formatter.print_csv(tbl_data, headers)
elif self.options.tsv:
Formatter.print_csv(tbl_data, headers, delim='\t')
else:
Formatter.print_table(tbl_data, headers)
def handle(self, *args, **options):
""" Main command handler. """
if options['job_list']:
# print out the id's instead of processing anything
columns = ['ID', 'PID', 'Table', 'Created', 'Touched', 'Sts',
'Refs', 'Progress', 'Data file']
data = []
for j in Job.objects.all().order_by('id'):
datafile = j.datafile()
if not os.path.exists(datafile):
datafile += " (missing)"
parent_id = j.parent.id if j.parent else '--'
data.append([j.id, parent_id, j.table.name, j.created,
j.touched, j.status, j.refcount, j.progress,
datafile])
Formatter.print_table(data, columns)
elif options['job_data']:
job = Job.objects.get(id=options['job_data'])
columns = [c.name for c in job.table.get_columns()]
if job.status == job.COMPLETE:
Formatter.print_table(job.values(), columns)
elif options['job_age']:
logger.debug('Aging all jobs.')
Job.age_jobs(force=True)
elif options['job_flush']:
logger.debug('Flushing all jobs.')
while Job.objects.count():
ids = Job.objects.values_list('pk', flat=True)[:100]
Job.objects.filter(pk__in=ids).delete()
def handle(self, *args, **options):
""" Main command handler
"""
self.options = options
if options['table_list']:
# print out the id's instead of processing anything
tables = Table.objects.all()
for t in tables:
self.console('%5d - %s' % (t.id, t))
elif options['table_list_by_report']:
# or print them out organized by report/widget/table
output = []
reports = Report.objects.all()
for report in reports:
for section in report.section_set.all():
for widget in section.widget_set.all():
for table in widget.tables.all():
line = [table.id, report.title, widget.title, table]
output.append(line)
Formatter.print_table(output, ['ID', 'Report', 'Widget', 'Table'])
elif options['criteria_list']:
if 'table_id' in options and options['table_id'] is not None:
table = Table.objects.get(id=options['table_id'])
elif 'table_name' in options:
table = Table.objects.get(name=options['table_name'])
else:
raise ValueError("Must specify either --table-id or "
"--table-name to run a table")
form = self.get_form(table)
output = [[c.keyword, c.label] for c in form._tablefields.values()]
Formatter.print_table(output, ['Keyword', 'Label'])
else:
if 'table_id' in options and options['table_id'] is not None:
table = Table.objects.get(id=options['table_id'])
elif 'table_name' in options:
table = Table.objects.get(name=options['table_name'])
else:
raise ValueError("Must specify either --table-id or "
"--table-name to run a table")
# Django gives us a nice error if we can't find the table
self.console('Table %s found.' % table)
# Parse criteria options
criteria_options = {}
if 'criteria' in options and options['criteria'] is not None:
for s in options['criteria']:
(k, v) = s.split(':', 1)
criteria_options[k] = v
form = self.get_form(table, data=criteria_options)
if not form.is_valid(check_unknown=True):
self.console('Invalid criteria:')
logger.error('Invalid criteria: %s' %
','.join('%s:%s' % (k, v)
for k, v in form.errors.iteritems()))
for k, v in form.errors.iteritems():
self.console(' %s: %s' % (k, ','.join(v)))
sys.exit(1)
criteria = form.criteria()
columns = [c.name for c in table.get_columns()]
if options['only_columns']:
print columns
return
job = Job.create(table=table, criteria=criteria)
job.save()
self.console('Job created: %s' % job)
self.console('Criteria: %s' % criteria.print_details())
start_time = datetime.datetime.now()
job.start()
self.console('Job running . . ', ending='')
# wait for results
while not job.done():
#self.console('. ', ending='')
#self.stdout.flush()
time.sleep(1)
end_time = datetime.datetime.now()
delta = end_time - start_time
seconds = float(delta.microseconds +
(delta.seconds + delta.days * 24 * 3600) * 10**6) / 10**6
self.console('Done!! (elapsed time: %.2f seconds)' % seconds)
self.console('')
# Need to refresh the column list in case the job changed them
# (ephemeral cols)
columns = [c.name for c in table.get_columns()]
if job.status == job.COMPLETE:
if options['as_csv']:
if options['output_file']:
with open(options['output_file'], 'w') as f:
for line in Formatter.get_csv(job.values(), columns):
f.write(line)
f.write('\n')
else:
Formatter.print_csv(job.values(), columns)
else:
Formatter.print_table(job.values(), columns)
else:
self.console("Job completed with an error:")
self.console(job.message)
sys.exit(1)
def print_data(self, data, header):
if self.options.as_csv:
Formatter.print_csv(data, header)
else:
Formatter.print_table(data, header)
def handle(self, *args, **options):
""" Main command handler
"""
self.options = options
if options['table_list']:
# print out the id's instead of processing anything
tables = Table.objects.all()
for t in tables:
self.console('%5d - %s' % (t.id, t))
elif options['table_list_by_report']:
# or print them out organized by report/widget/table
output = []
reports = Report.objects.all()
for report in reports:
for section in report.section_set.all():
for widget in section.widget_set.all():
for table in widget.tables.all():
line = [
table.id, report.title, widget.title, table
]
output.append(line)
Formatter.print_table(output, ['ID', 'Report', 'Widget', 'Table'])
elif options['criteria_list']:
if 'table_id' in options and options['table_id'] is not None:
table = Table.objects.get(id=options['table_id'])
elif 'table_name' in options:
table = Table.objects.get(name=options['table_name'])
else:
raise ValueError("Must specify either --table-id or "
"--table-name to run a table")
form = self.get_form(table)
output = [[c.keyword, c.label] for c in form._tablefields.values()]
Formatter.print_table(output, ['Keyword', 'Label'])
else:
if 'table_id' in options and options['table_id'] is not None:
table = Table.objects.get(id=options['table_id'])
elif 'table_name' in options:
table = Table.objects.get(name=options['table_name'])
else:
raise ValueError("Must specify either --table-id or "
"--table-name to run a table")
# Django gives us a nice error if we can't find the table
self.console('Table %s found.' % table)
# Parse criteria options
criteria_options = {}
if 'criteria' in options and options['criteria'] is not None:
for s in options['criteria']:
(k, v) = s.split(':', 1)
criteria_options[k] = v
form = self.get_form(table, data=criteria_options)
if not form.is_valid(check_unknown=True):
self.console('Invalid criteria:')
logger.error('Invalid criteria: %s' %
','.join('%s:%s' % (k, v)
for k, v in form.errors.iteritems()))
for k, v in form.errors.iteritems():
self.console(' %s: %s' % (k, ','.join(v)))
sys.exit(1)
criteria = form.criteria()
columns = [c.name for c in table.get_columns()]
if options['only_columns']:
print columns
return
job = Job.create(table=table, criteria=criteria)
job.save()
self.console('Job created: %s' % job)
self.console('Criteria: %s' % criteria.print_details())
start_time = datetime.datetime.now()
job.start()
self.console('Job running . . ', ending='')
# wait for results
while not job.done():
#self.console('. ', ending='')
#self.stdout.flush()
time.sleep(1)
end_time = datetime.datetime.now()
delta = end_time - start_time
seconds = float(delta.microseconds +
(delta.seconds + delta.days * 24 * 3600) *
10**6) / 10**6
self.console('Done!! (elapsed time: %.2f seconds)' % seconds)
self.console('')
# Need to refresh the column list in case the job changed them
# (ephemeral cols)
columns = [c.name for c in table.get_columns()]
if job.status == job.COMPLETE:
if options['as_csv']:
if options['output_file']:
with open(options['output_file'], 'w') as f:
for line in Formatter.get_csv(
job.values(), columns):
f.write(line)
f.write('\n')
else:
Formatter.print_csv(job.values(), columns)
else:
Formatter.print_table(job.values(), columns)
else:
self.console("Job completed with an error:")
self.console(job.message)
sys.exit(1)
def handle(self, *args, **options):
""" Main command handler
"""
self.options = options
if options['table_list']:
# print out the id's instead of processing anything
tables = Table.objects.all()
for t in tables:
self.console('%5d - %s' % (t.id, t))
elif options['table_list_by_report']:
# or print them out organized by report/widget/table
output = []
reports = Report.objects.all()
for report in reports:
for widget in report.widget_set.all():
for table in widget.tables.all():
line = [table.id, report.title, widget.title, table]
output.append(line)
Formatter.print_table(output, ['ID', 'Report', 'Widget', 'Table'])
else:
if 'table_id' in options and options['table_id'] is not None:
table = Table.objects.get(id=options['table_id'])
elif 'table_name' in options:
table = Table.objects.get(name=options['table_name'])
else:
raise ValueError("Must specify either --table-id or --table-name to run a table")
# Django gives us a nice error if we can't find the table
self.console('Table %s found.' % table)
# Look for a related report
widgets = Widget.objects.filter(tables__in=[table])
if len(widgets) > 0:
report = widgets[0].report
form = create_report_criteria_form(report=report)
else:
form = None
add_options = {}
if 'criteria' in options and options['criteria'] is not None:
for s in options['criteria']:
(k,v) = s.split(':', 1)
add_options[k] = v
if 'endtime' in options and options['endtime'] is not None:
try:
endtime = form.fields['endtime'].clean(options['endtime'])
except ValidationError:
raise ValidationError("Could not parse endtime: %s, try MM/DD/YYYY HH:MM am" % options['endtime'])
tz = pytz.timezone("US/Eastern")
endtime = endtime.replace(tzinfo=tz)
else:
endtime = datetime.datetime.now()
criteria = Criteria(endtime=datetime_to_seconds(endtime),
duration=options['duration'],
filterexpr=options['filterexpr'],
table=table,
ignore_cache=options['ignore_cache'])
if form:
for k,field in form.fields.iteritems():
if not k.startswith('criteria_'): continue
tc = TableCriteria.objects.get(pk=k.split('_')[1])
if ( options['criteria'] is not None and
tc.keyword in add_options):
val = add_options[tc.keyword]
else:
val = field.initial
# handle table criteria and generate children objects
tc = TableCriteria.get_instance(k, val)
criteria[k] = tc
for child in tc.children.all():
child.value = val
criteria['criteria_%d' % child.id] = child
columns = [c.name for c in table.get_columns()]
if options['only_columns']:
print columns
return
job = Job.create(table=table, criteria=criteria)
job.save()
self.console('Job created: %s' % job)
self.console('Criteria: %s' % criteria.print_details())
start_time = datetime.datetime.now()
job.start()
self.console('Job running . . ', ending='')
# wait for results
while not job.done():
#self.console('. ', ending='')
#self.stdout.flush()
time.sleep(1)
end_time = datetime.datetime.now()
delta = end_time - start_time
seconds = float(delta.microseconds +
(delta.seconds + delta.days * 24 * 3600) * 10**6) / 10**6
self.console('Done!! (elapsed time: %.2f seconds)' % seconds)
self.console('')
# Need to refresh the column list in case the job changed them (ephemeral cols)
columns = [c.name for c in table.get_columns()]
if job.status == job.COMPLETE:
if options['as_csv']:
Formatter.print_csv(job.values(), columns)
else:
Formatter.print_table(job.values(), columns)
else:
self.console("Job completed with an error:")
self.console(job.message)
sys.exit(1)
def handle(self, *args, **options):
""" Main command handler
"""
self.options = options
if options['table_list']:
# print out the id's instead of processing anything
tables = Table.objects.all()
for t in tables:
self.console('%5d - %s' % (t.id, t))
elif options['table_list_by_report']:
# or print them out organized by report/widget/table
output = []
reports = Report.objects.all()
for report in reports:
for widget in report.widget_set.all():
for table in widget.tables.all():
line = [table.id, report.title, widget.title, table]
output.append(line)
Formatter.print_table(output, ['ID', 'Report', 'Widget', 'Table'])
else:
table = Table.objects.get(id=options['table_id'])
# Django gives us a nice error if we can't find the table
self.console('Table %s found.' % table)
criteria = Criteria(endtime=options['endtime'],
duration=options['duration'],
filterexpr=options['filterexpr'],
table=table,
ignore_cache=options['ignore_cache'])
job = Job(table=table, criteria=criteria)
job.save()
try:
self.console('Job created: %s' % job)
self.console('Criteria: %s' % criteria.print_details())
start_time = datetime.datetime.now()
job.start()
self.console('Job running . . ', ending='')
# wait for results
while not job.done():
self.console('. ', ending='')
self.stdout.flush()
time.sleep(1)
end_time = datetime.datetime.now()
delta = end_time - start_time
self.console('Done!! (took roughly %.2f seconds)' % delta.total_seconds())
self.console('')
columns = [c.name for c in table.get_columns()]
if not options['csv']:
Formatter.print_table(job.data(), columns)
else:
Formatter.print_csv(job.data(), columns)
finally:
job.delete()