def main(self):
self.tfilter1 = TrafficFilter(self.options.expr1)
self.report1 = TrafficSummaryReport(self.netprofiler)
self.columns_report1 = self.netprofiler.get_columns([self.options.column1])
self.trigger1 = safe_lambda(self.options.trigger1)
if self.options.trigger2 and self.options.column2 and self.options.expr2:
self.tfilter2 = TrafficFilter(self.options.expr2)
self.report2 = TrafficSummaryReport(self.netprofiler)
self.columns_report2 = self.netprofiler.get_columns([self.options.column2])
self.trigger2 = safe_lambda(self.options.trigger2)
else:
self.trigger2 = None
try:
while 1:
print('Running report 1 ...')
alert_flag = False
result = None
result2 = None
result = self.run_query(self.report1,
self.columns_report1,
self.tfilter1,
self.trigger1)
if result:
if self.trigger2:
print('Trigger 1 passed, running report 2 ...')
result2 = self.run_query(self.report2,
self.columns_report2,
self.tfilter2,
self.trigger2)
if result2:
print('Trigger 2 passed ...')
alert_flag = True
else:
print('Trigger 1 passed ...')
alert_flag = True
if alert_flag:
print('ALERT ALERT!')
self.update_description(result, result2)
self.send_trap()
time.sleep(self.options.refresh)
except KeyboardInterrupt:
print('Exiting ...')
sys.exit(0)
def test_traffic_overall_time_series_report(self):
columns = [
self.profiler.columns.key.time,
self.profiler.columns.value.avg_bytes,
self.profiler.columns.value.avg_pkts
]
timerange = TimeFilter.parse_range("last 1 h")
trafficexpr = TrafficFilter("host 10/8")
resolution = "15min"
report = TrafficOverallTimeSeriesReport(self.profiler)
report.run(columns, timerange, trafficexpr, resolution=resolution)
legend = report.get_legend()
keys = [c.key for c in legend]
data = report.get_data()
for item in data:
d = dict(zip(keys, item))
# resolution assumes 15-minute responses
self.assertTrue(
timerange.compare_time(d['time'], resolution=15 * 60))
report.delete()
def main(self):
# groupby validation should be part of validate_args, but there
# is no NetProfiler initialized at that part of the initialization
try:
self.groupby = self.netprofiler.groupbys[self.options.groupby]
except KeyError:
if self.options.groupby not in self.netprofiler.groupbys.values():
self.parser.error('Invalid groupby chosen.')
else:
self.groupby = self.options.groupby
self.timefilter = TimeFilter.parse_range(self.options.timefilter)
if self.options.trafficexpr:
self.trafficexpr = TrafficFilter(self.options.trafficexpr)
else:
self.trafficexpr = None
with TrafficSummaryReport(self.netprofiler) as report:
report.run(columns=self.options.columns.split(','),
groupby=self.groupby,
sort_col=self.options.sortby,
timefilter=self.timefilter,
trafficexpr=self.trafficexpr,
centricity=self.centricity,
resolution='auto')
data = report.get_data()
legend = [c.label for c in report.get_legend()]
self.print_data(data, legend)
def main(self):
self.timefilter = TimeFilter.parse_range(self.options.timefilter)
if self.options.trafficexpr:
self.trafficexpr = TrafficFilter(self.options.trafficexpr)
else:
self.trafficexpr = None
with MultiQueryReport(self.netprofiler) as report:
report.run(template_id=int(self.options.template_id),
timefilter=self.timefilter,
trafficexpr=self.trafficexpr)
print('Report Template {id} successfully run.'
.format(id=self.options.template_id))
self.netprofiler.conn.download(
'/api/profiler/1.6/reporting/reports/{id}/view.{fmt}'
.format(id=report.id, fmt=self.options.fmt),
path=self.options.pathname,
overwrite=True
)
print('Completed Report {id} downloaded to {path}.'
.format(id=report.id,
path=self.options.pathname))
def test_traffic_summary_report(self):
groupby = self.profiler.groupbys.host
columns = [self.profiler.columns.key.host_ip,
self.profiler.columns.value.avg_bytes,
self.profiler.columns.value.avg_pkts]
sort_col = self.profiler.columns.value.avg_bytes
timerange = self.yesterday
trafficexpr = TrafficFilter("host 10/8")
with TrafficSummaryReport(self.profiler) as rep:
rep.run(groupby, columns,
sort_col, timerange,
trafficexpr)
legend = rep.get_legend()
self.assertEqual(len(legend), 3)
legend = rep.get_legend(columns=[self.profiler.columns.key.host_ip,
self.profiler.columns.value.avg_bytes])
self.assertEqual(len(legend), 2)
self.assertEqual(legend[0].key, 'host_ip')
self.assertEqual(legend[1].key, 'avg_bytes')
data = rep.get_data()
if data:
self.assertEqual(len(data[0]), 3)
#check that data is refetched from cache
data = rep.get_data()
data = rep.get_data(columns=[self.profiler.columns.key.host_ip,
self.profiler.columns.value.avg_bytes])
if data:
self.assertEqual(len(data[0]), 2)
def test_resolution(self):
groupby = self.profiler.groupbys.host
columns = [self.profiler.columns.key.host_ip,
self.profiler.columns.value.avg_bytes,
self.profiler.columns.value.avg_pkts]
sort_col = self.profiler.columns.value.avg_bytes
trafficexpr = TrafficFilter("host 10/8")
resolutions = [["1min", "last 5 min"],
["15min", "last 1 hour"],
["hour", "last 4 hours"],
["6hour", "last 1 day"],
["day", "last 1 week"],
["3600", "last 4 hours"], # hour resolution
["60", "last 5 min"], # minute resolution
#["week", "last 4 weeks"],
#["month", "last 12 months"],
#Commented values blow up with a
#E RvbdHTTPException: 400 Unknown time resolution.
]
for (resolution, duration) in resolutions:
timerange = TimeFilter.parse_range(duration)
with TrafficSummaryReport(self.profiler) as rep:
rep.run(groupby, columns,
sort_col, timerange,
trafficexpr, resolution=resolution)
def main(self):
if self.options.listinterfacegroups:
self.list_interface_groups(self.options.host, self.options.sshusername,
self.options.sshpassword)
return
if self.options.listhostgroups:
self.list_host_groups(NetProfiler(self.options.host, auth=self.auth))
return
try:
timefilter = TimeFilter.parse_range(self.options.timefilter)
except ValueError:
print "Could not parse time filter expression."
return
profiler = NetProfiler(self.options.host, auth=self.auth)
if not self.options.clean:
print "Reporting on the period: {}".format(self.options.timefilter)
print "Using the traffic filter: {}".format(self.options.trafficfilter)
print "Calculating data at percentile {}".format(self.options.percentile)
print ("Averaging based on buckets of {} "
"minutes").format(self.options.buckettime)
if self.options.graph:
print "Saving a graph to {}".format(self.options.graph)
print
trafficfilter = TrafficFilter(self.options.trafficfilter)
self.report_item(profiler, timefilter, trafficfilter,
self.options.buckettime, self.options.percentile)
def main(self):
""" Setup query and run report with default column set
"""
if self.options.timerange:
timefilter = TimeFilter.parse_range(self.options.timerange)
else:
timefilter = TimeFilter(self.options.time0, self.options.time1)
trafficexpr = TrafficFilter(self.options.trafficexpr)
columns = [
self.netprofiler.columns.key.srv_host_ip,
self.netprofiler.columns.key.app_info,
self.netprofiler.columns.key.start_time,
self.netprofiler.columns.key.end_time,
self.netprofiler.columns.value.s2c_total_bytes,
self.netprofiler.columns.value.s2c_total_pkts,
self.netprofiler.columns.value.response_time,
self.netprofiler.columns.value.server_delay
]
report = TrafficFlowListReport(self.netprofiler)
report.run(columns, timefilter=timefilter, trafficexpr=trafficexpr)
data = report.get_data()
report.delete()
headers = [c.key for c in columns]
Formatter.print_table(data, headers)
def _prepare_report_args(self):
class Args(object):
pass
args = Args()
criteria = self.job.criteria
if criteria.netprofiler_device == '':
logger.debug('%s: No netprofiler device selected' % self.table)
self.job.mark_error("No NetProfiler Device Selected")
return False
args.profiler = DeviceManager.get_device(criteria.netprofiler_device)
args.columns = [
col.name for col in self.table.get_columns(synthetic=False)
]
args.sortcol = None
if self.table.sortcols is not None:
args.sortcol = self.table.sortcols[0]
args.timefilter = TimeFilter(start=criteria.starttime,
end=criteria.endtime)
logger.info("Running NetProfiler table %d report for timeframe %s" %
(self.table.id, str(args.timefilter)))
if ('datafilter' in criteria) and (criteria.datafilter is not None):
args.datafilter = criteria.datafilter.split(',')
else:
args.datafilter = None
args.trafficexpr = TrafficFilter(
self.job.combine_filterexprs(
exprs=criteria.netprofiler_filterexpr))
# Incoming criteria.resolution is a timedelta
logger.debug('NetProfiler report got criteria resolution %s (%s)' %
(criteria.resolution, type(criteria.resolution)))
if criteria.resolution != 'auto':
rsecs = int(timedelta_total_seconds(criteria.resolution))
args.resolution = Report.RESOLUTION_MAP[rsecs]
else:
args.resolution = 'auto'
logger.debug('NetProfiler report using resolution %s (%s)' %
(args.resolution, type(args.resolution)))
args.limit = (self.table.options.limit if hasattr(
self.table.options, 'limit') else None)
if getattr(self.table.options, 'interface', False):
args.centricity = 'int'
else:
args.centricity = 'hos'
return args
def test_report_with_area(self):
groupby = self.profiler.groupbys.host
columns = [self.profiler.columns.key.host_ip,
self.profiler.columns.value.avg_bytes,
self.profiler.columns.value.avg_pkts]
sort_col = self.profiler.columns.value.avg_bytes
timerange = self.yesterday
trafficexpr = TrafficFilter("host 10/8")
area = self.profiler.areas.vxlan_tenant
with TrafficSummaryReport(self.profiler) as rep:
rep.run(groupby, columns,
sort_col, timerange,
trafficexpr, area=area)
def main(self):
self.timefilter = TimeFilter.parse_range(self.options.timefilter)
if self.options.trafficexpr:
self.trafficexpr = TrafficFilter(self.options.trafficexpr)
else:
self.trafficexpr = None
with HostTimeSeriesReport(self.netprofiler) as report:
report.run(timefilter=self.timefilter,
trafficexpr=self.trafficexpr)
data = report.get_data()
legend = [c.label for c in report.get_legend()]
self.print_data(data, legend)
def traffic_report(self, host, timefilter, report_type):
""" Generate average statistics for host given `timefilter` time period
`report_type` is one of ('timeseries', 'summary')
"""
print 'Running %s report for %s over %s/%s' % (
report_type, host, timefilter.start, timefilter.end)
texpr = TrafficFilter('host %s' % host)
if report_type == 'timeseries':
columns = [c[0] for c in TCOLUMNS]
report = TrafficOverallTimeSeriesReport(self.netprofiler)
report.run(columns,
timefilter=timefilter,
trafficexpr=texpr,
resolution=self.options.resolution)
elif report_type == 'summary':
columns = [c[0] for c in SCOLUMNS]
report = TrafficSummaryReport(self.netprofiler)
if self.options.groupby_application:
columns.insert(0, 'app_name')
groupby = 'app'
elif self.options.groupby_interface:
columns.insert(0, 'interface_alias')
columns.insert(0, 'interface')
groupby = 'ifc'
else:
groupby = 'hos'
report.run(groupby,
columns,
timefilter=timefilter,
trafficexpr=texpr,
resolution=self.options.resolution)
else:
raise RuntimeError('unknown report type: %s' % report_type)
print 'Report complete, gathering data ...'
data = report.get_data()
if not data:
print "Empty data results."
elif len(data) == 10000:
print 'WARNING: data size exceeds max length of 10000 rows'
legend = report.get_legend()
report.delete()
return legend, data
def main(self):
self.timefilter = TimeFilter.parse_range(self.options.timefilter)
if self.options.trafficexpr:
self.trafficexpr = TrafficFilter(self.options.trafficexpr)
else:
self.trafficexpr = None
with TrafficFlowListReport(self.netprofiler) as report:
report.run(columns=self.options.columns.split(','),
sort_col=self.options.sortby,
timefilter=self.timefilter,
trafficexpr=self.trafficexpr)
data = report.get_data()
legend = [c.label for c in report.get_legend()]
self.print_data(data, legend)
def main(self):
self.timefilter = TimeFilter.parse_range(self.options.timefilter)
if self.options.trafficexpr:
self.trafficexpr = TrafficFilter(self.options.trafficexpr)
else:
self.trafficexpr = None
with TrafficOverallTimeSeriesReport(self.netprofiler) as report:
report.run(columns=self.options.columns.split(','),
timefilter=self.timefilter,
trafficexpr=self.trafficexpr,
centricity=self.centricity)
data = report.get_data()
legend = [c.label for c in report.get_legend()]
self.print_data(data, legend)
def main(self):
# Create and run a traffic summary report of all server ports in use
# by hosts in 10/8
report = TrafficOverallTimeSeriesReport(self.netprofiler)
# Run the report
report.run(columns=[
self.netprofiler.columns.key.time,
self.netprofiler.columns.value.avg_bytes,
self.netprofiler.columns.value.network_rtt
],
timefilter=TimeFilter.parse_range("last 15 m"),
trafficexpr=TrafficFilter("host 10/8"))
# Retrieve and print data
data = report.get_data()
printer = pprint.PrettyPrinter(2)
printer.pprint(data)
def test_unsupported_column(self):
groupby = self.profiler.groupbys.port
# host_ip shouldn't be included as part of 'port' groupby
columns = [self.profiler.columns.key.host_ip,
self.profiler.columns.value.avg_bytes,
self.profiler.columns.value.avg_pkts]
sort_col = self.profiler.columns.value.avg_bytes
timerange = self.yesterday
trafficexpr = TrafficFilter("host 10/8")
report = TrafficSummaryReport(self.profiler)
kwds = dict(groupby=groupby,
columns=columns,
sort_col=sort_col,
timefilter=timerange,
trafficexpr=trafficexpr)
self.assertRaises(RvbdException, report.run, None, kwds)
def main(self):
# Create and run a traffic summary report of all server ports in use
# by hosts in 10/8
report = TrafficSummaryReport(self.netprofiler)
# Run the report
report.run(groupby=self.netprofiler.groupbys.port,
columns=[
self.netprofiler.columns.key.protoport,
self.netprofiler.columns.key.protoport_name,
self.netprofiler.columns.value.avg_bytes,
self.netprofiler.columns.value.network_rtt
],
sort_col=self.netprofiler.columns.value.avg_bytes,
timefilter=TimeFilter.parse_range("last 15 m"),
trafficexpr=TrafficFilter("host 10/8"))
# Retrieve and print data
data = report.get_data()
printer = pprint.PrettyPrinter(2)
printer.pprint(data[:20])
def get_interfaces(self, device_ip):
""" Query netprofiler to attempt to automatically determine
LAN and WAN interface ids.
"""
cols = self.profiler.get_columns(['interface_dns', 'interface'])
super(WANReport, self).run(realm='traffic_summary',
groupby='ifc',
columns=cols,
timefilter=TimeFilter.parse_range('last 1 h'),
trafficexpr=TrafficFilter('device %s' % device_ip),
centricity='int',
resolution='auto',
sync=True)
interfaces = self._get_data()
lan = [address for name, address in interfaces if 'lan' in name]
wan = [address for name, address in interfaces if 'wan' in name]
if not lan or not wan:
raise RvbdException('Unable to determine LAN and WAN interfaces for device %s' %
device_ip)
return lan, wan
def test_traffic_flow_list_report(self):
columns = [self.profiler.columns.key.srv_host_ip,
self.profiler.columns.key.app_info,
self.profiler.columns.key.start_time,
self.profiler.columns.key.end_time,
self.profiler.columns.value.s2c_total_bytes,
self.profiler.columns.value.s2c_total_pkts,
self.profiler.columns.value.response_time,
self.profiler.columns.value.server_delay]
timerange = self.yesterday
trafficexpr = TrafficFilter("host 10/8")
with TrafficFlowListReport(self.profiler) as report:
report.run(columns, timefilter=timerange, trafficexpr=trafficexpr)
legend = report.get_legend()
keys = [c.key for c in legend]
self.assertTrue('app_info' in keys)
data = report.get_data()
if data:
self.assertEqual(len(data[0]), 8)
def main(self):
""" Setup query and run report with default column set
"""
if self.options.timerange:
timefilter = TimeFilter.parse_range(self.options.timerange)
else:
timefilter = TimeFilter(self.options.time0, self.options.time1)
if self.options.trafficexpr:
trafficexpr = TrafficFilter(self.options.trafficexpr)
else:
trafficexpr = None
legend_columns, all_data = self.identity_report(
timefilter=timefilter,
trafficexpr=trafficexpr,
testfile=self.options.testfile)
legend, activity = self.analyze_login_data(all_data, legend_columns)
if activity and self.options.timeseries_report:
headers, tbl_data = self.generate_traffic(activity, legend,
'timeseries')
elif activity and self.options.summary_report:
headers, tbl_data = self.generate_traffic(activity, legend,
'summary')
else:
headers = ('Host IP', 'Login Time', 'Logout Time', 'Duration')
tbl_data = [(x[0], format_time(x[1]), format_time(x[2]), x[3])
for x in activity]
if self.options.csv:
Formatter.print_csv(tbl_data, headers)
elif self.options.tsv:
Formatter.print_csv(tbl_data, headers, delim='\t')
else:
Formatter.print_table(tbl_data, headers)
def main(self):
# Create and run a traffic summary report of all hosts in use
# and then take that data and send it to a specified host group
report = TrafficSummaryReport(self.netprofiler)
# Run the report
report.run(groupby=self.netprofiler.groupbys.host,
columns=[
self.netprofiler.columns.key.host_ip,
self.netprofiler.columns.key.group_name
],
sort_col=self.netprofiler.columns.key.group_name,
timefilter=TimeFilter.parse_range(self.options.timefilter),
trafficexpr=TrafficFilter(self.options.trafficexpr))
# Store the report's data
data = report.get_data()
# Grab the type_name and group_name from options.group
(type_name, group_name) = self.options.group.split(':', 1)
# Create an array to store the new config data
new_config_entries = []
# Using data from the report, put it in config-readable format.
for i in range(len(data)):
new_config_entries.append({
'cidr': data[i][0] + '/32',
'name': group_name
})
# Make sure that if there were no entries returned,
# we don't overwrite the old data
if len(new_config_entries) == 0:
print('ERROR: Report returned zero hosts for supplied parameters')
return
# Get the ID of the host type specified by name
host_types = self.netprofiler.api.host_group_types.get_all()
target_type_id = -1
for i, host_type in enumerate(host_types):
if type_name == host_type['name']:
target_type_id = host_type['id']
break
# If target_type_id is still -1, then we didn't find that host
if target_type_id == -1:
print('ERROR: Host Group Type: "' + type_name + '" was not found.')
return
# Get the current config from the target host group
config = self.netprofiler.api.host_group_types.get_config(
target_type_id)
old_config_size = len(config)
# If the append flag is not true,
# remove all entries in config matching group_name
if self.options.append is False:
config = filter(lambda a: a['name'] != group_name, config)
config.extend(new_config_entries)
new_config_size = len(config)
self.netprofiler.api.host_group_types.set_config(
target_type_id, config)
print("Successfully updated type: " + type_name + ", group: " +
group_name)
print("The old config had " + str(old_config_size) +
" elements. It now has " + str(new_config_size) + " elements.\n")
def run(self):
""" Main execution method. """
criteria = self.job.criteria
if criteria.netprofiler_device == '':
logger.debug('%s: No netprofiler device selected' % self.table)
self.job.mark_error("No NetProfiler Device Selected")
return False
profiler = DeviceManager.get_device(criteria.netprofiler_device)
report = steelscript.netprofiler.core.report.MultiQueryReport(profiler)
tf = TimeFilter(start=criteria.starttime, end=criteria.endtime)
logger.info("Running NetProfilerTemplateTable table %d report "
"for timeframe %s" % (self.table.id, str(tf)))
trafficexpr = TrafficFilter(
self.job.combine_filterexprs(exprs=criteria.profiler_filterexpr))
# Incoming criteria.resolution is a timedelta
logger.debug('NetProfiler report got criteria resolution %s (%s)' %
(criteria.resolution, type(criteria.resolution)))
if criteria.resolution != 'auto':
rsecs = int(timedelta_total_seconds(criteria.resolution))
resolution = steelscript.netprofiler.core.report.Report.RESOLUTION_MAP[
rsecs]
else:
resolution = 'auto'
logger.debug('NetProfiler report using resolution %s (%s)' %
(resolution, type(resolution)))
with lock:
res = report.run(template_id=self.table.options.template_id,
timefilter=tf,
trafficexpr=trafficexpr,
resolution=resolution)
if res is True:
logger.info("Report template complete.")
self.job.safe_update(progress=100)
# Retrieve the data
with lock:
query = report.get_query_by_index(0)
data = query.get_data()
headers = report.get_legend()
tz = criteria.starttime.tzinfo
# Update criteria
criteria.starttime = (datetime.datetime.utcfromtimestamp(
query.actual_t0).replace(tzinfo=tz))
criteria.endtime = (datetime.datetime.utcfromtimestamp(
query.actual_t1).replace(tzinfo=tz))
self.job.safe_update(actual_criteria=criteria)
# create dataframe with all of the default headers
df = pandas.DataFrame(data, columns=[h.key for h in headers])
# now filter down to the columns requested by the table
columns = [col.name for col in self.table.get_columns(synthetic=False)]
self.data = df[columns]
logger.info("Report %s returned %s rows" % (self.job, len(self.data)))
return True