def main(self): self.tfilter1 = TrafficFilter(self.options.expr1) self.report1 = TrafficSummaryReport(self.netprofiler) self.columns_report1 = self.netprofiler.get_columns([self.options.column1]) self.trigger1 = safe_lambda(self.options.trigger1) if self.options.trigger2 and self.options.column2 and self.options.expr2: self.tfilter2 = TrafficFilter(self.options.expr2) self.report2 = TrafficSummaryReport(self.netprofiler) self.columns_report2 = self.netprofiler.get_columns([self.options.column2]) self.trigger2 = safe_lambda(self.options.trigger2) else: self.trigger2 = None try: while 1: print('Running report 1 ...') alert_flag = False result = None result2 = None result = self.run_query(self.report1, self.columns_report1, self.tfilter1, self.trigger1) if result: if self.trigger2: print('Trigger 1 passed, running report 2 ...') result2 = self.run_query(self.report2, self.columns_report2, self.tfilter2, self.trigger2) if result2: print('Trigger 2 passed ...') alert_flag = True else: print('Trigger 1 passed ...') alert_flag = True if alert_flag: print('ALERT ALERT!') self.update_description(result, result2) self.send_trap() time.sleep(self.options.refresh) except KeyboardInterrupt: print('Exiting ...') sys.exit(0)
def test_traffic_overall_time_series_report(self): columns = [ self.profiler.columns.key.time, self.profiler.columns.value.avg_bytes, self.profiler.columns.value.avg_pkts ] timerange = TimeFilter.parse_range("last 1 h") trafficexpr = TrafficFilter("host 10/8") resolution = "15min" report = TrafficOverallTimeSeriesReport(self.profiler) report.run(columns, timerange, trafficexpr, resolution=resolution) legend = report.get_legend() keys = [c.key for c in legend] data = report.get_data() for item in data: d = dict(zip(keys, item)) # resolution assumes 15-minute responses self.assertTrue( timerange.compare_time(d['time'], resolution=15 * 60)) report.delete()
def main(self): # groupby validation should be part of validate_args, but there # is no NetProfiler initialized at that part of the initialization try: self.groupby = self.netprofiler.groupbys[self.options.groupby] except KeyError: if self.options.groupby not in self.netprofiler.groupbys.values(): self.parser.error('Invalid groupby chosen.') else: self.groupby = self.options.groupby self.timefilter = TimeFilter.parse_range(self.options.timefilter) if self.options.trafficexpr: self.trafficexpr = TrafficFilter(self.options.trafficexpr) else: self.trafficexpr = None with TrafficSummaryReport(self.netprofiler) as report: report.run(columns=self.options.columns.split(','), groupby=self.groupby, sort_col=self.options.sortby, timefilter=self.timefilter, trafficexpr=self.trafficexpr, centricity=self.centricity, resolution='auto') data = report.get_data() legend = [c.label for c in report.get_legend()] self.print_data(data, legend)
def main(self): self.timefilter = TimeFilter.parse_range(self.options.timefilter) if self.options.trafficexpr: self.trafficexpr = TrafficFilter(self.options.trafficexpr) else: self.trafficexpr = None with MultiQueryReport(self.netprofiler) as report: report.run(template_id=int(self.options.template_id), timefilter=self.timefilter, trafficexpr=self.trafficexpr) print('Report Template {id} successfully run.' .format(id=self.options.template_id)) self.netprofiler.conn.download( '/api/profiler/1.6/reporting/reports/{id}/view.{fmt}' .format(id=report.id, fmt=self.options.fmt), path=self.options.pathname, overwrite=True ) print('Completed Report {id} downloaded to {path}.' .format(id=report.id, path=self.options.pathname))
def test_traffic_summary_report(self): groupby = self.profiler.groupbys.host columns = [self.profiler.columns.key.host_ip, self.profiler.columns.value.avg_bytes, self.profiler.columns.value.avg_pkts] sort_col = self.profiler.columns.value.avg_bytes timerange = self.yesterday trafficexpr = TrafficFilter("host 10/8") with TrafficSummaryReport(self.profiler) as rep: rep.run(groupby, columns, sort_col, timerange, trafficexpr) legend = rep.get_legend() self.assertEqual(len(legend), 3) legend = rep.get_legend(columns=[self.profiler.columns.key.host_ip, self.profiler.columns.value.avg_bytes]) self.assertEqual(len(legend), 2) self.assertEqual(legend[0].key, 'host_ip') self.assertEqual(legend[1].key, 'avg_bytes') data = rep.get_data() if data: self.assertEqual(len(data[0]), 3) #check that data is refetched from cache data = rep.get_data() data = rep.get_data(columns=[self.profiler.columns.key.host_ip, self.profiler.columns.value.avg_bytes]) if data: self.assertEqual(len(data[0]), 2)
def test_resolution(self): groupby = self.profiler.groupbys.host columns = [self.profiler.columns.key.host_ip, self.profiler.columns.value.avg_bytes, self.profiler.columns.value.avg_pkts] sort_col = self.profiler.columns.value.avg_bytes trafficexpr = TrafficFilter("host 10/8") resolutions = [["1min", "last 5 min"], ["15min", "last 1 hour"], ["hour", "last 4 hours"], ["6hour", "last 1 day"], ["day", "last 1 week"], ["3600", "last 4 hours"], # hour resolution ["60", "last 5 min"], # minute resolution #["week", "last 4 weeks"], #["month", "last 12 months"], #Commented values blow up with a #E RvbdHTTPException: 400 Unknown time resolution. ] for (resolution, duration) in resolutions: timerange = TimeFilter.parse_range(duration) with TrafficSummaryReport(self.profiler) as rep: rep.run(groupby, columns, sort_col, timerange, trafficexpr, resolution=resolution)
def main(self): if self.options.listinterfacegroups: self.list_interface_groups(self.options.host, self.options.sshusername, self.options.sshpassword) return if self.options.listhostgroups: self.list_host_groups(NetProfiler(self.options.host, auth=self.auth)) return try: timefilter = TimeFilter.parse_range(self.options.timefilter) except ValueError: print "Could not parse time filter expression." return profiler = NetProfiler(self.options.host, auth=self.auth) if not self.options.clean: print "Reporting on the period: {}".format(self.options.timefilter) print "Using the traffic filter: {}".format(self.options.trafficfilter) print "Calculating data at percentile {}".format(self.options.percentile) print ("Averaging based on buckets of {} " "minutes").format(self.options.buckettime) if self.options.graph: print "Saving a graph to {}".format(self.options.graph) print trafficfilter = TrafficFilter(self.options.trafficfilter) self.report_item(profiler, timefilter, trafficfilter, self.options.buckettime, self.options.percentile)
def main(self): """ Setup query and run report with default column set """ if self.options.timerange: timefilter = TimeFilter.parse_range(self.options.timerange) else: timefilter = TimeFilter(self.options.time0, self.options.time1) trafficexpr = TrafficFilter(self.options.trafficexpr) columns = [ self.netprofiler.columns.key.srv_host_ip, self.netprofiler.columns.key.app_info, self.netprofiler.columns.key.start_time, self.netprofiler.columns.key.end_time, self.netprofiler.columns.value.s2c_total_bytes, self.netprofiler.columns.value.s2c_total_pkts, self.netprofiler.columns.value.response_time, self.netprofiler.columns.value.server_delay ] report = TrafficFlowListReport(self.netprofiler) report.run(columns, timefilter=timefilter, trafficexpr=trafficexpr) data = report.get_data() report.delete() headers = [c.key for c in columns] Formatter.print_table(data, headers)
def _prepare_report_args(self): class Args(object): pass args = Args() criteria = self.job.criteria if criteria.netprofiler_device == '': logger.debug('%s: No netprofiler device selected' % self.table) self.job.mark_error("No NetProfiler Device Selected") return False args.profiler = DeviceManager.get_device(criteria.netprofiler_device) args.columns = [ col.name for col in self.table.get_columns(synthetic=False) ] args.sortcol = None if self.table.sortcols is not None: args.sortcol = self.table.sortcols[0] args.timefilter = TimeFilter(start=criteria.starttime, end=criteria.endtime) logger.info("Running NetProfiler table %d report for timeframe %s" % (self.table.id, str(args.timefilter))) if ('datafilter' in criteria) and (criteria.datafilter is not None): args.datafilter = criteria.datafilter.split(',') else: args.datafilter = None args.trafficexpr = TrafficFilter( self.job.combine_filterexprs( exprs=criteria.netprofiler_filterexpr)) # Incoming criteria.resolution is a timedelta logger.debug('NetProfiler report got criteria resolution %s (%s)' % (criteria.resolution, type(criteria.resolution))) if criteria.resolution != 'auto': rsecs = int(timedelta_total_seconds(criteria.resolution)) args.resolution = Report.RESOLUTION_MAP[rsecs] else: args.resolution = 'auto' logger.debug('NetProfiler report using resolution %s (%s)' % (args.resolution, type(args.resolution))) args.limit = (self.table.options.limit if hasattr( self.table.options, 'limit') else None) if getattr(self.table.options, 'interface', False): args.centricity = 'int' else: args.centricity = 'hos' return args
def test_report_with_area(self): groupby = self.profiler.groupbys.host columns = [self.profiler.columns.key.host_ip, self.profiler.columns.value.avg_bytes, self.profiler.columns.value.avg_pkts] sort_col = self.profiler.columns.value.avg_bytes timerange = self.yesterday trafficexpr = TrafficFilter("host 10/8") area = self.profiler.areas.vxlan_tenant with TrafficSummaryReport(self.profiler) as rep: rep.run(groupby, columns, sort_col, timerange, trafficexpr, area=area)
def main(self): self.timefilter = TimeFilter.parse_range(self.options.timefilter) if self.options.trafficexpr: self.trafficexpr = TrafficFilter(self.options.trafficexpr) else: self.trafficexpr = None with HostTimeSeriesReport(self.netprofiler) as report: report.run(timefilter=self.timefilter, trafficexpr=self.trafficexpr) data = report.get_data() legend = [c.label for c in report.get_legend()] self.print_data(data, legend)
def traffic_report(self, host, timefilter, report_type): """ Generate average statistics for host given `timefilter` time period `report_type` is one of ('timeseries', 'summary') """ print 'Running %s report for %s over %s/%s' % ( report_type, host, timefilter.start, timefilter.end) texpr = TrafficFilter('host %s' % host) if report_type == 'timeseries': columns = [c[0] for c in TCOLUMNS] report = TrafficOverallTimeSeriesReport(self.netprofiler) report.run(columns, timefilter=timefilter, trafficexpr=texpr, resolution=self.options.resolution) elif report_type == 'summary': columns = [c[0] for c in SCOLUMNS] report = TrafficSummaryReport(self.netprofiler) if self.options.groupby_application: columns.insert(0, 'app_name') groupby = 'app' elif self.options.groupby_interface: columns.insert(0, 'interface_alias') columns.insert(0, 'interface') groupby = 'ifc' else: groupby = 'hos' report.run(groupby, columns, timefilter=timefilter, trafficexpr=texpr, resolution=self.options.resolution) else: raise RuntimeError('unknown report type: %s' % report_type) print 'Report complete, gathering data ...' data = report.get_data() if not data: print "Empty data results." elif len(data) == 10000: print 'WARNING: data size exceeds max length of 10000 rows' legend = report.get_legend() report.delete() return legend, data
def main(self): self.timefilter = TimeFilter.parse_range(self.options.timefilter) if self.options.trafficexpr: self.trafficexpr = TrafficFilter(self.options.trafficexpr) else: self.trafficexpr = None with TrafficFlowListReport(self.netprofiler) as report: report.run(columns=self.options.columns.split(','), sort_col=self.options.sortby, timefilter=self.timefilter, trafficexpr=self.trafficexpr) data = report.get_data() legend = [c.label for c in report.get_legend()] self.print_data(data, legend)
def main(self): self.timefilter = TimeFilter.parse_range(self.options.timefilter) if self.options.trafficexpr: self.trafficexpr = TrafficFilter(self.options.trafficexpr) else: self.trafficexpr = None with TrafficOverallTimeSeriesReport(self.netprofiler) as report: report.run(columns=self.options.columns.split(','), timefilter=self.timefilter, trafficexpr=self.trafficexpr, centricity=self.centricity) data = report.get_data() legend = [c.label for c in report.get_legend()] self.print_data(data, legend)
def main(self): # Create and run a traffic summary report of all server ports in use # by hosts in 10/8 report = TrafficOverallTimeSeriesReport(self.netprofiler) # Run the report report.run(columns=[ self.netprofiler.columns.key.time, self.netprofiler.columns.value.avg_bytes, self.netprofiler.columns.value.network_rtt ], timefilter=TimeFilter.parse_range("last 15 m"), trafficexpr=TrafficFilter("host 10/8")) # Retrieve and print data data = report.get_data() printer = pprint.PrettyPrinter(2) printer.pprint(data)
def test_unsupported_column(self): groupby = self.profiler.groupbys.port # host_ip shouldn't be included as part of 'port' groupby columns = [self.profiler.columns.key.host_ip, self.profiler.columns.value.avg_bytes, self.profiler.columns.value.avg_pkts] sort_col = self.profiler.columns.value.avg_bytes timerange = self.yesterday trafficexpr = TrafficFilter("host 10/8") report = TrafficSummaryReport(self.profiler) kwds = dict(groupby=groupby, columns=columns, sort_col=sort_col, timefilter=timerange, trafficexpr=trafficexpr) self.assertRaises(RvbdException, report.run, None, kwds)
def main(self): # Create and run a traffic summary report of all server ports in use # by hosts in 10/8 report = TrafficSummaryReport(self.netprofiler) # Run the report report.run(groupby=self.netprofiler.groupbys.port, columns=[ self.netprofiler.columns.key.protoport, self.netprofiler.columns.key.protoport_name, self.netprofiler.columns.value.avg_bytes, self.netprofiler.columns.value.network_rtt ], sort_col=self.netprofiler.columns.value.avg_bytes, timefilter=TimeFilter.parse_range("last 15 m"), trafficexpr=TrafficFilter("host 10/8")) # Retrieve and print data data = report.get_data() printer = pprint.PrettyPrinter(2) printer.pprint(data[:20])
def get_interfaces(self, device_ip): """ Query netprofiler to attempt to automatically determine LAN and WAN interface ids. """ cols = self.profiler.get_columns(['interface_dns', 'interface']) super(WANReport, self).run(realm='traffic_summary', groupby='ifc', columns=cols, timefilter=TimeFilter.parse_range('last 1 h'), trafficexpr=TrafficFilter('device %s' % device_ip), centricity='int', resolution='auto', sync=True) interfaces = self._get_data() lan = [address for name, address in interfaces if 'lan' in name] wan = [address for name, address in interfaces if 'wan' in name] if not lan or not wan: raise RvbdException('Unable to determine LAN and WAN interfaces for device %s' % device_ip) return lan, wan
def test_traffic_flow_list_report(self): columns = [self.profiler.columns.key.srv_host_ip, self.profiler.columns.key.app_info, self.profiler.columns.key.start_time, self.profiler.columns.key.end_time, self.profiler.columns.value.s2c_total_bytes, self.profiler.columns.value.s2c_total_pkts, self.profiler.columns.value.response_time, self.profiler.columns.value.server_delay] timerange = self.yesterday trafficexpr = TrafficFilter("host 10/8") with TrafficFlowListReport(self.profiler) as report: report.run(columns, timefilter=timerange, trafficexpr=trafficexpr) legend = report.get_legend() keys = [c.key for c in legend] self.assertTrue('app_info' in keys) data = report.get_data() if data: self.assertEqual(len(data[0]), 8)
def main(self): """ Setup query and run report with default column set """ if self.options.timerange: timefilter = TimeFilter.parse_range(self.options.timerange) else: timefilter = TimeFilter(self.options.time0, self.options.time1) if self.options.trafficexpr: trafficexpr = TrafficFilter(self.options.trafficexpr) else: trafficexpr = None legend_columns, all_data = self.identity_report( timefilter=timefilter, trafficexpr=trafficexpr, testfile=self.options.testfile) legend, activity = self.analyze_login_data(all_data, legend_columns) if activity and self.options.timeseries_report: headers, tbl_data = self.generate_traffic(activity, legend, 'timeseries') elif activity and self.options.summary_report: headers, tbl_data = self.generate_traffic(activity, legend, 'summary') else: headers = ('Host IP', 'Login Time', 'Logout Time', 'Duration') tbl_data = [(x[0], format_time(x[1]), format_time(x[2]), x[3]) for x in activity] if self.options.csv: Formatter.print_csv(tbl_data, headers) elif self.options.tsv: Formatter.print_csv(tbl_data, headers, delim='\t') else: Formatter.print_table(tbl_data, headers)
def main(self): # Create and run a traffic summary report of all hosts in use # and then take that data and send it to a specified host group report = TrafficSummaryReport(self.netprofiler) # Run the report report.run(groupby=self.netprofiler.groupbys.host, columns=[ self.netprofiler.columns.key.host_ip, self.netprofiler.columns.key.group_name ], sort_col=self.netprofiler.columns.key.group_name, timefilter=TimeFilter.parse_range(self.options.timefilter), trafficexpr=TrafficFilter(self.options.trafficexpr)) # Store the report's data data = report.get_data() # Grab the type_name and group_name from options.group (type_name, group_name) = self.options.group.split(':', 1) # Create an array to store the new config data new_config_entries = [] # Using data from the report, put it in config-readable format. for i in range(len(data)): new_config_entries.append({ 'cidr': data[i][0] + '/32', 'name': group_name }) # Make sure that if there were no entries returned, # we don't overwrite the old data if len(new_config_entries) == 0: print('ERROR: Report returned zero hosts for supplied parameters') return # Get the ID of the host type specified by name host_types = self.netprofiler.api.host_group_types.get_all() target_type_id = -1 for i, host_type in enumerate(host_types): if type_name == host_type['name']: target_type_id = host_type['id'] break # If target_type_id is still -1, then we didn't find that host if target_type_id == -1: print('ERROR: Host Group Type: "' + type_name + '" was not found.') return # Get the current config from the target host group config = self.netprofiler.api.host_group_types.get_config( target_type_id) old_config_size = len(config) # If the append flag is not true, # remove all entries in config matching group_name if self.options.append is False: config = filter(lambda a: a['name'] != group_name, config) config.extend(new_config_entries) new_config_size = len(config) self.netprofiler.api.host_group_types.set_config( target_type_id, config) print("Successfully updated type: " + type_name + ", group: " + group_name) print("The old config had " + str(old_config_size) + " elements. It now has " + str(new_config_size) + " elements.\n")
def run(self): """ Main execution method. """ criteria = self.job.criteria if criteria.netprofiler_device == '': logger.debug('%s: No netprofiler device selected' % self.table) self.job.mark_error("No NetProfiler Device Selected") return False profiler = DeviceManager.get_device(criteria.netprofiler_device) report = steelscript.netprofiler.core.report.MultiQueryReport(profiler) tf = TimeFilter(start=criteria.starttime, end=criteria.endtime) logger.info("Running NetProfilerTemplateTable table %d report " "for timeframe %s" % (self.table.id, str(tf))) trafficexpr = TrafficFilter( self.job.combine_filterexprs(exprs=criteria.profiler_filterexpr)) # Incoming criteria.resolution is a timedelta logger.debug('NetProfiler report got criteria resolution %s (%s)' % (criteria.resolution, type(criteria.resolution))) if criteria.resolution != 'auto': rsecs = int(timedelta_total_seconds(criteria.resolution)) resolution = steelscript.netprofiler.core.report.Report.RESOLUTION_MAP[ rsecs] else: resolution = 'auto' logger.debug('NetProfiler report using resolution %s (%s)' % (resolution, type(resolution))) with lock: res = report.run(template_id=self.table.options.template_id, timefilter=tf, trafficexpr=trafficexpr, resolution=resolution) if res is True: logger.info("Report template complete.") self.job.safe_update(progress=100) # Retrieve the data with lock: query = report.get_query_by_index(0) data = query.get_data() headers = report.get_legend() tz = criteria.starttime.tzinfo # Update criteria criteria.starttime = (datetime.datetime.utcfromtimestamp( query.actual_t0).replace(tzinfo=tz)) criteria.endtime = (datetime.datetime.utcfromtimestamp( query.actual_t1).replace(tzinfo=tz)) self.job.safe_update(actual_criteria=criteria) # create dataframe with all of the default headers df = pandas.DataFrame(data, columns=[h.key for h in headers]) # now filter down to the columns requested by the table columns = [col.name for col in self.table.get_columns(synthetic=False)] self.data = df[columns] logger.info("Report %s returned %s rows" % (self.job, len(self.data))) return True