def main(self): if self.options.showsources: svcdef = self.appresponse.find_service('npm.reports') dr = svcdef.bind('source_names') source_names = dr.execute('get').data print('\n'.join(source_names)) return source = SourceProxy(name=self.options.sourcename) columns = [] headers = [] if self.options.keycolumns: for col in self.options.keycolumns.split(','): columns.append(Key(col)) headers.append(col) for col in self.options.valuecolumns.split(','): columns.append(Value(col)) headers.append(col) topbycolumns = [] if self.options.topbycolumns: for col in self.options.topbycolumns.split(','): topbycolumns.append(Key(col)) data_def = DataDef(source=source, columns=columns, granularity=self.options.granularity, resolution=self.options.resolution, time_range=self.options.timerange, limit=self.options.limit, topbycolumns=topbycolumns) if self.options.filterexpr: data_def.add_filter( TrafficFilter(type_='steelfilter', value=self.options.filterexpr)) report = Report(self.appresponse) report.add(data_def) report.run() data = report.get_data() headers = report.get_legend() report.delete() if self.options.csvfile: with open(self.options.csvfile, 'w') as f: for line in Formatter.get_csv(data, headers): f.write(line) f.write('\n') else: Formatter.print_csv(data, headers)
def main(self): if self.options.sourcetype == 'file': source = self.appresponse.fs.get_file_by_id(self.options.sourceid) elif self.options.sourcetype == 'job': source = self.appresponse.capture.\ get_job_by_name(self.options.sourceid) else: source = self.appresponse.clips.\ get_clip_by_id(self.options.sourceid) data_source = SourceProxy(source) columns = [] headers = [] if self.options.keycolumns: for col in self.options.keycolumns.split(','): columns.append(Key(col)) headers.append(col) for col in self.options.valuecolumns.split(','): columns.append(Value(col)) headers.append(col) data_def = DataDef(source=data_source, columns=columns, granularity=self.options.granularity, resolution=self.options.resolution, time_range=self.options.timerange) if self.options.filterexpr: data_def.add_filter( TrafficFilter(type_=self.options.filtertype, value=self.options.filterexpr)) report = Report(self.appresponse) report.add(data_def) report.run() data = report.get_data() headers = report.get_legend() report.delete() if self.options.csvfile: with open(self.options.csvfile, 'w') as f: for line in Formatter.get_csv(data, headers): f.write(line) f.write('\n') else: Formatter.print_csv(data, headers)
def get_column_objects(self, source_name, columns): """Return Key/Value objects for given set of string names.""" coldefs = self.sources[source_name]['columns'] def iskey(coldef): if 'grouped_by' in coldef and coldef['grouped_by'] is True: return True return False cols = [] for c in columns: obj = Key(c) if iskey(coldefs[c]) else Value(c) cols.append(obj) return cols
def run(self): criteria = self.job.criteria ar = DeviceManager.get_device(criteria.appresponse_device) if self.table.options.source == 'packets': source_name = criteria.appresponse_source if source_name.startswith(SourceProxy.JOB_PREFIX): job_id = source_name.lstrip(SourceProxy.JOB_PREFIX) source = SourceProxy(ar.capture.get_job_by_id(job_id)) else: file_id = source_name.lstrip(SourceProxy.FILE_PREFIX) source = SourceProxy(ar.fs.get_file_by_id(file_id)) else: source = SourceProxy(name=self.table.options.source) col_extractors, col_names = [], {} for col in self.table.get_columns(synthetic=False): col_names[col.options.extractor] = col.name if col.iskey: col_extractors.append(Key(col.options.extractor)) else: col_extractors.append(Value(col.options.extractor)) # If the data source is of file type and entire PCAP # is set True, then set start end times to None if isinstance(source, File) and criteria.entire_pcap: start = None end = None else: start = datetime_to_seconds(criteria.starttime) end = datetime_to_seconds(criteria.endtime) granularity = criteria.granularity.total_seconds() data_def = DataDef(source=source, columns=col_extractors, granularity=str(granularity), start=start, end=end) report = Report(ar) report.add(data_def) report.run() df = report.get_dataframe() df.columns = map(lambda x: col_names[x], df.columns) def to_int(x): return x if str(x).isdigit() else None def to_float(x): return x if str(x).replace('.', '', 1).isdigit() else None # Numerical columns can be returned as '#N/D' when not available # Thus convert them to None to help sorting for col in self.table.get_columns(synthetic=False): if col.datatype == Column.DATATYPE_FLOAT: df[col.name] = df[col.name].apply(lambda x: to_float(x)) elif col.datatype == Column.DATATYPE_INTEGER: df[col.name] = df[col.name].apply(lambda x: to_int(x)) elif col.datatype == Column.DATATYPE_TIME: if granularity < 1: # The fractional epoch time values are in string # Thus needs to be converted to float df[col.name] = df[col.name].apply(float) if self.table.options.sort_col_name: df.sort(columns=self.table.options.sort_col_name, ascending=self.table.options.ascending, inplace=True) return QueryComplete(df)
def packet_columns(): key_cols = [Key(key) for key in PACKETS_KEY_COLS] value_cols = [Value(key) for key in PACKETS_VALUE_COLS] return key_cols, value_cols
def main(self): if self.options.sourcename == 'packets': if self.options.sourceid is None: source = self.appresponse.capture.get_vifgs()[0] else: source = SourceProxy(name='packets', path=self.options.sourceid) else: source = SourceProxy(name='aggregates') columns = [] headers = [] for col in self.options.keycolumns.split(','): columns.append(Key(col)) headers.append(col) for col in self.options.valuecolumns.split(','): columns.append(Value(col)) headers.append(col) data_def = DataDef(source=source, columns=columns, granularity=self.options.granularity, resolution=self.options.resolution, live=True) if self.options.filterexpr: data_def.add_filter(TrafficFilter(type_='steelfilter', value=self.options.filterexpr)) print('Running report, press Ctrl-C to exit.') print('') report = self.appresponse.create_report(data_def) time.sleep(1) try: while 1: banner = '{} {}'.format(datetime.datetime.now(), '--' * 20) print(banner) try: data = report.get_data()['data'] headers = report.get_legend() if self.options.sortby: index = headers.index(self.options.sortby) data.sort(key=lambda x: x[index], reverse=True) if self.limit: total_rows = len(data) limit_string = ('Showing {} out of {} rows.' .format(self.limit, total_rows)) data = data[:self.limit] else: limit_string = None except KeyError: # something went wrong, print error and exit print('Error accessing data:') print(report.get_data()) raise KeyboardInterrupt if self.options.csvfile: with open(self.options.csvfile, 'a') as f: f.write(banner) f.write('\n') for line in Formatter.get_csv(data, headers): f.write(line) f.write('\n') if limit_string: f.write(limit_string) f.write('\n') else: Formatter.print_table(data, headers) if limit_string: print(limit_string) time.sleep(self.delay) except KeyboardInterrupt: print('Exiting ...') report.delete()
def run(self): criteria = self.job.criteria ar = DeviceManager.get_device(criteria.appresponse_device) if self.table.options.source == 'packets': source_name = criteria.appresponse_source if source_name.startswith(SourceProxy.JOB_PREFIX): job_id = source_name.lstrip(SourceProxy.JOB_PREFIX) source = SourceProxy(ar.capture.get_job_by_id(job_id)) else: file_id = source_name.lstrip(SourceProxy.FILE_PREFIX) source = SourceProxy(ar.fs.get_file_by_id(file_id)) else: source = SourceProxy(name=self.table.options.source) col_extractors = [] col_names = {} aliases = {} for col in self.table.get_columns(synthetic=False): col_names[col.options.extractor] = col.name if col.iskey: col_extractors.append(Key(col.options.extractor)) else: col_extractors.append(Value(col.options.extractor)) if col.options.alias: aliases[col.options.extractor] = col.options.alias col_extractors.append(Value(col.options.alias)) # If the data source is of file type and entire PCAP # is set True, then set start end times to None if (self.table.options.source == 'packets' and source.path.startswith(SourceProxy.FILE_PREFIX) and criteria.entire_pcap): start = None end = None else: start = datetime_to_seconds(criteria.starttime) end = datetime_to_seconds(criteria.endtime) granularity = criteria.granularity.total_seconds() resolution = None # temp fix for https://bugzilla.nbttech.com/show_bug.cgi?id=305478 # if we aren't asking for a timeseries, make sure the data gets # aggregated by making resolution greater than the report duration if (self.table.options.source == 'packets' and 'start_time' not in col_names.keys() and 'end_time' not in col_names.keys()): resolution = end - start + granularity data_def = DataDef( source=source, columns=col_extractors, granularity=granularity, resolution=resolution, start=start, end=end) if hasattr(criteria, 'appresponse_steelfilter'): logger.debug('calculating steelfilter expression ...') filterexpr = self.job.combine_filterexprs( exprs=criteria.appresponse_steelfilter ) if filterexpr: logger.debug('applying steelfilter expression: %s' % filterexpr) data_def.add_filter(TrafficFilter(type_='steelfilter', value=filterexpr)) report = Report(ar) report.add(data_def) report.run() df = report.get_dataframe() report.delete() if aliases: # overwrite columns with their alias values, then drop 'em for k, v in aliases.iteritems(): df[k] = df[v] df.drop(v, 1, inplace=True) df.columns = map(lambda x: col_names[x], df.columns) def to_int(x): return x if str(x).isdigit() else None def to_float(x): return x if str(x).replace('.', '', 1).isdigit() else None # Numerical columns can be returned as '#N/D' when not available # Thus convert them to None to help sorting for col in self.table.get_columns(synthetic=False): if col.datatype == Column.DATATYPE_FLOAT: df[col.name] = df[col.name].apply(lambda x: to_float(x)) elif col.datatype == Column.DATATYPE_INTEGER: df[col.name] = df[col.name].apply(lambda x: to_int(x)) elif col.datatype == Column.DATATYPE_TIME: if granularity < 1: # The fractional epoch time values are in string # Thus needs to be converted to float df[col.name] = df[col.name].apply(float) if self.table.options.sort_col_name: df.sort(columns=self.table.options.sort_col_name, ascending=self.table.options.ascending, inplace=True) return QueryComplete(df)