def post(self, request, device_type): if device_type == 'profiler': form = ProfilerInputForm(request.POST) elif device_type == 'shark': form = SharkInputForm(request.POST) else: raise Http404 results = None if form.is_valid(): data = form.cleaned_data if device_type == 'profiler': profiler = DeviceManager.get_device(data['device']) results = profiler.search_columns(realms=[data['realm']], centricities=[data['centricity']], groupbys=[data['groupby']]) results.sort(key=operator.attrgetter('key')) results.sort(key=operator.attrgetter('iskey'), reverse=True) results = [(c.iskey, c.key, c.label, c.id) for c in results] elif device_type == 'shark': shark = DeviceManager.get_device(data['device']) results = [(f.id, f.description, f.type) for f in shark.get_extractor_fields()] results.sort(key=operator.itemgetter(0)) return render_to_response('help.html', {'device': device_type.title(), 'form': form, 'results': results}, context_instance=RequestContext(request))
def run(self): """ Main execution method """ criteria = self.job.criteria if criteria.sharepoint_device == '': logger.debug('%s: No sharepoint device selected' % self.table) self.job.mark_error("No Sharepoint Device Selected") return False sp = DeviceManager.get_device(criteria.sharepoint_device) site = sp.get_site_object(self.table.options.site_url) site_instance = site.lists[self.table.options.list_name] fields = [tc.name for tc in self.table.get_columns(synthetic=False)] self.data = [] for row in site_instance.rows: d = [getattr(row, f) for f in fields] self.data.append(d) logger.info("SharepointTable job %s returning %s data" % (self.job, len(self.data))) return True
def shark_source_name_choices(form, id, field_kwargs, params): shark_device = form.get_field_value('shark_device', id) if shark_device == '': label = 'Source' choices = [('', '<No shark device>')] else: shark = DeviceManager.get_device(shark_device) #source_type = form.get_field_value('shark_source_type', id) source_type = 'job' choices = [] if source_type == 'job': for job in shark.get_capture_jobs(): choices.append(('jobs/' + job.name, job.name)) label = 'Capture Job' elif source_type == 'clip': # Not tested label = 'Trace Clip' for clip in shark.get_clips(): choices.append((clip, clip)) else: raise KeyError('Unknown source type: %s' % source_type) field_kwargs['label'] = label field_kwargs['choices'] = choices
def run(self): """ Main execution method """ criteria = self.job.criteria if criteria.sharepoint_device == '': logger.debug('%s: No sharepoint device selected' % self.table) self.job.mark_error("No Sharepoint Device Selected") return False sp = DeviceManager.get_device(criteria.sharepoint_device) site = sp.get_site_object(self.table.options.site_url) site_instance = site.lists[self.table.options.list_name] fields = [tc.name for tc in self.table.get_columns(synthetic=False)] self.data = [] for row in site_instance.rows: d = [getattr(row, f) for f in fields] self.data.append(d) logger.info("SharepointTable job %s returning %s data" % (self.job, len(self.data))) return True
def __init__(self, *args, **kwargs): super(DeviceDetailForm, self).__init__(*args, **kwargs) modules = DeviceManager.get_modules() choices = zip(modules, modules) self.fields['module'] = forms.ChoiceField(choices=choices)
def shark_source_name_choices(form, id, field_kwargs, params): shark_device = form.get_field_value('shark_device', id) if shark_device == '': label = 'Source' choices = [('', '<No shark device>')] else: shark = DeviceManager.get_device(shark_device) #source_type = form.get_field_value('shark_source_type', id) source_type = 'job' choices = [] if source_type == 'job': for job in shark.get_capture_jobs(): choices.append (('jobs/' + job.name, job.name)) label = 'Capture Job' elif source_type == 'clip': # Not tested label = 'Trace Clip' for clip in shark.get_clips(): choices.append ((clip, clip)) else: raise KeyError('Unknown source type: %s' % source_type) field_kwargs['label'] = label field_kwargs['choices'] = choices
def put(self, request, *args, **kwargs): DeviceFormSet = modelformset_factory(Device, form=DeviceListForm, extra=0) formset = DeviceFormSet(request.DATA) if formset.is_valid(): formset.save() DeviceManager.clear() profile = UserProfile.objects.get(user=request.user) if not profile.profile_seen: # only redirect if first login return HttpResponseRedirect( reverse('preferences') + '?next=/report') elif '/devices' not in request.META['HTTP_REFERER']: return HttpResponseRedirect(request.META['HTTP_REFERER']) else: return HttpResponseRedirect(reverse('device-list')) else: data = {'formset': formset} return Response(data, template_name='device_list.html')
def run(self): """ Main execution method """ criteria = self.job.criteria if criteria.solarwinds_device == "": logger.debug("%s: No solarwinds device selected" % self.table) self.job.mark_error("No Solarwinds Device Selected") return False sw = DeviceManager.get_device(criteria.solarwinds_device) # TODO add queries self.data = None logger.info("SolarwindsTable job %s returning %s data" % (self.job, len(self.data))) return True
def run(self): """ Main execution method """ criteria = self.job.criteria if criteria.solarwinds_device == '': logger.debug('%s: No solarwinds device selected' % self.table) self.job.mark_error("No Solarwinds Device Selected") return False sw = DeviceManager.get_device(criteria.profiler_device) # TODO add queries self.data = None logger.info("SolarwindsTable job %s returning %s data" % (self.job, len(self.data))) return True
def run(self): """ Main execution method """ criteria = self.job.criteria if criteria.profiler_device == '': logger.debug('%s: No profiler device selected' % (self.table)) self.job.mark_error("No Profiler Device Selected") return False profiler = DeviceManager.get_device(criteria.profiler_device) report = rvbd.profiler.report.SingleQueryReport(profiler) columns = [col.name for col in self.table.get_columns(synthetic=False)] # This returns an array of rows, one row per device # Each row is a dict containing elements such as: # id, ipaddr, name, type, type_id, and version with lock: devicedata = profiler.api.devices.get_all() # Convert to a DataFrame to make it easier to work with df = pandas.DataFrame(devicedata) for col in columns: if col not in df: raise KeyError("Devices table has no column '%s'" % col.name) df = df.ix[:, columns] if self.table.sortcol is not None: df = df.sort(self.table.sortcol.name) if self.table.rows > 0: df = df[:self.table.rows] self.data = df logger.info("DeviceTable job %s returning %d devices" % (self.job, len(self.data))) return True
def run(self): """ Main execution method """ criteria = self.job.criteria if criteria.profiler_device == '': logger.debug('%s: No profiler device selected' % (self.table)) self.job.mark_error("No Profiler Device Selected") return False profiler = DeviceManager.get_device(criteria.profiler_device) report = rvbd.profiler.report.SingleQueryReport(profiler) columns = [col.name for col in self.table.get_columns(synthetic=False)] # This returns an array of rows, one row per device # Each row is a dict containing elements such as: # id, ipaddr, name, type, type_id, and version with lock: devicedata = profiler.api.devices.get_all() # Convert to a DataFrame to make it easier to work with df = pandas.DataFrame(devicedata) for col in columns: if col not in df: raise KeyError("Devices table has no column '%s'" % col.name) df = df.ix[:,columns] if self.table.sortcol is not None: df = df.sort(self.table.sortcol.name) if self.table.rows > 0: df = df[:self.table.rows] self.data = df logger.info("DeviceTable job %s returning %d devices" % (self.job, len(self.data))) return True
def handle(self, *args, **options): self.stdout.write('Reloading report objects ... ') management.call_command('clean_pyc', path=settings.PROJECT_ROOT) management.call_command('syncdb', interactive=False) self.importer = Importer(buf=self.stdout) if options['report_id']: # single report report_id = options['report_id'] pk = int(report_id) report = Report.objects.get(pk=pk) management.call_command('clean', applications=False, report_id=report_id, clear_cache=False, clear_logs=False) DeviceManager.clear() self.import_module(report.sourcefile) elif options['report_name']: name = options['report_name'] try: report = Report.objects.get(sourcefile__endswith=name) report_id = report.id management.call_command('clean', applications=False, report_id=report_id, clear_cache=False, clear_logs=False) self.import_module(report.sourcefile) except ObjectDoesNotExist: self.import_module(name) DeviceManager.clear() elif options['namespace']: reports = Report.objects.filter(namespace=options['namespace']) for report in reports: management.call_command('clean', applications=False, report_id=report.id, clear_cache=False, clear_logs=False) self.import_module(report.sourcefile) else: # clear all data management.call_command('clean', applications=True, report_id=None, clear_cache=True, clear_logs=False) # start with fresh device instances DeviceManager.clear() report_dir = os.path.join(settings.PROJECT_ROOT, options['report_dir'] or 'config') self.importer.import_directory(report_dir, report_name=None) for plugin in plugins.enabled(): if plugin.reports: #from IPython import embed; embed() plugin.load_reports()
def run(self): # All user entered criteria is available directly from this object. # Values for any fields added to the table will appear as # attributes according to the field keyword. criteria = self.job.criteria # Check that a sample_device was selected if criteria.sample_device == "": logger.debug("%s: No sample device selected" % self.table) self.job.mark_error("No Sample Device Selected") return False sample_device = DeviceManager.get_device(criteria.sample_device) # Get the columns for this report columns = self.table.get_columns(synthetic=False) sortcol = None if self.table.sortcol is not None: sortcol = self.table.sortcol.name # Time selection is available via criterai.starttime and endtime. # These are both datetime objects. t0 = criteria.starttime t1 = criteria.endtime # Time resolution is a timedelta object resolution = criteria.resolution # Grab the custom min and max criteria cmin = float(criteria.min) cmax = float(criteria.max) # Grab the table options beta = self.table.options.beta # Now, do some computation -- create table with a 'time' column # ranging from t0 to t1 with the defined resolution. Then # for each additional column do some math function on the # data t = t0 rows = [] while t < t1: row = [] for col in columns: if col.name == "time": row.append(t) else: period_td = parse_timedelta(col.options.period) period_secs = timedelta_total_seconds(period_td) alpha = col.options.alpha funcname = col.options.func # seconds since the t0 secs = timedelta_total_seconds(t - t0) rad = (secs / period_secs) * 2 * math.pi funcmap = {"sin": math.sin, "cos": math.cos} # Compute! val = beta + alpha * funcmap[funcname](rad) # Clip by the min/max criteria val = max(cmin, val) val = min(cmax, val) # Add the value to the row row.append(val) # Add the row rows.append(row) # This function runs pretty fast, but this shows how to mark # progress self.job.mark_progress(100 * (timedelta_total_seconds(t - t0) / timedelta_total_seconds(t1 - t0))) t = t + resolution # Save the result in self.data self.data = rows if self.table.rows > 0: self.data = self.data[: self.table.rows] logger.info("Report %s returned %s rows" % (self.job, len(self.data))) return True
def run(self): """ Main execution method """ criteria = self.job.criteria if criteria.shark_device == '': logger.debug('%s: No shark device selected' % self.table) self.job.mark_error("No Shark Device Selected") return False #self.fake_run() #return True shark = DeviceManager.get_device(criteria.shark_device) logger.debug("Creating columns for Shark table %d" % self.table.id) # Create Key/Value Columns columns = [] for tc in self.table.get_columns(synthetic=False): tc_options = tc.options if (tc.iskey and tc.name == 'time' and tc_options.extractor == 'sample_time'): # don't create column, use the sample time for timeseries self.timeseries = True self.column_names.append('time') continue elif tc.iskey: c = Key(tc_options.extractor, description=tc.label, default_value=tc_options.default_value) else: if tc_options.operation: try: operation = getattr(Operation, tc_options.operation) except AttributeError: operation = Operation.sum print('ERROR: Unknown operation attribute ' '%s for column %s.' % (tc_options.operation, tc.name)) else: operation = Operation.none c = Value(tc_options.extractor, operation, description=tc.label, default_value=tc_options.default_value) self.column_names.append(tc.name) columns.append(c) # Identify Sort Column sortidx = None if self.table.sortcol is not None: sort_name = self.table.sortcol.options.extractor for i, c in enumerate(columns): if c.field == sort_name: sortidx = i break # Initialize filters criteria = self.job.criteria filters = [] filterexpr = self.job.combine_filterexprs( exprs=criteria.shark_filterexpr, joinstr="&") if filterexpr: filters.append(SharkFilter(filterexpr)) tf = TimeFilter(start=criteria.starttime, end=criteria.endtime) filters.append(tf) logger.info("Setting shark table %d timeframe to %s" % (self.table.id, str(tf))) # Get source type from options try: with lock: source = path_to_class(shark, self.job.criteria.shark_source_name) except RvbdHTTPException, e: source = None raise e
def run(self): """ Main execution method """ criteria = self.job.criteria if criteria.profiler_device == '': logger.debug('%s: No profiler device selected' % self.table) self.job.mark_error("No Profiler Device Selected") return False #self.fake_run() #return True profiler = DeviceManager.get_device(criteria.profiler_device) report = rvbd.profiler.report.SingleQueryReport(profiler) columns = [col.name for col in self.table.get_columns(synthetic=False)] sortcol = None if self.table.sortcol is not None: sortcol = self.table.sortcol.name tf = TimeFilter(start=criteria.starttime, end=criteria.endtime) logger.info("Running Profiler table %d report for timeframe %s" % (self.table.id, str(tf))) if ('datafilter' in criteria) and (criteria.datafilter is not None): datafilter = criteria.datafilter.split(',') else: datafilter = None trafficexpr = TrafficFilter( self.job.combine_filterexprs(exprs=criteria.profiler_filterexpr) ) # Incoming criteria.resolution is a timedelta logger.debug('Profiler report got criteria resolution %s (%s)' % (criteria.resolution, type(criteria.resolution))) if criteria.resolution != 'auto': rsecs = int(timedelta_total_seconds(criteria.resolution)) resolution = rvbd.profiler.report.Report.RESOLUTION_MAP[rsecs] else: resolution = 'auto' logger.debug('Profiler report using resolution %s (%s)' % (resolution, type(resolution))) with lock: report.run(realm=self.table.options.realm, groupby=profiler.groupbys[self.table.options.groupby], centricity=self.table.options.centricity, columns=columns, timefilter=tf, trafficexpr=trafficexpr, data_filter=datafilter, resolution=resolution, sort_col=sortcol, sync=False ) done = False logger.info("Waiting for report to complete") while not done: time.sleep(0.5) with lock: s = report.status() self.job.safe_update(progress=int(s['percent'])) done = (s['status'] == 'completed') # Retrieve the data with lock: query = report.get_query_by_index(0) self.data = query.get_data() tz = criteria.starttime.tzinfo # Update criteria criteria.starttime = (datetime.datetime .utcfromtimestamp(query.actual_t0) .replace(tzinfo=tz)) criteria.endtime = (datetime.datetime .utcfromtimestamp(query.actual_t1) .replace(tzinfo=tz)) self.job.safe_update(actual_criteria=criteria) if self.table.rows > 0: self.data = self.data[:self.table.rows] logger.info("Report %s returned %s rows" % (self.job, len(self.data))) return True
def run(self): """ Main execution method """ criteria = self.job.criteria if criteria.shark_device == '': logger.debug('%s: No shark device selected' % self.table) self.job.mark_error("No Shark Device Selected") return False #self.fake_run() #return True shark = DeviceManager.get_device(criteria.shark_device) logger.debug("Creating columns for Shark table %d" % self.table.id) # Create Key/Value Columns columns = [] for tc in self.table.get_columns(synthetic=False): tc_options = tc.options if ( tc.iskey and tc.name == 'time' and tc_options.extractor == 'sample_time'): # don't create column, use the sample time for timeseries self.timeseries = True self.column_names.append('time') continue elif tc.iskey: c = Key(tc_options.extractor, description=tc.label, default_value=tc_options.default_value) else: if tc_options.operation: try: operation = getattr(Operation, tc_options.operation) except AttributeError: operation = Operation.sum print ('ERROR: Unknown operation attribute ' '%s for column %s.' % (tc_options.operation, tc.name)) else: operation = Operation.none c = Value(tc_options.extractor, operation, description=tc.label, default_value=tc_options.default_value) self.column_names.append(tc.name) columns.append(c) # Identify Sort Column sortidx = None if self.table.sortcol is not None: sort_name = self.table.sortcol.options.extractor for i, c in enumerate(columns): if c.field == sort_name: sortidx = i break # Initialize filters criteria = self.job.criteria filters = [] filterexpr = self.job.combine_filterexprs( exprs=criteria.shark_filterexpr, joinstr="&" ) if filterexpr: filters.append(SharkFilter(filterexpr)) tf = TimeFilter(start=criteria.starttime, end=criteria.endtime) filters.append(tf) logger.info("Setting shark table %d timeframe to %s" % (self.table.id, str(tf))) # Get source type from options try: with lock: source = path_to_class(shark, self.job.criteria.shark_source_name) except RvbdHTTPException, e: source = None raise e
def handle(self, *args, **options): self.stdout.write('Reloading report objects ... ') management.call_command('clean_pyc', path=settings.PROJECT_ROOT) management.call_command('syncdb', interactive=False) self.importer = Importer(buf=self.stdout) if options['report_id']: # single report report_id = options['report_id'] pk = int(report_id) report = Report.objects.get(pk=pk) management.call_command('clean', applications=False, report_id=report_id, clear_cache=False, clear_logs=False) DeviceManager.clear() self.import_module(report.sourcefile) elif options['report_name']: name = options['report_name'] try: report = Report.objects.get(sourcefile__endswith=name) report_id = report.id management.call_command('clean', applications=False, report_id=report_id, clear_cache=False, clear_logs=False) self.import_module(report.sourcefile) except ObjectDoesNotExist: self.import_module(name) DeviceManager.clear() elif options['namespace']: reports = Report.objects.filter(namespace=options['namespace']) self.capture_enabled(reports) for report in reports: management.call_command('clean', applications=False, report_id=report.id, clear_cache=False, clear_logs=False) self.import_module(report.sourcefile) self.apply_enabled() else: self.capture_enabled() # clear all data management.call_command('clean', applications=True, report_id=None, clear_cache=True, clear_logs=False) # start with fresh device instances DeviceManager.clear() report_dir = os.path.join(settings.PROJECT_ROOT, options['report_dir'] or 'config') self.importer.import_directory(report_dir, report_name=None) for plugin in plugins.enabled(): if plugin.reports: #from IPython import embed; embed() plugin.load_reports() self.apply_enabled()
def run(self): # All user entered criteria is available directly from this object. # Values for any fields added to the table will appear as # attributes according to the field keyword. criteria = self.job.criteria # Check that a sample_device was selected if criteria.sample_device == '': logger.debug('%s: No sample device selected' % self.table) self.job.mark_error("No Sample Device Selected") return False sample_device = DeviceManager.get_device(criteria.sample_device) # Get the columns for this report columns = self.table.get_columns(synthetic=False) sortcol = None if self.table.sortcol is not None: sortcol = self.table.sortcol.name # Time selection is available via criterai.starttime and endtime. # These are both datetime objects. t0 = criteria.starttime t1 = criteria.endtime # Time resolution is a timedelta object resolution = criteria.resolution # Grab the custom min and max criteria cmin = float(criteria.min) cmax = float(criteria.max) # Grab the table options beta = self.table.options.beta # Now, do some computation -- create table with a 'time' column # ranging from t0 to t1 with the defined resolution. Then # for each additional column do some math function on the # data t = t0 rows = [] while t < t1: row = [] for col in columns: if col.name == 'time': row.append(t) else: period_td = parse_timedelta(col.options.period) period_secs = timedelta_total_seconds(period_td) alpha = col.options.alpha funcname = col.options.func # seconds since the t0 secs = timedelta_total_seconds(t - t0) rad = (secs / period_secs) * 2 * math.pi funcmap = { 'sin': math.sin, 'cos': math.cos, } # Compute! val = beta + alpha * funcmap[funcname](rad) # Clip by the min/max criteria val = max(cmin, val) val = min(cmax, val) # Add the value to the row row.append(val) # Add the row rows.append(row) # This function runs pretty fast, but this shows how to mark # progress self.job.mark_progress(100 * (timedelta_total_seconds(t - t0) / timedelta_total_seconds(t1 - t0))) t = t + resolution # Save the result in self.data self.data = rows if self.table.rows > 0: self.data = self.data[:self.table.rows] logger.info("Report %s returned %s rows" % (self.job, len(self.data))) return True
def run(self): """ Main execution method. """ criteria = self.job.criteria if criteria.profiler_device == '': logger.debug('%s: No profiler device selected' % self.table) self.job.mark_error("No Profiler Device Selected") return False profiler = DeviceManager.get_device(criteria.profiler_device) report = rvbd.profiler.report.MultiQueryReport(profiler) tf = TimeFilter(start=criteria.starttime, end=criteria.endtime) logger.info("Running ProfilerTemplateTable table %d report " "for timeframe %s" % (self.table.id, str(tf))) trafficexpr = TrafficFilter( self.job.combine_filterexprs(exprs=criteria.profiler_filterexpr) ) # Incoming criteria.resolution is a timedelta logger.debug('Profiler report got criteria resolution %s (%s)' % (criteria.resolution, type(criteria.resolution))) if criteria.resolution != 'auto': rsecs = int(timedelta_total_seconds(criteria.resolution)) resolution = rvbd.profiler.report.Report.RESOLUTION_MAP[rsecs] else: resolution = 'auto' logger.debug('Profiler report using resolution %s (%s)' % (resolution, type(resolution))) with lock: res = report.run(template_id=self.table.options.template_id, timefilter=tf, trafficexpr=trafficexpr, resolution=resolution) if res is True: logger.info("Report template complete.") self.job.safe_update(progress=100) # Retrieve the data with lock: query = report.get_query_by_index(0) data = query.get_data() headers = report.get_legend() tz = criteria.starttime.tzinfo # Update criteria criteria.starttime = (datetime.datetime .utcfromtimestamp(query.actual_t0) .replace(tzinfo=tz)) criteria.endtime = (datetime.datetime .utcfromtimestamp(query.actual_t1) .replace(tzinfo=tz)) self.job.safe_update(actual_criteria=criteria) # create dataframe with all of the default headers df = pandas.DataFrame(data, columns=[h.key for h in headers]) # now filter down to the columns requested by the table columns = [col.name for col in self.table.get_columns(synthetic=False)] self.data = df[columns] if self.table.sortcol is not None: self.data = self.data.sort(self.table.sortcol.name) if self.table.rows > 0: self.data = self.data[:self.table.rows] logger.info("Report %s returned %s rows" % (self.job, len(self.data))) return True