def put(self, request, *args, **kwargs):
""" Function to save changes to multiple devices once.
This function is called only when the "Save Changes" button is
clicked on /devices/ page. However, it only supports enable/disable
device(s). The url sent out will only include 'enable' field.
"""
DeviceFormSet = modelformset_factory(Device,
form=DeviceListForm,
extra=0)
formset = DeviceFormSet(request.DATA)
if formset.is_valid():
formset.save()
DeviceManager.clear()
messages.success(request._request,
'Changes successfully saved')
if '/devices' not in request.META['HTTP_REFERER']:
return HttpResponseRedirect(request.META['HTTP_REFERER'])
else:
return HttpResponseRedirect(reverse('device-list'))
else:
data = {'formset': formset, 'auth': Auth}
return Response(data, template_name='device_list.html')
def post(self, request):
form = DeviceBatchForm(data=request.POST,
files=request.FILES)
if not form.is_valid():
return Response({'form': form},
template_name='device_batch.html')
data = form.cleaned_data
try:
msg = StringIO()
management.call_command('device', batch_file=data['batch_file'],
stdout=msg)
messages.add_message(request._request, messages.INFO,
msg.getvalue())
except CommandError as e:
form._errors['batch_file'] = form.error_class([e])
return Response({'form': form},
template_name='device_batch.html')
DeviceManager.clear()
return HttpResponseRedirect(reverse('device-list'))
def post(self, request, device_type):
if device_type == 'netprofiler':
device = 'NetProfiler'
form = NetProfilerInputForm(request.POST)
elif device_type == 'netshark':
device = 'NetShark'
form = NetSharkInputForm(request.POST)
else:
raise Http404
results = None
if form.is_valid():
data = form.cleaned_data
if device_type == 'netprofiler':
profiler = DeviceManager.get_device(data['device'])
results = profiler.search_columns(realms=[data['realm']],
centricities=[data['centricity']],
groupbys=[data['groupby']])
results.sort(key=operator.attrgetter('key'))
results.sort(key=operator.attrgetter('iskey'), reverse=True)
results = [(c.iskey, c.key, c.label, c.id) for c in results]
elif device_type == 'netshark':
shark = DeviceManager.get_device(data['device'])
results = [(f.id, f.description, f.type) for f in shark.get_extractor_fields()]
results.sort(key=operator.itemgetter(0))
return render_to_response('help.html',
{'device': device,
'form': form,
'results': results},
context_instance=RequestContext(request))
def test_get_devices_with_unknown_module(self):
dev = copy.copy(self.dev)
dev['id'] = 2
dev['module'] = 'unknown_module'
dev_obj = Device(**dev)
dev_obj.save()
with self.assertRaises(DeviceModuleNotFound):
DeviceManager.get_device(2)
def post(self, request, device_id=None):
if device_id:
device = get_object_or_404(Device, pk=device_id)
form = DeviceDetailForm(request.DATA, instance=device)
else:
form = DeviceDetailForm(request.DATA)
if form.is_valid():
form.save()
DeviceManager.clear()
return HttpResponseRedirect(reverse('device-list'))
else:
return Response({'form': form, 'auth': Auth},
template_name='device_detail.html')
def __init__(self, *args, **kwargs):
super(DeviceDetailForm, self).__init__(*args, **kwargs)
modules = DeviceManager.get_modules()
choices = zip(modules, modules)
self.fields['module'] = forms.ChoiceField(choices=choices)
def netshark_source_name_choices(form, id_, field_kwargs, params):
""" Query netshark for available capture jobs / trace clips. """
netshark_device = form.get_field_value('netshark_device', id_)
if netshark_device == '':
choices = [('', '<No netshark device>')]
else:
netshark = DeviceManager.get_device(netshark_device)
choices = []
for job in netshark.get_capture_jobs():
choices.append((job.source_path, job.name))
for clip in netshark.get_clips():
choices.append((clip.source_path, 'Clip: ' + clip.description))
if params['include_files']:
for f in netshark.get_files():
choices.append((f.source_path, 'File: ' + f.path))
if params['include_interfaces']:
for iface in netshark.get_interfaces():
choices.append((iface.source_path, 'If: ' + iface.description))
field_kwargs['label'] = 'Source'
field_kwargs['choices'] = choices
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
if criteria.netprofiler_device == '':
logger.debug('%s: No netprofiler device selected' % (self.table))
self.job.mark_error("No NetProfiler Device Selected")
return False
profiler = DeviceManager.get_device(criteria.netprofiler_device)
columns = [col.name for col in self.table.get_columns(synthetic=False)]
# This returns an array of rows, one row per device
# Each row is a dict containing elements such as:
# id, ipaddr, name, type, type_id, and version
with lock:
devicedata = profiler.api.devices.get_all()
# Convert to a DataFrame to make it easier to work with
df = pandas.DataFrame(devicedata)
for col in columns:
if col not in df:
raise KeyError("Devices table has no column '%s'" % col.name)
df = df.ix[:,columns]
self.data = df
logger.info("DeviceTable job %s returning %d devices" % (self.job, len(self.data)))
return True
def netshark_source_name_choices(form, id, field_kwargs, params):
""" Query netshark for available capture jobs / trace clips. """
netshark_device = form.get_field_value('netshark_device', id)
if netshark_device == '':
label = 'Source'
choices = [('', '<No netshark device>')]
else:
netshark = DeviceManager.get_device(netshark_device)
#source_type = form.get_field_value('shark_source_type', id)
source_type = 'job'
choices = []
if source_type == 'job':
for job in netshark.get_capture_jobs():
choices.append(('jobs/' + job.name, job.name))
label = 'Capture Job'
elif source_type == 'clip':
# Not tested
label = 'Trace Clip'
for clip in netshark.get_clips():
choices.append((clip, clip))
else:
raise KeyError('Unknown source type: %s' % source_type)
field_kwargs['label'] = label
field_kwargs['choices'] = choices
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
if criteria.sharepoint_device == '':
logger.debug('%s: No sharepoint device selected' % self.table)
self.job.mark_error("No Sharepoint Device Selected")
return False
sp = DeviceManager.get_device(criteria.sharepoint_device)
site = sp.get_site_object(self.table.options.site_url)
site_instance = site.lists[self.table.options.list_name]
fields = [tc.name for tc in self.table.get_columns(synthetic=False)]
self.data = []
for row in site_instance.rows:
d = [getattr(row, f) for f in fields]
self.data.append(d)
logger.info("SharepointTable job %s returning %s data" %
(self.job, len(self.data)))
return True
def analyze(self, jobs):
criteria = self.job.criteria
sharks_query_table = Table.from_ref(
self.table.options.related_tables['basetable'])
depjobs = {}
# For every (shark, job), we spin off a new job to grab the data, then
# merge everything into one dataframe at the end.
for s in Device.objects.filter(module='netshark', enabled=True):
shark = DeviceManager.get_device(s.id)
for capjob in shark.get_capture_jobs():
# Start with criteria from the primary table -- this gives us
# endtime, duration and netshark_filterexpr.
bytes_criteria = copy.copy(criteria)
bytes_criteria.netshark_device = s.id
bytes_criteria.netshark_source_name = 'jobs/' + capjob.name
bytes_criteria.resolution = datetime.timedelta(0, 1)
bytes_criteria.aggregated = True
job = Job.create(table=sharks_query_table,
criteria=bytes_criteria)
depjobs[job.id] = job
return QueryContinue(self.collect, depjobs)
def add_widgets_to_live_report(report, template_id, widget_query_ids,
netprofiler_name=None):
if netprofiler_name:
netprofiler_id = Device.objects.filter(name=netprofiler_name)[0].id
else:
netprofiler_id = Device.objects.\
filter(enabled=True, module='netprofiler')[0].id
profiler = DeviceManager.get_device(netprofiler_id)
lr = LiveReport(profiler, template_id)
for wid, qid in widget_query_ids.items():
q = [q for q in lr.queries if q.id.endswith(qid)][0]
t = NetProfilerLiveTable.create(
'live-{0}-{1}'.format(template_id, wid),
netprofiler_id=netprofiler_id,
template_id=template_id,
query_id=q.id,
widget_id=wid,
cacheable=False)
if q.is_time_series:
widget_cls = yui3.TimeSeriesWidget
t.add_column('time', 'Time', datatype='time', iskey=True)
else:
widget_cls = yui3.TableWidget
widget_title = 'Template %s Widget %s' % (template_id, wid)
report.add_widget(widget_cls, t, widget_title, width=12)
def appresponse_source_choices(form, id_, field_kwargs, params):
""" Query AppResponse for available capture jobs / files."""
# most of these results will be cached by the underlying AR object
ar_id = form.get_field_value('appresponse_device', id_)
with lock:
if ar_id == '':
choices = [('', '<No AppResponse Device>')]
else:
ar = DeviceManager.get_device(ar_id)
choices = []
for job in ar.capture.get_jobs():
if job.status == 'RUNNING':
choices.append((SourceProxy(job).path, job.name))
for clip in ar.clips.get_clips():
choices.append((SourceProxy(clip).path, clip.name))
if params['include_files']:
for f in ar.fs.get_files():
choices.append((SourceProxy(f).path, f.id))
if params['include_msa_files_only']:
choices = []
for f in ar.fs.get_files(force=True):
if f.is_msa():
choices.append((SourceProxy(f).path, f.id))
field_kwargs['label'] = 'Source'
field_kwargs['choices'] = choices
def analyze(self, jobs):
criteria = self.job.criteria
sharks_query_table = Table.from_ref(
self.table.options.related_tables['basetable'])
depjobs = {}
# For every (shark, job), we spin off a new job to grab the data, then
# merge everything into one dataframe at the end.
for s in Device.objects.filter(module='netshark', enabled=True):
shark = DeviceManager.get_device(s.id)
for capjob in shark.get_capture_jobs():
# Start with criteria from the primary table -- this gives us
# endtime, duration and netshark_filterexpr.
bytes_criteria = copy.copy(criteria)
bytes_criteria.netshark_device = s.id
bytes_criteria.netshark_source_name = 'jobs/' + capjob.name
bytes_criteria.resolution = datetime.timedelta(0, 1)
bytes_criteria.aggregated = True
job = Job.create(table=sharks_query_table,
criteria=bytes_criteria)
depjobs[job.id] = job
return QueryContinue(self.collect, depjobs)
def add_widgets_to_live_report(report,
template_id,
widget_query_ids,
netprofiler_name=None):
if netprofiler_name:
netprofiler_id = Device.objects.filter(name=netprofiler_name)[0].id
else:
netprofiler_id = Device.objects.\
filter(enabled=True, module='netprofiler')[0].id
profiler = DeviceManager.get_device(netprofiler_id)
lr = LiveReport(profiler, template_id)
for wid, qid in widget_query_ids.iteritems():
q = [q for q in lr.queries if q.id.endswith(qid)][0]
t = NetProfilerLiveTable.create('live-{0}-{1}'.format(
template_id, wid),
netprofiler_id=netprofiler_id,
template_id=template_id,
query_id=q.id,
widget_id=wid,
cacheable=False)
if q.is_time_series:
widget_cls = yui3.TimeSeriesWidget
t.add_column('time', 'Time', datatype='time', iskey=True)
else:
widget_cls = yui3.TableWidget
widget_title = 'Template %s Widget %s' % (template_id, wid)
report.add_widget(widget_cls, t, widget_title, width=12)
def run(self):
criteria = self.job.criteria
netshark = DeviceManager.get_device(criteria.netshark_device)
self.export_name = str(path_to_class(netshark,
criteria.netshark_source_name))
source = netshark.get_capture_job_by_name(self.export_name)
timefilter = TimeFilter(criteria.starttime, criteria.endtime)
handle = Job._compute_handle(self.table, criteria)
# check if pcaps directory exists, if not make the directory
if not os.path.exists(PCAP_DIR):
os.mkdir(PCAP_DIR)
while self.all_pcap_size > settings.PCAP_SIZE_LIMIT:
self.delete_oldest_pcap()
self.filename = add_pcap_dir('%s.pcap' % handle)
filters = ([BpfFilter(filt) for filt in self.table.options.filters]
or None)
with netshark.create_export(
source, timefilter, filters=filters,
wait_for_data=self.table.options.wait_for_data,
wait_duration=self.table.options.wait_duration) as e:
self.download(e)
return QueryComplete(pandas.DataFrame([dict(filename=self.filename)]))
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
if criteria.netprofiler_device == '':
logger.debug('%s: No netprofiler device selected' % (self.table))
self.job.mark_error("No NetProfiler Device Selected")
return False
profiler = DeviceManager.get_device(criteria.netprofiler_device)
columns = [col.name for col in self.table.get_columns(synthetic=False)]
# This returns an array of rows, one row per device
# Each row is a dict containing elements such as:
# id, ipaddr, name, type, type_id, and version
with lock:
devicedata = profiler.api.devices.get_all()
# Convert to a DataFrame to make it easier to work with
df = pandas.DataFrame(devicedata)
for col in columns:
if col not in df:
raise KeyError("Devices table has no column '%s'" % col.name)
df = df.ix[:, columns]
self.data = df
logger.info("DeviceTable job %s returning %d devices" %
(self.job, len(self.data)))
return True
def run(self):
# For each of the widget, get all the data
profiler = DeviceManager.get_device(self.table.options.netprofiler_id)
lr = LiveReport(profiler, template_id=self.table.options.template_id)
# Figure out columns by querying the widget
# cols = lr.get_columns(self.table.options.widget_id)
# Find the query object
query_idx = lr.get_query_names().index(self.table.options.query_id)
# refresh the columns of the table
self._refresh_columns(profiler, report=lr, query=lr.queries[query_idx])
data = lr.get_data(index=query_idx)
col_names = [
col.label if col.ephemeral else col.key
for col in lr.queries[query_idx].columns
]
df = pd.DataFrame(columns=col_names, data=data)
return QueryComplete(df)
def __init__(self, *args, **kwargs):
super(DeviceDetailForm, self).__init__(*args, **kwargs)
modules = DeviceManager.get_modules()
choices = zip(modules, modules)
self.fields['module'] = forms.ChoiceField(choices=choices)
def analyze(self, jobs):
criteria = self.job.criteria
ar_query_table = Table.from_ref(
self.table.options.related_tables['basetable']
)
depjobs = {}
# For every (ar, job), we spin off a new job to grab the data, then
# merge everything into one dataframe at the end.
for s in Device.objects.filter(module='appresponse', enabled=True):
ar = DeviceManager.get_device(s.id)
for job in ar.capture.get_jobs():
# Start with criteria from the primary table -- this gives us
# endtime, duration and filterexpr.
bytes_criteria = copy.copy(criteria)
bytes_criteria.appresponse_device = s.id
bytes_criteria.appresponse_source = 'jobs/' + job.id
bytes_criteria.granularity = datetime.timedelta(0, 1)
newjob = Job.create(table=ar_query_table,
criteria=bytes_criteria)
depjobs[newjob.id] = newjob
return QueryContinue(self.collect, depjobs)
def run(self):
criteria = self.job.criteria
netshark = DeviceManager.get_device(criteria.netshark_device)
self.export_name = str(
path_to_class(netshark, criteria.netshark_source_name))
source = netshark.get_capture_job_by_name(self.export_name)
timefilter = TimeFilter(criteria.starttime, criteria.endtime)
handle = Job._compute_handle(self.table, criteria)
# check if pcaps directory exists, if not make the directory
if not os.path.exists(PCAP_DIR):
os.mkdir(PCAP_DIR)
while self.all_pcap_size > settings.PCAP_SIZE_LIMIT:
self.delete_oldest_pcap()
self.filename = add_pcap_dir('%s.pcap' % handle)
filters = ([BpfFilter(filt) for filt in self.table.options.filters]
or None)
with netshark.create_export(
source,
timefilter,
filters=filters,
wait_for_data=self.table.options.wait_for_data,
wait_duration=self.table.options.wait_duration) as e:
self.download(e)
return QueryComplete(pandas.DataFrame([dict(filename=self.filename)]))
def netshark_source_name_choices(form, id, field_kwargs, params):
""" Query netshark for available capture jobs / trace clips. """
netshark_device = form.get_field_value('netshark_device', id)
if netshark_device == '':
choices = [('', '<No netshark device>')]
else:
netshark = DeviceManager.get_device(netshark_device)
choices = []
for job in netshark.get_capture_jobs():
choices.append((job.source_path, job.name))
for clip in netshark.get_clips():
choices.append((clip.source_path, 'Clip: ' + clip.description))
if params['include_files']:
for f in netshark.get_files():
choices.append((f.source_path, 'File: ' + f.path))
if params['include_interfaces']:
for iface in netshark.get_interfaces():
choices.append((iface.source_path, 'If: ' + iface.description))
field_kwargs['label'] = 'Source'
field_kwargs['choices'] = choices
def _prepare_report_args(self):
class Args(object):
pass
args = Args()
criteria = self.job.criteria
if criteria.netprofiler_device == '':
logger.debug('%s: No netprofiler device selected' % self.table)
self.job.mark_error("No NetProfiler Device Selected")
return False
args.profiler = DeviceManager.get_device(criteria.netprofiler_device)
args.columns = [
col.name for col in self.table.get_columns(synthetic=False)
]
args.sortcol = None
if self.table.sortcols is not None:
args.sortcol = self.table.sortcols[0]
args.timefilter = TimeFilter(start=criteria.starttime,
end=criteria.endtime)
logger.info("Running NetProfiler table %d report for timeframe %s" %
(self.table.id, str(args.timefilter)))
if ('datafilter' in criteria) and (criteria.datafilter is not None):
args.datafilter = criteria.datafilter.split(',')
else:
args.datafilter = None
args.trafficexpr = TrafficFilter(
self.job.combine_filterexprs(
exprs=criteria.netprofiler_filterexpr))
# Incoming criteria.resolution is a timedelta
logger.debug('NetProfiler report got criteria resolution %s (%s)' %
(criteria.resolution, type(criteria.resolution)))
if criteria.resolution != 'auto':
rsecs = int(timedelta_total_seconds(criteria.resolution))
args.resolution = Report.RESOLUTION_MAP[rsecs]
else:
args.resolution = 'auto'
logger.debug('NetProfiler report using resolution %s (%s)' %
(args.resolution, type(args.resolution)))
args.limit = (self.table.options.limit if hasattr(
self.table.options, 'limit') else None)
if getattr(self.table.options, 'interface', False):
args.centricity = 'int'
else:
args.centricity = 'hos'
return args
def test_get_devices(self):
with patch("dev_pkg.new_device_instance", MockDevice):
device = DeviceManager.get_device(1)
self.assertEqual(device.host, self.dev['host'])
self.assertEqual(device.port, self.dev['port'])
self.assertEqual(device.auth.username, self.dev['username'])
self.assertEqual(device.auth.password, self.dev['password'])
def _prepare_report_args(self):
class Args(object):
pass
args = Args()
criteria = self.job.criteria
if criteria.netprofiler_device == '':
logger.debug('%s: No netprofiler device selected' % self.table)
self.job.mark_error("No NetProfiler Device Selected")
return False
args.profiler = DeviceManager.get_device(criteria.netprofiler_device)
args.columns = [col.name for col
in self.table.get_columns(synthetic=False)]
args.sortcol = None
if self.table.sortcols is not None:
args.sortcol = self.table.sortcols[0]
args.timefilter = TimeFilter(start=criteria.starttime,
end=criteria.endtime)
logger.info("Running NetProfiler table %d report for timeframe %s" %
(self.table.id, str(args.timefilter)))
if ('datafilter' in criteria) and (criteria.datafilter is not None):
args.datafilter = criteria.datafilter.split(',')
else:
args.datafilter = None
args.trafficexpr = TrafficFilter(
self.job.combine_filterexprs(exprs=criteria.netprofiler_filterexpr)
)
# Incoming criteria.resolution is a timedelta
logger.debug('NetProfiler report got criteria resolution %s (%s)' %
(criteria.resolution, type(criteria.resolution)))
if criteria.resolution != 'auto':
rsecs = int(timedelta_total_seconds(criteria.resolution))
args.resolution = Report.RESOLUTION_MAP[rsecs]
else:
args.resolution = 'auto'
logger.debug('NetProfiler report using resolution %s (%s)' %
(args.resolution, type(args.resolution)))
args.limit = (self.table.options.limit
if hasattr(self.table.options, 'limit') else None)
if getattr(self.table.options, 'interface', False):
args.centricity = 'int'
else:
args.centricity = 'hos'
return args
def __init__(self, *args, **kwargs):
super(DeviceDetailForm, self).__init__(*args, **kwargs)
modules = DeviceManager.get_modules()
choices = zip(modules, modules)
self.fields['module'] = forms.ChoiceField(
choices=choices,
widget=forms.Select(attrs={"onChange": 'changeMod()'}))
def post(self, request, data_type):
if data_type not in ['columns', 'sources']:
raise Http404
device = 'AppResponse'
if data_type == 'columns':
form = AppResponseColumnsInputForm(request.POST)
else:
form = AppResponseInputForm(request.POST)
results = None
if form.is_valid():
data = form.cleaned_data
ar = DeviceManager.get_device(data['device'])
if data_type == 'columns':
rawcols = ar.reports.sources[data['source']]['columns']
for col in rawcols.values():
if 'synthesized' in col:
synth = col['synthesized']
if isinstance(synth, dict):
col['synthesized'] = \
(', '.join(['{}: {}'.format(k, v)
for k, v in synth.iteritems()]))
colkeys = [
'id', 'field', 'label', 'metric', 'type', 'unit',
'description', 'synthesized', 'grouped_by'
]
coldf = pandas.DataFrame(rawcols.values(), columns=colkeys)
coldf.fillna('---', inplace=True)
coldf['iskey'] = coldf['grouped_by'].apply(
lambda x: True if x is True else '---')
coldf.sort_values(by='id', inplace=True)
results = list(coldf.to_records(index=False))
else:
colkeys = [
'name', 'filters_on_metrics', 'granularities', 'groups'
]
coldf = pandas.DataFrame(ar.reports.sources.values(),
columns=colkeys)
coldf['groups'] = coldf['name'].apply(
lambda x: ', '.join(report_source_to_groups[x]))
coldf.sort_values(by='name', inplace=True)
results = list(coldf.to_records(index=False))
serialized_sources = json.dumps(report_sources)
return render_to_response('help.html', {
'device': device,
'report_sources': serialized_sources,
'data_type': data_type,
'form': form,
'results': results
},
context_instance=RequestContext(request))
def run(self):
sh = DeviceManager.get_device(self.job.criteria.steelhead_device)
flows = Model.get(sh, feature='flows')
res = flows.show_flows('all')
for k, v in res['flows_summary'].iteritems():
v['category'] = k
return QueryComplete(res['flows_summary'].values())
def run(self):
sh_db = self.job.criteria.dev
cmd = self.job.criteria.command
sh = DeviceManager.get_device(sh_db.id)
output = sh.cli.exec_command(cmd, mode=CLIMode.ENABLE)
return QueryComplete([dict(dev_name=sh_db.name, output=output)])
def post(self, request, data_type):
if data_type not in ['columns', 'sources']:
raise Http404
device = 'AppResponse'
if data_type == 'columns':
form = AppResponseColumnsInputForm(request.POST)
else:
form = AppResponseInputForm(request.POST)
results = None
if form.is_valid():
data = form.cleaned_data
ar = DeviceManager.get_device(data['device'])
if data_type == 'columns':
rawcols = ar.reports.sources[data['source']]['columns']
for col in rawcols.values():
if 'synthesized' in col:
synth = col['synthesized']
if isinstance(synth, dict):
col['synthesized'] = \
(', '.join(['{}: {}'.format(k, v)
for k, v in synth.iteritems()]))
colkeys = ['id', 'field', 'label', 'metric', 'type',
'unit', 'description', 'synthesized', 'grouped_by']
coldf = pandas.DataFrame(rawcols.values(), columns=colkeys)
coldf.fillna('---', inplace=True)
coldf['iskey'] = coldf['grouped_by'].apply(
lambda x: True if x is True else '---')
coldf.sort_values(by='id', inplace=True)
results = list(coldf.to_records(index=False))
else:
colkeys = ['name', 'filters_on_metrics', 'granularities',
'groups']
coldf = pandas.DataFrame(ar.reports.sources.values(),
columns=colkeys)
coldf['groups'] = coldf['name'].apply(
lambda x: ', '.join(report_source_to_groups[x]))
coldf.sort_values(by='name', inplace=True)
results = list(coldf.to_records(index=False))
serialized_sources = json.dumps(report_sources)
return render_to_response('help.html',
{'device': device,
'report_sources': serialized_sources,
'data_type': data_type,
'form': form,
'results': results},
context_instance=RequestContext(request))
def run(self):
criteria = self.job.criteria
if criteria.scc_device == '':
logger.debug('%s: No scc device selected' % (self.table))
self.job.mark_error("No SCC Device Selected")
return False
columns = [col.name for col in self.table.get_columns(synthetic=False)]
scc = DeviceManager.get_device(criteria.scc_device)
# obtain the report class definition
report_cls = get_scc_report_class(self.service, self.resource)
# instatiate a report object
report_obj = report_cls(scc)
# Build criteria kwargs
kwargs = {}
for name in set(report_obj.required_fields +
report_obj.non_required_fields):
# criteria has attrs as starttime, endtime
# which maps to start_time and end_time
# referenced in a SCC service
if name in ['start_time', 'end_time']:
name_in_criteria = name.replace('_', '')
else:
name_in_criteria = name
if hasattr(criteria, name_in_criteria):
kwargs[name] = getattr(criteria, name_in_criteria)
report_obj.run(**kwargs)
df = self.extract_dataframe(report_obj.data)
if df is not None:
for col in columns:
if col not in df:
raise KeyError("Table %s has no column '%s'" %
(self.job.table.name, col))
df = df.ix[:, columns]
self.data = df
logger.info("SCC job %s returning %d rows of data" %
(self.job, len(self.data)))
else:
self.data = None
return QueryComplete(self.data)
def run(self):
criteria = self.job.criteria
if criteria.scc_device == '':
logger.debug('%s: No scc device selected' % (self.table))
self.job.mark_error("No SCC Device Selected")
return False
columns = [col.name for col in self.table.get_columns(synthetic=False)]
scc = DeviceManager.get_device(criteria.scc_device)
# obtain the report class definition
report_cls = get_scc_report_class(self.service, self.resource)
# instatiate a report object
report_obj = report_cls(scc)
# Build criteria kwargs
kwargs = {}
for name in set(report_obj.required_fields +
report_obj.non_required_fields):
# criteria has attrs as starttime, endtime
# which maps to start_time and end_time
# referenced in a SCC service
if name in ['start_time', 'end_time']:
name_in_criteria = name.replace('_', '')
else:
name_in_criteria = name
if hasattr(criteria, name_in_criteria):
kwargs[name] = getattr(criteria, name_in_criteria)
report_obj.run(**kwargs)
df = self.extract_dataframe(report_obj.data)
if df is not None:
for col in columns:
if col not in df:
raise KeyError("Table %s has no column '%s'" %
(self.job.table.name, col))
df = df.ix[:, columns]
self.data = df
logger.info("SCC job %s returning %d rows of data" %
(self.job, len(self.data)))
else:
self.data = None
return QueryComplete(self.data)
def netprofiler_live_templates(form, id, field_kwargs):
"""Query netprofiler for available live templates. """
netprofiler_device = form.get_field_value('netprofiler_device', id)
if netprofiler_device == '':
choices = [('', '<No netprofiler device>')]
else:
netprofiler = DeviceManager.get_device(netprofiler_device)
choices = [(t['id'], t['name'])
for t in netprofiler.api.templates.get_live_templates()]
field_kwargs['choices'] = choices
field_kwargs['label'] = 'Live Template'
def netprofiler_live_templates(form, id, field_kwargs):
"""Query netprofiler for available live templates. """
netprofiler_device = form.get_field_value('netprofiler_device', id)
if netprofiler_device == '':
choices = [('', '<No netprofiler device>')]
else:
netprofiler = DeviceManager.get_device(netprofiler_device)
choices = [(t['id'], t['name'])
for t in netprofiler.api.templates.get_live_templates()]
field_kwargs['choices'] = choices
field_kwargs['label'] = 'Live Template'
def post(self, request, device_type):
if device_type == 'netprofiler':
device = 'NetProfiler'
form = NetProfilerInputForm(request.POST)
elif device_type == 'netshark':
device = 'NetShark'
form = NetSharkInputForm(request.POST)
else:
raise Http404
results = None
if form.is_valid():
data = form.cleaned_data
if device_type == 'netprofiler':
profiler = DeviceManager.get_device(data['device'])
results = profiler.search_columns(
realms=[data['realm']],
centricities=[data['centricity']],
groupbys=[data['groupby']])
results.sort(key=operator.attrgetter('key'))
results.sort(key=operator.attrgetter('iskey'), reverse=True)
results = [(c.iskey, c.key, c.label, c.id) for c in results]
elif device_type == 'netshark':
shark = DeviceManager.get_device(data['device'])
results = [(f.id, f.description, f.type)
for f in shark.get_extractor_fields()]
results.sort(key=operator.itemgetter(0))
return render_to_response('help.html', {
'device': device,
'form': form,
'results': results
},
context_instance=RequestContext(request))
def run(self):
criteria = self.job.criteria
profiler = DeviceManager.get_device(criteria.netprofiler_device)
widget_config = profiler.api.templates.get_config(criteria.template_id)
recs = []
for w in widget_config:
dict0 = {'template_id': str(criteria.template_id)}
dict1 = dict((k, w[k]) for k in ['widget_id', 'title'])
dict2 = dict((k, w['config'][k]) for k in
['widget_type', 'visualization', 'datasource'])
recs.append(dict((k, v) for d in [dict0, dict1, dict2]
for k, v in d.items()))
return QueryComplete(pd.DataFrame(recs))
def run(self):
sh = DeviceManager.get_device(self.job.criteria.steelhead_device)
stats = Model.get(sh, feature='stats')
duration = self.job.criteria.duration
directions = ['lan-to-wan', 'wan-to-lan', 'bi-directional']
total = []
for d in directions:
res = stats.show_stats_bandwidth('all', d, duration)
res['direction'] = d
total.append(res)
return QueryComplete(total)
def netprofiler_application_choices(form, id, field_kwargs, params):
# let's get all the applications and store them
netprofiler_device = form.get_field_value('netprofiler_device', id)
if netprofiler_device == '':
choices = [('', '<No netprofiler device>')]
else:
netprofiler = DeviceManager.get_device(netprofiler_device)
apps = get_netprofiler_apps(netprofiler)
# now we've got the apps return just name and id
choices = [(x['name'], x['name']) for x in apps]
field_kwargs['label'] = 'Application'
field_kwargs['choices'] = choices
def netprofiler_hostgroup_types(form, id, field_kwargs, params):
""" Query netprofiler for all hostgroup types. """
netprofiler_device = form.get_field_value('netprofiler_device', id)
if netprofiler_device == '':
choices = [('', '<No netprofiler device>')]
else:
netprofiler = DeviceManager.get_device(netprofiler_device)
choices = []
for hgt in netprofiler.api.host_group_types.get_all():
choices.append((hgt['name'], hgt['name']))
field_kwargs['label'] = 'HostGroupType'
field_kwargs['choices'] = choices
def netprofiler_hostgroup_types(form, id, field_kwargs, params):
""" Query netprofiler for all hostgroup types. """
netprofiler_device = form.get_field_value('netprofiler_device', id)
if netprofiler_device == '':
choices = [('', '<No netprofiler device>')]
else:
netprofiler = DeviceManager.get_device(netprofiler_device)
choices = []
for hgt in netprofiler.api.host_group_types.get_all():
choices.append((hgt['name'], hgt['name']))
field_kwargs['label'] = 'HostGroupType'
field_kwargs['choices'] = choices
def run(self):
criteria = self.job.criteria
profiler = DeviceManager.get_device(criteria.netprofiler_device)
widget_config = profiler.api.templates.get_config(criteria.template_id)
recs = []
for w in widget_config:
dict0 = {'template_id': str(criteria.template_id)}
dict1 = dict((k, w[k]) for k in ['widget_id', 'title'])
dict2 = dict(
(k, w['config'][k])
for k in ['widget_type', 'visualization', 'datasource'])
recs.append(
dict((k, v) for d in [dict0, dict1, dict2]
for k, v in d.iteritems()))
return QueryComplete(pd.DataFrame(recs))
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
if criteria.solarwinds_device == '':
logger.debug('%s: No solarwinds device selected' % self.table)
self.job.mark_error("No Solarwinds Device Selected")
return False
sw = DeviceManager.get_device(criteria.solarwinds_device)
# TODO add queries
self.data = None
logger.info("SolarwindsTable job %s returning %s data" %
(self.job, len(self.data)))
return True
def run(self):
sks = Device.objects.filter(enabled=True, module='netshark')
res = []
for sk in sks:
sk_dev = DeviceManager.get_device(sk.id)
for job in sk_dev.get_capture_jobs():
if_name = job.data['config']['interface_name']
start = str(nsec_string_to_datetime(job.packet_start_time))
end = str(nsec_string_to_datetime(job.packet_end_time))
bpf_filter = job.data['config'].get('bpf_filter', '')
if len(bpf_filter) > self.MAX_LENGTH:
bpf_filter = bpf_filter[:self.MAX_LENGTH - 2] + '...'
pkts_dropped = job.get_stats()['packets_dropped']
pkts_written = job.get_stats()['packets_written']
job_data = dict(netshark=sk.name,
job_id=job.data['id'],
job_name=job.data['config']['name'],
interface=if_name,
state=job.data['status']['state'],
size=job.data['status']['packet_size'],
start_time=start,
end_time=end,
bpf_filter=bpf_filter,
dpi_enabled=str(job.dpi_enabled),
index_enabled=str(job.index_enabled),
last_sec_dropped=pkts_dropped['last_second'],
last_min_dropped=pkts_dropped['last_minute'],
last_hr_dropped=pkts_dropped['last_hour'],
last_sec_written=pkts_written['last_second'],
last_min_written=pkts_written['last_minute'],
last_hr_written=pkts_written['last_hour']
)
res.append(job_data)
return QueryComplete(pandas.DataFrame(res))
def run(self):
obj_class = self.table.options.obj_class
feature = self.table.options.feature
method = self.table.options.method
args = self.table.options.args
sh = DeviceManager.get_device(self.job.criteria.steelhead_device)
obj = obj_class.get(sh, feature=feature)
res = getattr(obj, method)(*args)
if not isinstance(res, list):
res = [res]
for e in res:
for k, v in e.iteritems():
e[k] = str(v)
return QueryComplete(res)
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
if criteria.solarwinds_device == '':
logger.debug('%s: No solarwinds device selected' % self.table)
self.job.mark_error("No Solarwinds Device Selected")
return False
sw = DeviceManager.get_device(criteria.solarwinds_device)
# TODO add queries
self.data = None
logger.info("SolarwindsTable job %s returning %s data" %
(self.job, len(self.data)))
return True
def run(self):
sks = Device.objects.filter(enabled=True, module='netshark')
res = []
for sk in sks:
sk_dev = DeviceManager.get_device(sk.id)
for job in sk_dev.get_capture_jobs():
if_name = job.data['config']['interface_name']
start = str(nsec_string_to_datetime(job.packet_start_time))
end = str(nsec_string_to_datetime(job.packet_end_time))
bpf_filter = job.data['config'].get('bpf_filter', '')
if len(bpf_filter) > self.MAX_LENGTH:
bpf_filter = bpf_filter[:self.MAX_LENGTH - 2] + '...'
pkts_dropped = job.get_stats()['packets_dropped']
pkts_written = job.get_stats()['packets_written']
job_data = dict(netshark=sk.name,
job_id=job.data['id'],
job_name=job.data['config']['name'],
interface=if_name,
state=job.data['status']['state'],
size=job.data['status']['packet_size'],
start_time=start,
end_time=end,
bpf_filter=bpf_filter,
dpi_enabled=str(job.dpi_enabled),
index_enabled=str(job.index_enabled),
last_sec_dropped=pkts_dropped['last_second'],
last_min_dropped=pkts_dropped['last_minute'],
last_hr_dropped=pkts_dropped['last_hour'],
last_sec_written=pkts_written['last_second'],
last_min_written=pkts_written['last_minute'],
last_hr_written=pkts_written['last_hour'])
res.append(job_data)
return QueryComplete(pandas.DataFrame(res))
def netshark_source_choices(form, id_, field_kwargs, params):
"""Query netshark for available capture jobs / trace clips."""
# simplified clone from base netshark datasource that allows for
# custom field names
netshark_device = form.get_field_value(params['field'], id_)
if netshark_device == '':
choices = [('', '<No netshark device>')]
else:
netshark = DeviceManager.get_device(netshark_device)
choices = []
for job in netshark.get_capture_jobs():
choices.append((job.source_path, job.name))
for clip in netshark.get_clips():
choices.append((clip.source_path, 'Clip: ' + clip.description))
field_kwargs['choices'] = choices
def netshark_msa_file_choices(form, id_, field_kwargs, params):
"""Query netshark for available MSA files."""
netshark_device = form.get_field_value('netshark_device', id_)
if netshark_device == '':
choices = [('', '<No netshark device>')]
else:
netshark = DeviceManager.get_device(netshark_device)
choices = []
# TODO - this enables automatic refresh
# whenever the report or criteria get reloaded, but
# it will cause some extra load when the report runs
# since each time a TableForm gets created all the files
# will be re-queried
for f in netshark.get_files(force_refetch=True):
if hasattr(f, 'list_linked_files'):
choices.append((f.source_path, 'MSA File: ' + f.path))
field_kwargs['choices'] = choices
def netprofiler_hostgroups(form, id, field_kwargs, params):
""" Query netprofiler for groups within a given hostgroup. """
netprofiler_device = form.get_field_value('netprofiler_device', id)
if netprofiler_device == '':
choices = [('', '<No netprofiler device>')]
else:
netprofiler = DeviceManager.get_device(netprofiler_device)
if params is not None and 'hostgroup_type' in params:
hgt = HostGroupType.find_by_name(netprofiler,
params['hostgroup_type'])
else:
hostgroup_type = form.get_field_value('hostgroup_type', id)
hgt = HostGroupType.find_by_name(netprofiler,
hostgroup_type)
choices = [(group, group) for group in hgt.groups.keys()]
field_kwargs['label'] = 'HostGroup'
field_kwargs['choices'] = choices
def run(self):
# For each of the widget, get all the data
profiler = DeviceManager.get_device(self.table.options.netprofiler_id)
lr = LiveReport(profiler, template_id=self.table.options.template_id)
# Figure out columns by querying the widget
# cols = lr.get_columns(self.table.options.widget_id)
# Find the query object
query_idx = lr.get_query_names().index(self.table.options.query_id)
# refresh the columns of the table
self._refresh_columns(profiler, report=lr, query=lr.queries[query_idx])
data = lr.get_data(index=query_idx)
col_names = [col.label if col.ephemeral else col.key
for col in lr.queries[query_idx].columns]
df = pd.DataFrame(columns=col_names, data=data)
return QueryComplete(df)
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
self.timeseries = False # if key column called 'time' is created
self.column_names = []
# Resolution comes in as a time_delta
resolution = timedelta_total_seconds(criteria.resolution)
default_delta = 1000000000 # one second
self.delta = int(default_delta * resolution) # sample size interval
if criteria.netshark_device == '':
logger.debug('%s: No netshark device selected' % self.table)
self.job.mark_error("No NetShark Device Selected")
return False
#self.fake_run()
#return True
shark = DeviceManager.get_device(criteria.netshark_device)
logger.debug("Creating columns for NetShark table %d" % self.table.id)
# Create Key/Value Columns
columns = []
for tc in self.table.get_columns(synthetic=False):
tc_options = tc.options
if ( tc.iskey and tc.name == 'time' and
tc_options.extractor == 'sample_time'):
# don't create column, use the sample time for timeseries
self.timeseries = True
self.column_names.append('time')
continue
elif tc.iskey:
c = Key(tc_options.extractor,
description=tc.label,
default_value=tc_options.default_value)
else:
if tc_options.operation:
try:
operation = getattr(Operation, tc_options.operation)
except AttributeError:
operation = Operation.sum
print ('ERROR: Unknown operation attribute '
'%s for column %s.' %
(tc_options.operation, tc.name))
else:
operation = Operation.none
c = Value(tc_options.extractor,
operation,
description=tc.label,
default_value=tc_options.default_value)
self.column_names.append(tc.name)
columns.append(c)
# Identify Sort Column
sortidx = None
if self.table.sortcols is not None:
sortcol = Column.objects.get(table=self.table,
name=self.table.sortcols[0])
sort_name = sortcol.options.extractor
for i, c in enumerate(columns):
if c.field == sort_name:
sortidx = i
break
# Initialize filters
criteria = self.job.criteria
filters = []
filterexpr = self.job.combine_filterexprs(
exprs=criteria.netshark_filterexpr,
joinstr="&"
)
if filterexpr:
filters.append(NetSharkFilter(filterexpr))
tf = TimeFilter(start=criteria.starttime, end=criteria.endtime)
filters.append(tf)
logger.info("Setting netshark table %d timeframe to %s" % (self.table.id,
str(tf)))
# Get source type from options
try:
with lock:
source = path_to_class(shark,
self.job.criteria.netshark_source_name)
except RvbdHTTPException, e:
source = None
raise e
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
if criteria.netprofiler_device == '':
logger.debug('%s: No netprofiler device selected' % self.table)
self.job.mark_error("No NetProfiler Device Selected")
return False
profiler = DeviceManager.get_device(criteria.netprofiler_device)
report = ServiceLocationReport(profiler)
tf = TimeFilter(start=criteria.starttime,
end=criteria.endtime)
logger.info(
'Running NetProfilerServiceByLocTable %d report for timeframe %s' %
(self.table.id, str(tf)))
with lock:
report.run(timefilter=tf, sync=False)
done = False
logger.info("Waiting for report to complete")
while not done:
time.sleep(0.5)
with lock:
s = report.status()
self.job.mark_progress(progress=int(s['percent']))
done = (s['status'] == 'completed')
# Retrieve the data
with lock:
data = report.get_data()
query = report.get_query_by_index(0)
tz = criteria.starttime.tzinfo
# Update criteria
criteria.starttime = (datetime.datetime
.utcfromtimestamp(query.actual_t0)
.replace(tzinfo=tz))
criteria.endtime = (datetime.datetime
.utcfromtimestamp(query.actual_t1)
.replace(tzinfo=tz))
self.job.safe_update(actual_criteria=criteria)
if len(data) == 0:
return QueryComplete(None)
# Add ephemeral columns for everything
Column.create(self.job.table, 'location', 'Location',
ephemeral=self.job, datatype='string')
for k in data[0].keys():
if k == 'location':
continue
Column.create(self.job.table, k, k,
ephemeral=self.job, datatype='string',
formatter='rvbd.formatHealth')
df = pandas.DataFrame(data)
if self.job.table.options.rgb:
state_map = {Service.SVC_NOT_AVAILABLE: 'gray',
Service.SVC_DISABLED: 'gray',
Service.SVC_INIT: 'gray',
Service.SVC_NORMAL: 'green',
Service.SVC_LOW: 'yellow',
Service.SVC_MED: 'yellow',
Service.SVC_HIGH: 'red',
Service.SVC_NODATA: 'gray'}
df = df.replace(state_map.keys(),
state_map.values())
return QueryComplete(df)
