def post(self, request, device_type):
if device_type == 'profiler':
form = ProfilerInputForm(request.POST)
elif device_type == 'shark':
form = SharkInputForm(request.POST)
else:
raise Http404
results = None
if form.is_valid():
data = form.cleaned_data
if device_type == 'profiler':
profiler = DeviceManager.get_device(data['device'])
results = profiler.search_columns(realms=[data['realm']],
centricities=[data['centricity']],
groupbys=[data['groupby']])
results.sort(key=operator.attrgetter('key'))
results.sort(key=operator.attrgetter('iskey'), reverse=True)
results = [(c.iskey, c.key, c.label, c.id) for c in results]
elif device_type == 'shark':
shark = DeviceManager.get_device(data['device'])
results = [(f.id, f.description, f.type) for f in shark.get_extractor_fields()]
results.sort(key=operator.itemgetter(0))
return render_to_response('help.html',
{'device': device_type.title(),
'form': form,
'results': results},
context_instance=RequestContext(request))
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
if criteria.sharepoint_device == '':
logger.debug('%s: No sharepoint device selected' % self.table)
self.job.mark_error("No Sharepoint Device Selected")
return False
sp = DeviceManager.get_device(criteria.sharepoint_device)
site = sp.get_site_object(self.table.options.site_url)
site_instance = site.lists[self.table.options.list_name]
fields = [tc.name for tc in self.table.get_columns(synthetic=False)]
self.data = []
for row in site_instance.rows:
d = [getattr(row, f) for f in fields]
self.data.append(d)
logger.info("SharepointTable job %s returning %s data" %
(self.job, len(self.data)))
return True
def shark_source_name_choices(form, id, field_kwargs, params):
shark_device = form.get_field_value('shark_device', id)
if shark_device == '':
label = 'Source'
choices = [('', '<No shark device>')]
else:
shark = DeviceManager.get_device(shark_device)
#source_type = form.get_field_value('shark_source_type', id)
source_type = 'job'
choices = []
if source_type == 'job':
for job in shark.get_capture_jobs():
choices.append(('jobs/' + job.name, job.name))
label = 'Capture Job'
elif source_type == 'clip':
# Not tested
label = 'Trace Clip'
for clip in shark.get_clips():
choices.append((clip, clip))
else:
raise KeyError('Unknown source type: %s' % source_type)
field_kwargs['label'] = label
field_kwargs['choices'] = choices
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
if criteria.sharepoint_device == '':
logger.debug('%s: No sharepoint device selected' % self.table)
self.job.mark_error("No Sharepoint Device Selected")
return False
sp = DeviceManager.get_device(criteria.sharepoint_device)
site = sp.get_site_object(self.table.options.site_url)
site_instance = site.lists[self.table.options.list_name]
fields = [tc.name for tc in self.table.get_columns(synthetic=False)]
self.data = []
for row in site_instance.rows:
d = [getattr(row, f) for f in fields]
self.data.append(d)
logger.info("SharepointTable job %s returning %s data" %
(self.job, len(self.data)))
return True
def __init__(self, *args, **kwargs):
super(DeviceDetailForm, self).__init__(*args, **kwargs)
modules = DeviceManager.get_modules()
choices = zip(modules, modules)
self.fields['module'] = forms.ChoiceField(choices=choices)
def shark_source_name_choices(form, id, field_kwargs, params):
shark_device = form.get_field_value('shark_device', id)
if shark_device == '':
label = 'Source'
choices = [('', '<No shark device>')]
else:
shark = DeviceManager.get_device(shark_device)
#source_type = form.get_field_value('shark_source_type', id)
source_type = 'job'
choices = []
if source_type == 'job':
for job in shark.get_capture_jobs():
choices.append (('jobs/' + job.name, job.name))
label = 'Capture Job'
elif source_type == 'clip':
# Not tested
label = 'Trace Clip'
for clip in shark.get_clips():
choices.append ((clip, clip))
else:
raise KeyError('Unknown source type: %s' % source_type)
field_kwargs['label'] = label
field_kwargs['choices'] = choices
def put(self, request, *args, **kwargs):
DeviceFormSet = modelformset_factory(Device,
form=DeviceListForm,
extra=0)
formset = DeviceFormSet(request.DATA)
if formset.is_valid():
formset.save()
DeviceManager.clear()
profile = UserProfile.objects.get(user=request.user)
if not profile.profile_seen:
# only redirect if first login
return HttpResponseRedirect(
reverse('preferences') + '?next=/report')
elif '/devices' not in request.META['HTTP_REFERER']:
return HttpResponseRedirect(request.META['HTTP_REFERER'])
else:
return HttpResponseRedirect(reverse('device-list'))
else:
data = {'formset': formset}
return Response(data, template_name='device_list.html')
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
if criteria.solarwinds_device == "":
logger.debug("%s: No solarwinds device selected" % self.table)
self.job.mark_error("No Solarwinds Device Selected")
return False
sw = DeviceManager.get_device(criteria.solarwinds_device)
# TODO add queries
self.data = None
logger.info("SolarwindsTable job %s returning %s data" % (self.job, len(self.data)))
return True
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
if criteria.solarwinds_device == '':
logger.debug('%s: No solarwinds device selected' % self.table)
self.job.mark_error("No Solarwinds Device Selected")
return False
sw = DeviceManager.get_device(criteria.profiler_device)
# TODO add queries
self.data = None
logger.info("SolarwindsTable job %s returning %s data" %
(self.job, len(self.data)))
return True
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
if criteria.profiler_device == '':
logger.debug('%s: No profiler device selected' % (self.table))
self.job.mark_error("No Profiler Device Selected")
return False
profiler = DeviceManager.get_device(criteria.profiler_device)
report = rvbd.profiler.report.SingleQueryReport(profiler)
columns = [col.name for col in self.table.get_columns(synthetic=False)]
# This returns an array of rows, one row per device
# Each row is a dict containing elements such as:
# id, ipaddr, name, type, type_id, and version
with lock:
devicedata = profiler.api.devices.get_all()
# Convert to a DataFrame to make it easier to work with
df = pandas.DataFrame(devicedata)
for col in columns:
if col not in df:
raise KeyError("Devices table has no column '%s'" % col.name)
df = df.ix[:, columns]
if self.table.sortcol is not None:
df = df.sort(self.table.sortcol.name)
if self.table.rows > 0:
df = df[:self.table.rows]
self.data = df
logger.info("DeviceTable job %s returning %d devices" %
(self.job, len(self.data)))
return True
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
if criteria.profiler_device == '':
logger.debug('%s: No profiler device selected' % (self.table))
self.job.mark_error("No Profiler Device Selected")
return False
profiler = DeviceManager.get_device(criteria.profiler_device)
report = rvbd.profiler.report.SingleQueryReport(profiler)
columns = [col.name for col in self.table.get_columns(synthetic=False)]
# This returns an array of rows, one row per device
# Each row is a dict containing elements such as:
# id, ipaddr, name, type, type_id, and version
with lock:
devicedata = profiler.api.devices.get_all()
# Convert to a DataFrame to make it easier to work with
df = pandas.DataFrame(devicedata)
for col in columns:
if col not in df:
raise KeyError("Devices table has no column '%s'" % col.name)
df = df.ix[:,columns]
if self.table.sortcol is not None:
df = df.sort(self.table.sortcol.name)
if self.table.rows > 0:
df = df[:self.table.rows]
self.data = df
logger.info("DeviceTable job %s returning %d devices" % (self.job, len(self.data)))
return True
def handle(self, *args, **options):
self.stdout.write('Reloading report objects ... ')
management.call_command('clean_pyc', path=settings.PROJECT_ROOT)
management.call_command('syncdb', interactive=False)
self.importer = Importer(buf=self.stdout)
if options['report_id']:
# single report
report_id = options['report_id']
pk = int(report_id)
report = Report.objects.get(pk=pk)
management.call_command('clean',
applications=False,
report_id=report_id,
clear_cache=False,
clear_logs=False)
DeviceManager.clear()
self.import_module(report.sourcefile)
elif options['report_name']:
name = options['report_name']
try:
report = Report.objects.get(sourcefile__endswith=name)
report_id = report.id
management.call_command('clean',
applications=False,
report_id=report_id,
clear_cache=False,
clear_logs=False)
self.import_module(report.sourcefile)
except ObjectDoesNotExist:
self.import_module(name)
DeviceManager.clear()
elif options['namespace']:
reports = Report.objects.filter(namespace=options['namespace'])
for report in reports:
management.call_command('clean',
applications=False,
report_id=report.id,
clear_cache=False,
clear_logs=False)
self.import_module(report.sourcefile)
else:
# clear all data
management.call_command('clean',
applications=True,
report_id=None,
clear_cache=True,
clear_logs=False)
# start with fresh device instances
DeviceManager.clear()
report_dir = os.path.join(settings.PROJECT_ROOT,
options['report_dir'] or 'config')
self.importer.import_directory(report_dir, report_name=None)
for plugin in plugins.enabled():
if plugin.reports:
#from IPython import embed; embed()
plugin.load_reports()
def run(self):
# All user entered criteria is available directly from this object.
# Values for any fields added to the table will appear as
# attributes according to the field keyword.
criteria = self.job.criteria
# Check that a sample_device was selected
if criteria.sample_device == "":
logger.debug("%s: No sample device selected" % self.table)
self.job.mark_error("No Sample Device Selected")
return False
sample_device = DeviceManager.get_device(criteria.sample_device)
# Get the columns for this report
columns = self.table.get_columns(synthetic=False)
sortcol = None
if self.table.sortcol is not None:
sortcol = self.table.sortcol.name
# Time selection is available via criterai.starttime and endtime.
# These are both datetime objects.
t0 = criteria.starttime
t1 = criteria.endtime
# Time resolution is a timedelta object
resolution = criteria.resolution
# Grab the custom min and max criteria
cmin = float(criteria.min)
cmax = float(criteria.max)
# Grab the table options
beta = self.table.options.beta
# Now, do some computation -- create table with a 'time' column
# ranging from t0 to t1 with the defined resolution. Then
# for each additional column do some math function on the
# data
t = t0
rows = []
while t < t1:
row = []
for col in columns:
if col.name == "time":
row.append(t)
else:
period_td = parse_timedelta(col.options.period)
period_secs = timedelta_total_seconds(period_td)
alpha = col.options.alpha
funcname = col.options.func
# seconds since the t0
secs = timedelta_total_seconds(t - t0)
rad = (secs / period_secs) * 2 * math.pi
funcmap = {"sin": math.sin, "cos": math.cos}
# Compute!
val = beta + alpha * funcmap[funcname](rad)
# Clip by the min/max criteria
val = max(cmin, val)
val = min(cmax, val)
# Add the value to the row
row.append(val)
# Add the row
rows.append(row)
# This function runs pretty fast, but this shows how to mark
# progress
self.job.mark_progress(100 * (timedelta_total_seconds(t - t0) / timedelta_total_seconds(t1 - t0)))
t = t + resolution
# Save the result in self.data
self.data = rows
if self.table.rows > 0:
self.data = self.data[: self.table.rows]
logger.info("Report %s returned %s rows" % (self.job, len(self.data)))
return True
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
if criteria.shark_device == '':
logger.debug('%s: No shark device selected' % self.table)
self.job.mark_error("No Shark Device Selected")
return False
#self.fake_run()
#return True
shark = DeviceManager.get_device(criteria.shark_device)
logger.debug("Creating columns for Shark table %d" % self.table.id)
# Create Key/Value Columns
columns = []
for tc in self.table.get_columns(synthetic=False):
tc_options = tc.options
if (tc.iskey and tc.name == 'time'
and tc_options.extractor == 'sample_time'):
# don't create column, use the sample time for timeseries
self.timeseries = True
self.column_names.append('time')
continue
elif tc.iskey:
c = Key(tc_options.extractor,
description=tc.label,
default_value=tc_options.default_value)
else:
if tc_options.operation:
try:
operation = getattr(Operation, tc_options.operation)
except AttributeError:
operation = Operation.sum
print('ERROR: Unknown operation attribute '
'%s for column %s.' %
(tc_options.operation, tc.name))
else:
operation = Operation.none
c = Value(tc_options.extractor,
operation,
description=tc.label,
default_value=tc_options.default_value)
self.column_names.append(tc.name)
columns.append(c)
# Identify Sort Column
sortidx = None
if self.table.sortcol is not None:
sort_name = self.table.sortcol.options.extractor
for i, c in enumerate(columns):
if c.field == sort_name:
sortidx = i
break
# Initialize filters
criteria = self.job.criteria
filters = []
filterexpr = self.job.combine_filterexprs(
exprs=criteria.shark_filterexpr, joinstr="&")
if filterexpr:
filters.append(SharkFilter(filterexpr))
tf = TimeFilter(start=criteria.starttime, end=criteria.endtime)
filters.append(tf)
logger.info("Setting shark table %d timeframe to %s" %
(self.table.id, str(tf)))
# Get source type from options
try:
with lock:
source = path_to_class(shark,
self.job.criteria.shark_source_name)
except RvbdHTTPException, e:
source = None
raise e
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
if criteria.profiler_device == '':
logger.debug('%s: No profiler device selected' % self.table)
self.job.mark_error("No Profiler Device Selected")
return False
#self.fake_run()
#return True
profiler = DeviceManager.get_device(criteria.profiler_device)
report = rvbd.profiler.report.SingleQueryReport(profiler)
columns = [col.name for col in self.table.get_columns(synthetic=False)]
sortcol = None
if self.table.sortcol is not None:
sortcol = self.table.sortcol.name
tf = TimeFilter(start=criteria.starttime,
end=criteria.endtime)
logger.info("Running Profiler table %d report for timeframe %s" %
(self.table.id, str(tf)))
if ('datafilter' in criteria) and (criteria.datafilter is not None):
datafilter = criteria.datafilter.split(',')
else:
datafilter = None
trafficexpr = TrafficFilter(
self.job.combine_filterexprs(exprs=criteria.profiler_filterexpr)
)
# Incoming criteria.resolution is a timedelta
logger.debug('Profiler report got criteria resolution %s (%s)' %
(criteria.resolution, type(criteria.resolution)))
if criteria.resolution != 'auto':
rsecs = int(timedelta_total_seconds(criteria.resolution))
resolution = rvbd.profiler.report.Report.RESOLUTION_MAP[rsecs]
else:
resolution = 'auto'
logger.debug('Profiler report using resolution %s (%s)' %
(resolution, type(resolution)))
with lock:
report.run(realm=self.table.options.realm,
groupby=profiler.groupbys[self.table.options.groupby],
centricity=self.table.options.centricity,
columns=columns,
timefilter=tf,
trafficexpr=trafficexpr,
data_filter=datafilter,
resolution=resolution,
sort_col=sortcol,
sync=False
)
done = False
logger.info("Waiting for report to complete")
while not done:
time.sleep(0.5)
with lock:
s = report.status()
self.job.safe_update(progress=int(s['percent']))
done = (s['status'] == 'completed')
# Retrieve the data
with lock:
query = report.get_query_by_index(0)
self.data = query.get_data()
tz = criteria.starttime.tzinfo
# Update criteria
criteria.starttime = (datetime.datetime
.utcfromtimestamp(query.actual_t0)
.replace(tzinfo=tz))
criteria.endtime = (datetime.datetime
.utcfromtimestamp(query.actual_t1)
.replace(tzinfo=tz))
self.job.safe_update(actual_criteria=criteria)
if self.table.rows > 0:
self.data = self.data[:self.table.rows]
logger.info("Report %s returned %s rows" % (self.job, len(self.data)))
return True
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
if criteria.shark_device == '':
logger.debug('%s: No shark device selected' % self.table)
self.job.mark_error("No Shark Device Selected")
return False
#self.fake_run()
#return True
shark = DeviceManager.get_device(criteria.shark_device)
logger.debug("Creating columns for Shark table %d" % self.table.id)
# Create Key/Value Columns
columns = []
for tc in self.table.get_columns(synthetic=False):
tc_options = tc.options
if ( tc.iskey and tc.name == 'time' and
tc_options.extractor == 'sample_time'):
# don't create column, use the sample time for timeseries
self.timeseries = True
self.column_names.append('time')
continue
elif tc.iskey:
c = Key(tc_options.extractor,
description=tc.label,
default_value=tc_options.default_value)
else:
if tc_options.operation:
try:
operation = getattr(Operation, tc_options.operation)
except AttributeError:
operation = Operation.sum
print ('ERROR: Unknown operation attribute '
'%s for column %s.' %
(tc_options.operation, tc.name))
else:
operation = Operation.none
c = Value(tc_options.extractor,
operation,
description=tc.label,
default_value=tc_options.default_value)
self.column_names.append(tc.name)
columns.append(c)
# Identify Sort Column
sortidx = None
if self.table.sortcol is not None:
sort_name = self.table.sortcol.options.extractor
for i, c in enumerate(columns):
if c.field == sort_name:
sortidx = i
break
# Initialize filters
criteria = self.job.criteria
filters = []
filterexpr = self.job.combine_filterexprs(
exprs=criteria.shark_filterexpr,
joinstr="&"
)
if filterexpr:
filters.append(SharkFilter(filterexpr))
tf = TimeFilter(start=criteria.starttime, end=criteria.endtime)
filters.append(tf)
logger.info("Setting shark table %d timeframe to %s" % (self.table.id,
str(tf)))
# Get source type from options
try:
with lock:
source = path_to_class(shark,
self.job.criteria.shark_source_name)
except RvbdHTTPException, e:
source = None
raise e
def handle(self, *args, **options):
self.stdout.write('Reloading report objects ... ')
management.call_command('clean_pyc', path=settings.PROJECT_ROOT)
management.call_command('syncdb', interactive=False)
self.importer = Importer(buf=self.stdout)
if options['report_id']:
# single report
report_id = options['report_id']
pk = int(report_id)
report = Report.objects.get(pk=pk)
management.call_command('clean',
applications=False,
report_id=report_id,
clear_cache=False,
clear_logs=False)
DeviceManager.clear()
self.import_module(report.sourcefile)
elif options['report_name']:
name = options['report_name']
try:
report = Report.objects.get(sourcefile__endswith=name)
report_id = report.id
management.call_command('clean',
applications=False,
report_id=report_id,
clear_cache=False,
clear_logs=False)
self.import_module(report.sourcefile)
except ObjectDoesNotExist:
self.import_module(name)
DeviceManager.clear()
elif options['namespace']:
reports = Report.objects.filter(namespace=options['namespace'])
self.capture_enabled(reports)
for report in reports:
management.call_command('clean',
applications=False,
report_id=report.id,
clear_cache=False,
clear_logs=False)
self.import_module(report.sourcefile)
self.apply_enabled()
else:
self.capture_enabled()
# clear all data
management.call_command('clean',
applications=True,
report_id=None,
clear_cache=True,
clear_logs=False)
# start with fresh device instances
DeviceManager.clear()
report_dir = os.path.join(settings.PROJECT_ROOT,
options['report_dir'] or 'config')
self.importer.import_directory(report_dir, report_name=None)
for plugin in plugins.enabled():
if plugin.reports:
#from IPython import embed; embed()
plugin.load_reports()
self.apply_enabled()
def run(self):
# All user entered criteria is available directly from this object.
# Values for any fields added to the table will appear as
# attributes according to the field keyword.
criteria = self.job.criteria
# Check that a sample_device was selected
if criteria.sample_device == '':
logger.debug('%s: No sample device selected' % self.table)
self.job.mark_error("No Sample Device Selected")
return False
sample_device = DeviceManager.get_device(criteria.sample_device)
# Get the columns for this report
columns = self.table.get_columns(synthetic=False)
sortcol = None
if self.table.sortcol is not None:
sortcol = self.table.sortcol.name
# Time selection is available via criterai.starttime and endtime.
# These are both datetime objects.
t0 = criteria.starttime
t1 = criteria.endtime
# Time resolution is a timedelta object
resolution = criteria.resolution
# Grab the custom min and max criteria
cmin = float(criteria.min)
cmax = float(criteria.max)
# Grab the table options
beta = self.table.options.beta
# Now, do some computation -- create table with a 'time' column
# ranging from t0 to t1 with the defined resolution. Then
# for each additional column do some math function on the
# data
t = t0
rows = []
while t < t1:
row = []
for col in columns:
if col.name == 'time':
row.append(t)
else:
period_td = parse_timedelta(col.options.period)
period_secs = timedelta_total_seconds(period_td)
alpha = col.options.alpha
funcname = col.options.func
# seconds since the t0
secs = timedelta_total_seconds(t - t0)
rad = (secs / period_secs) * 2 * math.pi
funcmap = {
'sin': math.sin,
'cos': math.cos,
}
# Compute!
val = beta + alpha * funcmap[funcname](rad)
# Clip by the min/max criteria
val = max(cmin, val)
val = min(cmax, val)
# Add the value to the row
row.append(val)
# Add the row
rows.append(row)
# This function runs pretty fast, but this shows how to mark
# progress
self.job.mark_progress(100 * (timedelta_total_seconds(t - t0) /
timedelta_total_seconds(t1 - t0)))
t = t + resolution
# Save the result in self.data
self.data = rows
if self.table.rows > 0:
self.data = self.data[:self.table.rows]
logger.info("Report %s returned %s rows" % (self.job, len(self.data)))
return True
def run(self):
""" Main execution method. """
criteria = self.job.criteria
if criteria.profiler_device == '':
logger.debug('%s: No profiler device selected' % self.table)
self.job.mark_error("No Profiler Device Selected")
return False
profiler = DeviceManager.get_device(criteria.profiler_device)
report = rvbd.profiler.report.MultiQueryReport(profiler)
tf = TimeFilter(start=criteria.starttime,
end=criteria.endtime)
logger.info("Running ProfilerTemplateTable table %d report "
"for timeframe %s" % (self.table.id, str(tf)))
trafficexpr = TrafficFilter(
self.job.combine_filterexprs(exprs=criteria.profiler_filterexpr)
)
# Incoming criteria.resolution is a timedelta
logger.debug('Profiler report got criteria resolution %s (%s)' %
(criteria.resolution, type(criteria.resolution)))
if criteria.resolution != 'auto':
rsecs = int(timedelta_total_seconds(criteria.resolution))
resolution = rvbd.profiler.report.Report.RESOLUTION_MAP[rsecs]
else:
resolution = 'auto'
logger.debug('Profiler report using resolution %s (%s)' %
(resolution, type(resolution)))
with lock:
res = report.run(template_id=self.table.options.template_id,
timefilter=tf,
trafficexpr=trafficexpr,
resolution=resolution)
if res is True:
logger.info("Report template complete.")
self.job.safe_update(progress=100)
# Retrieve the data
with lock:
query = report.get_query_by_index(0)
data = query.get_data()
headers = report.get_legend()
tz = criteria.starttime.tzinfo
# Update criteria
criteria.starttime = (datetime.datetime
.utcfromtimestamp(query.actual_t0)
.replace(tzinfo=tz))
criteria.endtime = (datetime.datetime
.utcfromtimestamp(query.actual_t1)
.replace(tzinfo=tz))
self.job.safe_update(actual_criteria=criteria)
# create dataframe with all of the default headers
df = pandas.DataFrame(data, columns=[h.key for h in headers])
# now filter down to the columns requested by the table
columns = [col.name for col in self.table.get_columns(synthetic=False)]
self.data = df[columns]
if self.table.sortcol is not None:
self.data = self.data.sort(self.table.sortcol.name)
if self.table.rows > 0:
self.data = self.data[:self.table.rows]
logger.info("Report %s returned %s rows" % (self.job, len(self.data)))
return True
