def __init__(self, packets_obj=None, name=None, path=None):
"""Initialize a data source for reports to run against.
:param packets_obj: Clip or File or Job object
:param str name: Name of general report sources
:param str path: Path of the packets data source
"""
if not packets_obj and not name:
raise AppResponseException("Either packets_obj or name is "
"required to be a valid data source.")
if packets_obj:
if isinstance(packets_obj, Clip):
path = '{}{}'.format(self.CLIP_PREFIX, packets_obj.id)
elif isinstance(packets_obj, File):
path = '{}{}'.format(self.FILE_PREFIX, packets_obj.id)
elif isinstance(packets_obj, Job):
path = '{}{}'.format(self.JOB_PREFIX, packets_obj.id)
else:
raise AppResponseException(
'Can only support job or clip or file packet source')
self.name = 'packets'
self.path = path
else:
self.name = name
self.path = path
def create_instance(self, data_defs):
"""Create a report instance with multiple data definition requests.
:param data_defs: list of DataDef objects
:return: one ReportInstance object
"""
if not data_defs:
msg = 'No data definitions are provided.'
raise AppResponseException(msg)
if (any(dd.source.name == 'packets' for dd in data_defs)
and any(dd.source.name != 'packets' for dd in data_defs)):
# Two report instance needs to be created, one uses 'npm.reports'
# service, the other one uses 'npm.probe.reports' service
# it would create unnecessary complexity to support this case
# thus report error and let the user to create two separate
# report instances
msg = ('Both packets data source and non-packets data source are '
'being queried in this report, which is not supported. The '
'data source names include {}'.format(', '.join(
set([dd.source.name for dd in data_defs]))))
raise AppResponseException(msg)
def _create_and_run(service_name, data_defs):
config = dict(data_defs=[dd.to_dict() for dd in data_defs])
logger.debug("Creating instance with data definitions %s" % config)
svcdef = self.appresponse.find_service(service_name)
datarep = svcdef.bind('instances')
resp = datarep.execute('create', _data=config)
instance = ReportInstance(data=resp.data, datarep=resp)
while not instance.is_complete():
time.sleep(1)
if instance.errors:
err_msgs = ';\n'.join(instance.errors)
raise AppResponseException(err_msgs)
return instance
if data_defs[0].source.name == 'packets':
# Needs to create a clip for for capture job packets source
# Keep the clip till the instance is completed
with self.appresponse.clips.create_clips(data_defs):
# capture job data_defs are modified in place
instance = _create_and_run(PACKETS_REPORT_SERVICE_NAME,
data_defs)
else:
instance = _create_and_run(GENERAL_REPORT_SERVICE_NAME, data_defs)
return instance
def create_clips(self, data_defs):
"""Create a Clips object from a list of data definition requests.
When some DataDef objects are using sources other than capture jobs,
then those sources will stay the same. The capture job sources will
be converted into Clip objects.
:param data_defs: list of DataDef objects
:return: a Clips object
"""
clips = []
for dd in data_defs:
if isinstance(dd.source, Job):
logger.debug("Creating a Clip object for one data def request "
"with capture job '{}'"
.format(dd.source.name))
clip = self.create_clip(dd.source, dd.timefilter,
from_job=True)
if clip.data.status.packets_written == 0:
msg = ('No packets found for job {} and {}'
.format(dd.source.name, dd.timefilter))
raise AppResponseException(msg)
clips.append(clip)
else:
clips.append(dd.source)
return Clips(clips)
def get_job_by_id(self, id_):
try:
logger.debug("Obtaining Job object with id '{}'".format(id_))
return next((j for j in self.get_jobs() if j.id == id_))
except StopIteration:
raise AppResponseException(
"No capture job found with ID '{}'".format(id_))
def get_job_by_name(self, name):
try:
logger.debug("Obtaining Job object with name '{}'".format(name))
return next((j for j in self.get_jobs() if j.name == name))
except StopIteration:
raise AppResponseException(
"No capture job found with name '{}'".format(name))
def analyze(self, jobs):
# Based on input pivot column names, i.e. CIFS, RTP, Facebook
# using dataframe keyed by Application ID, and start time
# derive dataframe keyed by start_time, with each row as
# a dictionary keyed by input pivot values
df = jobs['base'].data()
# First clear all the dynamic columns that were associated with
# the table last time the report is run
# do not delete the time column
for col in self.table.get_columns():
if col.name == 'time':
continue
col.delete()
base_table = Table.from_ref(self.table.options.tables.base)
time_col_name = None
for col in base_table.get_columns():
if col.datatype == Column.DATATYPE_TIME and col.iskey:
time_col_name = col.name
break
if not time_col_name:
raise AppResponseException("No key 'time' column defined "
"in base table")
pivot_column = self.table.options.pivot_column_name
sub_dfs = []
for pivot in self.job.criteria.pivot_column_names.split(','):
# Add pivot column to the table
pivot = pivot.strip()
AppResponseColumn.create(self.table, pivot, pivot)
# Add pivot column to the data frame
sub_df = df[df[pivot_column] == pivot]
# extract time column and value column
sub_df = sub_df[[
time_col_name, self.table.options.value_column_name
]]
# Rename columns to 'time' and the pivot column name
sub_df.rename(columns={
time_col_name: u'time',
self.table.options.value_column_name: pivot
},
inplace=True)
sub_dfs.append(sub_df)
df_final = reduce(
lambda df1, df2: pandas.merge(df1, df2, on=u'time', how='outer'),
sub_dfs)
return QueryComplete(df_final)
def check_for_errors(self):
"""Raise exception if any errors found."""
# Check errors when all queries have completed
for item in self.status:
if item['state'] == 'error':
for m in item['messages']:
self.errors.append(m['text'])
logger.error("Error msg from status: {}".format(m['text']))
if self.errors:
err_msgs = ';\n'.join(self.errors)
raise AppResponseException(err_msgs)
def _create_and_run(service_name, data_defs):
config = dict(data_defs=[dd.to_dict() for dd in data_defs])
logger.debug("Creating instance with data definitions %s" % config)
svcdef = self.appresponse.find_service(service_name)
datarep = svcdef.bind('instances')
resp = datarep.execute('create', _data=config)
instance = ReportInstance(data=resp.data, datarep=resp)
while not instance.is_complete():
time.sleep(1)
if instance.errors:
err_msgs = ';\n'.join(instance.errors)
raise AppResponseException(err_msgs)
return instance
def get_data(self, index=0):
"""Return data for the indexed data definition requests.
Note for live data objects `index` cannot be None, only
explicit requests are allowed. If multiple data_defs
in a report need to collect data, query them individually.
Also, the object returned from a live query will be a
`data_def_results` object
(https://support.riverbed.com/apis/npm.probe.reports/1.0/service.html#types_data_def_results)
The data can be referenced via data['data'] but meta data
about the results including endtime and startime can be
found at data['meta']
:param int index: Set to None to return data from all data
definitions, defaults to returning the data from just
the first data def.
"""
if not self._instance.live:
# get the already retrieved data
if index is None:
return [dd.data for dd in self._data_defs]
return self._data_defs[index].data
else:
# need to do some special processing
# first check for existing meta data, otherwise
# do our first collection
if index is None:
msg = 'index must be a value for live reports'
raise AppResponseException(msg)
else:
resp = self._instance.get_datadef_data(index)
if not self._data_defs[index]._data_columns:
self._data_defs[index]._data_columns = resp['columns']
source_name = self._data_defs[index].source.name
if 'data' in resp:
data = self._cast_number(resp, source_name)
else:
data = None
return {'data': data, 'meta': resp['meta']}
def create_instance(self, data_defs):
"""Create a report instance with multiple data definition requests.
:param data_defs: list of DataDef objects
:return: one ReportInstance object
"""
if not data_defs:
msg = 'No data definitions are provided.'
raise AppResponseException(msg)
if (any(dd.source.name == 'packets' for dd in data_defs)
and any(dd.source.name != 'packets' for dd in data_defs)):
# Two report instance needs to be created, one uses 'npm.reports'
# service, the other one uses 'npm.probe.reports' service
# it would create unnecessary complexity to support this case
# thus report error and let the user to create two separate
# report instances
msg = ('Both packets data source and non-packets data source are '
'being queried in this report, which is not supported. The '
'data source names include {}'.format(', '.join(
set([dd.source.name for dd in data_defs]))))
raise AppResponseException(msg)
live = all(dd.live for dd in data_defs)
if not live and any(dd.live for dd in data_defs):
msg = ('Incompatible DataDefs for report: live and non-live '
'cannot be mixed.')
raise AppResponseException(msg)
def _create_instance(service_name, data_defs, live):
config = dict(data_defs=[dd.to_dict() for dd in data_defs],
live=live)
logger.debug("Creating instance with data definitions %s" % config)
svcdef = self.appresponse.find_service(service_name)
datarep = svcdef.bind('instances')
resp = datarep.execute('create', _data=config)
# XXX sleepwalker bug? resp is actually an `instances`
# resource with the data for a single `instance`. Doing a
# `.pull()` will fill data with /instances collection instead
# here we cast result to actual `instance` instance
report_instance = svcdef.bind('instance', id=resp.data['id'])
instance = ReportInstance(data=resp.data,
datarep=report_instance,
live=live)
return instance
if data_defs[0].source.name == 'packets':
# Create clip for for capture job sources only
# Keep the clip till the instance is completed
if data_defs[0].source.path.startswith(SourceProxy.JOB_PREFIX):
with self.appresponse.clips.create_clips(data_defs):
instance = _create_instance(PACKETS_REPORT_SERVICE_NAME,
data_defs, False)
else:
instance = _create_instance(PACKETS_REPORT_SERVICE_NAME,
data_defs, live)
else:
instance = _create_instance(GENERAL_REPORT_SERVICE_NAME, data_defs,
live)
return instance
