def collect(self, jobs=None):
logger.debug("%s: bizhours.collect: %s" % (self, jobs))
basetable = Table.from_ref(
self.table.options.related_tables['basetable']
)
# collect all key names
keynames = []
istime = False
for key in basetable.get_columns(iskey=True):
keynames.append(key.name)
if key.istime():
istime = True
# Now collect the data
total_secs = 0
dfs = []
idx = 0
for jid, job in jobs.iteritems():
if job.status == Job.ERROR:
raise AnalysisException("%s for %s-%s failed: %s" %
(job, job.criteria.starttime,
job.criteria.endtime,
job.message))
subdf = job.data()
logger.debug("%s: returned %d rows" %
(job, len(subdf) if subdf is not None else 0))
if subdf is None:
continue
logger.debug("%s: actual_criteria %s" % (job, job.actual_criteria))
t0 = job.actual_criteria.starttime
t1 = job.actual_criteria.endtime
if not istime:
subdf['__secs__'] = timedelta_total_seconds(t1 - t0)
total_secs += timedelta_total_seconds(t1 - t0)
idx += 1
dfs.append(subdf)
if len(dfs) == 0:
return QueryComplete(None)
df = pandas.concat(dfs, ignore_index=True)
if not istime:
if 'aggregate' in self.table.options:
ops = self.table.options['aggregate']
for col in basetable.get_columns(iskey=False):
if col.name not in ops:
ops[col.name] = 'sum'
else:
ops = 'sum'
df = avg_groupby_aggregate(df, keynames, ops,
'__secs__', total_secs)
return QueryComplete(df)
def render(self, name, value, attrs):
initial_time = attrs.get('initial_time', None)
if initial_time:
m = re.match("now *- *(.+)", initial_time)
if m:
secs = timedelta_total_seconds(parse_timedelta(m.group(1)))
initial_time = (
"d = new Date(); d.setSeconds(d.getSeconds()-%d);" % secs)
else:
initial_time = "d = new Date('{}');".format(initial_time)
else:
initial_time = "d = new Date();"
round_initial = attrs.get('round_initial', None)
if round_initial:
js = (' p = %d*1000; '
'd = new Date(Math.floor(d.getTime() / p) * p);' %
round_initial)
initial_time += js
# only pin manually entered times if we are rounding above a minute
if round_initial >= 60:
step = int(round_initial) / 60
force_round_time = 'true'
else:
step = 15
force_round_time = 'false'
else:
step = 15
force_round_time = 'false'
msg = '''
<span class="input-group-addon">
<span id="timenow_{name}" class="glyphicon glyphicon-time" title="Set time/date to now"> </span>
</span>
{0}
<script type="text/javascript">
$("#id_{name}").timepicker({{
step: {step},
scrollDefaultNow:true,
forceRoundTime: {force_round_time},
timeFormat:"g:i:s a"
}});
$("#timenow_{name}").click(function() {{
$("#id_{name}").timepicker("setTime", new Date());
}});
{initial_time}
// align to timezone
var offset = ($('#tz').html()/100)*60*60*1000;
d = rvbd.timeutil.convertDateToUTC(d);
d.setTime(d.getTime() + offset);
$("#id_{name}").timepicker("setTime", d);
</script>
'''
return msg.format(super(TimeWidget, self).render(name, value, attrs),
name=name,
initial_time=initial_time,
step=step,
force_round_time=force_round_time)
def as_text(self):
"""Return certain field values as a dict for simple json parsing."""
result = {}
for k, v in self.cleaned_data.iteritems():
if isinstance(v, datetime.datetime):
result[k] = v.isoformat()
elif isinstance(v, datetime.timedelta):
result[k] = str(timedelta_total_seconds(v)) + " seconds"
elif isinstance(v, UploadedFile):
# look for uploaded files, save them off to another
# temporary file and return the path for use in JSON
# consumers of this file will need to clean them up
# TODO this will be replaced by the File Storage App
newtemp = tempfile.NamedTemporaryFile(delete=False)
v.seek(0)
shutil.copyfileobj(v, newtemp)
v.close()
newtemp.close()
result[k] = newtemp.name
else:
result[k] = v
return result
def schedule_job(self, name, job_config):
job_options, interval = self.parse_config(job_config)
if interval['offset'] > datetime.timedelta(0):
delta = interval['delta']
offset = interval['offset']
now = datetime.datetime.now(pytz.UTC)
# this gives the latest rounded time in the past
# so we add interval to it to get a future run time
delta_secs = timeutils.timedelta_total_seconds(delta)
next_run_time = timeutils.round_time(now,
round_to=delta_secs,
trim=True)
next_run_time += (delta + offset)
logger.debug(
'Setting next run time to %s (delta: %s, offset: %s)' %
(next_run_time, delta, offset))
job_options['next_run_time'] = next_run_time
logger.debug('Scheduling job named %s with kwargs: %s' %
(name, job_options))
self.scheduler.add_job(name=name, trigger='interval', **job_options)
def render(self, name, value, attrs):
initial_time = attrs.get('initial_time', None)
if initial_time:
m = re.match("now *- *(.+)", initial_time)
if m:
secs = timedelta_total_seconds(parse_timedelta(m.group(1)))
initial_time = (
"d = new Date(); d.setSeconds(d.getSeconds()-%d);" % secs)
else:
initial_time = "d = '%s';" % initial_time
else:
initial_time = "d = new Date();"
msg = '''
{0} <span id="timenow_{name}" class="icon-time" title="Set time/date to now"> </span>
<script type="text/javascript">
$("#id_{name}").timepicker({{
step: 15,
scrollDefaultNow:true,
timeFormat:"g:i:s a"
}});
$("#timenow_{name}").click(function() {{
$("#id_{name}").timepicker("setTime", new Date());
}});
{initial_time}
$("#id_{name}").timepicker("setTime", d);
</script>
'''
#'$("#id_{name}").timepicker("setTime", new Date());'
return msg.format(super(TimeWidget, self).render(name, value, attrs),
name=name,
initial_time=initial_time)
def as_text(self):
"""Return certain field values as a dict for simple json parsing."""
result = {}
for k, v in self.cleaned_data.iteritems():
if isinstance(v, datetime.datetime):
result[k] = v.isoformat()
elif isinstance(v, datetime.timedelta):
result[k] = str(timedelta_total_seconds(v)) + " seconds"
elif isinstance(v, UploadedFile):
# look for uploaded files, save them off to another
# temporary file and return the path for use in JSON
# consumers of this file will need to clean them up
# TODO this will be replaced by the File Storage App
newtemp = tempfile.NamedTemporaryFile(delete=False)
v.seek(0)
shutil.copyfileobj(v, newtemp)
v.close()
newtemp.close()
result[k] = newtemp.name
else:
result[k] = v
return result
def post_process_table(self, field_options):
resolution = field_options['resolution']
if resolution != 'auto':
if isinstance(resolution, int):
res = resolution
else:
res = int(timedelta_total_seconds(parse_timedelta(resolution)))
resolution = Report.RESOLUTION_MAP[res]
field_options['resolution'] = resolution
fields_add_device_selection(self, keyword='netprofiler_device',
label='NetProfiler', module='netprofiler',
enabled=True)
duration = field_options['duration']
if isinstance(duration, int):
duration = "%d min" % duration
fields_add_time_selection(self,
initial_duration=duration,
durations=field_options['durations'])
fields_add_resolution(self,
initial=field_options['resolution'],
resolutions=field_options['resolutions'],
special_values=['auto'])
self.fields_add_filterexpr()
def _round_times(self):
"""Round the start/end time in criteria to reflect what data will be
returned from data source and saved in db (based on investigation of
NetProfiler/NetShark query results).
Start/End time needs to round down to closest time in integer
resolutions. However, if either starttime or endtime in criteria is
already in integer resolutions, it should remain the same.
:return: start time, end time, resolution
"""
self.job.criteria.compute_times()
resolution = getattr(self.job.criteria, 'resolution', None)
if not resolution:
raise AttributeError("The data source table '%s' needs to have "
"'resolution' field." % self.ds_table.name)
resolution_sec = timedelta_total_seconds(self.job.criteria.resolution)
starttime = round_time(self.job.criteria.starttime,
round_to=resolution_sec,
trim=True)
endtime = round_time(self.job.criteria.endtime,
round_to=resolution_sec,
trim=True)
return starttime, endtime, resolution
def render(self, name, value, attrs):
initial_date = attrs.get('initial_date', None)
if initial_date:
m = re.match("now *- *(.+)", initial_date)
if m:
secs = timedelta_total_seconds(parse_timedelta(m.group(1)))
initial_date = (
"d = new Date(); d.setSeconds(d.getSeconds()-%d);" \
% secs)
else:
initial_date = "d = '%s';" % initial_date
else:
initial_date = "d = new Date();"
msg = '''
{0} <span id="datenow_{name}" class="icon-calendar" title="Set date to today"> </span>
<script type="text/javascript">
$("#id_{name}").datepicker({{
format: "mm/dd/YY",
defaultDate: +2,
autoclose: true
}});
{initial_date}
$("#id_{name}").datepicker("setDate", d);
$("#datenow_{name}").click(function() {{ $("#id_{name}").datepicker("setDate", new Date()); }});
</script>
'''
return msg.format(super(DateWidget, self).render(name, value, attrs),
name=name,
initial_date=initial_date)
def post_process_table(self, field_options):
resolution = field_options['resolution']
if resolution != 'auto':
if isinstance(resolution, int):
res = resolution
else:
res = int(timedelta_total_seconds(parse_timedelta(resolution)))
resolution = Report.RESOLUTION_MAP[res]
field_options['resolution'] = resolution
fields_add_device_selection(self, keyword='netprofiler_device',
label='NetProfiler', module='netprofiler',
enabled=True)
duration = field_options['duration']
if isinstance(duration, int):
duration = "%d min" % duration
fields_add_time_selection(self,
initial_duration=duration,
durations=field_options['durations'])
fields_add_resolution(self,
initial=field_options['resolution'],
resolutions=field_options['resolutions'],
special_values=['auto'])
self.fields_add_filterexpr()
def _round_times(self):
"""Round the start/end time in criteria to reflect what data will be
returned from data source and saved in db (based on investigation of
NetProfiler/NetShark query results).
Start/End time needs to round down to closest time in integer
resolutions. However, if either starttime or endtime in criteria is
already in integer resolutions, it should remain the same.
:return: start time, end time, resolution
"""
self.job.criteria.compute_times()
resolution = getattr(self.job.criteria, 'resolution', None)
if not resolution:
raise AttributeError("The data source table '%s' needs to have "
"'resolution' field." % self.ds_table.name)
resolution_sec = timedelta_total_seconds(self.job.criteria.resolution)
starttime = round_time(self.job.criteria.starttime,
round_to=resolution_sec,
trim=True)
endtime = round_time(self.job.criteria.endtime,
round_to=resolution_sec,
trim=True)
return starttime, endtime, resolution
def _prepare_report_args(self):
class Args(object):
pass
args = Args()
criteria = self.job.criteria
if criteria.netprofiler_device == '':
logger.debug('%s: No netprofiler device selected' % self.table)
self.job.mark_error("No NetProfiler Device Selected")
return False
args.profiler = DeviceManager.get_device(criteria.netprofiler_device)
args.columns = [
col.name for col in self.table.get_columns(synthetic=False)
]
args.sortcol = None
if self.table.sortcols is not None:
args.sortcol = self.table.sortcols[0]
args.timefilter = TimeFilter(start=criteria.starttime,
end=criteria.endtime)
logger.info("Running NetProfiler table %d report for timeframe %s" %
(self.table.id, str(args.timefilter)))
if ('datafilter' in criteria) and (criteria.datafilter is not None):
args.datafilter = criteria.datafilter.split(',')
else:
args.datafilter = None
args.trafficexpr = TrafficFilter(
self.job.combine_filterexprs(
exprs=criteria.netprofiler_filterexpr))
# Incoming criteria.resolution is a timedelta
logger.debug('NetProfiler report got criteria resolution %s (%s)' %
(criteria.resolution, type(criteria.resolution)))
if criteria.resolution != 'auto':
rsecs = int(timedelta_total_seconds(criteria.resolution))
args.resolution = Report.RESOLUTION_MAP[rsecs]
else:
args.resolution = 'auto'
logger.debug('NetProfiler report using resolution %s (%s)' %
(args.resolution, type(args.resolution)))
args.limit = (self.table.options.limit if hasattr(
self.table.options, 'limit') else None)
if getattr(self.table.options, 'interface', False):
args.centricity = 'int'
else:
args.centricity = 'hos'
return args
def _prepare_report_args(self):
class Args(object):
pass
args = Args()
criteria = self.job.criteria
if criteria.netprofiler_device == '':
logger.debug('%s: No netprofiler device selected' % self.table)
self.job.mark_error("No NetProfiler Device Selected")
return False
args.profiler = DeviceManager.get_device(criteria.netprofiler_device)
args.columns = [col.name for col
in self.table.get_columns(synthetic=False)]
args.sortcol = None
if self.table.sortcols is not None:
args.sortcol = self.table.sortcols[0]
args.timefilter = TimeFilter(start=criteria.starttime,
end=criteria.endtime)
logger.info("Running NetProfiler table %d report for timeframe %s" %
(self.table.id, str(args.timefilter)))
if ('datafilter' in criteria) and (criteria.datafilter is not None):
args.datafilter = criteria.datafilter.split(',')
else:
args.datafilter = None
args.trafficexpr = TrafficFilter(
self.job.combine_filterexprs(exprs=criteria.netprofiler_filterexpr)
)
# Incoming criteria.resolution is a timedelta
logger.debug('NetProfiler report got criteria resolution %s (%s)' %
(criteria.resolution, type(criteria.resolution)))
if criteria.resolution != 'auto':
rsecs = int(timedelta_total_seconds(criteria.resolution))
args.resolution = Report.RESOLUTION_MAP[rsecs]
else:
args.resolution = 'auto'
logger.debug('NetProfiler report using resolution %s (%s)' %
(args.resolution, type(args.resolution)))
args.limit = (self.table.options.limit
if hasattr(self.table.options, 'limit') else None)
if getattr(self.table.options, 'interface', False):
args.centricity = 'int'
else:
args.centricity = 'hos'
return args
def decompress(self, value):
if isinstance(value, str) or isinstance(value, unicode):
value = timedelta_total_seconds(parse_timedelta(value))
if value:
m = [v for v in self.choices if v[0] == value]
if len(m) == 1:
return m[0]
else:
return [0, '%d min' % (value / 60)]
return [None, None]
def decompress(self, value):
if isinstance(value, str) or isinstance(value, unicode):
value = timedelta_total_seconds(parse_timedelta(value))
if value:
m = [v for v in self.choices if v[0] == value]
if len(m) == 1:
return m[0]
else:
return [0, '%d min' % (value / 60)]
return [None, None]
def _get_url_fields(flds):
for k, v in flds.iteritems():
if k in ['starttime', 'endtime']:
yield (k, str(datetime_to_seconds(v)))
elif k in ['duration', 'resolution']:
try:
yield (k, str(int(timedelta_total_seconds(v))))
except AttributeError:
# v is of special value, not a string of some duration
yield (k, v.replace(' ', '+'))
else:
# use + as encoded white space
yield (k, str(v).replace(' ', '+'))
yield ('auto_run', 'true')
def _get_url_fields(flds):
for k, v in flds.iteritems():
if k in ['starttime', 'endtime']:
yield (k, str(datetime_to_seconds(v)))
elif k in ['duration', 'resolution']:
try:
yield (k, str(int(timedelta_total_seconds(v))))
except AttributeError:
# v is of special value, not a string of some duration
yield (k, v.replace(' ', '+'))
else:
# use + as encoded white space
yield (k, str(v).replace(' ', '+'))
yield ('auto_run', 'true')
def resample(df, timecol, interval, how):
"""Resample the input dataframe.
:param str timecol: the name of the column containing the row time
:param timedelta,str interval: the new interval
:param how: method for down or resampling (see pandas.Dataframe.resample)
"""
df[timecol] = pandas.DatetimeIndex(df[timecol])
df.set_index(timecol, inplace=True)
if isinstance(interval, timedelta):
interval = '%ss' % (timedelta_total_seconds(parse_timedelta(interval)))
df = df.resample(interval, how=how).reset_index()
return df
def resample(df, timecol, interval, how):
"""Resample the input dataframe.
:param str timecol: the name of the column containing the row time
:param timedelta,str interval: the new interval
:param how: method for down or resampling (see pandas.Dataframe.resample)
"""
df[timecol] = pandas.DatetimeIndex(df[timecol])
df.set_index(timecol, inplace=True)
if isinstance(interval, timedelta):
interval = '%ss' % (timedelta_total_seconds(parse_timedelta(interval)))
df = df.resample(interval, how=how).reset_index()
return df
def resample(df, timecol, interval, how='sum'):
"""Resample the input dataframe.
:param str timecol: the name of the column containing the row time
:param timedelta,str interval: the new interval
:param how: method for down or resampling (see pandas.Dataframe.resample)
"""
df[timecol] = pandas.DatetimeIndex(df[timecol])
df.set_index(timecol, inplace=True)
if isinstance(interval, timedelta):
interval = '%ss' % (timedelta_total_seconds(parse_timedelta(interval)))
# use new pandas reasmple API
# http://pandas.pydata.org/pandas-docs/stable/whatsnew.html#resample-api
r = df.resample(interval)
df = getattr(r, how)()
df.reset_index(inplace=True)
return df
def resample(df, timecol, interval, how='sum'):
"""Resample the input dataframe.
:param str timecol: the name of the column containing the row time
:param timedelta,str interval: the new interval
:param how: method for down or resampling (see pandas.Dataframe.resample)
"""
df[timecol] = pandas.DatetimeIndex(df[timecol])
df.set_index(timecol, inplace=True)
if isinstance(interval, timedelta):
interval = '%ss' % (timedelta_total_seconds(parse_timedelta(interval)))
# use new pandas reasmple API
# http://pandas.pydata.org/pandas-docs/stable/whatsnew.html#resample-api
r = df.resample(interval)
df = getattr(r, how)()
df.reset_index(inplace=True)
return df
def render(self, name, value, attrs):
initial_date = attrs.get('initial_date', None)
if initial_date:
m = re.match("now *- *(.+)", initial_date)
if m:
secs = timedelta_total_seconds(parse_timedelta(m.group(1)))
initial_date = (
"d = new Date(); d.setSeconds(d.getSeconds()-%d);" % secs
)
else:
initial_date = "d = '%s';" % initial_date
else:
initial_date = "d = new Date();"
round_initial = attrs.get('round_initial', None)
if round_initial:
js = (' p = %d*1000; '
'd = new Date(Math.floor(d.getTime() / p) * p);'
% round_initial)
initial_date += js
msg = '''
<span class="input-group-addon">
<span id="datenow_{name}" class="glyphicon glyphicon-calendar" title="Set date to today"> </span>
</span>
{0}
<script type="text/javascript">
$("#id_{name}").datepicker({{
format: "mm/dd/YY",
defaultDate: +2,
autoclose: true
}});
{initial_date}
$("#id_{name}").datepicker("setDate", d);
$("#datenow_{name}").click(function() {{ $("#id_{name}").datepicker("setDate", new Date()); }});
</script>
'''
return msg.format(
super(DateWidget, self).render(name, value, attrs),
name=name, initial_date=initial_date
)
def schedule_job(self, name, job_config):
job_options, interval = self.parse_config(job_config)
if interval['offset'] > datetime.timedelta(0):
delta = interval['delta']
offset = interval['offset']
now = datetime.datetime.now(pytz.UTC)
# this gives the latest rounded time in the past
# so we add interval to it to get a future run time
total_offset = timeutils.timedelta_total_seconds(delta + offset)
next_run_time = timeutils.round_time(now,
round_to=total_offset,
trim=True)
next_run_time += (delta + offset)
logger.debug('Setting next run time to %s (delta: %s, offset: %s)'
% (next_run_time, delta, offset))
job_options['next_run_time'] = next_run_time
logger.debug('Scheduling job named %s with kwargs: %s' % (name,
job_options))
self.scheduler.add_job(name=name, trigger='interval', **job_options)
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
if criteria.netprofiler_device == '':
logger.debug('%s: No netprofiler device selected' % self.table)
self.job.mark_error("No NetProfiler Device Selected")
return False
#self.fake_run()
#return True
profiler = DeviceManager.get_device(criteria.netprofiler_device)
report = steelscript.netprofiler.core.report.SingleQueryReport(profiler)
columns = [col.name for col in self.table.get_columns(synthetic=False)]
sortcol = None
if self.table.sortcols is not None:
sortcol = self.table.sortcols[0]
tf = TimeFilter(start=criteria.starttime,
end=criteria.endtime)
logger.info("Running NetProfiler table %d report for timeframe %s" %
(self.table.id, str(tf)))
if ('datafilter' in criteria) and (criteria.datafilter is not None):
datafilter = criteria.datafilter.split(',')
else:
datafilter = None
trafficexpr = TrafficFilter(
self.job.combine_filterexprs(exprs=criteria.netprofiler_filterexpr)
)
# Incoming criteria.resolution is a timedelta
logger.debug('NetProfiler report got criteria resolution %s (%s)' %
(criteria.resolution, type(criteria.resolution)))
if criteria.resolution != 'auto':
rsecs = int(timedelta_total_seconds(criteria.resolution))
resolution = steelscript.netprofiler.core.report.Report.RESOLUTION_MAP[rsecs]
else:
resolution = 'auto'
logger.debug('NetProfiler report using resolution %s (%s)' %
(resolution, type(resolution)))
with lock:
centricity = 'int' if self.table.options.interface else 'hos'
report.run(realm=self.table.options.realm,
groupby=profiler.groupbys[self.table.options.groupby],
centricity=centricity,
columns=columns,
timefilter=tf,
trafficexpr=trafficexpr,
data_filter=datafilter,
resolution=resolution,
sort_col=sortcol,
sync=False
)
done = False
logger.info("Waiting for report to complete")
while not done:
time.sleep(0.5)
with lock:
s = report.status()
self.job.safe_update(progress=int(s['percent']))
done = (s['status'] == 'completed')
# Retrieve the data
with lock:
query = report.get_query_by_index(0)
self.data = query.get_data()
tz = criteria.starttime.tzinfo
# Update criteria
criteria.starttime = (datetime.datetime
.utcfromtimestamp(query.actual_t0)
.replace(tzinfo=tz))
criteria.endtime = (datetime.datetime
.utcfromtimestamp(query.actual_t1)
.replace(tzinfo=tz))
self.job.safe_update(actual_criteria=criteria)
if self.table.rows > 0:
self.data = self.data[:self.table.rows]
logger.info("Report %s returned %s rows" % (self.job, len(self.data)))
return True
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
self.timeseries = False # if key column called 'time' is created
self.column_names = []
# Resolution comes in as a time_delta
resolution = timedelta_total_seconds(criteria.resolution)
default_delta = 1000000000 # one second
self.delta = int(default_delta * resolution) # sample size interval
if criteria.netshark_device == '':
logger.debug('%s: No netshark device selected' % self.table)
self.job.mark_error("No NetShark Device Selected")
return False
shark = DeviceManager.get_device(criteria.netshark_device)
logger.debug("Creating columns for NetShark table %d" % self.table.id)
# Create Key/Value Columns
columns = []
for tc in self.table.get_columns(synthetic=False):
tc_options = tc.options
if (tc.iskey and tc.name == 'time'
and tc_options.extractor == 'sample_time'):
# don't create column, use the sample time for timeseries
self.timeseries = True
self.column_names.append('time')
continue
elif tc.iskey:
c = Key(tc_options.extractor,
description=tc.label,
default_value=tc_options.default_value)
else:
if tc_options.operation:
try:
operation = getattr(Operation, tc_options.operation)
except AttributeError:
operation = Operation.sum
print('ERROR: Unknown operation attribute '
'%s for column %s.' %
(tc_options.operation, tc.name))
else:
operation = Operation.none
c = Value(tc_options.extractor,
operation,
description=tc.label,
default_value=tc_options.default_value)
self.column_names.append(tc.name)
columns.append(c)
# Identify Sort Column
sortidx = None
if self.table.sortcols is not None:
sortcol = Column.objects.get(table=self.table,
name=self.table.sortcols[0])
sort_name = sortcol.options.extractor
for i, c in enumerate(columns):
if c.field == sort_name:
sortidx = i
break
# Initialize filters
criteria = self.job.criteria
filters = []
if hasattr(criteria, 'netshark_filterexpr'):
logger.debug('calculating netshark filter expression ...')
filterexpr = self.job.combine_filterexprs(
exprs=criteria.netshark_filterexpr, joinstr="&")
if filterexpr:
logger.debug('applying netshark filter expression: %s' %
filterexpr)
filters.append(NetSharkFilter(filterexpr))
if hasattr(criteria, 'netshark_bpf_filterexpr'):
# TODO evaluate how to combine multiple BPF filters
# this will just apply one at a time
filterexpr = criteria.netshark_bpf_filterexpr
logger.debug('applying netshark BPF filter expression: %s' %
filterexpr)
filters.append(BpfFilter(filterexpr))
resolution = criteria.resolution
if resolution.seconds == 1:
sampling_time_msec = 1000
elif resolution.microseconds == 1000:
sampling_time_msec = 1
if criteria.duration > parse_timedelta('1s'):
msg = ("Cannot run a millisecond report with a duration "
"longer than 1 second")
raise ValueError(msg)
else:
sampling_time_msec = 1000
# Get source type from options
logger.debug("NetShark Source: %s" %
self.job.criteria.netshark_source_name)
source = path_to_class(shark, self.job.criteria.netshark_source_name)
live = source.is_live()
persistent = criteria.get('netshark_persistent', False)
if live and not persistent:
raise ValueError("Live views must be run with persistent set")
view = None
if persistent:
# First, see a view by this title already exists
# Title is the table name plus a criteria hash including
# all criteria *except* the timeframe
h = hashlib.md5()
h.update('.'.join([c.name for c in self.table.get_columns()]))
for k, v in criteria.iteritems():
if criteria.is_timeframe_key(k):
continue
h.update('%s:%s' % (k, v))
title = '/'.join([
'steelscript-appfwk',
str(self.table.id), self.table.namespace, self.table.name,
h.hexdigest()
])
view = NetSharkViews.find_by_name(shark, title)
logger.debug("Persistent view title: %s" % title)
else:
# Only assign a title for persistent views
title = None
timefilter = TimeFilter(start=criteria.starttime, end=criteria.endtime)
if not view:
# Not persistent, or not yet created...
if not live:
# Cannot attach time filter to a live view,
# it will be added later at get_data() time
if criteria.starttime and criteria.endtime:
filters.append(timefilter)
logger.info("Setting netshark table %d timeframe to %s" %
(self.table.id, str(timefilter)))
else:
# if times are set to zero, don't add to filter
# this will process entire timeframe of source instead
logger.info("Not setting netshark table %d timeframe" %
self.table.id)
# Create it
with lock:
logger.debug("%s: Creating view for table %s" %
(str(self), str(self.table)))
view = shark.create_view(source,
columns,
filters=filters,
sync=False,
name=title,
sampling_time_msec=sampling_time_msec)
if not live:
done = False
logger.debug("Waiting for netshark table %d to complete" %
self.table.id)
while not done:
time.sleep(0.5)
with lock:
s = view.get_progress()
self.job.mark_progress(s)
self.job.save()
done = view.is_ready()
logger.debug("Retrieving data for timeframe: %s" % timefilter)
# Retrieve the data
with lock:
getdata_kwargs = {}
if sortidx:
getdata_kwargs['sortby'] = sortidx
if self.table.options.aggregated:
getdata_kwargs['aggregated'] = self.table.options.aggregated
else:
getdata_kwargs['delta'] = self.delta
if live:
# For live views, attach the time frame to the get_data()
getdata_kwargs['start'] = (datetime_to_nanoseconds(
criteria.starttime))
getdata_kwargs['end'] = (datetime_to_nanoseconds(
criteria.endtime))
self.data = view.get_data(**getdata_kwargs)
if not persistent:
view.close()
if self.table.rows > 0:
self.data = self.data[:self.table.rows]
self.parse_data()
logger.info("NetShark Report %s returned %s rows" %
(self.job, len(self.data)))
return QueryComplete(self.data)
def render(self, name, value, attrs):
initial_time = attrs.get('initial_time', None)
if initial_time:
m = re.match("now *- *(.+)", initial_time)
if m:
secs = timedelta_total_seconds(parse_timedelta(m.group(1)))
initial_time = (
"d = new Date(); d.setSeconds(d.getSeconds()-%d);" % secs
)
else:
initial_time = "d = new Date('{}');".format(initial_time)
else:
initial_time = "d = new Date();"
round_initial = attrs.get('round_initial', None)
if round_initial:
js = (' p = %d*1000; '
'd = new Date(Math.floor(d.getTime() / p) * p);'
% round_initial)
initial_time += js
# only pin manually entered times if we are rounding above a minute
if round_initial >= 60:
step = int(round_initial) / 60
force_round_time = 'true'
else:
step = 15
force_round_time = 'false'
else:
step = 15
force_round_time = 'false'
msg = '''
<span class="input-group-addon">
<span id="timenow_{name}" class="glyphicon glyphicon-time" title="Set time/date to now"> </span>
</span>
{0}
<script type="text/javascript">
$("#id_{name}").timepicker({{
step: {step},
scrollDefaultNow:true,
forceRoundTime: {force_round_time},
timeFormat:"g:i:s a"
}});
$("#timenow_{name}").click(function() {{
$("#id_{name}").timepicker("setTime", new Date());
}});
{initial_time}
// align to timezone
var offset = ($('#tz').html()/100)*60*60*1000;
d = rvbd.timeutil.convertDateToUTC(d);
d.setTime(d.getTime() + offset);
$("#id_{name}").timepicker("setTime", d);
</script>
'''
return msg.format(
super(TimeWidget, self).render(name, value, attrs),
name=name, initial_time=initial_time, step=step,
force_round_time=force_round_time
)
def compute_synthetic(self, job, df):
""" Compute the synthetic columns from DF a two-dimensional array
of the non-synthetic columns.
Synthesis occurs as follows:
1. Compute all synthetic columns where compute_post_resample
is False
2. If the table is a time-based table with a defined resolution,
the result is resampled.
3. Any remaining columns are computed.
"""
if df is None:
return None
all_columns = job.get_columns()
all_col_names = [c.name for c in all_columns]
def compute(df, syncols):
for syncol in syncols:
expr = syncol.compute_expression
g = tokenize.generate_tokens(StringIO(expr).readline)
newexpr = ""
getvalue = False
getclose = False
for ttype, tvalue, _, _, _ in g:
if getvalue:
if ttype != tokenize.NAME:
msg = "Invalid syntax, expected {name}: %s" % tvalue
raise ValueError(msg)
elif tvalue in all_col_names:
newexpr += "df['%s']" % tvalue
elif tvalue in job.criteria:
newexpr += '"%s"' % str(job.criteria.get(tvalue))
else:
raise ValueError("Invalid variable name: %s" %
tvalue)
getclose = True
getvalue = False
elif getclose:
if ttype != tokenize.OP and tvalue != "}":
msg = "Invalid syntax, expected {name}: %s" % tvalue
raise ValueError(msg)
getclose = False
elif ttype == tokenize.OP and tvalue == "{":
getvalue = True
else:
newexpr += tvalue
newexpr += ' '
try:
df[syncol.name] = eval(newexpr)
except NameError as e:
m = (('%s: expression failed: %s, check '
'APPFWK_SYNTHETIC_MODULES: %s') %
(self, newexpr, str(e)))
logger.exception(m)
raise TableComputeSyntheticError(m)
# 1. Compute synthetic columns where post_resample is False
compute(df, [
col for col in all_columns
if (col.synthetic and col.compute_post_resample is False)
])
# 2. Resample
colmap = {}
timecol = None
for col in all_columns:
colmap[col.name] = col
if col.istime():
timecol = col.name
if self.resample:
if timecol is None:
raise (TableComputeSyntheticError(
"%s: 'resample' is set but no 'time' column'" % self))
if (('resolution' not in job.criteria)
and ('resample_resolution' not in job.criteria)):
raise (TableComputeSyntheticError(
("%s: 'resample' is set but criteria missing " +
"'resolution' or 'resample_resolution'") % self))
how = {}
for k in df.keys():
if k == timecol or k not in colmap:
continue
how[k] = colmap[k].resample_operation
if 'resample_resolution' in job.criteria:
resolution = job.criteria.resample_resolution
else:
resolution = job.criteria.resolution
resolution = timedelta_total_seconds(resolution)
if resolution < 1:
raise (TableComputeSyntheticError(
("Table %s cannot resample at a resolution " +
"less than 1 second") % self))
logger.debug('%s: resampling to %ss' % (self, int(resolution)))
indexed = df.set_index(timecol)
resampled = indexed.resample('%ss' % int(resolution),
how,
convention='end').reset_index()
df = resampled
# 3. Compute remaining synthetic columns (post_resample is True)
compute(df, [
c for c in all_columns
if (c.synthetic and c.compute_post_resample is True)
])
return df
def process(cls, widget, job, data):
class ColInfo:
def __init__(self,
col,
dataindex,
axis,
istime=False,
isdate=False):
self.col = col
self.key = cleankey(col.name)
self.dataindex = dataindex
self.axis = axis
self.istime = istime
self.isdate = isdate
t_cols = job.get_columns()
colinfo = {} # map by widget key
# columns of None is a special case, just use all
# defined columns other than time
if widget.options.columns is None:
valuecolnames = [
col.name for col in t_cols
if not col.istime() and not col.isdate()
]
else:
valuecolnames = widget.options.columns
# Column keys are the 'cleaned' column names
w_keys = [cleankey(n) for n in valuecolnames]
# Retrieve the desired value columns
# ...and the indices for the value values
# (as the 'data' has *all* columns)
time_colinfo = None
for i, c in enumerate(t_cols):
if c.istime():
ci = ColInfo(c, i, -1, istime=True)
time_colinfo = ci
elif c.isdate():
ci = ColInfo(c, i, -1, isdate=True)
time_colinfo = ci
elif c.name in valuecolnames:
if c.isnumeric():
ci = ColInfo(c, i, -1, istime=False, isdate=False)
else:
raise KeyError(
"Cannot graph non-numeric data in timeseries widget: "
"column {0}".format(c.name))
colinfo[ci.key] = ci
if widget.options.altaxis:
altaxis = widget.options.altaxis
axes_def = {
'0': {
'position': 'left',
'columns':
[col for col in valuecolnames if col not in altaxis]
},
'1': {
'position': 'right',
'columns':
[col for col in valuecolnames if col in altaxis]
}
}
else:
axes_def = {'0': {'position': 'left', 'columns': valuecolnames}}
w_series = []
axes = Axes(axes_def)
# Setup the time axis
w_axes = {
"time": {
"keys": ["time"],
"position": "bottom",
"type": "time",
"styles": {
"label": {
"fontSize": "8pt",
"rotation": "-45"
}
}
}
}
# Create a better time format depending on t0/t1
t_dataindex = time_colinfo.dataindex
t0 = data[0][t_dataindex]
t1 = data[-1][t_dataindex]
if not hasattr(t0, 'utcfromtimestamp'):
t0 = timeutils.sec_string_to_datetime(t0)
t1 = timeutils.sec_string_to_datetime(t1)
total_seconds = timeutils.timedelta_total_seconds(t1 - t0)
if total_seconds < 2:
w_axes['time']['formatter'] = 'formatTimeMs'
elif total_seconds < 120:
w_axes['time']['labelFormat'] = '%k:%M:%S'
elif total_seconds < (24 * 60 * 60):
w_axes['time']['labelFormat'] = '%k:%M'
elif time_colinfo.isdate:
w_axes['time']['formatter'] = 'formatDate'
else:
w_axes['time']['labelFormat'] = '%D %k:%M'
# Setup the other axes, checking the axis for each column
for w_key in w_keys:
# Need to interate the valuecolnames array to preserve order
ci = colinfo[w_key]
w_series.append({
"xKey": "time",
"xDisplayName": "Time",
"yKey": ci.key,
"yDisplayName": ci.col.label,
"styles": {
"line": {
"weight": 1
},
"marker": {
"height": 3,
"width": 3
}
}
})
ci.axis = axes.getaxis(ci.col.name)
axis_name = 'axis' + str(ci.axis)
if axis_name not in w_axes:
w_axes[axis_name] = {
"type": "numeric",
"position": axes.position(ci.axis),
"keys": []
}
w_axes[axis_name]['keys'].append(ci.key)
# Output row data
rows = []
# min/max values by axis 0/1
minval = {}
maxval = {}
stacked = widget.options.stacked
# Iterate through all rows if input data
for rawrow in data:
t = rawrow[t_dataindex]
try:
t = timeutils.datetime_to_microseconds(t) / 1000
except AttributeError:
t = t * 1000
row = {'time': t}
rowmin = {}
rowmax = {}
for ci in colinfo.values():
if ci.istime or ci.isdate:
continue
a = ci.axis
val = rawrow[ci.dataindex]
row[ci.key] = val if val != '' else None
# If stacked and there is only one value, use that
# value as the rowmin. If stacked and there is more than
# one value for the axis, use a floor of 0 to give proper
# context.
if a not in rowmin:
rowmin[a] = val if val != '' else 0
rowmax[a] = val if val != '' else 0
else:
rowmin[a] = (0 if stacked else min(rowmin[a], val))
rowmax[a] = ((rowmax[a] +
val) if stacked else max(rowmax[a], val))
for a in rowmin.keys():
minval[a] = rowmin[a] if (a not in minval) else min(
minval[a], rowmin[a])
maxval[a] = rowmax[a] if (a not in maxval) else max(
maxval[a], rowmax[a])
rows.append(row)
# Setup the scale values for the axes
for ci in colinfo.values():
if ci.istime or ci.isdate:
continue
axis_name = 'axis' + str(ci.axis)
if minval and maxval:
n = NiceScale(minval[ci.axis], maxval[ci.axis])
w_axes[axis_name]['minimum'] = "%.10f" % n.nicemin
w_axes[axis_name]['maximum'] = "%.10f" % n.nicemax
w_axes[axis_name]['tickExponent'] = math.log10(n.tickspacing)
w_axes[axis_name]['styles'] = {
'majorUnit': {
'count': n.numticks
}
}
else:
# empty data which would result in keyError above
w_axes[axis_name]['minimum'] = "0"
w_axes[axis_name]['maximum'] = "1"
w_axes[axis_name]['tickExponent'] = 1
w_axes[axis_name]['styles'] = {'majorUnit': {'count': 1}}
if ci.col.units == ci.col.UNITS_PCT:
w_axes[axis_name]['formatter'] = 'formatPct'
else:
w_axes[axis_name]['formatter'] = 'formatMetric'
if stacked:
charttype = "area"
elif widget.options.bar:
charttype = "column"
else:
charttype = "combo"
data = {
"chartTitle": widget.title.format(**job.actual_criteria),
"type": charttype,
"stacked": stacked,
"dataProvider": rows,
"seriesCollection": w_series,
"axes": w_axes,
"legend": {
"position": "bottom",
"fontSize": "8pt",
"styles": {
"gap": 0
}
},
"interactionType": "planar"
}
# logger.debug("data:\n\n%s\n" % data)
return data
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
self.timeseries = False # if key column called 'time' is created
self.column_names = []
# Resolution comes in as a time_delta
resolution = timedelta_total_seconds(criteria.resolution)
default_delta = 1000000000 # one second
self.delta = int(default_delta * resolution) # sample size interval
if criteria.netshark_device == '':
logger.debug('%s: No netshark device selected' % self.table)
self.job.mark_error("No NetShark Device Selected")
return False
#self.fake_run()
#return True
shark = DeviceManager.get_device(criteria.netshark_device)
logger.debug("Creating columns for NetShark table %d" % self.table.id)
# Create Key/Value Columns
columns = []
for tc in self.table.get_columns(synthetic=False):
tc_options = tc.options
if ( tc.iskey and tc.name == 'time' and
tc_options.extractor == 'sample_time'):
# don't create column, use the sample time for timeseries
self.timeseries = True
self.column_names.append('time')
continue
elif tc.iskey:
c = Key(tc_options.extractor,
description=tc.label,
default_value=tc_options.default_value)
else:
if tc_options.operation:
try:
operation = getattr(Operation, tc_options.operation)
except AttributeError:
operation = Operation.sum
print ('ERROR: Unknown operation attribute '
'%s for column %s.' %
(tc_options.operation, tc.name))
else:
operation = Operation.none
c = Value(tc_options.extractor,
operation,
description=tc.label,
default_value=tc_options.default_value)
self.column_names.append(tc.name)
columns.append(c)
# Identify Sort Column
sortidx = None
if self.table.sortcols is not None:
sortcol = Column.objects.get(table=self.table,
name=self.table.sortcols[0])
sort_name = sortcol.options.extractor
for i, c in enumerate(columns):
if c.field == sort_name:
sortidx = i
break
# Initialize filters
criteria = self.job.criteria
filters = []
filterexpr = self.job.combine_filterexprs(
exprs=criteria.netshark_filterexpr,
joinstr="&"
)
if filterexpr:
filters.append(NetSharkFilter(filterexpr))
tf = TimeFilter(start=criteria.starttime, end=criteria.endtime)
filters.append(tf)
logger.info("Setting netshark table %d timeframe to %s" % (self.table.id,
str(tf)))
# Get source type from options
try:
with lock:
source = path_to_class(shark,
self.job.criteria.netshark_source_name)
except RvbdHTTPException, e:
source = None
raise e
def run(self):
criteria = self.job.criteria
tzname = criteria.business_hours_tzname
logger.debug("%s: timezone: %s" % (self.job, tzname))
tz = pytz.timezone(tzname)
# Convert to datetime objects in the requested timezone
st = criteria.starttime.astimezone(tz)
et = criteria.endtime.astimezone(tz)
logger.debug("%s: times: %s - %s" % (self.job, st, et))
# Business hours start/end, as string "HH:MMam" like 8:00am
sb = parse_time(criteria.business_hours_start)
eb = parse_time(criteria.business_hours_end)
weekends = criteria.business_hours_weekends
# Iterate from st to et until
times = []
t = st
while t <= et:
# Set t0/t1 to date of t but time of sb/eb
t0 = replace_time(t, sb, tz)
t1 = replace_time(t, eb, tz)
# Advance t by 1 day
t = t + datetime.timedelta(days=1)
# Strip timezone, then re-add to account for DST
t = tz.localize(t.replace(tzinfo=None))
# Skip weekends
if not weekends and t0.weekday() >= 5:
continue
# Now see if we have any overlap of biz hours for today
if et < t0:
# Report end time is today before biz hours start, all done
break
if et < t1:
# Report end time is today in the middle of biz hours, adjust
t1 = et
if t1 < st:
# Report start time occurs today *after* biz end, nothing today
continue
if t0 < st:
# Report start time occurs today in the middle of the biz hours
# Adjust t0
t0 = st
logger.debug("%s: start: %s, end: %s, duration: %s" %
(self.job, str(t0), str(t1),
str(timedelta_total_seconds(t1-t0))))
times.append((t0, t1, timedelta_total_seconds(t1-t0)))
if len(times) == 0:
data = None
else:
# manually assign data types to avoid Python 2.6 issues
# when converting date times
columns = ['starttime', 'endtime', 'totalsecs']
s1 = pandas.Series([x[0] for x in times], dtype='datetime64[ns]')
s2 = pandas.Series([x[1] for x in times], dtype='datetime64[ns]')
tt = pandas.Series([x[2] for x in times], dtype='int')
# create dataframe then assign using correct column ordering
df = pandas.DataFrame(dict(zip(columns, [s1, s2, tt])))
data = df[columns]
return QueryComplete(data)
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
self.timeseries = False # if key column called 'time' is created
self.column_names = []
# Resolution comes in as a time_delta
resolution = timedelta_total_seconds(criteria.resolution)
default_delta = 1000000000 # one second
self.delta = int(default_delta * resolution) # sample size interval
if criteria.netshark_device == '':
logger.debug('%s: No netshark device selected' % self.table)
self.job.mark_error("No NetShark Device Selected")
return False
shark = DeviceManager.get_device(criteria.netshark_device)
logger.debug("Creating columns for NetShark table %d" % self.table.id)
# Create Key/Value Columns
columns = []
for tc in self.table.get_columns(synthetic=False):
tc_options = tc.options
if (tc.iskey and tc.name == 'time' and
tc_options.extractor == 'sample_time'):
# don't create column, use the sample time for timeseries
self.timeseries = True
self.column_names.append('time')
continue
elif tc.iskey:
c = Key(tc_options.extractor,
description=tc.label,
default_value=tc_options.default_value)
else:
if tc_options.operation:
try:
operation = getattr(Operation, tc_options.operation)
except AttributeError:
operation = Operation.sum
print ('ERROR: Unknown operation attribute '
'%s for column %s.' %
(tc_options.operation, tc.name))
else:
operation = Operation.none
c = Value(tc_options.extractor,
operation,
description=tc.label,
default_value=tc_options.default_value)
self.column_names.append(tc.name)
columns.append(c)
# Identify Sort Column
sortidx = None
if self.table.sortcols is not None:
sortcol = Column.objects.get(table=self.table,
name=self.table.sortcols[0])
sort_name = sortcol.options.extractor
for i, c in enumerate(columns):
if c.field == sort_name:
sortidx = i
break
# Initialize filters
criteria = self.job.criteria
filters = []
if hasattr(criteria, 'netshark_filterexpr'):
logger.debug('calculating netshark filter expression ...')
filterexpr = self.job.combine_filterexprs(
exprs=criteria.netshark_filterexpr,
joinstr="&"
)
if filterexpr:
logger.debug('applying netshark filter expression: %s'
% filterexpr)
filters.append(NetSharkFilter(filterexpr))
if hasattr(criteria, 'netshark_bpf_filterexpr'):
# TODO evaluate how to combine multiple BPF filters
# this will just apply one at a time
filterexpr = criteria.netshark_bpf_filterexpr
logger.debug('applying netshark BPF filter expression: %s'
% filterexpr)
filters.append(BpfFilter(filterexpr))
resolution = criteria.resolution
if resolution.seconds == 1:
sampling_time_msec = 1000
elif resolution.microseconds == 1000:
sampling_time_msec = 1
if criteria.duration > parse_timedelta('1s'):
msg = ("Cannot run a millisecond report with a duration "
"longer than 1 second")
raise ValueError(msg)
else:
sampling_time_msec = 1000
# Get source type from options
logger.debug("NetShark Source: %s" %
self.job.criteria.netshark_source_name)
source = path_to_class(
shark, self.job.criteria.netshark_source_name)
live = source.is_live()
persistent = criteria.get('netshark_persistent', False)
if live and not persistent:
raise ValueError("Live views must be run with persistent set")
view = None
if persistent:
# First, see a view by this title already exists
# Title is the table name plus a criteria hash including
# all criteria *except* the timeframe
h = hashlib.md5()
h.update('.'.join([c.name for c in
self.table.get_columns()]))
for k, v in criteria.iteritems():
if criteria.is_timeframe_key(k):
continue
h.update('%s:%s' % (k, v))
title = '/'.join(['steelscript-appfwk', str(self.table.id),
self.table.namespace, self.table.name,
h.hexdigest()])
view = NetSharkViews.find_by_name(shark, title)
logger.debug("Persistent view title: %s" % title)
else:
# Only assign a title for persistent views
title = None
if not view:
# Not persistent, or not yet created...
if not live:
# Cannot attach time filter to a live view,
# it will be added later at get_data() time
tf = TimeFilter(start=criteria.starttime,
end=criteria.endtime)
filters.append(tf)
logger.info("Setting netshark table %d timeframe to %s" %
(self.table.id, str(tf)))
# Create it
with lock:
logger.debug("%s: Creating view for table %s" %
(str(self), str(self.table)))
view = shark.create_view(
source, columns, filters=filters, sync=False,
name=title, sampling_time_msec=sampling_time_msec)
if not live:
done = False
logger.debug("Waiting for netshark table %d to complete" %
self.table.id)
while not done:
time.sleep(0.5)
with lock:
s = view.get_progress()
self.job.mark_progress(s)
self.job.save()
done = view.is_ready()
logger.debug("Retrieving data for timeframe: %s - %s" %
(datetime_to_nanoseconds(criteria.starttime),
datetime_to_nanoseconds(criteria.endtime)))
# Retrieve the data
with lock:
getdata_kwargs = {}
if sortidx:
getdata_kwargs['sortby'] = sortidx
if self.table.options.aggregated:
getdata_kwargs['aggregated'] = self.table.options.aggregated
else:
getdata_kwargs['delta'] = self.delta
if live:
# For live views, attach the time frame to the get_data()
getdata_kwargs['start'] = (
datetime_to_nanoseconds(criteria.starttime))
getdata_kwargs['end'] = (
datetime_to_nanoseconds(criteria.endtime))
self.data = view.get_data(**getdata_kwargs)
if not persistent:
view.close()
if self.table.rows > 0:
self.data = self.data[:self.table.rows]
self.parse_data()
logger.info("NetShark Report %s returned %s rows" %
(self.job, len(self.data)))
return QueryComplete(self.data)
def run(self):
""" Main execution method. """
criteria = self.job.criteria
if criteria.netprofiler_device == '':
logger.debug('%s: No netprofiler device selected' % self.table)
self.job.mark_error("No NetProfiler Device Selected")
return False
profiler = DeviceManager.get_device(criteria.netprofiler_device)
report = steelscript.netprofiler.core.report.MultiQueryReport(profiler)
tf = TimeFilter(start=criteria.starttime, end=criteria.endtime)
logger.info("Running NetProfilerTemplateTable table %d report "
"for timeframe %s" % (self.table.id, str(tf)))
trafficexpr = TrafficFilter(
self.job.combine_filterexprs(exprs=criteria.profiler_filterexpr))
# Incoming criteria.resolution is a timedelta
logger.debug('NetProfiler report got criteria resolution %s (%s)' %
(criteria.resolution, type(criteria.resolution)))
if criteria.resolution != 'auto':
rsecs = int(timedelta_total_seconds(criteria.resolution))
resolution = steelscript.netprofiler.core.report.Report.RESOLUTION_MAP[
rsecs]
else:
resolution = 'auto'
logger.debug('NetProfiler report using resolution %s (%s)' %
(resolution, type(resolution)))
with lock:
res = report.run(template_id=self.table.options.template_id,
timefilter=tf,
trafficexpr=trafficexpr,
resolution=resolution)
if res is True:
logger.info("Report template complete.")
self.job.safe_update(progress=100)
# Retrieve the data
with lock:
query = report.get_query_by_index(0)
data = query.get_data()
headers = report.get_legend()
tz = criteria.starttime.tzinfo
# Update criteria
criteria.starttime = (datetime.datetime.utcfromtimestamp(
query.actual_t0).replace(tzinfo=tz))
criteria.endtime = (datetime.datetime.utcfromtimestamp(
query.actual_t1).replace(tzinfo=tz))
self.job.safe_update(actual_criteria=criteria)
# create dataframe with all of the default headers
df = pandas.DataFrame(data, columns=[h.key for h in headers])
# now filter down to the columns requested by the table
columns = [col.name for col in self.table.get_columns(synthetic=False)]
self.data = df[columns]
logger.info("Report %s returned %s rows" % (self.job, len(self.data)))
return True
def compute_synthetic(self, job, df):
""" Compute the synthetic columns from DF a two-dimensional array
of the non-synthetic columns.
Synthesis occurs as follows:
1. Compute all synthetic columns where compute_post_resample
is False
2. If the table is a time-based table with a defined resolution,
the result is resampled.
3. Any remaining columns are computed.
"""
if df is None:
return None
all_columns = job.get_columns()
all_col_names = [c.name for c in all_columns]
def compute(df, syncols):
for syncol in syncols:
expr = syncol.compute_expression
g = tokenize.generate_tokens(StringIO(expr).readline)
newexpr = ""
getvalue = False
getclose = False
for ttype, tvalue, _, _, _ in g:
if getvalue:
if ttype != tokenize.NAME:
msg = "Invalid syntax, expected {name}: %s" % tvalue
raise ValueError(msg)
elif tvalue in all_col_names:
newexpr += "df['%s']" % tvalue
elif tvalue in job.criteria:
newexpr += '"%s"' % str(job.criteria.get(tvalue))
else:
raise ValueError("Invalid variable name: %s" % tvalue)
getclose = True
getvalue = False
elif getclose:
if ttype != tokenize.OP and tvalue != "}":
msg = "Invalid syntax, expected {name}: %s" % tvalue
raise ValueError(msg)
getclose = False
elif ttype == tokenize.OP and tvalue == "{":
getvalue = True
else:
newexpr += tvalue
newexpr += ' '
try:
df[syncol.name] = eval(newexpr)
except NameError as e:
m = (('%s: expression failed: %s, check '
'APPFWK_SYNTHETIC_MODULES: %s') %
(self, newexpr, str(e)))
logger.exception(m)
raise TableComputeSyntheticError(m)
# 1. Compute synthetic columns where post_resample is False
compute(df, [col for col in all_columns if
(col.synthetic and col.compute_post_resample is False)])
# 2. Resample
colmap = {}
timecol = None
for col in all_columns:
colmap[col.name] = col
if col.istime():
timecol = col.name
if self.resample:
if timecol is None:
raise (TableComputeSyntheticError
("%s: 'resample' is set but no 'time' column'" %
self))
if (('resolution' not in job.criteria) and
('resample_resolution' not in job.criteria)):
raise (TableComputeSyntheticError
(("%s: 'resample' is set but criteria missing " +
"'resolution' or 'resample_resolution'") % self))
how = {}
for k in df.keys():
if k == timecol or k not in colmap:
continue
how[k] = colmap[k].resample_operation
if 'resample_resolution' in job.criteria:
resolution = job.criteria.resample_resolution
else:
resolution = job.criteria.resolution
resolution = timedelta_total_seconds(resolution)
if resolution < 1:
raise (TableComputeSyntheticError
(("Table %s cannot resample at a resolution " +
"less than 1 second") % self))
logger.debug('%s: resampling to %ss' % (self, int(resolution)))
indexed = df.set_index(timecol)
resampled = indexed.resample('%ss' % int(resolution), how,
convention='end').reset_index()
df = resampled
# 3. Compute remaining synthetic columns (post_resample is True)
compute(df, [c for c in all_columns
if (c.synthetic and c.compute_post_resample is True)])
return df
def create(cls, shark, interface, name,
packet_retention_size_limit,
packet_retention_packet_limit=None,
packet_retention_time_limit=None,
bpf_filter=None,
snap_length=65525,
indexing_size_limit=None,
indexing_synced=False,
indexing_time_limit=None,
start_immediately=True,
requested_start_time=None,
requested_stop_time=None,
stop_rule_size_limit=None,
stop_rule_packet_limit=None,
stop_rule_time_limit=None):
"""Create a new capture job"""
def _calc_size(size, total):
if isinstance(size, str) and size[-1] == '%':
size = total * int(size[:-1]) / 100
elif not isinstance(size, (int, long)) and size is not None:
size = utils.human2bytes(size)
return size
stats = shark.get_stats()
packet_storage_total = stats['storage']['packet_storage']['total']
index_storage_total = stats['storage']['os_storage']['index_storage']['total']
packet_retention_size_limit = _calc_size(packet_retention_size_limit, packet_storage_total)
stop_rule_size_limit = _calc_size(stop_rule_size_limit, packet_storage_total) if stop_rule_size_limit else None
indexing_size_limit = _calc_size(indexing_size_limit, index_storage_total) if indexing_size_limit else None
packet_retention_packet_limit = int(packet_retention_packet_limit) if packet_retention_packet_limit else None
stop_rule_size_limit = int(stop_rule_size_limit) if stop_rule_size_limit else None
stop_rule_packet_limit = int(stop_rule_packet_limit) if stop_rule_packet_limit else None
snap_length = int(snap_length) if snap_length else 65535
indexing_size_limit = int(indexing_size_limit) if indexing_size_limit else None
if indexing_time_limit:
indexing_time_limit = int(timeutils.timedelta_total_seconds(indexing_time_limit))
if packet_retention_time_limit:
packet_retention_time_limit = int(timeutils.timedelta_total_seconds(packet_retention_time_limit))
if stop_rule_time_limit:
stop_rule_time_limit = int(timeutils.timedelta_total_seconds(stop_rule_time_limit))
if requested_start_time:
requested_start_time = requested_start_time.strftime(cls._timefmt)
if requested_stop_time:
requested_stop_time = requested_stop_time.strftime(cls._timefmt)
jobrequest = {'interface_name': interface.id}
jobrequest['name'] = name
if packet_retention_size_limit or packet_retention_packet_limit or packet_retention_time_limit:
jobrequest['packet_retention'] = dict()
if packet_retention_size_limit:
jobrequest['packet_retention']['size_limit'] = packet_retention_size_limit
if packet_retention_packet_limit:
jobrequest['packet_retention']['packet_limit'] = packet_retention_packet_limit
if packet_retention_time_limit:
jobrequest['packet_retention']['time_limit'] = packet_retention_time_limit
if bpf_filter:
jobrequest['bpf_filter'] = bpf_filter
if requested_start_time:
jobrequest['requested_start_time'] = requested_start_time
if requested_stop_time:
jobrequest['requested_stop_time'] = requested_stop_time
if stop_rule_size_limit or stop_rule_packet_limit or stop_rule_time_limit:
jobrequest['stop_rule'] = dict()
if stop_rule_size_limit:
jobrequest['stop_rule']['size_limit'] = stop_rule_size_limit
if stop_rule_packet_limit:
jobrequest['stop_rule']['packet_limit'] = stop_rule_packet_limit
if stop_rule_time_limit:
jobrequest['stop_rule']['time_limit'] = stop_rule_time_limit
if snap_length:
jobrequest['snap_length'] = int(snap_length)
# Indexing logic table
# case size synced time valid?
# 1 None False None valid - index is disabled
# 2 None False int error
# 3 None True None error
# 4 None True int error
# 5 int False None valid
# 6 int False int valid
# 7 int True None valid
# 8 int True int valid
if (indexing_synced or indexing_time_limit) and not indexing_size_limit:
raise NetSharkException(
'indexing_size_limit must be specified '
'with indexing_synced or indexing_time_limit')
elif indexing_size_limit:
jobrequest['indexing'] = dict()
jobrequest['indexing']['size_limit'] = indexing_size_limit
if indexing_synced:
jobrequest['indexing']['synced'] = indexing_synced
if indexing_time_limit:
jobrequest['indexing']['time_limit'] = indexing_time_limit
jobrequest['start_immediately'] = start_immediately
job_id = shark.api.jobs.add(jobrequest)
job = cls(shark, job_id)
return job
def run(self, template_id, timefilter=None, resolution="auto",
query=None, trafficexpr=None, data_filter=None, sync=True,
custom_criteria=None):
"""Create the report and begin running the report on NetProfiler.
If the `sync` option is True, periodically poll until the report is
complete, otherwise return immediately.
:param int template_id: numeric id of the template to use for the report
:param timefilter: range of time to query,
instance of :class:`TimeFilter`
:param str resolution: data resolution, such as (1min, 15min, etc.),
defaults to 'auto'
:param str query: query object containing criteria
:param trafficexpr: instance of :class:`TrafficFilter`
:param str data_filter: deprecated filter to run against report data
:param bool sync: if True, poll for status until the report is complete
"""
self.template_id = template_id
if timefilter is None:
self.timefilter = TimeFilter.parse_range("last 5 min")
else:
self.timefilter = timefilter
self.query = query
self.trafficexpr = trafficexpr
self.data_filter = data_filter
self.id = None
self.queries = list()
self.last_status = None
if resolution not in ["auto", "1min", "15min", "hour",
"6hour", "day", "week", "month"]:
rd = parse_timedelta(resolution)
resolution = self.RESOLUTION_MAP[int(timedelta_total_seconds(rd))]
self.resolution = resolution
start = datetime_to_seconds(self.timefilter.start)
end = datetime_to_seconds(self.timefilter.end)
criteria = RecursiveUpdateDict(**{"time_frame": {"start": int(start),
"end": int(end)}
})
if self.query is not None:
criteria["query"] = self.query
if self.resolution != "auto":
criteria["time_frame"]["resolution"] = self.resolution
if self.data_filter:
criteria['deprecated'] = {self.data_filter[0]: self.data_filter[1]}
if self.trafficexpr is not None:
criteria["traffic_expression"] = self.trafficexpr.filter
if custom_criteria:
for k, v in custom_criteria.iteritems():
criteria[k] = v
to_post = {"template_id": self.template_id,
"criteria": criteria}
logger.debug("Posting JSON: %s" % to_post)
response = self.profiler.api.report.reports(data=to_post)
try:
self.id = int(response['id'])
except KeyError:
raise ValueError(
"failed to retrieve report id from report creation response: %s"
% response)
logger.info("Created report %d" % self.id)
if sync:
self.wait_for_complete()
def run(self):
""" Main execution method. """
criteria = self.job.criteria
if criteria.netprofiler_device == '':
logger.debug('%s: No netprofiler device selected' % self.table)
self.job.mark_error("No NetProfiler Device Selected")
return False
profiler = DeviceManager.get_device(criteria.netprofiler_device)
report = steelscript.netprofiler.core.report.MultiQueryReport(profiler)
tf = TimeFilter(start=criteria.starttime,
end=criteria.endtime)
logger.info("Running NetProfilerTemplateTable table %d report "
"for timeframe %s" % (self.table.id, str(tf)))
trafficexpr = TrafficFilter(
self.job.combine_filterexprs(exprs=criteria.profiler_filterexpr)
)
# Incoming criteria.resolution is a timedelta
logger.debug('NetProfiler report got criteria resolution %s (%s)' %
(criteria.resolution, type(criteria.resolution)))
if criteria.resolution != 'auto':
rsecs = int(timedelta_total_seconds(criteria.resolution))
resolution = steelscript.netprofiler.core.report.Report.RESOLUTION_MAP[rsecs]
else:
resolution = 'auto'
logger.debug('NetProfiler report using resolution %s (%s)' %
(resolution, type(resolution)))
with lock:
res = report.run(template_id=self.table.options.template_id,
timefilter=tf,
trafficexpr=trafficexpr,
resolution=resolution)
if res is True:
logger.info("Report template complete.")
self.job.safe_update(progress=100)
# Retrieve the data
with lock:
query = report.get_query_by_index(0)
data = query.get_data()
headers = report.get_legend()
tz = criteria.starttime.tzinfo
# Update criteria
criteria.starttime = (datetime.datetime
.utcfromtimestamp(query.actual_t0)
.replace(tzinfo=tz))
criteria.endtime = (datetime.datetime
.utcfromtimestamp(query.actual_t1)
.replace(tzinfo=tz))
self.job.safe_update(actual_criteria=criteria)
# create dataframe with all of the default headers
df = pandas.DataFrame(data, columns=[h.key for h in headers])
# now filter down to the columns requested by the table
columns = [col.name for col in self.table.get_columns(synthetic=False)]
self.data = df[columns]
logger.info("Report %s returned %s rows" % (self.job, len(self.data)))
return True
def process(cls, widget, job, data):
class ColInfo:
def __init__(self, col, dataindex, axis,
istime=False, isdate=False):
self.col = col
self.key = cleankey(col.name)
self.dataindex = dataindex
self.axis = axis
self.istime = istime
self.isdate = isdate
t_cols = job.get_columns()
colinfo = {} # map by widget key
# columns of None is a special case, just use all
# defined columns other than time
if widget.options.columns is None:
valuecolnames = [col.name for col in t_cols
if not col.istime() and not col.isdate()]
else:
valuecolnames = widget.options.columns
# Column keys are the 'cleaned' column names
w_keys = [cleankey(n) for n in valuecolnames]
# Retrieve the desired value columns
# ...and the indices for the value values
# (as the 'data' has *all* columns)
time_colinfo = None
for i, c in enumerate(t_cols):
if c.istime():
ci = ColInfo(c, i, -1, istime=True)
time_colinfo = ci
elif c.isdate():
ci = ColInfo(c, i, -1, isdate=True)
time_colinfo = ci
elif c.name in valuecolnames:
if c.isnumeric():
ci = ColInfo(c, i, -1, istime=False, isdate=False)
else:
raise KeyError(
"Cannot graph non-numeric data in timeseries widget: "
"column {0}".format(c.name))
colinfo[ci.key] = ci
if widget.options.altaxis:
altaxis = widget.options.altaxis
axes_def = {'0': {'position': 'left',
'columns': [col for col in valuecolnames
if col not in altaxis]},
'1': {'position': 'right',
'columns': [col for col in valuecolnames
if col in altaxis]}
}
else:
axes_def = {'0': {'position': 'left',
'columns': valuecolnames}}
w_series = []
axes = Axes(axes_def)
# Setup the time axis
w_axes = {"time": {"keys": ["time"],
"position": "bottom",
"type": "time",
"styles": {"label": {"fontSize": "8pt",
"rotation": "-45"}}}}
# Create a better time format depending on t0/t1
t_dataindex = time_colinfo.dataindex
t0 = data[0][t_dataindex]
t1 = data[-1][t_dataindex]
if not hasattr(t0, 'utcfromtimestamp'):
t0 = timeutils.sec_string_to_datetime(t0)
t1 = timeutils.sec_string_to_datetime(t1)
total_seconds = timeutils.timedelta_total_seconds(t1 - t0)
if total_seconds < 2:
w_axes['time']['formatter'] = 'formatTimeMs'
elif total_seconds < 120:
w_axes['time']['labelFormat'] = '%k:%M:%S'
elif total_seconds < (24 * 60 * 60):
w_axes['time']['labelFormat'] = '%k:%M'
elif time_colinfo.isdate:
w_axes['time']['formatter'] = 'formatDate'
else:
w_axes['time']['labelFormat'] = '%D %k:%M'
# Setup the other axes, checking the axis for each column
for w_key in w_keys:
# Need to interate the valuecolnames array to preserve order
ci = colinfo[w_key]
w_series.append({"xKey": "time",
"xDisplayName": "Time",
"yKey": ci.key,
"yDisplayName": ci.col.label,
"styles": {"line": {"weight": 1},
"marker": {"height": 3,
"width": 3}}})
ci.axis = axes.getaxis(ci.col.name)
axis_name = 'axis' + str(ci.axis)
if axis_name not in w_axes:
w_axes[axis_name] = {"type": "numeric",
"position": axes.position(ci.axis),
"keys": []
}
w_axes[axis_name]['keys'].append(ci.key)
# Output row data
rows = []
# min/max values by axis 0/1
minval = {}
maxval = {}
stacked = widget.options.stacked
# Iterate through all rows if input data
for rawrow in data:
t = rawrow[t_dataindex]
try:
t = timeutils.datetime_to_microseconds(t) / 1000
except AttributeError:
t = t * 1000
row = {'time': t}
rowmin = {}
rowmax = {}
for ci in colinfo.values():
if ci.istime or ci.isdate:
continue
a = ci.axis
val = rawrow[ci.dataindex]
row[ci.key] = val if val != '' else None
# If stacked and there is only one value, use that
# value as the rowmin. If stacked and there is more than
# one value for the axis, use a floor of 0 to give proper
# context.
if a not in rowmin:
rowmin[a] = val if val != '' else 0
rowmax[a] = val if val != '' else 0
else:
rowmin[a] = (0 if stacked
else min(rowmin[a], val))
rowmax[a] = ((rowmax[a] + val) if stacked
else max(rowmax[a], val))
for a in rowmin.keys():
minval[a] = rowmin[a] if (a not in minval) else min(minval[a],
rowmin[a])
maxval[a] = rowmax[a] if (a not in maxval) else max(maxval[a],
rowmax[a])
rows.append(row)
# Setup the scale values for the axes
for ci in colinfo.values():
if ci.istime or ci.isdate:
continue
axis_name = 'axis' + str(ci.axis)
if minval and maxval:
n = NiceScale(minval[ci.axis], maxval[ci.axis])
w_axes[axis_name]['minimum'] = "%.10f" % n.nicemin
w_axes[axis_name]['maximum'] = "%.10f" % n.nicemax
w_axes[axis_name]['tickExponent'] = math.log10(n.tickspacing)
w_axes[axis_name]['styles'] = {
'majorUnit': {'count': n.numticks}
}
else:
# empty data which would result in keyError above
w_axes[axis_name]['minimum'] = "0"
w_axes[axis_name]['maximum'] = "1"
w_axes[axis_name]['tickExponent'] = 1
w_axes[axis_name]['styles'] = {'majorUnit': {'count': 1}}
if ci.col.units == ci.col.UNITS_PCT:
w_axes[axis_name]['formatter'] = 'formatPct'
else:
w_axes[axis_name]['formatter'] = 'formatMetric'
if stacked:
charttype = "area"
elif widget.options.bar:
charttype = "column"
else:
charttype = "combo"
data = {
"chartTitle": widget.title.format(**job.actual_criteria),
"type": charttype,
"stacked": stacked,
"dataProvider": rows,
"seriesCollection": w_series,
"axes": w_axes,
"legend": {"position": "bottom",
"fontSize": "8pt",
"styles": {"gap": 0}},
"interactionType": "planar"
}
# logger.debug("data:\n\n%s\n" % data)
return data
def create(cls,
shark,
interface,
name,
packet_retention_size_limit,
packet_retention_packet_limit=None,
packet_retention_time_limit=None,
bpf_filter=None,
snap_length=65525,
indexing_size_limit=None,
indexing_synced=False,
indexing_time_limit=None,
start_immediately=True,
requested_start_time=None,
requested_stop_time=None,
stop_rule_size_limit=None,
stop_rule_packet_limit=None,
stop_rule_time_limit=None):
"""Create a new capture job"""
def _calc_size(size, total):
if isinstance(size, str) and size[-1] == '%':
size = total * int(size[:-1]) / 100
elif not isinstance(size, (int, long)) and size is not None:
size = datautils.human2bytes(size)
return size
stats = shark.get_stats()
packet_storage_total = stats['storage']['packet_storage']['total']
index_storage_total = stats['storage']['os_storage']['index_storage'][
'total']
packet_retention_size_limit = _calc_size(packet_retention_size_limit,
packet_storage_total)
stop_rule_size_limit = _calc_size(
stop_rule_size_limit,
packet_storage_total) if stop_rule_size_limit else None
indexing_size_limit = _calc_size(
indexing_size_limit,
index_storage_total) if indexing_size_limit else None
packet_retention_packet_limit = int(
packet_retention_packet_limit
) if packet_retention_packet_limit else None
stop_rule_size_limit = int(
stop_rule_size_limit) if stop_rule_size_limit else None
stop_rule_packet_limit = int(
stop_rule_packet_limit) if stop_rule_packet_limit else None
snap_length = int(snap_length) if snap_length else 65535
indexing_size_limit = int(
indexing_size_limit) if indexing_size_limit else None
if indexing_time_limit:
indexing_time_limit = int(
timeutils.timedelta_total_seconds(indexing_time_limit))
if packet_retention_time_limit:
packet_retention_time_limit = int(
timeutils.timedelta_total_seconds(packet_retention_time_limit))
if stop_rule_time_limit:
stop_rule_time_limit = int(
timeutils.timedelta_total_seconds(stop_rule_time_limit))
if requested_start_time:
requested_start_time = requested_start_time.strftime(cls._timefmt)
if requested_stop_time:
requested_stop_time = requested_stop_time.strftime(cls._timefmt)
jobrequest = {'interface_name': interface.id}
jobrequest['name'] = name
if packet_retention_size_limit or packet_retention_packet_limit or packet_retention_time_limit:
jobrequest['packet_retention'] = dict()
if packet_retention_size_limit:
jobrequest['packet_retention'][
'size_limit'] = packet_retention_size_limit
if packet_retention_packet_limit:
jobrequest['packet_retention'][
'packet_limit'] = packet_retention_packet_limit
if packet_retention_time_limit:
jobrequest['packet_retention'][
'time_limit'] = packet_retention_time_limit
if bpf_filter:
jobrequest['bpf_filter'] = bpf_filter
if requested_start_time:
jobrequest['requested_start_time'] = requested_start_time
if requested_stop_time:
jobrequest['requested_stop_time'] = requested_stop_time
if stop_rule_size_limit or stop_rule_packet_limit or stop_rule_time_limit:
jobrequest['stop_rule'] = dict()
if stop_rule_size_limit:
jobrequest['stop_rule']['size_limit'] = stop_rule_size_limit
if stop_rule_packet_limit:
jobrequest['stop_rule'][
'packet_limit'] = stop_rule_packet_limit
if stop_rule_time_limit:
jobrequest['stop_rule']['time_limit'] = stop_rule_time_limit
if snap_length:
jobrequest['snap_length'] = int(snap_length)
# Indexing logic table
# case size synced time valid?
# 1 None False None valid - index is disabled
# 2 None False int error
# 3 None True None error
# 4 None True int error
# 5 int False None valid
# 6 int False int valid
# 7 int True None valid
# 8 int True int valid
if (indexing_synced
or indexing_time_limit) and not indexing_size_limit:
raise NetSharkException(
'indexing_size_limit must be specified '
'with indexing_synced or indexing_time_limit')
elif indexing_size_limit:
jobrequest['indexing'] = dict()
jobrequest['indexing']['size_limit'] = indexing_size_limit
if indexing_synced:
jobrequest['indexing']['synced'] = indexing_synced
if indexing_time_limit:
jobrequest['indexing']['time_limit'] = indexing_time_limit
jobrequest['start_immediately'] = start_immediately
job_id = shark.api.jobs.add(jobrequest)
job = cls(shark, job_id)
return job
def run(self,
template_id,
timefilter=None,
resolution="auto",
query=None,
trafficexpr=None,
data_filter=None,
sync=True,
custom_criteria=None):
"""Create the report and begin running the report on NetProfiler.
If the `sync` option is True, periodically poll until the report is
complete, otherwise return immediately.
:param int template_id: numeric id of the template to use for the report
:param timefilter: range of time to query,
instance of :class:`TimeFilter`
:param str resolution: data resolution, such as (1min, 15min, etc.),
defaults to 'auto'
:param str query: query object containing criteria
:param trafficexpr: instance of :class:`TrafficFilter`
:param str data_filter: deprecated filter to run against report data
:param bool sync: if True, poll for status until the report is complete
"""
self.template_id = template_id
if timefilter is None:
self.timefilter = TimeFilter.parse_range("last 5 min")
else:
self.timefilter = timefilter
self.query = query
self.trafficexpr = trafficexpr
self.data_filter = data_filter
self.id = None
self.queries = list()
self.last_status = None
if resolution not in [
"auto", "1min", "15min", "hour", "6hour", "day", "week",
"month"
]:
rd = parse_timedelta(resolution)
resolution = self.RESOLUTION_MAP[int(timedelta_total_seconds(rd))]
self.resolution = resolution
start = datetime_to_seconds(self.timefilter.start)
end = datetime_to_seconds(self.timefilter.end)
criteria = RecursiveUpdateDict(
**{"time_frame": {
"start": int(start),
"end": int(end)
}})
if self.query is not None:
criteria["query"] = self.query
if self.resolution != "auto":
criteria["time_frame"]["resolution"] = self.resolution
if self.data_filter:
criteria['deprecated'] = {self.data_filter[0]: self.data_filter[1]}
if self.trafficexpr is not None:
criteria["traffic_expression"] = self.trafficexpr.filter
if custom_criteria:
for k, v in custom_criteria.iteritems():
criteria[k] = v
to_post = {"template_id": self.template_id, "criteria": criteria}
logger.debug("Posting JSON: %s" % to_post)
response = self.profiler.api.report.reports(data=to_post)
try:
self.id = int(response['id'])
except KeyError:
raise ValueError(
"failed to retrieve report id from report creation response: %s"
% response)
logger.info("Created report %d" % self.id)
if sync:
self.wait_for_complete()
