def __init__(self, **kwargs):
self._special_values = kwargs.pop('special_values', None)
initial = kwargs.pop('initial', None)
if (initial is not None and
self._special_values is None or
initial not in self._special_values):
initial_td = parse_timedelta(initial)
initial_valid = False
else:
initial_td = None
initial_valid = True
choices = []
# Rebuild the choices list to ensure that
# the value is normalized using timedelta_str
for choice in kwargs.pop('choices'):
td = None
if not (isinstance(choice, list) or isinstance(choice, tuple)):
if (self._special_values is None or
choice not in self._special_values):
td = parse_timedelta(choice)
td_str = timedelta_str(td)
value = td_str
label = td_str
else:
value = choice
label = choice
else:
(value, label) = choice
if (self._special_values is None or
value not in self._special_values):
td = parse_timedelta(value)
value = timedelta_str(td)
choice = (value, label)
choices.append(choice)
if initial_td is not None and initial_td == td:
initial = value
initial_valid = True
kwargs['choices'] = choices
logger.debug("Choices: %s" % choices)
if not initial_valid:
raise KeyError('Initial duration is invalid: %s' % initial)
super(DurationField, self).__init__(initial=initial, **kwargs)
def render(self, name, value, attrs):
initial_date = attrs.get('initial_date', None)
if initial_date:
m = re.match("now *- *(.+)", initial_date)
if m:
secs = timedelta_total_seconds(parse_timedelta(m.group(1)))
initial_date = (
"d = new Date(); d.setSeconds(d.getSeconds()-%d);" \
% secs)
else:
initial_date = "d = '%s';" % initial_date
else:
initial_date = "d = new Date();"
msg = '''
{0} <span id="datenow_{name}" class="icon-calendar" title="Set date to today"> </span>
<script type="text/javascript">
$("#id_{name}").datepicker({{
format: "mm/dd/YY",
defaultDate: +2,
autoclose: true
}});
{initial_date}
$("#id_{name}").datepicker("setDate", d);
$("#datenow_{name}").click(function() {{ $("#id_{name}").datepicker("setDate", new Date()); }});
</script>
'''
return msg.format(
super(DateWidget, self).render(name, value, attrs),
name=name, initial_date=initial_date
)
def render(self, name, value, attrs):
initial_time = attrs.get('initial_time', None)
if initial_time:
m = re.match("now *- *(.+)", initial_time)
if m:
secs = timedelta_total_seconds(parse_timedelta(m.group(1)))
initial_time = (
"d = new Date(); d.setSeconds(d.getSeconds()-%d);" \
% secs)
else:
initial_time = "d = '%s';" % initial_time
else:
initial_time = "d = new Date();"
msg = '''
{0} <span id="timenow_{name}" class="icon-time" title="Set time/date to now"> </span>
<script type="text/javascript">
$("#id_{name}").timepicker({{
step: 15,
scrollDefaultNow:true,
timeFormat:"g:i:s a"
}});
$("#timenow_{name}").click(function() {{
$("#id_{name}").timepicker("setTime", new Date());
}});
{initial_time}
$("#id_{name}").timepicker("setTime", d);
</script>
'''
#'$("#id_{name}").timepicker("setTime", new Date());'
return msg.format(
super(TimeWidget, self).render(name, value, attrs),
name=name, initial_time=initial_time
)
def create(cls, name, groupby, realm, duration,
resolution='auto', filterexpr=None, interface=False,
**kwargs):
logger.debug('Creating ProfilerTable table %s (%s) - %s/%s' %
(name, duration, groupby, realm))
options = TableOptions(groupby=groupby,
realm=realm,
centricity='int' if interface else 'hos')
t = Table(name=name, module=__name__,
filterexpr=filterexpr, options=options, **kwargs)
t.save()
if resolution != 'auto':
if isinstance(resolution, int):
res = resolution
else:
res = int(timedelta_total_seconds(parse_timedelta(resolution)))
resolution = rvbd.profiler.report.Report.RESOLUTION_MAP[res]
if isinstance(duration, int):
duration = "%d min" % duration
fields_add_device_selection(t, keyword='profiler_device',
label='Profiler', module='profiler',
enabled=True)
fields_add_time_selection(t, initial_duration=duration)
fields_add_filterexpr(t)
fields_add_resolution(t, initial=resolution,
resolutions=[('auto', 'Automatic'),
'1min', '15min', 'hour', '6hour'],
special_values=['auto'])
return t
def __init__(self, **kwargs):
self._special_values = kwargs.pop('special_values', None)
initial = kwargs.pop('initial', None)
if ((initial is not None) and
(self._special_values is None or
initial not in self._special_values)):
initial_td = parse_timedelta(initial)
initial_valid = False
else:
initial_td = None
initial_valid = True
choices = []
# Rebuild the choices list to ensure that
# the value is normalized using timedelta_str
for choice in kwargs.pop('choices'):
td = None
if not (isinstance(choice, list) or isinstance(choice, tuple)):
if (self._special_values is None or
choice not in self._special_values):
td = parse_timedelta(choice)
td_str = timedelta_str(td)
value = td_str
label = td_str
else:
value = choice
label = choice
else:
(value, label) = choice
if (self._special_values is None or
value not in self._special_values):
td = parse_timedelta(value)
value = timedelta_str(td)
choice = (value, label)
choices.append(choice)
if initial_td is not None and initial_td == td:
initial = value
initial_valid = True
kwargs['choices'] = choices
if not initial_valid:
raise KeyError('Initial duration is invalid: %s' % initial)
super(DurationField, self).__init__(initial=initial, **kwargs)
def to_python(self, value):
if value in validators.EMPTY_VALUES:
v = None
elif self._special_values and value in self._special_values:
v = value
else:
try:
v = parse_timedelta(value)
except:
raise ValidationError('Invalid duration string: %s' % value)
return v
def decompress(self, value):
if isinstance(value, str) or isinstance(value, unicode):
value = timedelta_total_seconds(parse_timedelta(value))
if value:
m = [v for v in self.choices if v[0] == value]
if len(m) == 1:
return m[0]
else:
return [0, '%d min' % (value / 60)]
return [None, None]
def decompress(self, value):
if isinstance(value, str) or isinstance(value, unicode):
value = timedelta_total_seconds(parse_timedelta(value))
if value:
m = [v for v in self.choices if v[0] == value]
if len(m) == 1:
return m[0]
else:
return [0, '%d min' % (value / 60)]
return [None, None]
def to_python(self, value):
if value in validators.EMPTY_VALUES:
v = None
elif self._special_values and value in self._special_values:
v = value
else:
try:
v = parse_timedelta(value)
except:
raise ValidationError('Invalid duration string: %s' % value)
logger.debug("DurationField.to_python: %s" % v)
return v
def create(cls, name, template_id, duration,
resolution='auto', filterexpr=None, **kwargs):
""" Create a ProfilerTemplateTable.
This queries a Profiler saved report template rather than creating
a new report from scratch.
`template_id` is a saved-report template ID
`duration` is in minutes or a string like '15min'
"""
logger.debug('Creating ProfilerTemplateTable table %s (%s) - %s/%s' %
(name, template_id, duration, resolution))
options = TableOptions(template_id=template_id)
t = Table(name=name, module=__name__,
filterexpr=filterexpr, options=options, **kwargs)
t.save()
if resolution != 'auto':
if isinstance(resolution, int):
res = resolution
else:
res = int(timedelta_total_seconds(parse_timedelta(resolution)))
resolution = rvbd.profiler.report.Report.RESOLUTION_MAP[res]
if isinstance(duration, int):
duration = "%d min" % duration
fields_add_device_selection(t, keyword='profiler_device',
label='Profiler', module='profiler',
enabled=True)
fields_add_time_selection(t, initial_duration=duration)
fields_add_filterexpr(t)
fields_add_resolution(t, initial=resolution,
resolutions=[('auto', 'Automatic'),
'1min', '15min', 'hour', '6hour'],
special_values=['auto'])
return t
def run(self):
# All user entered criteria is available directly from this object.
# Values for any fields added to the table will appear as
# attributes according to the field keyword.
criteria = self.job.criteria
# Check that a sample_device was selected
if criteria.sample_device == "":
logger.debug("%s: No sample device selected" % self.table)
self.job.mark_error("No Sample Device Selected")
return False
sample_device = DeviceManager.get_device(criteria.sample_device)
# Get the columns for this report
columns = self.table.get_columns(synthetic=False)
sortcol = None
if self.table.sortcol is not None:
sortcol = self.table.sortcol.name
# Time selection is available via criterai.starttime and endtime.
# These are both datetime objects.
t0 = criteria.starttime
t1 = criteria.endtime
# Time resolution is a timedelta object
resolution = criteria.resolution
# Grab the custom min and max criteria
cmin = float(criteria.min)
cmax = float(criteria.max)
# Grab the table options
beta = self.table.options.beta
# Now, do some computation -- create table with a 'time' column
# ranging from t0 to t1 with the defined resolution. Then
# for each additional column do some math function on the
# data
t = t0
rows = []
while t < t1:
row = []
for col in columns:
if col.name == "time":
row.append(t)
else:
period_td = parse_timedelta(col.options.period)
period_secs = timedelta_total_seconds(period_td)
alpha = col.options.alpha
funcname = col.options.func
# seconds since the t0
secs = timedelta_total_seconds(t - t0)
rad = (secs / period_secs) * 2 * math.pi
funcmap = {"sin": math.sin, "cos": math.cos}
# Compute!
val = beta + alpha * funcmap[funcname](rad)
# Clip by the min/max criteria
val = max(cmin, val)
val = min(cmax, val)
# Add the value to the row
row.append(val)
# Add the row
rows.append(row)
# This function runs pretty fast, but this shows how to mark
# progress
self.job.mark_progress(100 * (timedelta_total_seconds(t - t0) / timedelta_total_seconds(t1 - t0)))
t = t + resolution
# Save the result in self.data
self.data = rows
if self.table.rows > 0:
self.data = self.data[: self.table.rows]
logger.info("Report %s returned %s rows" % (self.job, len(self.data)))
return True
def run(self):
# All user entered criteria is available directly from this object.
# Values for any fields added to the table will appear as
# attributes according to the field keyword.
criteria = self.job.criteria
# Check that a sample_device was selected
if criteria.sample_device == '':
logger.debug('%s: No sample device selected' % self.table)
self.job.mark_error("No Sample Device Selected")
return False
sample_device = DeviceManager.get_device(criteria.sample_device)
# Get the columns for this report
columns = self.table.get_columns(synthetic=False)
sortcol = None
if self.table.sortcol is not None:
sortcol = self.table.sortcol.name
# Time selection is available via criterai.starttime and endtime.
# These are both datetime objects.
t0 = criteria.starttime
t1 = criteria.endtime
# Time resolution is a timedelta object
resolution = criteria.resolution
# Grab the custom min and max criteria
cmin = float(criteria.min)
cmax = float(criteria.max)
# Grab the table options
beta = self.table.options.beta
# Now, do some computation -- create table with a 'time' column
# ranging from t0 to t1 with the defined resolution. Then
# for each additional column do some math function on the
# data
t = t0
rows = []
while t < t1:
row = []
for col in columns:
if col.name == 'time':
row.append(t)
else:
period_td = parse_timedelta(col.options.period)
period_secs = timedelta_total_seconds(period_td)
alpha = col.options.alpha
funcname = col.options.func
# seconds since the t0
secs = timedelta_total_seconds(t - t0)
rad = (secs / period_secs) * 2 * math.pi
funcmap = {
'sin': math.sin,
'cos': math.cos,
}
# Compute!
val = beta + alpha * funcmap[funcname](rad)
# Clip by the min/max criteria
val = max(cmin, val)
val = min(cmax, val)
# Add the value to the row
row.append(val)
# Add the row
rows.append(row)
# This function runs pretty fast, but this shows how to mark
# progress
self.job.mark_progress(100 * (timedelta_total_seconds(t - t0) /
timedelta_total_seconds(t1 - t0)))
t = t + resolution
# Save the result in self.data
self.data = rows
if self.table.rows > 0:
self.data = self.data[:self.table.rows]
logger.info("Report %s returned %s rows" % (self.job, len(self.data)))
return True
class TableQuery:
# Used by Table to actually run a query
def __init__(self, table, job):
self.table = table
self.job = job
self.timeseries = False # if key column called 'time' is created
self.column_names = []
# Resolution comes in as a time_delta
resolution = timedelta_total_seconds(job.criteria.resolution)
default_delta = 1000000000 # one second
self.delta = int(default_delta * resolution) # sample size interval
def fake_run(self):
import fake_data
self.data = fake_data.make_data(self.table, self.job)
def run(self):
""" Main execution method
"""
criteria = self.job.criteria
if criteria.shark_device == '':
logger.debug('%s: No shark device selected' % self.table)
self.job.mark_error("No Shark Device Selected")
return False
#self.fake_run()
#return True
shark = DeviceManager.get_device(criteria.shark_device)
logger.debug("Creating columns for Shark table %d" % self.table.id)
# Create Key/Value Columns
columns = []
for tc in self.table.get_columns(synthetic=False):
tc_options = tc.options
if (tc.iskey and tc.name == 'time'
and tc_options.extractor == 'sample_time'):
# don't create column, use the sample time for timeseries
self.timeseries = True
self.column_names.append('time')
continue
elif tc.iskey:
c = Key(tc_options.extractor,
description=tc.label,
default_value=tc_options.default_value)
else:
if tc_options.operation:
try:
operation = getattr(Operation, tc_options.operation)
except AttributeError:
operation = Operation.sum
print('ERROR: Unknown operation attribute '
'%s for column %s.' %
(tc_options.operation, tc.name))
else:
operation = Operation.none
c = Value(tc_options.extractor,
operation,
description=tc.label,
default_value=tc_options.default_value)
self.column_names.append(tc.name)
columns.append(c)
# Identify Sort Column
sortidx = None
if self.table.sortcol is not None:
sort_name = self.table.sortcol.options.extractor
for i, c in enumerate(columns):
if c.field == sort_name:
sortidx = i
break
# Initialize filters
criteria = self.job.criteria
filters = []
filterexpr = self.job.combine_filterexprs(
exprs=criteria.shark_filterexpr, joinstr="&")
if filterexpr:
filters.append(SharkFilter(filterexpr))
tf = TimeFilter(start=criteria.starttime, end=criteria.endtime)
filters.append(tf)
logger.info("Setting shark table %d timeframe to %s" %
(self.table.id, str(tf)))
# Get source type from options
try:
with lock:
source = path_to_class(shark,
self.job.criteria.shark_source_name)
except RvbdHTTPException, e:
source = None
raise e
resolution = criteria.resolution
if resolution.seconds == 1:
sampling_time_msec = 1000
elif resolution.microseconds == 1000:
sampling_time_msec = 1
if criteria.duration > parse_timedelta('1s'):
msg = ("Cannot run a millisecond report with a duration "
"longer than 1 second")
raise ValueError(msg)
else:
sampling_time_msec = 1000
# Setup the view
if source is not None:
with lock:
view = shark.create_view(source,
columns,
filters=filters,
sync=False,
sampling_time_msec=sampling_time_msec)
else:
# XXX raise other exception
return None
done = False
logger.debug("Waiting for shark table %d to complete" % self.table.id)
while not done:
time.sleep(0.5)
with lock:
s = view.get_progress()
self.job.progress = s
self.job.save()
done = (s == 100)
# Retrieve the data
with lock:
if self.table.options.aggregated:
self.data = view.get_data(
aggregated=self.table.options.aggregated, sortby=sortidx)
else:
self.data = view.get_data(delta=self.delta, sortby=sortidx)
view.close()
if self.table.rows > 0:
self.data = self.data[:self.table.rows]
self.parse_data()
logger.info("Shark Report %s returned %s rows" %
(self.job, len(self.data)))
return True
def run(
self, template_id, timefilter=None, resolution="auto", query=None, trafficexpr=None, data_filter=None, sync=True
):
"""Create the report on Profiler and begin running
the report. If the `sync` option is True, periodically
poll until the report is complete, otherwise return
immediately.
`template_id` is the numeric id of the template to use for the report
`timefilter` is the range of time to query, a TimeFilter object
`resolution` is the data resolution (1min, 15min, etc.), defaults to 'auto'
`query` is the query object containing criteria
`trafficexpr` is a TrafficFilter object
`data_filter` is a deprecated filter to run against report data
`sync` if True, poll for status until the report is complete
"""
self.template_id = template_id
self.custom_columns = False
if self.template_id != 184:
# the columns in this report won't match, use custom columns instead
self.custom_columns = True
if timefilter is None:
self.timefilter = TimeFilter.parse_range("last 5 min")
else:
self.timefilter = timefilter
self.query = query
self.trafficexpr = trafficexpr
self.data_filter = data_filter
self.id = None
self.queries = list()
self.last_status = None
if resolution not in ["auto", "1min", "15min", "hour", "6hour", "day", "week", "month"]:
rd = parse_timedelta(resolution)
resolution = self.RESOLUTION_MAP[int(timedelta_total_seconds(rd))]
self.resolution = resolution
start = datetime_to_seconds(self.timefilter.start)
end = datetime_to_seconds(self.timefilter.end)
# using a RecursiveUpdateDict
criteria = RecursiveUpdateDict(**{"time_frame": {"start": int(start), "end": int(end)}})
if self.query is not None:
criteria["query"] = self.query
if self.resolution != "auto":
criteria["time_frame"]["resolution"] = self.resolution
if self.data_filter:
criteria["deprecated"] = {self.data_filter[0]: self.data_filter[1]}
if self.trafficexpr is not None:
criteria["traffic_expression"] = self.trafficexpr.filter
to_post = {"template_id": self.template_id, "criteria": criteria}
logger.debug("Posting JSON: %s" % to_post)
response = self.profiler.api.report.reports(data=to_post)
try:
self.id = int(response["id"])
except KeyError:
raise ValueError("failed to retrieve report id from report creation response: %s" % response)
logger.info("Created report %d" % self.id)
if sync:
self.wait_for_complete()
