def _round_times(self):
"""Round the start/end time in criteria to reflect what data will be
returned from data source and saved in db (based on investigation of
NetProfiler/NetShark query results).
Start/End time needs to round down to closest time in integer
resolutions. However, if either starttime or endtime in criteria is
already in integer resolutions, it should remain the same.
:return: start time, end time, resolution
"""
self.job.criteria.compute_times()
resolution = getattr(self.job.criteria, 'resolution', None)
if not resolution:
raise AttributeError("The data source table '%s' needs to have "
"'resolution' field." % self.ds_table.name)
resolution_sec = timedelta_total_seconds(self.job.criteria.resolution)
starttime = round_time(self.job.criteria.starttime,
round_to=resolution_sec,
trim=True)
endtime = round_time(self.job.criteria.endtime,
round_to=resolution_sec,
trim=True)
return starttime, endtime, resolution
def _round_times(self):
"""Round the start/end time in criteria to reflect what data will be
returned from data source and saved in db (based on investigation of
NetProfiler/NetShark query results).
Start/End time needs to round down to closest time in integer
resolutions. However, if either starttime or endtime in criteria is
already in integer resolutions, it should remain the same.
:return: start time, end time, resolution
"""
self.job.criteria.compute_times()
resolution = getattr(self.job.criteria, 'resolution', None)
if not resolution:
raise AttributeError("The data source table '%s' needs to have "
"'resolution' field." % self.ds_table.name)
resolution_sec = timedelta_total_seconds(self.job.criteria.resolution)
starttime = round_time(self.job.criteria.starttime,
round_to=resolution_sec,
trim=True)
endtime = round_time(self.job.criteria.endtime,
round_to=resolution_sec,
trim=True)
return starttime, endtime, resolution
def schedule_job(self, name, job_config):
job_options, interval = self.parse_config(job_config)
if interval['offset'] > datetime.timedelta(0):
delta = interval['delta']
offset = interval['offset']
now = datetime.datetime.now(pytz.UTC)
# this gives the latest rounded time in the past
# so we add interval to it to get a future run time
delta_secs = timeutils.timedelta_total_seconds(delta)
next_run_time = timeutils.round_time(now,
round_to=delta_secs,
trim=True)
next_run_time += (delta + offset)
logger.debug(
'Setting next run time to %s (delta: %s, offset: %s)' %
(next_run_time, delta, offset))
job_options['next_run_time'] = next_run_time
logger.debug('Scheduling job named %s with kwargs: %s' %
(name, job_options))
self.scheduler.add_job(name=name, trigger='interval', **job_options)
def process_criteria(kws):
"""Process criteria options into separate dict."""
# get runtime parameters
interval = kws.pop('interval')
if interval['offset'] > datetime.timedelta(0):
# round now to nearest whole second
now = datetime.datetime.now(pytz.UTC)
roundnow = timeutils.round_time(now, round_to=1)
endtime = roundnow - interval['offset']
logger.debug('Setting end time to %s (via offset: %s)' %
(endtime, interval['offset']))
else:
endtime = None
# if we have criteria, parse and append to base
criteria = kws.pop('criteria', None)
if criteria:
crits = re.split(',\s|,', criteria)
critdict = dict(c.split(':', 1) for c in crits)
if endtime:
critdict['endtime'] = endtime
else:
critdict = dict()
kws['offset'] = interval['offset']
kws['delta'] = interval['delta']
return critdict, kws
def process_criteria(kws):
"""Process criteria options into separate dict."""
# get runtime parameters
interval = kws.pop('interval')
timezone = kws.pop('timezone', None)
if timezone:
tz = pytz.timezone(timezone)
else:
logger.info('No timezone specified, using UTC.')
tz = pytz.UTC
if interval['offset'] > datetime.timedelta(0):
# round now to nearest whole second
now = datetime.datetime.now(tz)
roundnow = timeutils.round_time(now, round_to=1)
endtime = roundnow - interval['offset']
endtime = endtime.isoformat()
logger.debug('Setting end time to %s (via offset: %s)' %
(endtime, interval['offset']))
else:
endtime = None
# if we have criteria, parse and append to base
criteria = kws.pop('criteria', None)
if criteria:
crits = re.split(',\s|,', criteria)
critdict = dict(c.split(':', 1) for c in crits)
if endtime:
critdict['endtime'] = endtime
else:
critdict = dict()
kws['offset'] = interval['offset']
kws['delta'] = interval['delta']
return critdict, kws
def schedule_job(self, name, job_config):
job_options, interval = self.parse_config(job_config)
if interval['offset'] > datetime.timedelta(0):
delta = interval['delta']
offset = interval['offset']
now = datetime.datetime.now(pytz.UTC)
# this gives the latest rounded time in the past
# so we add interval to it to get a future run time
total_offset = timeutils.timedelta_total_seconds(delta + offset)
next_run_time = timeutils.round_time(now,
round_to=total_offset,
trim=True)
next_run_time += (delta + offset)
logger.debug('Setting next run time to %s (delta: %s, offset: %s)'
% (next_run_time, delta, offset))
job_options['next_run_time'] = next_run_time
logger.debug('Scheduling job named %s with kwargs: %s' % (name,
job_options))
self.scheduler.add_job(name=name, trigger='interval', **job_options)
def get(self, request, namespace=None, report_slug=None, widget_slug=None):
try:
report = Report.objects.get(namespace=namespace,
slug=report_slug)
except:
raise Http404
logger.debug("Received GET for report %s widget definition" %
report_slug)
if widget_slug:
w = get_object_or_404(
Widget,
slug=widget_slug,
section__in=Section.objects.filter(report=report)
)
widgets = [w]
else:
# Add 'id' to order_by so that stacked widgets will return
# with the same order as created
widgets = report.widgets().order_by('row', 'col', 'id')
# parse time and localize to user profile timezone
timezone = get_timezone(request)
now = datetime.datetime.now(timezone)
# pin the endtime to a round interval if we are set to
# reload periodically
minutes = report.reload_minutes
offset = report.reload_offset
if minutes:
# avoid case of long duration reloads to have large reload gap
# e.g. 24-hour report will consider 12:15 am or later a valid time
# to roll-over the time time values, rather than waiting
# until 12:00 pm
trimmed = round_time(dt=now, round_to=60*minutes, trim=True)
if now - trimmed > datetime.timedelta(seconds=offset):
now = trimmed
else:
now = round_time(dt=now, round_to=60*minutes)
widget_defs = []
for w in widgets:
# get default criteria values for widget
# and set endtime to now, if applicable
widget_fields = w.collect_fields()
form = TableFieldForm(widget_fields, use_widgets=False)
# create object from the tablefield keywords
# and populate it with initial data generated by default
keys = form._tablefields.keys()
criteria = dict(zip(keys, [None]*len(keys)))
criteria.update(form.data)
# calculate time offsets
if 'endtime' in criteria:
criteria['endtime'] = now.isoformat()
# only consider starttime if its paired with an endtime
if 'starttime' in criteria:
start = now
field = form.fields['starttime']
initial = field.widget.attrs.get('initial_time', None)
if initial:
m = re.match("now *- *(.+)", initial)
if m:
delta = parse_timedelta(m.group(1))
start = now - delta
criteria['starttime'] = start.isoformat()
# setup json definition object
widget_def = w.get_definition(criteria)
widget_defs.append(widget_def)
# Build the primary key corresponding to static data for this
# widget
if report.static:
rw_id = '-'.join([namespace, report_slug,
widget_def['widgetslug']])
# Add cached widget data if available.
try:
data_cache = WidgetDataCache.objects.get(
report_widget_id=rw_id)
widget_def['dataCache'] = data_cache.data
except WidgetDataCache.DoesNotExist:
msg = "No widget data cache available with id %s." % rw_id
resp = {'message': msg,
'status': 'error',
'exception': ''}
widget_def['dataCache'] = json.dumps(resp)
report_def = self.report_def(widget_defs, now)
return JsonResponse(report_def, safe=False)
def get(self, request, namespace=None, report_slug=None, widget_slug=None):
try:
report = Report.objects.get(namespace=namespace, slug=report_slug)
except:
raise Http404
logger.debug("Received GET for report %s widget definition" %
report_slug)
if widget_slug:
w = get_object_or_404(
Widget,
slug=widget_slug,
section__in=Section.objects.filter(report=report))
widgets = [w]
else:
widgets = report.widgets().order_by('row', 'col')
# parse time and localize to user profile timezone
timezone = pytz.timezone(request.user.timezone)
now = datetime.datetime.now(timezone)
# pin the endtime to a round interval if we are set to
# reload periodically
minutes = report.reload_minutes
if minutes:
# avoid case of long duration reloads to have large reload gap
# e.g. 24-hour report will consider 12:15 am or later a valid time
# to roll-over the time time values, rather than waiting
# until 12:00 pm
trimmed = round_time(dt=now, round_to=60 * minutes, trim=True)
if now - trimmed > datetime.timedelta(minutes=15):
now = trimmed
else:
now = round_time(dt=now, round_to=60 * minutes)
widget_defs = []
for w in widgets:
# get default criteria values for widget
# and set endtime to now, if applicable
widget_fields = w.collect_fields()
form = TableFieldForm(widget_fields, use_widgets=False)
# create object from the tablefield keywords
# and populate it with initial data generated by default
keys = form._tablefields.keys()
criteria = dict(zip(keys, [None] * len(keys)))
criteria.update(form.data)
# calculate time offsets
if 'endtime' in criteria:
criteria['endtime'] = now.isoformat()
# only consider starttime if its paired with an endtime
if 'starttime' in criteria:
start = now
field = form.fields['starttime']
initial = field.widget.attrs.get('initial_time', None)
if initial:
m = re.match("now *- *(.+)", initial)
if m:
delta = parse_timedelta(m.group(1))
start = now - delta
criteria['starttime'] = start.isoformat()
# setup json definition object
widget_def = w.get_definition(criteria)
widget_defs.append(widget_def)
report_def = self.report_def(widget_defs, now)
return JsonResponse(report_def, safe=False)
def get(self, request, namespace=None, report_slug=None, widget_slug=None):
try:
report = Report.objects.get(namespace=namespace,
slug=report_slug)
except:
raise Http404
logger.debug("Received GET for report %s widget definition" %
report_slug)
if widget_slug:
w = get_object_or_404(
Widget,
slug=widget_slug,
section__in=Section.objects.filter(report=report)
)
widgets = [w]
else:
widgets = report.widgets().order_by('row', 'col')
# parse time and localize to user profile timezone
timezone = pytz.timezone(request.user.timezone)
now = datetime.datetime.now(timezone)
# pin the endtime to a round interval if we are set to
# reload periodically
minutes = report.reload_minutes
if minutes:
# avoid case of long duration reloads to have large reload gap
# e.g. 24-hour report will consider 12:15 am or later a valid time
# to roll-over the time time values, rather than waiting
# until 12:00 pm
trimmed = round_time(dt=now, round_to=60*minutes, trim=True)
if now - trimmed > datetime.timedelta(minutes=15):
now = trimmed
else:
now = round_time(dt=now, round_to=60*minutes)
widget_defs = []
for w in widgets:
# get default criteria values for widget
# and set endtime to now, if applicable
widget_fields = w.collect_fields()
form = TableFieldForm(widget_fields, use_widgets=False)
# create object from the tablefield keywords
# and populate it with initial data generated by default
keys = form._tablefields.keys()
criteria = dict(zip(keys, [None]*len(keys)))
criteria.update(form.data)
# calculate time offsets
if 'endtime' in criteria:
criteria['endtime'] = now.isoformat()
# only consider starttime if its paired with an endtime
if 'starttime' in criteria:
start = now
field = form.fields['starttime']
initial = field.widget.attrs.get('initial_time', None)
if initial:
m = re.match("now *- *(.+)", initial)
if m:
delta = parse_timedelta(m.group(1))
start = now - delta
criteria['starttime'] = start.isoformat()
# setup json definition object
widget_def = w.get_definition(criteria)
widget_defs.append(widget_def)
report_def = self.report_def(widget_defs, now)
return JsonResponse(report_def, safe=False)
def get(self, request, namespace=None, report_slug=None, widget_slug=None):
try:
report = Report.objects.get(namespace=namespace,
slug=report_slug)
except:
raise Http404
logger.debug("Received GET for report %s widget definition" %
report_slug)
if widget_slug:
w = get_object_or_404(
Widget,
slug=widget_slug,
section__in=Section.objects.filter(report=report)
)
widgets = [w]
else:
# Add 'id' to order_by so that stacked widgets will return
# with the same order as created
widgets = report.widgets().order_by('row', 'col', 'id')
# parse time and localize to user profile timezone
timezone = get_timezone(request)
now = datetime.datetime.now(timezone)
# pin the endtime to a round interval if we are set to
# reload periodically
minutes = report.reload_minutes
offset = report.reload_offset
if minutes:
# avoid case of long duration reloads to have large reload gap
# e.g. 24-hour report will consider 12:15 am or later a valid time
# to roll-over the time time values, rather than waiting
# until 12:00 pm
trimmed = round_time(dt=now, round_to=60*minutes, trim=True)
if now - trimmed > datetime.timedelta(seconds=offset):
now = trimmed
else:
now = round_time(dt=now, round_to=60*minutes)
widget_defs = []
for w in widgets:
# get default criteria values for widget
# and set endtime to now, if applicable
widget_fields = w.collect_fields()
form = TableFieldForm(widget_fields, use_widgets=False)
# create object from the tablefield keywords
# and populate it with initial data generated by default
keys = form._tablefields.keys()
criteria = dict(zip(keys, [None]*len(keys)))
criteria.update(form.data)
# calculate time offsets
if 'endtime' in criteria:
criteria['endtime'] = now.isoformat()
# only consider starttime if its paired with an endtime
if 'starttime' in criteria:
start = now
field = form.fields['starttime']
initial = field.widget.attrs.get('initial_time', None)
if initial:
m = re.match("now *- *(.+)", initial)
if m:
delta = parse_timedelta(m.group(1))
start = now - delta
criteria['starttime'] = start.isoformat()
# Check for "Meta Widget" criteria items
system_settings = SystemSettings.get_system_settings()
if system_settings.ignore_cache:
criteria['ignore_cache'] = system_settings.ignore_cache
if system_settings.developer:
criteria['debug'] = system_settings.developer
# setup json definition object
widget_def = w.get_definition(criteria)
widget_defs.append(widget_def)
# Build the primary key corresponding to static data for this
# widget
if report.static:
rw_id = '-'.join([namespace, report_slug,
widget_def['widgetslug']])
# Add cached widget data if available.
try:
data_cache = WidgetDataCache.objects.get(
report_widget_id=rw_id)
widget_def['dataCache'] = data_cache.data
except WidgetDataCache.DoesNotExist:
msg = "No widget data cache available with id %s." % rw_id
resp = {'message': msg,
'status': 'error',
'exception': ''}
widget_def['dataCache'] = json.dumps(resp)
report_def = self.report_def(widget_defs, now)
return JsonResponse(report_def, safe=False)