def post(self, request, namespace=None, report_slug=None):
# handle REST calls
if report_slug is None:
return self.http_method_not_allowed(request)
logger.debug("Received POST for report %s, with params: %s" %
(report_slug, request.POST))
report = get_object_or_404(Report,
namespace=namespace,
slug=report_slug)
fields_by_section = report.collect_fields_by_section()
all_fields = SortedDict()
[all_fields.update(c) for c in fields_by_section.values()]
form = TableFieldForm(all_fields, hidden_fields=report.hidden_fields,
data=request.POST, files=request.FILES)
response = []
for field in form.dynamic_fields():
response.append({'id': field.auto_id,
'html': str(field)})
return JsonResponse(response, safe=False)
def post(self, request, pk):
"""Create new Job for the specified table using POSTed criteria."""
table = Table.objects.get(pk=pk)
all_fields = dict((f.keyword, f) for f in table.fields.all())
# data needs to be not-None or form will be created as unbound
data = self.request.POST or {}
form = TableFieldForm(all_fields,
use_widgets=False,
include_hidden=True,
data=data)
if form.is_valid(check_unknown=True):
criteria = form.criteria()
else:
return Response(form.errors, status=status.HTTP_400_BAD_REQUEST)
try:
job = Job.create(table, criteria)
job.start()
serializer = JobSerializer(job, many=False)
return Response(serializer.data,
status=status.HTTP_201_CREATED,
headers=self.get_success_headers(job))
except Exception as e:
msg = 'Error processing Job: %s' % e.message
raise JobCreationError(msg)
def post(self, request, namespace=None, report_slug=None):
# handle REST calls
if report_slug is None:
return self.http_method_not_allowed(request)
logger.debug("Received POST for report %s, with params: %s" %
(report_slug, request.POST))
report = get_object_or_404(Report,
namespace=namespace,
slug=report_slug)
fields_by_section = report.collect_fields_by_section()
all_fields = SortedDict()
[all_fields.update(c) for c in fields_by_section.values()]
form = TableFieldForm(all_fields,
hidden_fields=report.hidden_fields,
data=request.POST,
files=request.FILES)
response = []
for field in form.dynamic_fields():
response.append({'id': field.auto_id, 'html': str(field)})
return JsonResponse(response, safe=False)
def post(self, request, pk):
"""Create new Job for the specified table using POSTed criteria."""
table = Table.objects.get(pk=pk)
all_fields = dict((f.keyword, f) for f in table.fields.all())
# data needs to be not-None or form will be created as unbound
data = self.request.POST or {}
form = TableFieldForm(all_fields, use_widgets=False,
data=data)
if form.is_valid(check_unknown=True):
criteria = form.criteria()
else:
return Response(form.errors, status=status.HTTP_400_BAD_REQUEST)
try:
job = Job.create(table, criteria)
job.start()
serializer = JobSerializer(job, many=False)
return Response(serializer.data, status=status.HTTP_201_CREATED,
headers=self.get_success_headers(job))
except Exception as e:
msg = 'Error processing Job: %s' % e.message
raise JobCreationError(msg)
def get_form(self, table, data=None):
# First see if there's a single report associated with this table,
# and if so, use it to get the field set
widgets = Widget.objects.filter(tables__in=[table])
sections = set()
for w in widgets:
sections.add(w.section)
if len(sections) == 1:
all_fields = widgets[0].collect_fields()
else:
all_fields = {}
for f in table.fields.all():
all_fields[f.keyword] = f
return TableFieldForm(all_fields, use_widgets=False, data=data)
def post(self, request, namespace=None, report_slug=None):
if namespace is None or report_slug is None:
return self.http_method_not_allowed(request)
logger.debug("**")
logger.debug("**")
logger.debug("** REPORT - %s" % report_slug)
logger.debug("**")
logger.debug("**")
logger.debug("Received POST for report %s, with params: %s" %
(report_slug, request.POST))
report = get_object_or_404(Report, namespace=namespace,
slug=report_slug)
fields_by_section = report.collect_fields_by_section()
all_fields = OrderedDict()
[all_fields.update(c) for c in fields_by_section.values()]
form = TableFieldForm(all_fields, hidden_fields=report.hidden_fields,
data=request.POST, files=request.FILES)
if form.is_valid():
logger.debug('Form passed validation: %s' % form)
formdata = form.cleaned_data
logger.debug('Form cleaned data: %s' % formdata)
# parse time and localize to user profile timezone
timezone = get_timezone(request)
form.apply_timezone(timezone)
if formdata['debug']:
logger.debug("Debugging report and rotating logs now ...")
management.call_command('rotate_logs')
logger.debug("Report %s validated form: %s" %
(report_slug, formdata))
# construct report definition
now = datetime.datetime.now(timezone)
widgets = report.widget_definitions(form.as_text())
report_def = self.report_def(widgets, now, formdata['debug'])
logger.debug("Sending widget definitions for report %s: %s" %
(report_slug, report_def))
if settings.REPORT_HISTORY_ENABLED:
create_report_history(request, report, widgets)
return JsonResponse(report_def, safe=False)
else:
# return form with errors attached in a HTTP 400 Error response
return HttpResponse(str(form.errors), status=400)
def post(self, request, namespace, report_slug, widget_slug, format=None):
logger.debug("Received POST for report %s, widget %s: %s" %
(report_slug, widget_slug, request.POST))
report = get_object_or_404(Report, namespace=namespace,
slug=report_slug)
widget = get_object_or_404(
Widget,
slug=widget_slug,
section__in=Section.objects.filter(report=report)
)
req_json = json.loads(request.POST['criteria'])
fields = widget.collect_fields()
form = TableFieldForm(fields, use_widgets=False,
hidden_fields=report.hidden_fields,
include_hidden=True,
data=req_json, files=request.FILES)
if not form.is_valid():
raise ValueError("Widget internal criteria form is invalid:\n%s" %
(form.errors.as_text()))
if form.is_valid():
logger.debug('Form passed validation: %s' % form)
formdata = form.cleaned_data
logger.debug('Form cleaned data: %s' % formdata)
# parse time and localize to user profile timezone
timezone = get_timezone(request)
form.apply_timezone(timezone)
try:
form_criteria = form.criteria()
logger.debug('Form_criteria: %s' % form_criteria)
job = Job.create(table=widget.table(),
criteria=form_criteria)
job.start()
wjob = WidgetJob(widget=widget, job=job)
wjob.save()
logger.debug("Created WidgetJob %s for report %s (handle %s)" %
(str(wjob), report_slug, job.handle))
return Response({"joburl": reverse('report-job-detail',
args=[namespace,
report_slug,
widget_slug,
wjob.id])})
except Exception as e:
logger.exception("Failed to start job, an exception occurred")
ei = sys.exc_info()
resp = {}
resp['message'] = "".join(
traceback.format_exception_only(*sys.exc_info()[0:2])),
resp['exception'] = "".join(
traceback.format_exception(*sys.exc_info()))
return JsonResponse(resp, status=400)
else:
logger.error("form is invalid, entering debugger")
from IPython import embed; embed()
def create_report_history(request, report, widgets):
"""Create a report history object.
:param request: request object
:param report: Report object
:param widgets: List of widget definitions
"""
# create the form to derive criteria for bookmark only
# the form in the calling context can not be used
# because it does not include hidden fields
fields_by_section = report.collect_fields_by_section()
all_fields = SortedDict()
[all_fields.update(c) for c in fields_by_section.values()]
form = TableFieldForm(all_fields,
hidden_fields=report.hidden_fields,
include_hidden=True,
data=request.POST,
files=request.FILES)
# parse time and localize to user profile timezone
timezone = get_timezone(request)
form.apply_timezone(timezone)
form_data = form.cleaned_data
url = request._request.path + '?'
# form_data contains fields that don't belong to url
# e.g. ignore_cache, debug
# thus use compute_field_precedence method to filter those
table_fields = {k: form_data[k] for k in form.compute_field_precedence()}
# Convert field values into strings suitable for bookmark
def _get_url_fields(flds):
for k, v in flds.iteritems():
if k in ['starttime', 'endtime']:
yield (k, str(datetime_to_seconds(v)))
elif k in ['duration', 'resolution']:
try:
yield (k, str(int(timedelta_total_seconds(v))))
except AttributeError:
# v is of special value, not a string of some duration
yield (k, v.replace(' ', '+'))
else:
# use + as encoded white space
yield (k, str(v).replace(' ', '+'))
yield ('auto_run', 'true')
url_fields = ['='.join([k, v]) for k, v in _get_url_fields(table_fields)]
# Form the bookmark link
url += '&'.join(url_fields)
last_run = datetime.datetime.now(timezone)
# iterate over the passed in widget definitions and use those
# to calculate the actual job handles
# since these will mimic what gets used to create the actual jobs,
# the criteria will match more closely than using the report-level
# criteria data
handles = []
for widget in widgets:
wobj = Widget.objects.get(
slug=widget['widgetslug'],
section__in=Section.objects.filter(report=report)
)
fields = wobj.collect_fields()
form = TableFieldForm(fields, use_widgets=False,
hidden_fields=report.hidden_fields,
include_hidden=True,
data=widget['criteria'], files=request.FILES)
if form.is_valid():
# parse time and localize to user profile timezone
timezone = get_timezone(request)
form.apply_timezone(timezone)
form_criteria = form.criteria()
widget_table = wobj.table()
form_criteria = form_criteria.build_for_table(widget_table)
try:
form_criteria.compute_times()
except ValueError:
pass
handle = Job._compute_handle(widget_table, form_criteria)
logger.debug('ReportHistory: adding handle %s for widget_table %s'
% (handle, widget_table))
handles.append(handle)
else:
# log error, but don't worry about it for adding to RH
logger.warning("Error while calculating job handle for Widget %s, "
"internal criteria form is invalid: %s" %
(wobj, form.errors.as_text()))
job_handles = ','.join(handles)
if request.user.is_authenticated():
user = request.user.username
else:
user = settings.GUEST_USER_NAME
logger.debug('Creating ReportHistory for user %s at URL %s' % (user, url))
ReportHistory.create(namespace=report.namespace,
slug=report.slug,
bookmark=url,
first_run=last_run,
last_run=last_run,
job_handles=job_handles,
user=user,
criteria=table_fields,
run_count=1)
def post(self, request, namespace, report_slug, widget_slug, format=None):
logger.debug("Received POST for report %s, widget %s: %s" %
(report_slug, widget_slug, request.POST))
report = get_object_or_404(Report,
namespace=namespace,
slug=report_slug)
widget = get_object_or_404(
Widget,
slug=widget_slug,
section__in=Section.objects.filter(report=report))
req_json = json.loads(request.POST['criteria'])
fields = widget.collect_fields()
form = TableFieldForm(fields,
use_widgets=False,
hidden_fields=report.hidden_fields,
include_hidden=True,
data=req_json,
files=request.FILES)
if not form.is_valid():
raise ValueError("Widget internal criteria form is invalid:\n%s" %
(form.errors.as_text()))
if form.is_valid():
logger.debug('Form passed validation: %s' % form)
formdata = form.cleaned_data
logger.debug('Form cleaned data: %s' % formdata)
# parse time and localize to user profile timezone
timezone = pytz.timezone(request.user.timezone)
form.apply_timezone(timezone)
try:
form_criteria = form.criteria()
logger.debug('Form_criteria: %s' % form_criteria)
job = Job.create(table=widget.table(), criteria=form_criteria)
job.start()
wjob = WidgetJob(widget=widget, job=job)
wjob.save()
logger.debug("Created WidgetJob %s for report %s (handle %s)" %
(str(wjob), report_slug, job.handle))
return Response({
"joburl":
reverse(
'report-job-detail',
args=[namespace, report_slug, widget_slug, wjob.id])
})
except Exception as e:
logger.exception("Failed to start job, an exception occurred")
ei = sys.exc_info()
resp = {}
resp['message'] = "".join(
traceback.format_exception_only(*sys.exc_info()[0:2])),
resp['exception'] = "".join(
traceback.format_exception(*sys.exc_info()))
return JsonResponse(resp, status=400)
else:
logger.error("form is invalid, entering debugger")
from IPython import embed
embed()
def get(self, request, namespace=None, report_slug=None, widget_slug=None):
try:
report = Report.objects.get(namespace=namespace, slug=report_slug)
except:
raise Http404
logger.debug("Received GET for report %s widget definition" %
report_slug)
if widget_slug:
w = get_object_or_404(
Widget,
slug=widget_slug,
section__in=Section.objects.filter(report=report))
widgets = [w]
else:
widgets = report.widgets().order_by('row', 'col')
# parse time and localize to user profile timezone
timezone = pytz.timezone(request.user.timezone)
now = datetime.datetime.now(timezone)
# pin the endtime to a round interval if we are set to
# reload periodically
minutes = report.reload_minutes
if minutes:
# avoid case of long duration reloads to have large reload gap
# e.g. 24-hour report will consider 12:15 am or later a valid time
# to roll-over the time time values, rather than waiting
# until 12:00 pm
trimmed = round_time(dt=now, round_to=60 * minutes, trim=True)
if now - trimmed > datetime.timedelta(minutes=15):
now = trimmed
else:
now = round_time(dt=now, round_to=60 * minutes)
widget_defs = []
for w in widgets:
# get default criteria values for widget
# and set endtime to now, if applicable
widget_fields = w.collect_fields()
form = TableFieldForm(widget_fields, use_widgets=False)
# create object from the tablefield keywords
# and populate it with initial data generated by default
keys = form._tablefields.keys()
criteria = dict(zip(keys, [None] * len(keys)))
criteria.update(form.data)
# calculate time offsets
if 'endtime' in criteria:
criteria['endtime'] = now.isoformat()
# only consider starttime if its paired with an endtime
if 'starttime' in criteria:
start = now
field = form.fields['starttime']
initial = field.widget.attrs.get('initial_time', None)
if initial:
m = re.match("now *- *(.+)", initial)
if m:
delta = parse_timedelta(m.group(1))
start = now - delta
criteria['starttime'] = start.isoformat()
# setup json definition object
widget_def = w.get_definition(criteria)
widget_defs.append(widget_def)
report_def = self.report_def(widget_defs, now)
return JsonResponse(report_def, safe=False)
def render_html(self, report, request, namespace, report_slug, isprint):
""" Render HTML response
"""
logging.debug('Received request for report page: %s' % report_slug)
if not request.user.profile_seen:
# only redirect if first login
return HttpResponseRedirect(
reverse('preferences') + '?next=/report')
devices = Device.objects.filter(enabled=True)
device_modules = [obj.module for obj in devices]
# iterate through all sections of the report, for each section,
# iterate through the fields, and if any field's
# pre_process_func function is device_selection_preprocess
# then the field is a device field, then fetch the module and check
# the module is included in Device objects in database
missing_devices = set()
for _id, fields in report.collect_fields_by_section().iteritems():
for _name, field_obj in fields.iteritems():
func = field_obj.pre_process_func
if (func and func.function == 'device_selection_preprocess'):
# This field is a device field,
# check if the device is configured
module = func.params['module']
if module not in device_modules:
missing_devices.add(module)
if missing_devices:
missing_devices = ', '.join(list(missing_devices))
if not request.user.profile_seen:
# only redirect if first login
return HttpResponseRedirect(
reverse('preferences') + '?next=/report')
# Setup default criteria for the report based on underlying tables
system_settings = SystemSettings.get_system_settings()
form_init = {'ignore_cache': system_settings.ignore_cache}
for table in report.tables():
if table.criteria:
form_init.update(table.criteria)
# Collect all fields organized by section, with section id 0
# representing common report level fields
fields_by_section = report.collect_fields_by_section()
# Merge fields into a single dict for use by the Django Form # logic
all_fields = SortedDict()
[all_fields.update(c) for c in fields_by_section.values()]
form = TableFieldForm(all_fields,
hidden_fields=report.hidden_fields,
initial=form_init)
# Build a section map that indicates which section each field
# belongs in when displayed
section_map = []
if fields_by_section[0]:
section_map.append({
'title': 'Common',
'parameters': fields_by_section[0]
})
for s in Section.objects.filter(report=report).order_by(
'position', 'title'):
show = False
for v in fields_by_section[s.id].values():
if v.keyword not in (report.hidden_fields or []):
show = True
break
if show:
section_map.append({
'title': s.title,
'parameters': fields_by_section[s.id]
})
template, criteria, expand_tables = self.get_media_params(request)
return render_to_response(template, {
'report': report,
'developer': system_settings.developer,
'maps_version': system_settings.maps_version,
'maps_api_key': system_settings.maps_api_key,
'endtime': 'endtime' in form.fields,
'form': form,
'section_map': section_map,
'show_sections': (len(section_map) > 1),
'criteria': criteria,
'expand_tables': expand_tables,
'missing_devices': missing_devices,
'is_superuser': request.user.is_superuser
},
context_instance=RequestContext(request))
def create_report_history(request, report, widgets):
"""Create a report history object.
:param request: request object
:param report: Report object
:param widgets: List of widget definitions
"""
# create the form to derive criteria for bookmark only
# the form in the calling context can not be used
# because it does not include hidden fields
fields_by_section = report.collect_fields_by_section()
all_fields = OrderedDict()
[all_fields.update(c) for c in fields_by_section.values()]
form = TableFieldForm(all_fields,
hidden_fields=report.hidden_fields,
include_hidden=True,
data=request.POST,
files=request.FILES)
# parse time and localize to user profile timezone
timezone = get_timezone(request)
form.apply_timezone(timezone)
form_data = form.cleaned_data
url = request._request.path + '?'
# form_data contains fields that don't belong to url
# e.g. ignore_cache, debug
# thus use compute_field_precedence method to filter those
table_fields = {k: form_data[k] for k in form.compute_field_precedence()}
# Convert field values into strings suitable for bookmark
def _get_url_fields(flds):
for k, v in flds.iteritems():
if k in ['starttime', 'endtime']:
yield (k, str(datetime_to_seconds(v)))
elif k in ['duration', 'resolution']:
try:
yield (k, str(int(timedelta_total_seconds(v))))
except AttributeError:
# v is of special value, not a string of some duration
yield (k, v.replace(' ', '+'))
else:
# use + as encoded white space
yield (k, str(v).replace(' ', '+'))
yield ('auto_run', 'true')
url_fields = ['='.join([k, v]) for k, v in _get_url_fields(table_fields)]
# Form the bookmark link
url += '&'.join(url_fields)
last_run = datetime.datetime.now(timezone)
# iterate over the passed in widget definitions and use those
# to calculate the actual job handles
# since these will mimic what gets used to create the actual jobs,
# the criteria will match more closely than using the report-level
# criteria data
handles = []
for widget in widgets:
wobj = Widget.objects.get(
slug=widget['widgetslug'],
section__in=Section.objects.filter(report=report)
)
fields = wobj.collect_fields()
form = TableFieldForm(fields, use_widgets=False,
hidden_fields=report.hidden_fields,
include_hidden=True,
data=widget['criteria'], files=request.FILES)
if form.is_valid():
# parse time and localize to user profile timezone
timezone = get_timezone(request)
form.apply_timezone(timezone)
form_criteria = form.criteria()
widget_table = wobj.table()
form_criteria = form_criteria.build_for_table(widget_table)
try:
form_criteria.compute_times()
except ValueError:
pass
handle = Job._compute_handle(widget_table, form_criteria)
logger.debug('ReportHistory: adding handle %s for widget_table %s'
% (handle, widget_table))
handles.append(handle)
else:
# log error, but don't worry about it for adding to RH
logger.warning("Error while calculating job handle for Widget %s, "
"internal criteria form is invalid: %s" %
(wobj, form.errors.as_text()))
job_handles = ','.join(handles)
if request.user.is_authenticated():
user = request.user.username
else:
user = settings.GUEST_USER_NAME
logger.debug('Creating ReportHistory for user %s at URL %s' % (user, url))
ReportHistory.create(namespace=report.namespace,
slug=report.slug,
bookmark=url,
first_run=last_run,
last_run=last_run,
job_handles=job_handles,
user=user,
criteria=table_fields,
run_count=1)
def get(self, request, namespace=None, report_slug=None, widget_slug=None):
try:
report = Report.objects.get(namespace=namespace,
slug=report_slug)
except:
raise Http404
logger.debug("Received GET for report %s widget definition" %
report_slug)
if widget_slug:
w = get_object_or_404(
Widget,
slug=widget_slug,
section__in=Section.objects.filter(report=report)
)
widgets = [w]
else:
# Add 'id' to order_by so that stacked widgets will return
# with the same order as created
widgets = report.widgets().order_by('row', 'col', 'id')
# parse time and localize to user profile timezone
timezone = get_timezone(request)
now = datetime.datetime.now(timezone)
# pin the endtime to a round interval if we are set to
# reload periodically
minutes = report.reload_minutes
offset = report.reload_offset
if minutes:
# avoid case of long duration reloads to have large reload gap
# e.g. 24-hour report will consider 12:15 am or later a valid time
# to roll-over the time time values, rather than waiting
# until 12:00 pm
trimmed = round_time(dt=now, round_to=60*minutes, trim=True)
if now - trimmed > datetime.timedelta(seconds=offset):
now = trimmed
else:
now = round_time(dt=now, round_to=60*minutes)
widget_defs = []
for w in widgets:
# get default criteria values for widget
# and set endtime to now, if applicable
widget_fields = w.collect_fields()
form = TableFieldForm(widget_fields, use_widgets=False)
# create object from the tablefield keywords
# and populate it with initial data generated by default
keys = form._tablefields.keys()
criteria = dict(zip(keys, [None]*len(keys)))
criteria.update(form.data)
# calculate time offsets
if 'endtime' in criteria:
criteria['endtime'] = now.isoformat()
# only consider starttime if its paired with an endtime
if 'starttime' in criteria:
start = now
field = form.fields['starttime']
initial = field.widget.attrs.get('initial_time', None)
if initial:
m = re.match("now *- *(.+)", initial)
if m:
delta = parse_timedelta(m.group(1))
start = now - delta
criteria['starttime'] = start.isoformat()
# Check for "Meta Widget" criteria items
system_settings = SystemSettings.get_system_settings()
if system_settings.ignore_cache:
criteria['ignore_cache'] = system_settings.ignore_cache
if system_settings.developer:
criteria['debug'] = system_settings.developer
# setup json definition object
widget_def = w.get_definition(criteria)
widget_defs.append(widget_def)
# Build the primary key corresponding to static data for this
# widget
if report.static:
rw_id = '-'.join([namespace, report_slug,
widget_def['widgetslug']])
# Add cached widget data if available.
try:
data_cache = WidgetDataCache.objects.get(
report_widget_id=rw_id)
widget_def['dataCache'] = data_cache.data
except WidgetDataCache.DoesNotExist:
msg = "No widget data cache available with id %s." % rw_id
resp = {'message': msg,
'status': 'error',
'exception': ''}
widget_def['dataCache'] = json.dumps(resp)
report_def = self.report_def(widget_defs, now)
return JsonResponse(report_def, safe=False)