def post(self, request, namespace=None, report_slug=None): # handle REST calls if report_slug is None: return self.http_method_not_allowed(request) logger.debug("Received POST for report %s, with params: %s" % (report_slug, request.POST)) report = get_object_or_404(Report, namespace=namespace, slug=report_slug) fields_by_section = report.collect_fields_by_section() all_fields = SortedDict() [all_fields.update(c) for c in fields_by_section.values()] form = TableFieldForm(all_fields, hidden_fields=report.hidden_fields, data=request.POST, files=request.FILES) response = [] for field in form.dynamic_fields(): response.append({'id': field.auto_id, 'html': str(field)}) return JsonResponse(response, safe=False)
def post(self, request, pk): """Create new Job for the specified table using POSTed criteria.""" table = Table.objects.get(pk=pk) all_fields = dict((f.keyword, f) for f in table.fields.all()) # data needs to be not-None or form will be created as unbound data = self.request.POST or {} form = TableFieldForm(all_fields, use_widgets=False, include_hidden=True, data=data) if form.is_valid(check_unknown=True): criteria = form.criteria() else: return Response(form.errors, status=status.HTTP_400_BAD_REQUEST) try: job = Job.create(table, criteria) job.start() serializer = JobSerializer(job, many=False) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=self.get_success_headers(job)) except Exception as e: msg = 'Error processing Job: %s' % e.message raise JobCreationError(msg)
def post(self, request, pk): """Create new Job for the specified table using POSTed criteria.""" table = Table.objects.get(pk=pk) all_fields = dict((f.keyword, f) for f in table.fields.all()) # data needs to be not-None or form will be created as unbound data = self.request.POST or {} form = TableFieldForm(all_fields, use_widgets=False, data=data) if form.is_valid(check_unknown=True): criteria = form.criteria() else: return Response(form.errors, status=status.HTTP_400_BAD_REQUEST) try: job = Job.create(table, criteria) job.start() serializer = JobSerializer(job, many=False) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=self.get_success_headers(job)) except Exception as e: msg = 'Error processing Job: %s' % e.message raise JobCreationError(msg)
def get_form(self, table, data=None): # First see if there's a single report associated with this table, # and if so, use it to get the field set widgets = Widget.objects.filter(tables__in=[table]) sections = set() for w in widgets: sections.add(w.section) if len(sections) == 1: all_fields = widgets[0].collect_fields() else: all_fields = {} for f in table.fields.all(): all_fields[f.keyword] = f return TableFieldForm(all_fields, use_widgets=False, data=data)
def post(self, request, namespace=None, report_slug=None): if namespace is None or report_slug is None: return self.http_method_not_allowed(request) logger.debug("**") logger.debug("**") logger.debug("** REPORT - %s" % report_slug) logger.debug("**") logger.debug("**") logger.debug("Received POST for report %s, with params: %s" % (report_slug, request.POST)) report = get_object_or_404(Report, namespace=namespace, slug=report_slug) fields_by_section = report.collect_fields_by_section() all_fields = OrderedDict() [all_fields.update(c) for c in fields_by_section.values()] form = TableFieldForm(all_fields, hidden_fields=report.hidden_fields, data=request.POST, files=request.FILES) if form.is_valid(): logger.debug('Form passed validation: %s' % form) formdata = form.cleaned_data logger.debug('Form cleaned data: %s' % formdata) # parse time and localize to user profile timezone timezone = get_timezone(request) form.apply_timezone(timezone) if formdata['debug']: logger.debug("Debugging report and rotating logs now ...") management.call_command('rotate_logs') logger.debug("Report %s validated form: %s" % (report_slug, formdata)) # construct report definition now = datetime.datetime.now(timezone) widgets = report.widget_definitions(form.as_text()) report_def = self.report_def(widgets, now, formdata['debug']) logger.debug("Sending widget definitions for report %s: %s" % (report_slug, report_def)) if settings.REPORT_HISTORY_ENABLED: create_report_history(request, report, widgets) return JsonResponse(report_def, safe=False) else: # return form with errors attached in a HTTP 400 Error response return HttpResponse(str(form.errors), status=400)
def post(self, request, namespace, report_slug, widget_slug, format=None): logger.debug("Received POST for report %s, widget %s: %s" % (report_slug, widget_slug, request.POST)) report = get_object_or_404(Report, namespace=namespace, slug=report_slug) widget = get_object_or_404( Widget, slug=widget_slug, section__in=Section.objects.filter(report=report) ) req_json = json.loads(request.POST['criteria']) fields = widget.collect_fields() form = TableFieldForm(fields, use_widgets=False, hidden_fields=report.hidden_fields, include_hidden=True, data=req_json, files=request.FILES) if not form.is_valid(): raise ValueError("Widget internal criteria form is invalid:\n%s" % (form.errors.as_text())) if form.is_valid(): logger.debug('Form passed validation: %s' % form) formdata = form.cleaned_data logger.debug('Form cleaned data: %s' % formdata) # parse time and localize to user profile timezone timezone = get_timezone(request) form.apply_timezone(timezone) try: form_criteria = form.criteria() logger.debug('Form_criteria: %s' % form_criteria) job = Job.create(table=widget.table(), criteria=form_criteria) job.start() wjob = WidgetJob(widget=widget, job=job) wjob.save() logger.debug("Created WidgetJob %s for report %s (handle %s)" % (str(wjob), report_slug, job.handle)) return Response({"joburl": reverse('report-job-detail', args=[namespace, report_slug, widget_slug, wjob.id])}) except Exception as e: logger.exception("Failed to start job, an exception occurred") ei = sys.exc_info() resp = {} resp['message'] = "".join( traceback.format_exception_only(*sys.exc_info()[0:2])), resp['exception'] = "".join( traceback.format_exception(*sys.exc_info())) return JsonResponse(resp, status=400) else: logger.error("form is invalid, entering debugger") from IPython import embed; embed()
def create_report_history(request, report, widgets): """Create a report history object. :param request: request object :param report: Report object :param widgets: List of widget definitions """ # create the form to derive criteria for bookmark only # the form in the calling context can not be used # because it does not include hidden fields fields_by_section = report.collect_fields_by_section() all_fields = SortedDict() [all_fields.update(c) for c in fields_by_section.values()] form = TableFieldForm(all_fields, hidden_fields=report.hidden_fields, include_hidden=True, data=request.POST, files=request.FILES) # parse time and localize to user profile timezone timezone = get_timezone(request) form.apply_timezone(timezone) form_data = form.cleaned_data url = request._request.path + '?' # form_data contains fields that don't belong to url # e.g. ignore_cache, debug # thus use compute_field_precedence method to filter those table_fields = {k: form_data[k] for k in form.compute_field_precedence()} # Convert field values into strings suitable for bookmark def _get_url_fields(flds): for k, v in flds.iteritems(): if k in ['starttime', 'endtime']: yield (k, str(datetime_to_seconds(v))) elif k in ['duration', 'resolution']: try: yield (k, str(int(timedelta_total_seconds(v)))) except AttributeError: # v is of special value, not a string of some duration yield (k, v.replace(' ', '+')) else: # use + as encoded white space yield (k, str(v).replace(' ', '+')) yield ('auto_run', 'true') url_fields = ['='.join([k, v]) for k, v in _get_url_fields(table_fields)] # Form the bookmark link url += '&'.join(url_fields) last_run = datetime.datetime.now(timezone) # iterate over the passed in widget definitions and use those # to calculate the actual job handles # since these will mimic what gets used to create the actual jobs, # the criteria will match more closely than using the report-level # criteria data handles = [] for widget in widgets: wobj = Widget.objects.get( slug=widget['widgetslug'], section__in=Section.objects.filter(report=report) ) fields = wobj.collect_fields() form = TableFieldForm(fields, use_widgets=False, hidden_fields=report.hidden_fields, include_hidden=True, data=widget['criteria'], files=request.FILES) if form.is_valid(): # parse time and localize to user profile timezone timezone = get_timezone(request) form.apply_timezone(timezone) form_criteria = form.criteria() widget_table = wobj.table() form_criteria = form_criteria.build_for_table(widget_table) try: form_criteria.compute_times() except ValueError: pass handle = Job._compute_handle(widget_table, form_criteria) logger.debug('ReportHistory: adding handle %s for widget_table %s' % (handle, widget_table)) handles.append(handle) else: # log error, but don't worry about it for adding to RH logger.warning("Error while calculating job handle for Widget %s, " "internal criteria form is invalid: %s" % (wobj, form.errors.as_text())) job_handles = ','.join(handles) if request.user.is_authenticated(): user = request.user.username else: user = settings.GUEST_USER_NAME logger.debug('Creating ReportHistory for user %s at URL %s' % (user, url)) ReportHistory.create(namespace=report.namespace, slug=report.slug, bookmark=url, first_run=last_run, last_run=last_run, job_handles=job_handles, user=user, criteria=table_fields, run_count=1)
def post(self, request, namespace, report_slug, widget_slug, format=None): logger.debug("Received POST for report %s, widget %s: %s" % (report_slug, widget_slug, request.POST)) report = get_object_or_404(Report, namespace=namespace, slug=report_slug) widget = get_object_or_404( Widget, slug=widget_slug, section__in=Section.objects.filter(report=report)) req_json = json.loads(request.POST['criteria']) fields = widget.collect_fields() form = TableFieldForm(fields, use_widgets=False, hidden_fields=report.hidden_fields, include_hidden=True, data=req_json, files=request.FILES) if not form.is_valid(): raise ValueError("Widget internal criteria form is invalid:\n%s" % (form.errors.as_text())) if form.is_valid(): logger.debug('Form passed validation: %s' % form) formdata = form.cleaned_data logger.debug('Form cleaned data: %s' % formdata) # parse time and localize to user profile timezone timezone = pytz.timezone(request.user.timezone) form.apply_timezone(timezone) try: form_criteria = form.criteria() logger.debug('Form_criteria: %s' % form_criteria) job = Job.create(table=widget.table(), criteria=form_criteria) job.start() wjob = WidgetJob(widget=widget, job=job) wjob.save() logger.debug("Created WidgetJob %s for report %s (handle %s)" % (str(wjob), report_slug, job.handle)) return Response({ "joburl": reverse( 'report-job-detail', args=[namespace, report_slug, widget_slug, wjob.id]) }) except Exception as e: logger.exception("Failed to start job, an exception occurred") ei = sys.exc_info() resp = {} resp['message'] = "".join( traceback.format_exception_only(*sys.exc_info()[0:2])), resp['exception'] = "".join( traceback.format_exception(*sys.exc_info())) return JsonResponse(resp, status=400) else: logger.error("form is invalid, entering debugger") from IPython import embed embed()
def get(self, request, namespace=None, report_slug=None, widget_slug=None): try: report = Report.objects.get(namespace=namespace, slug=report_slug) except: raise Http404 logger.debug("Received GET for report %s widget definition" % report_slug) if widget_slug: w = get_object_or_404( Widget, slug=widget_slug, section__in=Section.objects.filter(report=report)) widgets = [w] else: widgets = report.widgets().order_by('row', 'col') # parse time and localize to user profile timezone timezone = pytz.timezone(request.user.timezone) now = datetime.datetime.now(timezone) # pin the endtime to a round interval if we are set to # reload periodically minutes = report.reload_minutes if minutes: # avoid case of long duration reloads to have large reload gap # e.g. 24-hour report will consider 12:15 am or later a valid time # to roll-over the time time values, rather than waiting # until 12:00 pm trimmed = round_time(dt=now, round_to=60 * minutes, trim=True) if now - trimmed > datetime.timedelta(minutes=15): now = trimmed else: now = round_time(dt=now, round_to=60 * minutes) widget_defs = [] for w in widgets: # get default criteria values for widget # and set endtime to now, if applicable widget_fields = w.collect_fields() form = TableFieldForm(widget_fields, use_widgets=False) # create object from the tablefield keywords # and populate it with initial data generated by default keys = form._tablefields.keys() criteria = dict(zip(keys, [None] * len(keys))) criteria.update(form.data) # calculate time offsets if 'endtime' in criteria: criteria['endtime'] = now.isoformat() # only consider starttime if its paired with an endtime if 'starttime' in criteria: start = now field = form.fields['starttime'] initial = field.widget.attrs.get('initial_time', None) if initial: m = re.match("now *- *(.+)", initial) if m: delta = parse_timedelta(m.group(1)) start = now - delta criteria['starttime'] = start.isoformat() # setup json definition object widget_def = w.get_definition(criteria) widget_defs.append(widget_def) report_def = self.report_def(widget_defs, now) return JsonResponse(report_def, safe=False)
def render_html(self, report, request, namespace, report_slug, isprint): """ Render HTML response """ logging.debug('Received request for report page: %s' % report_slug) if not request.user.profile_seen: # only redirect if first login return HttpResponseRedirect( reverse('preferences') + '?next=/report') devices = Device.objects.filter(enabled=True) device_modules = [obj.module for obj in devices] # iterate through all sections of the report, for each section, # iterate through the fields, and if any field's # pre_process_func function is device_selection_preprocess # then the field is a device field, then fetch the module and check # the module is included in Device objects in database missing_devices = set() for _id, fields in report.collect_fields_by_section().iteritems(): for _name, field_obj in fields.iteritems(): func = field_obj.pre_process_func if (func and func.function == 'device_selection_preprocess'): # This field is a device field, # check if the device is configured module = func.params['module'] if module not in device_modules: missing_devices.add(module) if missing_devices: missing_devices = ', '.join(list(missing_devices)) if not request.user.profile_seen: # only redirect if first login return HttpResponseRedirect( reverse('preferences') + '?next=/report') # Setup default criteria for the report based on underlying tables system_settings = SystemSettings.get_system_settings() form_init = {'ignore_cache': system_settings.ignore_cache} for table in report.tables(): if table.criteria: form_init.update(table.criteria) # Collect all fields organized by section, with section id 0 # representing common report level fields fields_by_section = report.collect_fields_by_section() # Merge fields into a single dict for use by the Django Form # logic all_fields = SortedDict() [all_fields.update(c) for c in fields_by_section.values()] form = TableFieldForm(all_fields, hidden_fields=report.hidden_fields, initial=form_init) # Build a section map that indicates which section each field # belongs in when displayed section_map = [] if fields_by_section[0]: section_map.append({ 'title': 'Common', 'parameters': fields_by_section[0] }) for s in Section.objects.filter(report=report).order_by( 'position', 'title'): show = False for v in fields_by_section[s.id].values(): if v.keyword not in (report.hidden_fields or []): show = True break if show: section_map.append({ 'title': s.title, 'parameters': fields_by_section[s.id] }) template, criteria, expand_tables = self.get_media_params(request) return render_to_response(template, { 'report': report, 'developer': system_settings.developer, 'maps_version': system_settings.maps_version, 'maps_api_key': system_settings.maps_api_key, 'endtime': 'endtime' in form.fields, 'form': form, 'section_map': section_map, 'show_sections': (len(section_map) > 1), 'criteria': criteria, 'expand_tables': expand_tables, 'missing_devices': missing_devices, 'is_superuser': request.user.is_superuser }, context_instance=RequestContext(request))
def create_report_history(request, report, widgets): """Create a report history object. :param request: request object :param report: Report object :param widgets: List of widget definitions """ # create the form to derive criteria for bookmark only # the form in the calling context can not be used # because it does not include hidden fields fields_by_section = report.collect_fields_by_section() all_fields = OrderedDict() [all_fields.update(c) for c in fields_by_section.values()] form = TableFieldForm(all_fields, hidden_fields=report.hidden_fields, include_hidden=True, data=request.POST, files=request.FILES) # parse time and localize to user profile timezone timezone = get_timezone(request) form.apply_timezone(timezone) form_data = form.cleaned_data url = request._request.path + '?' # form_data contains fields that don't belong to url # e.g. ignore_cache, debug # thus use compute_field_precedence method to filter those table_fields = {k: form_data[k] for k in form.compute_field_precedence()} # Convert field values into strings suitable for bookmark def _get_url_fields(flds): for k, v in flds.iteritems(): if k in ['starttime', 'endtime']: yield (k, str(datetime_to_seconds(v))) elif k in ['duration', 'resolution']: try: yield (k, str(int(timedelta_total_seconds(v)))) except AttributeError: # v is of special value, not a string of some duration yield (k, v.replace(' ', '+')) else: # use + as encoded white space yield (k, str(v).replace(' ', '+')) yield ('auto_run', 'true') url_fields = ['='.join([k, v]) for k, v in _get_url_fields(table_fields)] # Form the bookmark link url += '&'.join(url_fields) last_run = datetime.datetime.now(timezone) # iterate over the passed in widget definitions and use those # to calculate the actual job handles # since these will mimic what gets used to create the actual jobs, # the criteria will match more closely than using the report-level # criteria data handles = [] for widget in widgets: wobj = Widget.objects.get( slug=widget['widgetslug'], section__in=Section.objects.filter(report=report) ) fields = wobj.collect_fields() form = TableFieldForm(fields, use_widgets=False, hidden_fields=report.hidden_fields, include_hidden=True, data=widget['criteria'], files=request.FILES) if form.is_valid(): # parse time and localize to user profile timezone timezone = get_timezone(request) form.apply_timezone(timezone) form_criteria = form.criteria() widget_table = wobj.table() form_criteria = form_criteria.build_for_table(widget_table) try: form_criteria.compute_times() except ValueError: pass handle = Job._compute_handle(widget_table, form_criteria) logger.debug('ReportHistory: adding handle %s for widget_table %s' % (handle, widget_table)) handles.append(handle) else: # log error, but don't worry about it for adding to RH logger.warning("Error while calculating job handle for Widget %s, " "internal criteria form is invalid: %s" % (wobj, form.errors.as_text())) job_handles = ','.join(handles) if request.user.is_authenticated(): user = request.user.username else: user = settings.GUEST_USER_NAME logger.debug('Creating ReportHistory for user %s at URL %s' % (user, url)) ReportHistory.create(namespace=report.namespace, slug=report.slug, bookmark=url, first_run=last_run, last_run=last_run, job_handles=job_handles, user=user, criteria=table_fields, run_count=1)
def get(self, request, namespace=None, report_slug=None, widget_slug=None): try: report = Report.objects.get(namespace=namespace, slug=report_slug) except: raise Http404 logger.debug("Received GET for report %s widget definition" % report_slug) if widget_slug: w = get_object_or_404( Widget, slug=widget_slug, section__in=Section.objects.filter(report=report) ) widgets = [w] else: # Add 'id' to order_by so that stacked widgets will return # with the same order as created widgets = report.widgets().order_by('row', 'col', 'id') # parse time and localize to user profile timezone timezone = get_timezone(request) now = datetime.datetime.now(timezone) # pin the endtime to a round interval if we are set to # reload periodically minutes = report.reload_minutes offset = report.reload_offset if minutes: # avoid case of long duration reloads to have large reload gap # e.g. 24-hour report will consider 12:15 am or later a valid time # to roll-over the time time values, rather than waiting # until 12:00 pm trimmed = round_time(dt=now, round_to=60*minutes, trim=True) if now - trimmed > datetime.timedelta(seconds=offset): now = trimmed else: now = round_time(dt=now, round_to=60*minutes) widget_defs = [] for w in widgets: # get default criteria values for widget # and set endtime to now, if applicable widget_fields = w.collect_fields() form = TableFieldForm(widget_fields, use_widgets=False) # create object from the tablefield keywords # and populate it with initial data generated by default keys = form._tablefields.keys() criteria = dict(zip(keys, [None]*len(keys))) criteria.update(form.data) # calculate time offsets if 'endtime' in criteria: criteria['endtime'] = now.isoformat() # only consider starttime if its paired with an endtime if 'starttime' in criteria: start = now field = form.fields['starttime'] initial = field.widget.attrs.get('initial_time', None) if initial: m = re.match("now *- *(.+)", initial) if m: delta = parse_timedelta(m.group(1)) start = now - delta criteria['starttime'] = start.isoformat() # Check for "Meta Widget" criteria items system_settings = SystemSettings.get_system_settings() if system_settings.ignore_cache: criteria['ignore_cache'] = system_settings.ignore_cache if system_settings.developer: criteria['debug'] = system_settings.developer # setup json definition object widget_def = w.get_definition(criteria) widget_defs.append(widget_def) # Build the primary key corresponding to static data for this # widget if report.static: rw_id = '-'.join([namespace, report_slug, widget_def['widgetslug']]) # Add cached widget data if available. try: data_cache = WidgetDataCache.objects.get( report_widget_id=rw_id) widget_def['dataCache'] = data_cache.data except WidgetDataCache.DoesNotExist: msg = "No widget data cache available with id %s." % rw_id resp = {'message': msg, 'status': 'error', 'exception': ''} widget_def['dataCache'] = json.dumps(resp) report_def = self.report_def(widget_defs, now) return JsonResponse(report_def, safe=False)