def test_default(self):
obj = SystemSettings.get_system_settings()
self.assertEqual(obj.ignore_cache, False)
self.assertEqual(obj.developer, False)
self.assertEqual(obj.maps_version, 'OPEN_STREET_MAPS')
self.assertEqual(obj.maps_api_key, None)
self.assertEqual(obj.global_error_handler, True)
def process(cls, widget, job, data):
columns = job.get_columns()
col_names = [c.name for c in columns]
catcol = [c for c in columns if c.name == widget.options.key][0]
col = [c for c in columns if c.name == widget.options.value][0]
# For each slice, catcol will be the label, col will be the value
rows = []
if data:
for rawrow in data:
row = dict(zip(col_names, rawrow))
r = [row[catcol.name], row[col.name]]
rows.append(r)
else:
# create a "full" pie to show something
rows = [[1, 1]]
d_unit = SystemSettings.get_system_settings().data_units
data = {
'chartTitle': format_labels(
widget.title.format(**job.actual_criteria),
d_unit, columns),
'rows': rows,
'type': 'pie',
}
return data
def process(cls, widget, job, data):
columns = job.get_columns()
col_names = [c.name for c in columns]
catcol = [c for c in columns if c.name == widget.options.key][0]
col = [c for c in columns if c.name == widget.options.value][0]
# For each slice, catcol will be the label, col will be the value
rows = []
if data:
for rawrow in data:
row = dict(zip(col_names, rawrow))
r = [row[catcol.name], row[col.name]]
rows.append(r)
else:
# create a "full" pie to show something
rows = [[1, 1]]
d_unit = SystemSettings.get_system_settings().data_units
data = {
'chartTitle':
format_labels(widget.title.format(**job.actual_criteria), d_unit,
columns),
'rows':
rows,
'type':
'pie',
}
return data
def process(cls, widget, job, data):
helper = UIWidgetHelper(widget, job)
d_unit = SystemSettings.get_system_settings().data_units
# create composite name for key label
keyname = '-'.join([k.name for k in helper.keycols])
# For each slice, keyname will be the label
rows = []
for rawrow in data:
row = dict(zip([c.name for c in helper.all_cols], rawrow))
# populate values first
r = {c.name: format_single_value(c, row[c.name], d_unit)
for c in helper.valcols}
# now add the key
key = '-'.join([row[k.name] for k in helper.keycols])
r[keyname] = key
rows.append(r)
data = {
'chartTitle': format_labels(
widget.title.format(**job.actual_criteria),
d_unit, helper.valcols),
'rows': rows,
'keyname': keyname,
'values': [c.name for c in helper.valcols],
'names': {c.col.name: c.col.label for c in helper.colmap.values()},
'type': widget.options.charttype,
}
return data
def post(self, request):
instance = SystemSettings.get_system_settings()
form = SystemSettingsForm(request.DATA, instance=instance)
if form.is_valid():
form.save()
try:
return HttpResponseRedirect(request.QUERY_PARAMS['next'])
except KeyError:
return HttpResponseRedirect(request.META['HTTP_REFERER'])
else:
return Response({'form': form}, template_name='system_settings.html')
def get(self, request, namespace=None, report_slug=None, widget_slug=None):
request_data = request.GET.dict()
token = request_data['auth']
del request_data['auth']
token_obj = WidgetAuthToken.objects.get(token=token)
criteria_dict = token_obj.criteria
# request_data carries fields to override the criteria of the widget,
# each token maps to a list fields that are allowed to be modified,
# check to ensure each field in request_data:
# 1. is a valid field included in criteria
# 2. is included in the editable fields mapping the token in the url
for field in request_data:
if field not in criteria_dict:
msg = "Field '%s' is invalid" % field
logger.error(msg)
return HttpResponse(msg, status=403)
if not token_obj.edit_fields or field not in token_obj.edit_fields:
msg = "Field '%s' is not allowed to change" % field
logger.error(msg)
return HttpResponse(msg, status=403)
criteria_dict[field] = request_data[field]
criteria_str = json.dumps(criteria_dict)
report = get_object_or_404(Report,
namespace=namespace,
slug=report_slug)
# widget slugs aren't unique globally, but should be unique within
# any given report
w = get_object_or_404(
Widget,
slug=widget_slug,
section__in=Section.objects.filter(report=report))
system_settings = SystemSettings.get_system_settings()
widget_def = w.get_definition(mark_safe(criteria_str))
return render_to_response('widget.html', {
"widget":
widget_def,
"widget_url":
reverse('report-auto-view', args=(namespace, report_slug)),
"widget_authtoken":
token,
'maps_version':
system_settings.maps_version,
'maps_api_key':
system_settings.maps_api_key
},
context_instance=RequestContext(request))
def appfwk_vars(request):
return {
'django_version': django.VERSION,
'offline_js': settings.OFFLINE_JS,
'appfwk_version': settings.VERSION,
'js_versions': settings.JS_VERSIONS,
'js_files': settings.ONLINE_JS_FILES,
'css_files': settings.ONLINE_CSS_FILES,
'developer': SystemSettings.get_system_settings().developer,
'guest_enabled': settings.GUEST_USER_ENABLED,
'guest_show_button': settings.GUEST_SHOW_BUTTON,
'report_history_enabled': settings.REPORT_HISTORY_ENABLED,
}
def post(self, request):
instance = SystemSettings.get_system_settings()
form = SystemSettingsForm(request.DATA, instance=instance)
if form.is_valid():
form.save()
try:
return HttpResponseRedirect(request.QUERY_PARAMS['next'])
except KeyError:
return HttpResponseRedirect(request.META['HTTP_REFERER'])
else:
return Response({'form': form},
template_name='system_settings.html')
def test_post(self):
setting = {'ignore_cache': True,
'developer': True,
'maps_version': 'FREE',
'maps_api_key': 'key',
'global_error_handler': False
}
self.post_escape_keyerror(setting)
setting = SystemSettings.get_system_settings()
self.assertTrue(setting.ignore_cache)
self.assertTrue(setting.developer)
self.assertEqual(setting.maps_version, 'FREE')
self.assertEqual(setting.maps_api_key, 'key')
self.assertFalse(setting.global_error_handler)
def get(self, request, namespace=None, report_slug=None, widget_slug=None):
request_data = request.GET.dict()
token = request_data['auth']
del request_data['auth']
token_obj = WidgetAuthToken.objects.get(token=token)
criteria_dict = token_obj.criteria
# request_data carries fields to override the criteria of the widget,
# each token maps to a list fields that are allowed to be modified,
# check to ensure each field in request_data:
# 1. is a valid field included in criteria
# 2. is included in the editable fields mapping the token in the url
for field in request_data:
if field not in criteria_dict:
msg = "Field '%s' is invalid" % field
logger.error(msg)
return HttpResponse(msg, status=403)
if not token_obj.edit_fields or field not in token_obj.edit_fields:
msg = "Field '%s' is not allowed to change" % field
logger.error(msg)
return HttpResponse(msg, status=403)
criteria_dict[field] = request_data[field]
criteria_str = json.dumps(criteria_dict)
report = get_object_or_404(Report, namespace=namespace,
slug=report_slug)
# widget slugs aren't unique globally, but should be unique within
# any given report
w = get_object_or_404(
Widget,
slug=widget_slug,
section__in=Section.objects.filter(report=report)
)
system_settings = SystemSettings.get_system_settings()
widget_def = w.get_definition(mark_safe(criteria_str))
return render_to_response(
'widget.html',
{"widget": widget_def,
"widget_url": reverse('report-auto-view', args=(namespace,
report_slug)),
"widget_authtoken": token,
'maps_version': system_settings.maps_version,
'maps_api_key': system_settings.maps_api_key},
context_instance=RequestContext(request)
)
def base_process(cls, widget, job, data):
helper = UIWidgetHelper(widget, job)
d_unit = SystemSettings.get_system_settings().data_units
rows = []
allcols = helper.colmap.values()
if widget.options.get('columns', None):
cols = [c for c in allcols if c.key in widget.options['columns']]
else:
cols = allcols
for rawrow in data:
row = {}
for col in cols:
if col.istime or col.isdate:
t = rawrow[col.dataindex]
try:
val = timeutils.datetime_to_microseconds(t) / 1000
except AttributeError:
val = t * 1000
else:
col.label = format_labels(col.label,
d_unit,
helper.valcols)
val = format_single_value(col.col,
rawrow[col.dataindex],
d_unit)
row[col.key] = val
rows.append(row)
column_defs = [
c.to_json('key', 'label', 'sortable', 'formatter', 'allow_html')
for c in cols
]
data = {
"chartTitle": format_labels(
widget.title.format(**job.actual_criteria),
d_unit, helper.valcols),
"columns": column_defs,
"data": rows
}
return data
def base_process(cls, widget, job, data):
helper = UIWidgetHelper(widget, job)
d_unit = SystemSettings.get_system_settings().data_units
rows = []
allcols = helper.colmap.values()
if widget.options.get('columns', None):
cols = [c for c in allcols if c.key in widget.options['columns']]
else:
cols = allcols
for rawrow in data:
row = {}
for col in cols:
if col.istime or col.isdate:
t = rawrow[col.dataindex]
try:
val = timeutils.datetime_to_microseconds(t) / 1000
except AttributeError:
val = t * 1000
else:
col.label = format_labels(col.label, d_unit,
helper.valcols)
val = format_single_value(col.col, rawrow[col.dataindex],
d_unit)
row[col.key] = val
rows.append(row)
column_defs = [
c.to_json('key', 'label', 'sortable', 'formatter', 'allow_html')
for c in cols
]
data = {
"chartTitle":
format_labels(widget.title.format(**job.actual_criteria), d_unit,
helper.valcols),
"columns":
column_defs,
"data":
rows
}
return data
def authorized():
""" Verifies the Maps API can be used given the version selected
and the API key supplied.
Returns True/False, and an error message if applicable
"""
settings = SystemSettings.get_system_settings()
if settings.maps_version == 'DISABLED':
msg = (u'Maps API has been disabled.\n' 'See Admin User for changes.')
return False, msg
elif (settings.maps_version in ('FREE', 'BUSINESS')
and not settings.maps_api_key):
msg = (u'A valid API_KEY must be provided for either \n'
'"Free" or "Business" Google Maps API choices.\n'
'See Configure->Preferences to update.')
return False, msg
else:
return True, ''
def authorized():
""" Verifies the Maps API can be used given the version selected
and the API key supplied.
Returns True/False, and an error message if applicable
"""
settings = SystemSettings.get_system_settings()
if settings.maps_version == 'DISABLED':
msg = (u'Maps API has been disabled.\n'
'See Admin User for changes.')
return False, msg
elif (settings.maps_version in ('FREE', 'BUSINESS') and
not settings.maps_api_key):
msg = (u'A valid API_KEY must be provided for either \n'
'"Free" or "Business" Google Maps API choices.\n'
'See Configure->Preferences to update.')
return False, msg
else:
return True, ''
def process_error(sender, **kwargs):
context = kwargs.pop('context')
source = Source.get(context)
logger.debug('Received error_signal from %s with context %s' %
(sender, context))
handlers = ErrorHandlerCache.filter(Source.encode(source))
logger.debug('Found %d handlers.' % len(handlers))
event = Event(severity=ERROR_SEVERITY, context=context,
trigger_result=None)
event.save()
logger.debug('New Event created: %s' % event)
for h in handlers:
DestinationThread(h.destination, event, is_error=True).start()
system_settings = SystemSettings.get_system_settings()
if system_settings.global_error_handler:
if not handlers or any([h.allow_global for h in handlers]):
for d in GlobalErrorHandlerCache.data():
DestinationThread(d, event, is_error=True).start()
def process_error(sender, **kwargs):
context = kwargs.pop('context')
source = Source.get(context)
logger.debug('Received error_signal from %s with context %s' %
(sender, context))
handlers = ErrorHandlerCache.filter(Source.encode(source))
logger.debug('Found %d handlers.' % len(handlers))
event = Event(severity=ERROR_SEVERITY, context=context,
trigger_result=None)
event.save()
logger.debug('New Event created: %s' % event)
for h in handlers:
DestinationThread(h.destination, event, is_error=True).start()
system_settings = SystemSettings.get_system_settings()
if system_settings.global_error_handler:
if not handlers or any([h.allow_global for h in handlers]):
for d in GlobalErrorHandlerCache.data():
DestinationThread(d, event, is_error=True).start()
def process(cls, widget, job, data):
helper = UIWidgetHelper(widget, job)
d_unit = SystemSettings.get_system_settings().data_units
# create composite name for key label
keyname = '-'.join([k.name for k in helper.keycols])
# For each slice, keyname will be the label
rows = []
for rawrow in data:
row = dict(zip([c.name for c in helper.all_cols], rawrow))
# populate values first
r = {
c.name: format_single_value(c, row[c.name], d_unit)
for c in helper.valcols
}
# now add the key
key = '-'.join([row[k.name] for k in helper.keycols])
r[keyname] = key
rows.append(r)
data = {
'chartTitle':
format_labels(widget.title.format(**job.actual_criteria), d_unit,
helper.valcols),
'rows':
rows,
'keyname':
keyname,
'values': [c.name for c in helper.valcols],
'names': {c.col.name: c.col.label
for c in helper.colmap.values()},
'type':
widget.options.charttype,
}
return data
def get(self, request):
instance = SystemSettings.get_system_settings()
form = SystemSettingsForm(instance=instance)
return Response({'form': form}, template_name='system_settings.html')
def handle(self, *args, **options):
if not options['force']:
msg = ('You have requested to reset portal, this will delete\n'
'everything from the database and start from scratch.\n'
'Are you sure?\n'
"Type 'yes' to continue, or 'no' to cancel: ")
confirm = raw_input(msg)
else:
confirm = 'yes'
if confirm != 'yes':
self.stdout.write('Aborting.')
return
# Iterating keys in buffers and save data based on options
for name in self.buffer_names:
if not options['drop_' + name]:
self.save_data(name)
# lets clear it
self.stdout.write('Resetting database ... ', ending='')
management.call_command('reset_db',
interactive=False,
router='default')
self.stdout.write('done.')
management.call_command('clean',
applications=False,
report_id=None,
clear_cache=True,
clear_logs=True)
management.call_command('clean_pyc', path=settings.PROJECT_ROOT)
# some chain of migration dependencies requires this first
# https://code.djangoproject.com/ticket/24524
management.call_command('migrate', 'preferences', interactive=False)
# now we can do the full migrate (previously syncdb)
management.call_command('migrate', interactive=False)
self.stdout.write('Loading initial data ... ', ending='')
initial_data = glob.glob(os.path.join(settings.INITIAL_DATA, '*.json'))
initial_data.sort()
if not options['drop_users']:
# filter out default admin user fixture and reload previous users
initial_data = [f for f in initial_data if 'admin_user' not in f]
if initial_data:
management.call_command('loaddata', *initial_data)
for name in self.buffer_names:
self.load_data(name)
# if we don't have a settings fixture, create new default item
if not SystemSettings.objects.all():
SystemSettings().save()
management.call_command('reload', report_id=None)
if (not options['drop_users']
and (self.buffers['users']['buffer'] is None
or len(AppfwkUser.objects.all()) == 0)):
self.stdout.write('WARNING: No users added to database. '
'If you would like to include the default '
'admin user, rerun this command with the '
"'--drop-users' option.")
# reset progressd cache
self.stdout.write('Resetting progressd ...', ending='')
try:
from steelscript.appfwk.apps.jobs.progress import progressd
progressd.reset()
except RvbdException:
self.stdout.write(' unable to connect to progressd, skipping ...',
ending='')
self.stdout.write(' done.')
def process(cls, widget, job, data):
helper = UIWidgetHelper(widget, job)
catname = '-'.join([k.name for k in helper.keycols])
timecol = [c for c in helper.keycols if c.istime() or c.isdate()][0]
if widget.options.altaxis:
altcols = [
c.name for c in helper.all_cols
if c.name in widget.options.altaxis
]
else:
altcols = []
def c3datefmt(d):
return '%s.%s' % (force_to_utc(d).strftime('%Y-%m-%dT%H:%M:%S'),
"%03dZ" % int(d.microsecond / 1000))
df = pandas.DataFrame(data, columns=helper.col_names)
t0 = df[timecol.name].min()
t1 = df[timecol.name].max()
timeaxis = TimeAxis([t0, t1])
timeaxis.compute()
tickvalues = [c3datefmt(d) for d in timeaxis.ticks]
d_unit = SystemSettings.get_system_settings().data_units
[format_df_values(c, df, d_unit) for c in helper.valcols]
# Replace the '.' character with '*' from column names as '.'
# is be treated as object-attr access action in c3 0.4.11 version
df = df.rename(columns=dict(
zip(helper.col_names, map(replace_dot, helper.col_names))))
rows = json.loads(
df.to_json(orient='records', date_format='iso', date_unit='ms'))
data = {
'chartTitle':
format_labels(widget.title.format(**job.actual_criteria), d_unit,
helper.valcols),
'json':
rows,
'key':
catname,
'values': [replace_dot(c.name) for c in helper.valcols],
'names': {
replace_dot(c.col.name): format_labels(c.col.label, d_unit,
helper.valcols)
for c in helper.colmap.values()
},
'altaxis': {c: 'y2'
for c in altcols} or None,
'tickFormat':
timeaxis.best[1],
'tickValues':
tickvalues,
'type':
'line'
}
if widget.options.stacked:
data['groups'] = [[
c.col.label for c in helper.colmap.values() if not c.col.iskey
]]
data['type'] = 'area'
if widget.options.bar:
# can override 'area' type, or just be used as a plain bar display
data['type'] = 'bar'
return data
def process(cls, widget, job, data):
"""Class method to generate list of circle objects.
Subclass should manipulate this list into specific
JSON structures as needed.
"""
settings = SystemSettings.get_system_settings()
maps_version = settings.maps_version
post_process = POST_PROCESS_MAP[maps_version]
columns = job.get_columns()
ColInfo = namedtuple('ColInfo', ['col', 'dataindex'])
keycol = None
latcol = None
longcol = None
valuecol = None
labelcol = None
urlcol = None
for i, c in enumerate(columns):
if c.name == widget.options.key:
keycol = ColInfo(c, i)
elif c.name == widget.options.latitude:
latcol = ColInfo(c, i)
elif c.name == widget.options.longitude:
longcol = ColInfo(c, i)
elif c.name == widget.options.value:
valuecol = ColInfo(c, i)
elif c.name == widget.options.label:
labelcol = ColInfo(c, i)
elif c.name == widget.options.url:
urlcol = ColInfo(c, i)
# Array of circle objects for each data row
Circle = namedtuple('Circle', [
'title', 'lat', 'long', 'value', 'size', 'url', 'units',
'formatter'
])
circles = []
if data:
valmin = None
valmax = None
if valuecol.col.isnumeric():
values = zip(*data)[valuecol.dataindex]
filtered = filter(bool, values)
if filtered:
valmin = min(filtered)
valmax = max(filtered)
else:
valmin = 1
valmax = 1
geolookup = None
if valuecol.col.isnumeric():
formatter = 'formatMetric'
else:
formatter = None
for rawrow in data:
val = rawrow[valuecol.dataindex]
# skip empty result values which are not explicitly zero
if val is None or val == '':
continue
if valmax:
normalized = float(val - valmin) / float(valmax - valmin)
marker_size = 5 + normalized * 20
else:
marker_size = 5
if keycol:
key = rawrow[keycol.dataindex]
# XXXCJ - this is a hack for NetProfiler based host groups,
# need to generalize this, probably via options
if widget.table().options.get('groupby') == 'host_group':
try:
geo = Location.objects.get(name=key)
except ObjectDoesNotExist:
continue
else:
# Perform geolookup on the key
# (should be an IP address...)
if geolookup is None:
geolookup = LookupIP.instance()
geo = geolookup.lookup(key)
if geo:
# Found a match, create a Circle
url = rawrow[urlcol.dataindex] if urlcol else None
circle = Circle(title=geo.name,
lat=geo.latitude,
long=geo.longitude,
value=val,
size=marker_size,
units=valuecol.col.units_str(),
formatter=formatter,
url=url)
circles.append(circle)
else:
# use lat/long columns instead of lookups
lat = rawrow[latcol.dataindex]
long = rawrow[longcol.dataindex]
title = rawrow[labelcol.dataindex] if labelcol else val
url = rawrow[urlcol.dataindex] if urlcol else None
circle = Circle(title=title,
lat=lat,
long=long,
value=val,
size=marker_size,
units=valuecol.col.units_str(),
formatter=formatter,
url=url)
circles.append(circle)
else:
# no data just return empty circles list
pass
data = {
"chartTitle": widget.title.format(**job.actual_criteria),
"circles": circles,
"minbounds": widget.options.min_bounds
}
return post_process(data)
def render_html(self, report, request, namespace, report_slug, isprint):
""" Render HTML response
"""
logger.debug('Received request for report page: %s' % report_slug)
if request.user.is_authenticated() and not request.user.profile_seen:
# only redirect if first login
return HttpResponseRedirect(reverse('preferences')+'?next=/report')
devices = Device.objects.filter(enabled=True)
device_modules = [obj.module for obj in devices]
# iterate through all sections of the report, for each section,
# iterate through the fields, and if any field's
# pre_process_func function is device_selection_preprocess
# then the field is a device field, then fetch the module and check
# the module is included in Device objects in database
missing_devices = set()
for _id, fields in report.collect_fields_by_section().iteritems():
for _name, field_obj in fields.iteritems():
func = field_obj.pre_process_func
if (func and func.function == 'device_selection_preprocess'):
# This field is a device field,
# check if the device is configured
module = func.params['module']
if module not in device_modules:
missing_devices.add(module)
if missing_devices:
missing_devices = ', '.join(list(missing_devices))
if request.user.is_authenticated() and not request.user.profile_seen:
# only redirect if first login
return HttpResponseRedirect(reverse('preferences')+'?next=/report')
# Setup default criteria for the report based on underlying tables
system_settings = SystemSettings.get_system_settings()
form_init = {'ignore_cache': system_settings.ignore_cache}
for table in report.tables():
if table.criteria:
form_init.update(table.criteria)
# Collect all fields organized by section, with section id 0
# representing common report level fields
fields_by_section = report.collect_fields_by_section()
# Merge fields into a single dict for use by the Django Form # logic
all_fields = SortedDict()
[all_fields.update(c) for c in fields_by_section.values()]
self.update_criteria_from_bookmark(report, request, all_fields)
form = TableFieldForm(all_fields,
hidden_fields=report.hidden_fields,
initial=form_init)
# Build a section map that indicates which section each field
# belongs in when displayed
section_map = []
if fields_by_section[0]:
section_map.append({'title': 'Common',
'parameters': fields_by_section[0]})
for s in Section.objects.filter(report=report).order_by('position',
'title'):
show = False
for v in fields_by_section[s.id].values():
if v.keyword not in (report.hidden_fields or []):
show = True
break
if show:
section_map.append({'title': s.title,
'parameters': fields_by_section[s.id]})
template, criteria, expand_tables = self.get_media_params(request)
return render_to_response(
template,
{'report': report,
'developer': system_settings.developer,
'maps_version': system_settings.maps_version,
'maps_api_key': system_settings.maps_api_key,
'weather_enabled': system_settings.weather_enabled,
'weather_url': system_settings.weather_url,
'endtime': 'endtime' in form.fields,
'form': form,
'section_map': section_map,
'show_sections': (len(section_map) > 1),
'criteria': criteria,
'expand_tables': expand_tables,
'missing_devices': missing_devices,
'is_superuser': request.user.is_superuser},
context_instance=RequestContext(request)
)
def developer(request):
return {'developer': SystemSettings.get_system_settings().developer}
def render_html(self, report, request, namespace, report_slug, isprint):
""" Render HTML response
"""
logging.debug('Received request for report page: %s' % report_slug)
if not request.user.profile_seen:
# only redirect if first login
return HttpResponseRedirect(
reverse('preferences') + '?next=/report')
devices = Device.objects.filter(enabled=True)
device_modules = [obj.module for obj in devices]
# iterate through all sections of the report, for each section,
# iterate through the fields, and if any field's
# pre_process_func function is device_selection_preprocess
# then the field is a device field, then fetch the module and check
# the module is included in Device objects in database
missing_devices = set()
for _id, fields in report.collect_fields_by_section().iteritems():
for _name, field_obj in fields.iteritems():
func = field_obj.pre_process_func
if (func and func.function == 'device_selection_preprocess'):
# This field is a device field,
# check if the device is configured
module = func.params['module']
if module not in device_modules:
missing_devices.add(module)
if missing_devices:
missing_devices = ', '.join(list(missing_devices))
if not request.user.profile_seen:
# only redirect if first login
return HttpResponseRedirect(
reverse('preferences') + '?next=/report')
# Setup default criteria for the report based on underlying tables
system_settings = SystemSettings.get_system_settings()
form_init = {'ignore_cache': system_settings.ignore_cache}
for table in report.tables():
if table.criteria:
form_init.update(table.criteria)
# Collect all fields organized by section, with section id 0
# representing common report level fields
fields_by_section = report.collect_fields_by_section()
# Merge fields into a single dict for use by the Django Form # logic
all_fields = SortedDict()
[all_fields.update(c) for c in fields_by_section.values()]
form = TableFieldForm(all_fields,
hidden_fields=report.hidden_fields,
initial=form_init)
# Build a section map that indicates which section each field
# belongs in when displayed
section_map = []
if fields_by_section[0]:
section_map.append({
'title': 'Common',
'parameters': fields_by_section[0]
})
for s in Section.objects.filter(report=report).order_by(
'position', 'title'):
show = False
for v in fields_by_section[s.id].values():
if v.keyword not in (report.hidden_fields or []):
show = True
break
if show:
section_map.append({
'title': s.title,
'parameters': fields_by_section[s.id]
})
template, criteria, expand_tables = self.get_media_params(request)
return render_to_response(template, {
'report': report,
'developer': system_settings.developer,
'maps_version': system_settings.maps_version,
'maps_api_key': system_settings.maps_api_key,
'endtime': 'endtime' in form.fields,
'form': form,
'section_map': section_map,
'show_sections': (len(section_map) > 1),
'criteria': criteria,
'expand_tables': expand_tables,
'missing_devices': missing_devices,
'is_superuser': request.user.is_superuser
},
context_instance=RequestContext(request))
def test_singleton(self):
obj1 = SystemSettings.get_system_settings()
obj2 = SystemSettings.get_system_settings()
self.assertEqual(obj1, obj2)
def process(cls, widget, job, data):
"""Class method to generate list of circle objects.
Subclass should manipulate this list into specific
JSON structures as needed.
"""
settings = SystemSettings.get_system_settings()
maps_version = settings.maps_version
post_process = POST_PROCESS_MAP[maps_version]
columns = job.get_columns()
ColInfo = namedtuple('ColInfo', ['col', 'dataindex'])
keycol = None
latcol = None
longcol = None
valuecol = None
labelcol = None
urlcol = None
for i, c in enumerate(columns):
if c.name == widget.options.key:
keycol = ColInfo(c, i)
elif c.name == widget.options.latitude:
latcol = ColInfo(c, i)
elif c.name == widget.options.longitude:
longcol = ColInfo(c, i)
elif c.name == widget.options.value:
valuecol = ColInfo(c, i)
elif c.name == widget.options.label:
labelcol = ColInfo(c, i)
elif c.name == widget.options.url:
urlcol = ColInfo(c, i)
# Array of circle objects for each data row
Circle = namedtuple('Circle',
['title', 'lat', 'long', 'value', 'size', 'url',
'units', 'formatter'])
circles = []
if data:
valmin = None
valmax = None
if valuecol.col.isnumeric():
values = zip(*data)[valuecol.dataindex]
filtered = filter(bool, values)
if filtered:
valmin = min(filtered)
valmax = max(filtered)
else:
valmin = 1
valmax = 1
geolookup = None
if valuecol.col.isnumeric():
formatter = 'formatMetric'
else:
formatter = None
for rawrow in data:
val = rawrow[valuecol.dataindex]
# skip empty result values which are not explicitly zero
if val is None or val == '':
continue
if valmax:
normalized = float(val - valmin) / float(valmax - valmin)
marker_size = 5 + normalized * 20
else:
marker_size = 5
if keycol:
key = rawrow[keycol.dataindex]
# XXXCJ - this is a hack for NetProfiler based host groups,
# need to generalize this, probably via options
if widget.table().options.get('groupby') == 'host_group':
try:
geo = Location.objects.get(name=key)
except ObjectDoesNotExist:
continue
else:
# Perform geolookup on the key
# (should be an IP address...)
if geolookup is None:
geolookup = LookupIP.instance()
geo = geolookup.lookup(key)
if geo:
# Found a match, create a Circle
url = rawrow[urlcol.dataindex] if urlcol else None
circle = Circle(title=geo.name,
lat=geo.latitude,
long=geo.longitude,
value=val,
size=marker_size,
units=valuecol.col.units_str(),
formatter=formatter,
url=url)
circles.append(circle)
else:
# use lat/long columns instead of lookups
lat = rawrow[latcol.dataindex]
long = rawrow[longcol.dataindex]
title = rawrow[labelcol.dataindex] if labelcol else val
url = rawrow[urlcol.dataindex] if urlcol else None
circle = Circle(title=title,
lat=lat,
long=long,
value=val,
size=marker_size,
units=valuecol.col.units_str(),
formatter=formatter,
url=url)
circles.append(circle)
else:
# no data just return empty circles list
pass
data = {
"chartTitle": widget.title.format(**job.actual_criteria),
"circles": circles,
"minbounds": widget.options.min_bounds
}
return post_process(data)
def test_save(self):
SystemSettings().save()
objs = SystemSettings.objects.all()
self.assertEqual(len(objs), 1)
self.assertEqual(objs[0].pk, 1)
def process(cls, widget, job, data):
helper = UIWidgetHelper(widget, job)
catname = '-'.join([k.name for k in helper.keycols])
timecol = [c for c in helper.keycols if c.istime() or c.isdate()][0]
if widget.options.altaxis:
altcols = [c.name for c in helper.all_cols if
c.name in widget.options.altaxis]
else:
altcols = []
def c3datefmt(d):
return '%s.%s' % (force_to_utc(d).strftime('%Y-%m-%dT%H:%M:%S'),
"%03dZ" % int(d.microsecond / 1000))
df = pandas.DataFrame(data, columns=helper.col_names)
t0 = df[timecol.name].min()
t1 = df[timecol.name].max()
timeaxis = TimeAxis([t0, t1])
timeaxis.compute()
tickvalues = [c3datefmt(d) for d in timeaxis.ticks]
d_unit = SystemSettings.get_system_settings().data_units
[format_df_values(c, df, d_unit) for c in helper.valcols]
# Replace the '.' character with '*' from column names as '.'
# is be treated as object-attr access action in c3 0.4.11 version
df = df.rename(columns=dict(zip(helper.col_names,
map(replace_dot, helper.col_names))))
rows = json.loads(
df.to_json(orient='records', date_format='iso', date_unit='ms')
)
data = {
'chartTitle': format_labels(
widget.title.format(**job.actual_criteria),
d_unit, helper.valcols),
'json': rows,
'key': catname,
'values': [replace_dot(c.name) for c in helper.valcols],
'names': {replace_dot(c.col.name): format_labels(c.col.label,
d_unit,
helper.valcols)
for c in helper.colmap.values()},
'altaxis': {c: 'y2' for c in altcols} or None,
'tickFormat': timeaxis.best[1],
'tickValues': tickvalues,
'type': 'line'
}
if widget.options.stacked:
data['groups'] = [[c.col.label for c in helper.colmap.values()
if not c.col.iskey]]
data['type'] = 'area'
if widget.options.bar:
# can override 'area' type, or just be used as a plain bar display
data['type'] = 'bar'
return data
def get(self, request, namespace=None, report_slug=None, widget_slug=None):
try:
report = Report.objects.get(namespace=namespace,
slug=report_slug)
except:
raise Http404
logger.debug("Received GET for report %s widget definition" %
report_slug)
if widget_slug:
w = get_object_or_404(
Widget,
slug=widget_slug,
section__in=Section.objects.filter(report=report)
)
widgets = [w]
else:
# Add 'id' to order_by so that stacked widgets will return
# with the same order as created
widgets = report.widgets().order_by('row', 'col', 'id')
# parse time and localize to user profile timezone
timezone = get_timezone(request)
now = datetime.datetime.now(timezone)
# pin the endtime to a round interval if we are set to
# reload periodically
minutes = report.reload_minutes
offset = report.reload_offset
if minutes:
# avoid case of long duration reloads to have large reload gap
# e.g. 24-hour report will consider 12:15 am or later a valid time
# to roll-over the time time values, rather than waiting
# until 12:00 pm
trimmed = round_time(dt=now, round_to=60*minutes, trim=True)
if now - trimmed > datetime.timedelta(seconds=offset):
now = trimmed
else:
now = round_time(dt=now, round_to=60*minutes)
widget_defs = []
for w in widgets:
# get default criteria values for widget
# and set endtime to now, if applicable
widget_fields = w.collect_fields()
form = TableFieldForm(widget_fields, use_widgets=False)
# create object from the tablefield keywords
# and populate it with initial data generated by default
keys = form._tablefields.keys()
criteria = dict(zip(keys, [None]*len(keys)))
criteria.update(form.data)
# calculate time offsets
if 'endtime' in criteria:
criteria['endtime'] = now.isoformat()
# only consider starttime if its paired with an endtime
if 'starttime' in criteria:
start = now
field = form.fields['starttime']
initial = field.widget.attrs.get('initial_time', None)
if initial:
m = re.match("now *- *(.+)", initial)
if m:
delta = parse_timedelta(m.group(1))
start = now - delta
criteria['starttime'] = start.isoformat()
# Check for "Meta Widget" criteria items
system_settings = SystemSettings.get_system_settings()
if system_settings.ignore_cache:
criteria['ignore_cache'] = system_settings.ignore_cache
if system_settings.developer:
criteria['debug'] = system_settings.developer
# setup json definition object
widget_def = w.get_definition(criteria)
widget_defs.append(widget_def)
# Build the primary key corresponding to static data for this
# widget
if report.static:
rw_id = '-'.join([namespace, report_slug,
widget_def['widgetslug']])
# Add cached widget data if available.
try:
data_cache = WidgetDataCache.objects.get(
report_widget_id=rw_id)
widget_def['dataCache'] = data_cache.data
except WidgetDataCache.DoesNotExist:
msg = "No widget data cache available with id %s." % rw_id
resp = {'message': msg,
'status': 'error',
'exception': ''}
widget_def['dataCache'] = json.dumps(resp)
report_def = self.report_def(widget_defs, now)
return JsonResponse(report_def, safe=False)
def get(self, request):
instance = SystemSettings.get_system_settings()
form = SystemSettingsForm(instance=instance)
return Response({'form': form}, template_name='system_settings.html')
def developer(request):
return {'developer': SystemSettings.get_system_settings().developer}
