class WidgetAuthToken(models.Model):
""" Authentication token for each user per widget per report """
token = models.CharField(max_length=200)
user = models.ForeignKey(AppfwkUser)
pre_url = models.CharField(max_length=200, verbose_name='URL')
criteria = PickledObjectField()
edit_fields = SeparatedValuesField(null=True)
touched = models.DateTimeField(auto_now=True,
verbose_name='Last Time used')
def __unicode__(self):
return ("<Token %s, User %s, pre_url %s>" %
(self.token, self.user, self.pre_url))
class TableField(models.Model):
"""
Defines a single field associated with a table.
TableFields define the the parameters that are used by a Table
at run time. The Table.fields attribute associates one
or more fields with the table.
At run time, a Criteria object binds values to each field. The
Criteria object has an attribute matching each associated TableField
keyword.
When defining a TableField, the following model attributes
may be specified:
:param keyword: short identifier used like a variable name, this must
be unique per table
:param label: text label displayed in user interfaces
:param help_text: descriptive help text associated with this field
:param initial: starting or default value to use in user interfaces
:param required: boolean indicating if a non-null values must be provided
:param hidden: boolean indicating if this field should be hidden in
user interfaces, usually true when the value is computed from
other fields via post_process_func or post_process_template
:param field_cls: Django Form Field class to use for rendering.
If not specified, this defaults to CharField
:param field_kwargs: Dictionary of additional field specific
kwargs to pass to the field_cls constructor.
:param parents: List of parent keywords that this field depends on
for a final value. Used in conjunction with either
post_process_func or post_process_template.
:param pre_process_func: Function to call to perform any necessary
preprocessing before rendering a form field or accepting
user input.
:param post_process_func: Function to call to perform any post
submit processing. This may be additional value cleanup
or computation based on other form data.
:param post_process_template: Simple string format style template
to fill in based on other form criteria.
"""
keyword = models.CharField(max_length=100)
label = models.CharField(max_length=100, null=True, default=None)
help_text = models.CharField(blank=True,
null=True,
default=None,
max_length=400)
initial = PickledObjectField(blank=True, null=True)
required = models.BooleanField(default=False)
hidden = models.BooleanField(default=False)
field_cls = PickledObjectField(null=True)
field_kwargs = PickledObjectField(blank=True, null=True)
parent_keywords = SeparatedValuesField(null=True)
pre_process_func = FunctionField(null=True)
dynamic = models.BooleanField(default=False)
post_process_func = FunctionField(null=True)
post_process_template = models.CharField(null=True, max_length=500)
@classmethod
def create(cls, keyword, label=None, obj=None, **kwargs):
parent_keywords = kwargs.pop('parent_keywords', None)
if parent_keywords is None:
parent_keywords = []
field = cls(keyword=keyword, label=label, **kwargs)
field.save()
if field.post_process_template is not None:
f = string.Formatter()
for (_, parent_keyword, _,
_) in f.parse(field.post_process_template):
if parent_keyword is not None:
parent_keywords.append(parent_keyword)
field.parent_keywords = parent_keywords
field.save()
if obj is not None:
obj.fields.add(field)
return field
def __unicode__(self):
return "<TableField %s (%s)>" % (self.keyword, self.id)
def __repr__(self):
return unicode(self)
def is_report_criteria(self, table):
""" Runs through intersections of widgets to determine if this criteria
is applicable to the passed table
report <--> widgets <--> table
|
L- TableField (self)
"""
wset = set(table.widget_set.all())
rset = set(self.report_set.all())
return any(
wset.intersection(set(rwset.widget_set.all())) for rwset in rset)
@classmethod
def find_instance(cls, key):
""" Return instance given a keyword. """
params = TableField.objects.filter(keyword=key)
if len(params) == 0:
return None
elif len(params) > 1:
raise KeyError("Multiple TableField matches found for %s" % key)
param = params[0]
return param
class Report(models.Model):
""" Defines a Report as a collection of Sections and their Widgets. """
title = models.CharField(max_length=200)
description = models.TextField(null=True)
position = models.DecimalField(max_digits=7, decimal_places=3, default=10)
enabled = models.BooleanField(default=True)
slug = models.SlugField(unique=True, max_length=100)
namespace = models.CharField(max_length=100)
sourcefile = models.CharField(max_length=200)
filepath = models.FilePathField(max_length=200, path=settings.REPORTS_DIR)
fields = models.ManyToManyField(TableField, null=True, blank=True)
field_order = SeparatedValuesField(
null=True,
default=['starttime', 'endtime', 'duration', 'filterexpr'],
blank=True)
hidden_fields = SeparatedValuesField(null=True, blank=True)
# create an 'auto-load'-type report which uses default criteria
# values only, and optionally set a refresh timer
hide_criteria = models.BooleanField(default=False)
reload_minutes = models.IntegerField(default=0) # 0 means no reloads
@classmethod
def create(cls, title, **kwargs):
"""Create a new Report object and save it to the database.
:param str title: Title for the report
:param float position: Position in the menu for this report.
By default, all system reports have a ``position`` of 9 or
greater.
:param list field_order: List declaring the order to display
criteria fields. Any fields not list are displayed after
all listed fields.
:param list hidden_fields: List of criteria fields to hide from UI
:param bool hide_criteria: Set to true to hide criteria and run on load
:param int reload_minutes: If non-zero, report will be reloaded
automatically at the given duration in minutes
"""
logger.debug('Creating report %s' % title)
r = cls(title=title, **kwargs)
r.save()
return r
def __init__(self, *args, **kwargs):
super(Report, self).__init__(*args, **kwargs)
self._sections = []
def save(self, *args, **kwargs):
""" Apply sourcefile and namespaces to newly created Reports.
sourcefiles will be parsed into the following namespaces:
'config.reports.1_overall' --> 'default'
'steelscript.netshark.appfwk.reports.3_shark' --> 'netshark'
'steelscript.appfwk.business_hours.reports.x_rpt' --> 'business_hours'
"""
if not self.sourcefile:
mod = get_module()
self.filepath = mod.__file__
modname = get_module_name(mod)
self.sourcefile = get_sourcefile(modname)
if not self.namespace:
self.namespace = get_namespace(self.sourcefile)
if not self.slug:
self.slug = slugify(self.sourcefile.split('.')[-1])
super(Report, self).save(*args, **kwargs)
def __unicode__(self):
return "<Report %s (%s)>" % (self.title, self.id)
def __repr__(self):
return unicode(self)
def add_section(self, title=None, **kwargs):
"""Create a new section associated with this report.
:param str title: Title for the section. Defaults to
``section<n>``.
See :py:meth:`Section.create` for a complete description
and a list of valid ``kwargs``.
"""
if title is None:
title = 'section%d' % len(self._sections)
s = Section.create(report=self, title=title, **kwargs)
self._sections.append(s)
return s
def add_widget(self, cls, table, title, **kwargs):
"""Create a new widget associated with the last section added.
:param cls: UI class that will be used to render this widget
:param table: Table providing data for this widget
:param str title: Display title for this widget
See the specific ``cls.create()`` method for additional kwargs.
"""
if len(self._sections) == 0:
raise ValueError('Widgets can only be added to Sections. '
'Add section using "add_section" method first.')
s = kwargs.pop('section', self._sections[-1])
return cls.create(s, table, title, **kwargs)
def collect_fields_by_section(self):
""" Return a dict of all fields related to this report by section id.
"""
# map of section id to field dict
fields_by_section = SortedDict()
# section id=0 is the "common" section
# fields attached directly to the Report object are always added
# to the common section
fields_by_section[0] = SortedDict()
if self.fields:
report_fields = {}
for f in self.fields.all():
report_fields[f.keyword] = f
fields_by_section[0].update(report_fields)
# Pull in fields from each section (which may add fields to
# the common as well)
for s in Section.objects.filter(report=self):
for secid, fields in s.collect_fields_by_section().iteritems():
if secid not in fields_by_section:
fields_by_section[secid] = fields
else:
# update fields from fields_by_section so that the
# first definition of a field takes precedence.
# For example, if 'start' is defined at the report
# 'common' level, it's field will be used rather
# than that defined in the section. This is useful
# for custimizing the appearance/label/defaults of
# fields pulled in from tables
fields.update(fields_by_section[secid])
fields_by_section[secid] = fields
# Reorder fields in each section according to the field_order list
new_fields_by_section = {}
for i, fields in fields_by_section.iteritems():
# collect all field names
section_fields = fields_by_section[i]
section_field_names = set(section_fields.keys())
ordered_field_names = SortedDict()
# Iterate over the defined order list, which may not
# address all fields
for name in (self.field_order or []):
if name in section_field_names:
ordered_field_names[name] = section_fields[name]
section_field_names.remove(name)
# Preserve the order of any fields left
for name in section_field_names:
ordered_field_names[name] = section_fields[name]
new_fields_by_section[i] = ordered_field_names
return new_fields_by_section
def widgets(self):
return Widget.objects.filter(section__in=Section.objects.filter(
report=self)).order_by('id')
def tables(self, order_by='id'):
"""Return all tables from this report, ordered by `order_by`."""
return (Table.objects.filter(widget__in=Widget.objects.filter(
section__in=Section.objects.filter(
report=self))).distinct().order_by(order_by))
def widget_definitions(self, criteria):
"""Return list of widget definitions suitable for a JSON response.
"""
definitions = []
for w in self.widgets().order_by('row', 'col'):
widget_def = w.get_definition(criteria)
definitions.append(widget_def)
return definitions
class Table(models.Model):
name = models.CharField(max_length=200)
# Table data is produced by a queryclassname defined within the
# named module
module = models.CharField(max_length=200)
queryclassname = models.CharField(max_length=200)
namespace = models.CharField(max_length=100)
sourcefile = models.CharField(max_length=200)
# list of column names
sortcols = SeparatedValuesField(null=True)
# list of asc/desc - must match len of sortcols
sortdir = SeparatedValuesField(null=True)
# Valid values for sort kwarg
SORT_NONE = None
SORT_ASC = 'asc'
SORT_DESC = 'desc'
rows = models.IntegerField(default=-1)
filterexpr = models.CharField(null=True, max_length=400)
# resample flag -- resample to the criteria.resolution
# - this requires a "time" column
resample = models.BooleanField(default=False)
# options are typically fixed attributes defined at Table creation
options = PickledObjectField()
# list of fields that must be bound to values in criteria
# that this table needs to run
fields = models.ManyToManyField(TableField)
# Default values for fields associated with this table, these
# may be overridden by user criteria at run time
criteria = PickledObjectField()
# Function to call to tweak criteria for computing a job handle.
# This must return a dictionary of key/value pairs of values
# to use for computing a determining when a job must be rerun.
criteria_handle_func = FunctionField(null=True)
# Indicates if data can be cached
cacheable = models.BooleanField(default=True)
@classmethod
def to_ref(cls, arg):
""" Generate a table reference.
:param arg: may be either a Table object, table id,
or dictionary reference.
"""
if isinstance(arg, dict):
if 'namespace' not in arg or 'name' not in arg:
msg = 'Invalid table ref as dict, expected namespace/name'
raise KeyError(msg)
return arg
if isinstance(arg, Table):
table = arg
elif hasattr(arg, 'table'):
# Datasource table
table = arg.table
elif isinstance(arg, int):
table = Table.objects.get(id=arg)
else:
raise ValueError('No way to handle Table arg of type %s' %
type(arg))
return {
'sourcefile': table.sourcefile,
'namespace': table.namespace,
'name': table.name
}
@classmethod
def from_ref(cls, ref):
try:
return Table.objects.get(sourcefile=ref['sourcefile'],
namespace=ref['namespace'],
name=ref['name'])
except ObjectDoesNotExist:
logger.exception(
'Failed to resolve table ref: %s/%s/%s' %
(ref['sourcefile'], ref['namespace'], ref['name']))
raise
def __unicode__(self):
return "<Table %s (%s)>" % (str(self.id), self.name)
def __repr__(self):
return unicode(self)
@property
def queryclass(self):
# Lookup the query class for the table associated with this task
try:
i = importlib.import_module(self.module)
queryclass = i.__dict__[self.queryclassname]
except:
raise DatasourceException(
"Could not lookup queryclass %s in module %s" %
(self.queryclassname, self.module))
return queryclass
def get_columns(self, synthetic=None, ephemeral=None, iskey=None):
"""
Return the list of columns for this table.
`synthetic` is tri-state: None (default) is don't care,
True means only synthetic columns, False means
only non-synthetic columns
`ephemeral` is a job reference. If specified, include
ephemeral columns related to this job
`iskey` is tri-state: None (default) is don't care,
True means only key columns, False means
only non-key columns
"""
filtered = []
for c in Column.objects.filter(table=self).order_by(
'position', 'name'):
if synthetic is not None and c.synthetic != synthetic:
continue
if c.ephemeral is not None and c.ephemeral != ephemeral:
continue
if iskey is not None and c.iskey != iskey:
continue
filtered.append(c)
return filtered
def copy_columns(self,
table,
columns=None,
except_columns=None,
synthetic=None,
ephemeral=None):
""" Copy the columns from `table` into this table.
This method will copy all the columns from another table, including
all attributes as well as sorting.
"""
if not isinstance(table, Table):
table = Table.from_ref(table)
sortcols = []
sortdir = []
for c in table.get_columns(synthetic=synthetic, ephemeral=ephemeral):
if columns is not None and c.name not in columns:
continue
if except_columns is not None and c.name in except_columns:
continue
if table.sortcols and (c.name in table.sortcols):
sortcols.append(c.name)
sortdir.append(table.sortdir[table.sortcols.index(c.name)])
c.pk = None
c.table = self
c.save()
# Allocate an id, use that as the position
c.position = c.id
c.save()
if sortcols:
self.sortcols = sortcols
self.sortdir = sortdir
self.save()
def compute_synthetic(self, job, df):
""" Compute the synthetic columns from DF a two-dimensional array
of the non-synthetic columns.
Synthesis occurs as follows:
1. Compute all synthetic columns where compute_post_resample
is False
2. If the table is a time-based table with a defined resolution,
the result is resampled.
3. Any remaining columns are computed.
"""
if df is None:
return None
all_columns = job.get_columns()
all_col_names = [c.name for c in all_columns]
def compute(df, syncols):
for syncol in syncols:
expr = syncol.compute_expression
g = tokenize.generate_tokens(StringIO(expr).readline)
newexpr = ""
getvalue = False
getclose = False
for ttype, tvalue, _, _, _ in g:
if getvalue:
if ttype != tokenize.NAME:
msg = "Invalid syntax, expected {name}: %s" % tvalue
raise ValueError(msg)
elif tvalue in all_col_names:
newexpr += "df['%s']" % tvalue
elif tvalue in job.criteria:
newexpr += '"%s"' % str(job.criteria.get(tvalue))
else:
raise ValueError("Invalid variable name: %s" %
tvalue)
getclose = True
getvalue = False
elif getclose:
if ttype != tokenize.OP and tvalue != "}":
msg = "Invalid syntax, expected {name}: %s" % tvalue
raise ValueError(msg)
getclose = False
elif ttype == tokenize.OP and tvalue == "{":
getvalue = True
else:
newexpr += tvalue
newexpr += ' '
try:
df[syncol.name] = eval(newexpr)
except NameError as e:
m = (('%s: expression failed: %s, check '
'APPFWK_SYNTHETIC_MODULES: %s') %
(self, newexpr, str(e)))
logger.exception(m)
raise TableComputeSyntheticError(m)
# 1. Compute synthetic columns where post_resample is False
compute(df, [
col for col in all_columns
if (col.synthetic and col.compute_post_resample is False)
])
# 2. Resample
colmap = {}
timecol = None
for col in all_columns:
colmap[col.name] = col
if col.istime():
timecol = col.name
if self.resample:
if timecol is None:
raise (TableComputeSyntheticError(
"%s: 'resample' is set but no 'time' column'" % self))
if (('resolution' not in job.criteria)
and ('resample_resolution' not in job.criteria)):
raise (TableComputeSyntheticError(
("%s: 'resample' is set but criteria missing " +
"'resolution' or 'resample_resolution'") % self))
how = {}
for k in df.keys():
if k == timecol or k not in colmap:
continue
how[k] = colmap[k].resample_operation
if 'resample_resolution' in job.criteria:
resolution = job.criteria.resample_resolution
else:
resolution = job.criteria.resolution
resolution = timedelta_total_seconds(resolution)
if resolution < 1:
raise (TableComputeSyntheticError(
("Table %s cannot resample at a resolution " +
"less than 1 second") % self))
logger.debug('%s: resampling to %ss' % (self, int(resolution)))
indexed = df.set_index(timecol)
resampled = indexed.resample('%ss' % int(resolution),
how,
convention='end').reset_index()
df = resampled
# 3. Compute remaining synthetic columns (post_resample is True)
compute(df, [
c for c in all_columns
if (c.synthetic and c.compute_post_resample is True)
])
return df