class RegTestSuite(base.Structure):
"""A suite of regression tests.
"""
name_ = "regSuite"
_tests = base.StructListAttribute("tests",
childFactory=RegTest,
description="Tests making up this suite",
copyable=False)
_title = base.NWUnicodeAttribute("title",
description="A short, human-readable phrase describing what this"
" suite is about.")
_sequential = base.BooleanAttribute("sequential",
description="Set to true if the individual tests need to be run"
" in sequence.",
default=False)
def itertests(self, tags):
for test in self.tests:
if not test.tags or test.tags&tags:
yield test
def completeElement(self, ctx):
if self.title is None:
self.title = "Test suite from %s"%self.parent.sourceId
self._completeElementNext(base.Structure, ctx)
def expand(self, *args, **kwargs):
"""hand macro expansion to the RD.
"""
return self.parent.expand(*args, **kwargs)
class RDParameter(base.Structure):
"""A base class for parameters.
"""
_name = base.UnicodeAttribute("key", default=base.Undefined,
description="The name of the parameter", copyable=True, strip=True,
aliases=["name"])
_descr = base.NWUnicodeAttribute("description", default=None,
description="Some human-readable description of what the"
" parameter is about", copyable=True, strip=True)
_expr = base.DataContent(description="The default for the parameter."
" The special value __NULL__ indicates a NULL (python None) as usual."
" An empty content means a non-preset parameter, which must be filled"
" in applications. The magic value __EMPTY__ allows presetting an"
" empty string.",
copyable=True, strip=True, default=base.NotGiven)
_late = base.BooleanAttribute("late", default=False,
description="Bind the name not at setup time but at applying"
" time. In rowmaker procedures, for example, this allows you to"
" refer to variables like vars or rowIter in the bindings.")
def isDefaulted(self):
return self.content_ is not base.NotGiven
def validate(self):
self._validateNext(RDParameter)
if not utils.identifierPattern.match(self.key):
raise base.LiteralParseError("name", self.key, hint=
"The name you supplied was not defined by any procedure definition.")
def completeElement(self, ctx):
if self.content_=="__EMPTY__":
self.content_ = ""
self._completeElementNext(RDParameter, ctx)
class ColumnBase(base.Structure, base.MetaMixin):
"""A base class for columns, parameters, output fields, etc.
Actually, right now there's far too much cruft in here that
should go into Column proper or still somewhere else. Hence:
XXX TODO: Refactor.
See also Column for a docstring that still applies to all we've in
here.
"""
_name = ParamNameAttribute("name",
default=base.Undefined,
description="Name of the param",
copyable=True,
before="type")
_type = TypeNameAttribute(
"type",
default="real",
description="datatype for the column (SQL-like type system)",
copyable=True,
before="unit")
_unit = base.UnicodeAttribute("unit",
default="",
description="Unit of the values",
copyable=True,
before="ucd",
strip=True)
_ucd = base.UnicodeAttribute("ucd",
default="",
description="UCD of the column",
copyable=True,
before="description")
_description = base.NWUnicodeAttribute(
"description",
default="",
copyable=True,
description=
"A short (one-line) description of the values in this column.")
_tablehead = base.UnicodeAttribute(
"tablehead",
default=None,
description="Terse phrase to put into table headers for this"
" column",
copyable=True)
_utype = base.UnicodeAttribute("utype",
default=None,
description="utype for this column",
copyable=True)
_required = base.BooleanAttribute(
"required",
default=False,
description="Record becomes invalid when this column is NULL",
copyable=True)
_displayHint = DisplayHintAttribute(
"displayHint",
description="Suggested presentation; the format is "
" <kw>=<value>{,<kw>=<value>}, where what is interpreted depends"
" on the output format. See, e.g., documentation on HTML renderers"
" and the formatter child of outputFields.",
copyable=True)
_verbLevel = base.IntAttribute(
"verbLevel",
default=20,
description="Minimal verbosity level at which to include this column",
copyable=True)
_values = base.StructAttribute("values",
default=None,
childFactory=Values,
description="Specification of legal values",
copyable=True)
_fixup = base.UnicodeAttribute(
"fixup",
description=
"A python expression the value of which will replace this column's"
" value on database reads. Write a ___ to access the original"
' value. You can use macros for the embedding table.'
' This is for, e.g., simple URL generation'
' (fixup="\'\\internallink{/this/svc}\'+___").'
' It will *only* kick in when tuples are deserialized from the'
" database, i.e., *not* for values taken from tables in memory.",
default=None,
copyable=True)
_note = base.UnicodeAttribute(
"note",
description="Reference to a note meta"
" on this table explaining more about this column",
default=None,
copyable=True)
_xtype = base.UnicodeAttribute("xtype",
description="VOTable xtype giving"
" the serialization form",
default=None,
copyable=True)
_stc = TableManagedAttribute(
"stc",
description="Internally used"
" STC information for this column (do not assign to unless instructed"
" to do so)",
default=None,
copyable=True)
_stcUtype = TableManagedAttribute(
"stcUtype",
description="Internally used"
" STC information for this column (do not assign to)",
default=None,
copyable=True)
_properties = base.PropertyAttribute(copyable=True)
_original = base.OriginalAttribute()
restrictedMode = False
def __repr__(self):
return "<Column %s>" % repr(self.name)
def setMetaParent(self, parent):
# columns should *not* take part in meta inheritance. The reason is
# that there are usually many columns to a table, and there's no
# way I can see that any piece of metadata should be repeated in
# all of them. On the other hand, for votlinks (no name an example),
# meta inheritance would have disastrous consequences.
# So, we bend the rules a bit.
raise base.StructureError(
"Columns may not have meta parents.",
hint="The rationale for this is explained in rscdef/column.py,"
" look for setMetaParent.")
def onParentComplete(self):
# we need to resolve note on construction since columns are routinely
# copied to other tables and meta info does not necessarily follow.
if isinstance(self.note, basestring):
try:
self.note = self.parent.getNote(self.note)
except base.NotFoundError: # non-existing notes silently ignored
self.note = None
def completeElement(self, ctx):
self.restrictedMode = getattr(ctx, "restricted", False)
if isinstance(self.name, utils.QuotedName):
self.key = self.name.name
if ')' in self.key:
# No '()' allowed in key for that breaks the %()s syntax (sigh!).
# Work around with the following quick hack that would break
# if people carefully chose proper names. Anyone using delim.
# ids in SQL deserves a good spanking anyway.
self.key = self.key.replace(')', "__").replace('(', "__")
else:
self.key = self.name
self._completeElementNext(ColumnBase, ctx)
def isEnumerated(self):
return self.values and self.values.options
def validate(self):
self._validateNext(ColumnBase)
if self.restrictedMode and self.fixup:
raise base.RestrictedElement("fixup")
def validateValue(self, value):
"""raises a ValidationError if value does not match the constraints
given here.
"""
if value is None:
if self.required:
raise base.ValidationError(
"Field %s is empty but non-optional" % self.name,
self.name)
return
# Only validate these if we're not a database column
if not isinstance(self, Column):
vals = self.values
if vals:
if vals.options:
if value and not vals.validateOptions(value):
raise base.ValidationError(
"Value %s not consistent with"
" legal values %s" % (value, vals.options),
self.name)
else:
if vals.min and value < vals.min:
raise base.ValidationError(
"%s too small (must be at least %s)" %
(value, vals.min), self.name)
if vals.max and value > vals.max:
raise base.ValidationError(
"%s too large (must be less than %s)" %
(value, vals.max), self.name)
def isIndexed(self):
"""returns a guess as to whether this column is part of an index.
This may return True, False, or None (unknown).
"""
if self.parent and hasattr(self.parent, "indexedColumns"):
# parent is something like a TableDef
if self.name in self.parent.indexedColumns:
return True
else:
return False
def isPrimary(self):
"""returns a guess as to whether this column is a primary key of the
embedding table.
This may return True, False, or None (unknown).
"""
if self.parent and hasattr(self.parent, "primary"):
# parent is something like a TableDef
if self.name in self.parent.primary:
return True
else:
return False
_indexedCleartext = {
True: "indexed",
False: "notIndexed",
None: "unknown",
}
def asInfoDict(self):
"""returns a dictionary of certain, "user-interesting" properties
of the data field, in a dict of strings.
"""
return {
"name": unicode(self.name),
"description": self.description or "N/A",
"tablehead": self.getLabel(),
"unit": self.unit or "N/A",
"ucd": self.ucd or "N/A",
"verbLevel": self.verbLevel,
"indexState": self._indexedCleartext[self.isIndexed()],
"note": self.note,
}
def getDDL(self):
"""returns an SQL fragment describing this column ready for
inclusion in a DDL statement.
"""
type = self.type
# we have one "artificial" type, and it shouldn't become more than
# one; so, a simple hack should do it.
if type.upper() == "UNICODE":
type = "TEXT"
# The "str" does magic for delimited identifiers, so it's important.
items = [str(self.name), type]
if self.required:
items.append("NOT NULL")
return " ".join(items)
def getDisplayHintAsString(self):
return self._displayHint.unparse(self.displayHint)
def getLabel(self):
"""returns a short label for this column.
The label is either the tablehead or, missing it, the capitalized
column name.
"""
if self.tablehead is not None:
return self.tablehead
return str(self.name).capitalize()
def _getVOTableType(self):
"""returns the VOTable type, arraysize and xtype for this
column-like thing.
"""
type, arraysize, xtype = base.sqltypeToVOTable(self.type)
if self.type == "date":
xtype = "dachs:DATE"
return type, arraysize, xtype
class RegTest(procdef.ProcApp, unittest.TestCase):
"""A regression test.
"""
name_ = "regTest"
requiredType = "regTest"
formalArgs = "self"
runCount = 1
additionalNamesForProcs = {
"EqualingRE": EqualingRE}
_title = base.NWUnicodeAttribute("title",
default=base.Undefined,
description="A short, human-readable phrase describing what this"
" test is exercising.")
_url = base.StructAttribute("url",
childFactory=DataURL,
default=base.NotGiven,
description="The source from which to fetch the test data.")
_tags = base.StringSetAttribute("tags",
description="A list of (free-form) tags for this test. Tagged tests"
" are only run when the runner is constructed with at least one"
" of the tags given. This is mainly for restricting tags to production"
" or development servers.")
_rd = common.RDAttribute()
def __init__(self, *args, **kwargs):
unittest.TestCase.__init__(self, "fakeForPyUnit")
procdef.ProcApp.__init__(self, *args, **kwargs)
def fakeForPyUnit(self):
raise AssertionError("This is not a pyunit test right now")
@property
def description(self):
source = ""
if self.rd:
id = self.rd.sourceId
source = " (%s)"%id
return self.title+source
def retrieveData(self, serverURL, timeout):
"""returns headers and content when retrieving the resource at url.
Sets the headers and data attributes of the test instance.
"""
if self.url is base.NotGiven:
self.status, self.headers, self.data = None, None, None
else:
self.status, self.headers, self.data = self.url.retrieveResource(
serverURL, timeout=timeout)
def getDataSource(self):
"""returns a string pointing people to where data came from.
"""
if self.url is base.NotGiven:
return "(Unconditional)"
else:
return self.url.httpURL
def pointNextToLocation(self, addToPath=""):
"""arranges for the value of the location header to become the
base URL of the next test.
addToPath, if given, is appended to the location header.
If no location header was provided, the test fails.
All this of course only works for tests in sequential regSuites.
"""
if not hasattr(self, "followUp"):
raise AssertionError("pointNextToLocation only allowed within"
" sequential regSuites")
for key, value in self.headers:
if key=='location':
self.followUp.url.content_ = value+addToPath
break
else:
raise AssertionError("No location header in redirect")
@utils.document
def assertHasStrings(self, *strings):
"""checks that all its arguments are found within content.
"""
for phrase in strings:
assert phrase in self.data, "%s missing"%repr(phrase)
@utils.document
def assertLacksStrings(self, *strings):
"""checks that all its arguments are *not* found within content.
"""
for phrase in strings:
assert phrase not in self.data, "Unexpected: '%s'"%repr(phrase)
@utils.document
def assertHTTPStatus(self, expectedStatus):
assert expectedStatus==self.status, ("Bad status received, %s instead"
" of %s"%(self.status, expectedStatus))
@utils.document
def assertValidatesXSD(self):
"""checks whether the returned data are XSD valid.
As we've not yet found a python XSD validator capable enough to
deal with the complex web of schema files in the VO, this
requires a little piece of java (which also means that these tests
are fairly resource demanding). In a checkout of DaCHS, go to the
schemata subdirectory and run python makeValidator.py (this needs
a JDK as well as some external libraries; see the makeValidator source).
"""
from gavo.helpers import testtricks
msgs = testtricks.getXSDErrors(self.data)
if msgs:
raise AssertionError("Response not XSD valid. Validator output"
" starts with\n%s"%(msgs[:160]))
XPATH_NAMESPACE_MAP = {
"v": "http://www.ivoa.net/xml/VOTable/v1.3",
"v2": "http://www.ivoa.net/xml/VOTable/v1.2",
"v1": "http://www.ivoa.net/xml/VOTable/v1.1",
"o": "http://www.openarchives.org/OAI/2.0/",
"h": "http://www.w3.org/1999/xhtml",
}
@utils.document
def assertXpath(self, path, assertions):
"""checks an xpath assertion.
path is an xpath (as understood by lxml), with namespace
prefixes statically mapped; there's currently v2 (VOTable
1.2), v1 (VOTable 1.1), v (whatever VOTable version
is the current DaCHS default), h (the namespace of the
XHTML elements DaCHS generates), and o (OAI-PMH 2.0).
If you need more prefixes, hack the source and feed back
your changes (monkeypatching self.XPATH_NAMESPACE_MAP
is another option).
path must match exactly one element.
assertions is a dictionary mapping attribute names to
their expected value. Use the key None to check the
element content, and match for None if you expect an
empty element.
If you need an RE match rather than equality, there's
EqualingRE in your code's namespace.
This needs lxml (debian package python-lxml) installed.
As it's only a matter of time until lxml will become
a hard DaCHS dependency, installing it is a good idea
anyway.
"""
tree = lxtree.fromstring(self.data)
res = tree.xpath(path, namespaces=self.XPATH_NAMESPACE_MAP)
if len(res)==0:
raise AssertionError("Element not found: %s"%path)
elif len(res)!=1:
raise AssertionError("More than one item matched for %s"%path)
el = res[0]
for key, val in assertions.iteritems():
if key is None:
foundVal = el.text
else:
foundVal = el.attrib[key]
assert val==foundVal, "Trouble with %s: %s (%s, %s)"%(
key or "content", path, repr(val), repr(foundVal))
@utils.document
def assertHeader(self, key, value):
"""checks that header key has value in the response headers.
keys are compared case-insensitively, values are compared literally.
"""
try:
foundValue = getHeaderValue(self.headers, key)
self.assertEqual(foundValue, value)
except (KeyError, AssertionError):
raise AssertionError("Header %s=%s not found in %s"%(
key, value, self.headers))
@utils.document
def getFirstVOTableRow(self):
"""interprets data as a VOTable and returns the first row as a dictionary
In test use, make sure the VOTable returned is sorted, or you will get
randomly failing tests. Ideally, you'll constrain the results to just
one match; database-querying cores (which is where order is an
issue) also honor _DBOPTIONS_ORDER).
"""
data, metadata = votable.loads(self.data)
for row in metadata.iterDicts(data):
return row
@utils.document
def getVOTableRows(self):
"""parses the first table in a result VOTable and returns the contents
as a sequence of dictionaries.
"""
data, metadata = votable.loads(self.data)
return list(metadata.iterDicts(data))
class Group(base.Structure):
"""A group is a collection of columns, parameters and other groups
with a dash of metadata.
Within a group, you can refer to columns or params of the enclosing table
by their names. Nothing outside of the enclosing table can be
part of a group.
Rather than referring to params, you can also embed them into a group;
they will then *not* be present in the embedding table.
Groups may contain groups.
One application for this is grouping input keys for the form renderer.
For such groups, you probably want to give the label property (and
possibly cssClass).
"""
name_ = "group"
_name = column.ParamNameAttribute(
"name",
default=None,
description="Name of the column (must be SQL-valid for onDisk tables)",
copyable=True)
_ucd = base.UnicodeAttribute("ucd",
default=None,
description="The UCD of the group",
copyable=True)
_description = base.NWUnicodeAttribute(
"description",
default=None,
copyable=True,
description="A short (one-line) description of the group")
_utype = base.UnicodeAttribute("utype",
default=None,
description="A utype for the group",
copyable=True)
_columnRefs = base.StructListAttribute(
"columnRefs",
description="References to table columns belonging to this group",
childFactory=ColumnReference,
copyable=True)
_paramRefs = base.StructListAttribute(
"paramRefs",
description="Names of table parameters belonging to this group",
childFactory=ParameterReference,
copyable=True)
_params = common.ColumnListAttribute(
"params",
childFactory=column.Param,
description="Immediate param elements for this group (use paramref"
" to reference params defined in the parent table)",
copyable=True)
_groups = base.StructListAttribute(
"groups",
childFactory=attrdef.Recursive,
description="Sub-groups of this group (names are still referenced"
" from the enclosing table)",
copyable=True,
xmlName="group")
_props = base.PropertyAttribute(copyable=True)
@property
def table(self):
"""the table definition this group lives in.
For nested groups, this still is the ancestor table.
"""
try:
# (re) compute the table we belong to if there's no table cache
# or determination has failed so far.
if self.__tableCache is None:
raise AttributeError
except AttributeError:
# find something that has columns (presumably a table def) in our
# ancestors. I don't want to check for a TableDef instance
# since I don't want to import rscdef.table here (circular import)
# and things with column and params would work as well.
anc = self.parent
while anc:
if hasattr(anc, "columns"):
self.__tableCache = anc
break
anc = anc.parent
else:
self.__tableCache = None
return self.__tableCache
def onParentComplete(self):
"""checks that param and column names can be found in the parent table.
"""
# defer validation for sub-groups (parent group will cause validation)
if isinstance(self.parent, Group):
return
# forgo validation if the group doesn't have a table
if self.table is None:
return
try:
for col in self.iterColumns():
pass
for par in self.iterParams():
pass
except base.NotFoundError, msg:
raise base.StructureError(
"No param or field %s in found in table %s" %
(msg.what, self.table.id))
for group in self.groups:
group.onParentComplete()