class RegTestSuite(base.Structure):
"""A suite of regression tests.
"""
name_ = "regSuite"
_tests = base.StructListAttribute("tests",
childFactory=RegTest,
description="Tests making up this suite",
copyable=False)
_title = base.NWUnicodeAttribute("title",
description="A short, human-readable phrase describing what this"
" suite is about.")
_sequential = base.BooleanAttribute("sequential",
description="Set to true if the individual tests need to be run"
" in sequence.",
default=False)
def itertests(self, tags):
for test in self.tests:
if not test.tags or test.tags&tags:
yield test
def completeElement(self, ctx):
if self.title is None:
self.title = "Test suite from %s"%self.parent.sourceId
self._completeElementNext(base.Structure, ctx)
def expand(self, *args, **kwargs):
"""hand macro expansion to the RD.
"""
return self.parent.expand(*args, **kwargs)
class REGrammar(Grammar, FileRowAttributes):
"""A grammar that builds rowdicts from records and fields specified
via REs separating them.
There is also a simple facility for "cleaning up" records. This can be
used to remove standard shell-like comments; use
``recordCleaner="(?:#.*)?(.*)"``.
"""
name_ = "reGrammar"
rowIterator = REIterator
_til = base.IntAttribute(
"topIgnoredLines",
default=0,
description="Skip this many lines at the top of each source file.",
copyable=True)
_stopPat = REAttribute(
"stopPat",
default=None,
description="Stop parsing when a record *matches* this RE (this"
" is for skipping non-data footers",
copyable=True)
_recordSep = REAttribute(
"recordSep",
default=re.compile("\n"),
description="RE for separating two records in the source.",
copyable=True)
_fieldSep = REAttribute(
"fieldSep",
default=re.compile(r"\s+"),
description="RE for separating two fields in a record.",
copyable=True)
_commentPat = REAttribute(
"commentPat",
default=None,
description="RE inter-record material to be ignored (note: make this"
" match the entire comment, or you'll get random mess from partly-matched"
" comments. Use '(?m)^#.*$' for beginning-of-line hash-comments.",
copyable=True)
_recordCleaner = REAttribute(
"recordCleaner",
default=None,
description="A regular expression matched against each record."
" The matched groups in this RE are joined by blanks and used"
" as the new pattern. This can be used for simple cleaning jobs;"
" However, records not matching recordCleaner are rejected.",
copyable=True)
_names = base.StringListAttribute(
"names",
description="Names for the parsed fields, in matching sequence. You can"
r" use macros here, e.g., \\colNames{someTable}.",
expand=True,
copyable=True)
_lax = base.BooleanAttribute(
"lax",
description="allow more or less"
" fields in source records than there are names",
default=False,
copyable=True)
class KeyValueGrammar(Grammar):
"""A grammar to parse key-value pairs from files.
The default assumes one pair per line, with # comments and = as
separating character.
yieldPairs makes the grammar return an empty docdict
and {"key":, "value":} rowdicts.
Whitespace around key and value is ignored.
"""
name_ = "keyValueGrammar"
_kvSeps = base.UnicodeAttribute("kvSeparators", default=":=",
description="Characters accepted as separators between key and value")
_pairSeps = base.UnicodeAttribute("pairSeparators", default="\n",
description="Characters accepted as separators between pairs")
_cmtPat = REAttribute("commentPattern", default=re.compile("(?m)#.*"),
description="A regular expression describing comments.")
_yieldPairs = base.BooleanAttribute("yieldPairs", default=False,
description="Yield key-value pairs instead of complete records?")
_mapKeys = base.StructAttribute("mapKeys", childFactory=MapKeys,
default=None, description="Mappings to rename the keys coming from"
" the source files. Use this, in particular, if the keys are"
" not valid python identifiers.")
rowIterator = KVIterator
def onElementComplete(self):
self.recSplitter = re.compile("[%s]"%self.pairSeparators)
self.pairSplitter = re.compile("([^%s]+)[%s](.*)"%(
self.kvSeparators, self.kvSeparators))
if self.mapKeys is None:
self.mapKeys = base.makeStruct(MapKeys)
self._onElementCompleteNext(KeyValueGrammar)
class CustomGrammar(common.Grammar, base.RestrictionMixin):
"""A Grammar with a user-defined row iterator taken from a module.
See the `Writing Custom Grammars`_ (in the reference manual) for details.
"""
# To save on time when initializing the grammar (which happens at
# RD parsing time), we delay initializing the user grammar to when
# it's actually used (which happens much less frequently than loading
# the RD).
name_ = "customGrammar"
_module = rscdef.ResdirRelativeAttribute(
"module",
default=base.Undefined,
description="Path to module containing your row iterator.",
copyable=True)
_isDispatching = base.BooleanAttribute(
"isDispatching",
default=False,
description="Is this a dispatching grammar (i.e., does the row iterator"
" return pairs of role, row rather than only rows)?",
copyable=True)
def _initUserGrammar(self):
self.userModule, _ = utils.loadPythonModule(self.module)
self.rowIterator = self.userModule.RowIterator
if hasattr(self.userModule, "makeDataPack"):
self.dataPack = self.userModule.makeDataPack(self)
def parse(self, *args, **kwargs):
if not hasattr(self, "userModule"):
self._initUserGrammar()
return common.Grammar.parse(self, *args, **kwargs)
class RDParameter(base.Structure):
"""A base class for parameters.
"""
_name = base.UnicodeAttribute("key", default=base.Undefined,
description="The name of the parameter", copyable=True, strip=True,
aliases=["name"])
_descr = base.NWUnicodeAttribute("description", default=None,
description="Some human-readable description of what the"
" parameter is about", copyable=True, strip=True)
_expr = base.DataContent(description="The default for the parameter."
" The special value __NULL__ indicates a NULL (python None) as usual."
" An empty content means a non-preset parameter, which must be filled"
" in applications. The magic value __EMPTY__ allows presetting an"
" empty string.",
copyable=True, strip=True, default=base.NotGiven)
_late = base.BooleanAttribute("late", default=False,
description="Bind the name not at setup time but at applying"
" time. In rowmaker procedures, for example, this allows you to"
" refer to variables like vars or rowIter in the bindings.")
def isDefaulted(self):
return self.content_ is not base.NotGiven
def validate(self):
self._validateNext(RDParameter)
if not utils.identifierPattern.match(self.key):
raise base.LiteralParseError("name", self.key, hint=
"The name you supplied was not defined by any procedure definition.")
def completeElement(self, ctx):
if self.content_=="__EMPTY__":
self.content_ = ""
self._completeElementNext(RDParameter, ctx)
class CDFHeaderGrammar(Grammar):
"""A grammar that returns the header dictionary of a CDF file
(global attributes).
This grammar yields a single dictionary per file, which corresponds
to the global attributes. The values in this dictionary may have
complex structure; in particular, sequences are returned as lists.
To use this grammar, additional software is required that (by 2014)
is not packaged for Debian. See
http://spacepy.lanl.gov/doc/install_linux.html for installation
instructions. Note that you must install the CDF library itself as
described further down on that page; the default installation
instructions do not install the library in a public place, so if
you use these, you'll have to set CDF_LIB to the right value, too.
"""
name_ = "cdfHeaderGrammar"
_mapKeys = base.StructAttribute("mapKeys", childFactory=MapKeys,
default=None, copyable=True, description="Prescription for how to"
" map labels keys to grammar dictionary keys")
_autoAtomize = base.BooleanAttribute("autoAtomize",
default=False, copyable=True, description="Unpack 1-element"
" lists to their first value.")
rowIterator = CDFHeaderIterator
def onElementComplete(self):
if self.mapKeys is None:
self.mapKeys = base.makeStruct(MapKeys)
self._onElementCompleteNext(CDFHeaderGrammar)
class DictlistGrammar(Grammar):
"""A grammar that "parses" from lists of dicts.
Actually, it will just return the dicts as they are passed. This is
mostly useful internally, though it might come in handy in custom code.
"""
name_ = "dictlistGrammar"
rowIterator = ListIterator
_asPars = base.BooleanAttribute(
"asPars",
default=False,
description=
"Just return the first item of the list as parameters row and exit?")
class FixedQueryCore(core.Core, base.RestrictionMixin):
"""A core executing a predefined query.
This usually is not what you want, unless you want to expose the current
results of a specific query, e.g., for log or event data.
"""
name_ = "fixedQueryCore"
_timeout = base.FloatAttribute("timeout", default=15., description=
"Seconds until the query is aborted")
_query = base.UnicodeAttribute("query", default=base.Undefined,
description="The query to be executed. You must define the"
" output fields in the core's output table. The query will"
" be macro-expanded in the resource descriptor.")
_writable = base.BooleanAttribute("writable", default=False,
description="Use a writable DB connection?")
def completeElement(self, ctx):
if self.inputTable is base.NotGiven:
self.inputTable = base.makeStruct(inputdef.InputTable)
self._completeElementNext(FixedQueryCore, ctx)
def run(self, service, inputTable, queryMeta):
if self.writable:
connFactory = base.getWritableTableConn
else:
connFactory = base.getTableConn
with base.AdhocQuerier(connFactory) as querier:
try:
cursor = querier.query(
self.rd.expand(self.query), timeout=self.timeout)
if cursor.description is None:
return self._parseOutput([], queryMeta)
else:
return self._parseOutput(list(cursor), queryMeta)
except:
mapDBErrors(*sys.exc_info())
def _parseOutput(self, dbResponse, queryMeta):
"""builds an InternalDataSet out of dbResponse and the outputFields
of our service.
"""
if dbResponse is None:
dbResponse = []
queryMeta["Matched"] = len(dbResponse)
fieldNames = self.outputTable.dictKeys
return rsc.TableForDef(self.outputTable,
rows=[dict((k,v)
for k,v in itertools.izip(fieldNames, row))
for row in dbResponse])
class Script(base.Structure, base.RestrictionMixin):
"""A script, i.e., some executable item within a resource descriptor.
The content of scripts is given by their type -- usually, they are
either python scripts or SQL with special rules for breaking the
script into individual statements (which are basically like python's).
The special language AC_SQL is like SQL, but execution errors are
ignored. This is not what you want for most data RDs (it's intended
for housekeeping scripts).
See `Scripting`_.
"""
name_ = "script"
typeDesc_ = "Embedded executable code with a type definition"
_lang = base.EnumeratedUnicodeAttribute(
"lang",
default=base.Undefined,
description="Language of the script.",
validValues=["SQL", "python", "AC_SQL"],
copyable=True)
_type = base.EnumeratedUnicodeAttribute(
"type",
default=base.Undefined,
description="Point of time at which script is to run.",
validValues=[
"preImport", "newSource", "preIndex", "postCreation", "beforeDrop",
"sourceDone"
],
copyable=True)
_name = base.UnicodeAttribute(
"name",
default="anonymous",
description="A human-consumable designation of the script.",
copyable=True)
_notify = base.BooleanAttribute(
"notify",
default=True,
description="Send out a notification when running this"
" script.",
copyable=True)
_content = base.DataContent(copyable=True, description="The script body.")
_original = base.OriginalAttribute()
def getSource(self):
"""returns the content with all macros expanded.
"""
return self.parent.getExpander().expand(self.content_)
class FreeREGrammar(common.Grammar):
"""A grammar allowing "free" regular expressions to parse a document.
Basically, you give a rowProduction to match individual records in the
document. All matches of rowProduction will then be matched with
parseRE, which in turn must have named groups. The dictionary from
named groups to their matches makes up the input row.
For writing the parseRE, we recommend writing an element, using a
CDATA construct, and taking advantage of python's "verbose" regular
expressions. Here's an example::
<parseRE><![CDATA[(?xsm)^name::(?P<name>.*)
^query::(?P<query>.*)
^description::(?P<description>.*)\.\.
]]></parseRE>
"""
name_ = "freeREGrammar"
_rowProduction = regrammar.REAttribute("rowProduction",
default=re.compile(r"(?m)^.+$\n"),
description="RE matching a complete"
" record.")
_parseRE = regrammar.REAttribute(
"parseRE",
default=base.Undefined,
description="RE containing named groups matching a record")
_stripTokens = base.BooleanAttribute(
"stripTokens",
default=False,
description="Strip whitespace from result tokens?")
_ignoreJunk = base.BooleanAttribute(
"ignoreJunk",
default=False,
description="Ignore everything outside of the row production")
rowIterator = RowIterator
class VOTableGrammar(common.Grammar):
"""A grammar parsing from VOTables.
Currently, the PARAM fields are ignored, only the data rows are
returned.
voTableGrammars result in typed records, i.e., values normally come
in the types they are supposed to have.
"""
name_ = "voTableGrammar"
_gunzip = base.BooleanAttribute("gunzip",
description="Unzip sources"
" while reading?",
default=False)
rowIterator = VOTableRowIterator
class IgnoreOn(ConditionBase):
"""A condition on a row that, if true, causes the row to be dropped.
Here, you can set bail to abort an import when the condition is met
rather than just dropping the row.
"""
name_ = "ignoreOn"
_bail = base.BooleanAttribute("bail",
default=False,
description="Abort when condition is met?")
def __call__(self, row):
conditionMet = ConditionBase.__call__(self, row)
if self.bail and conditionMet:
raise TriggerPulled(
"Trigger %s satisfied and set to bail" % self.name, self.name)
return conditionMet
class CSVGrammar(Grammar, FileRowAttributes):
"""A grammar that uses python's csv module to parse files.
Note that these grammars by default interpret the first line of
the input file as the column names. When your files don't follow
that convention, you *must* give names (as in ``names='raj2000,
dej2000, magV'``), or you'll lose the first line and have silly
column names.
CSVGrammars currently do not support non-ASCII inputs.
Contact the authors if you need that.
"""
name_ = "csvGrammar"
_delimiter = base.UnicodeAttribute("delimiter",
description="CSV delimiter",
default=",",
copyable=True)
_names = base.StringListAttribute(
"names",
default=None,
description="Names for the parsed fields, in sequence of the"
" comma separated values. The default is to read the field names"
" from the first line of the csv file. You can use macros here,"
r" e.g., \\colNames{someTable}.",
expand=True,
copyable=True)
_strip = base.BooleanAttribute(
"strip",
default=False,
description="If True, whitespace immediately following a delimiter"
" is ignored.",
copyable=True)
_til = base.IntAttribute(
"topIgnoredLines",
default=0,
description="Skip this many lines at the top of each source file.")
rowIterator = CSVIterator
class ColumnBase(base.Structure, base.MetaMixin):
"""A base class for columns, parameters, output fields, etc.
Actually, right now there's far too much cruft in here that
should go into Column proper or still somewhere else. Hence:
XXX TODO: Refactor.
See also Column for a docstring that still applies to all we've in
here.
"""
_name = ParamNameAttribute("name",
default=base.Undefined,
description="Name of the param",
copyable=True,
before="type")
_type = TypeNameAttribute(
"type",
default="real",
description="datatype for the column (SQL-like type system)",
copyable=True,
before="unit")
_unit = base.UnicodeAttribute("unit",
default="",
description="Unit of the values",
copyable=True,
before="ucd",
strip=True)
_ucd = base.UnicodeAttribute("ucd",
default="",
description="UCD of the column",
copyable=True,
before="description")
_description = base.NWUnicodeAttribute(
"description",
default="",
copyable=True,
description=
"A short (one-line) description of the values in this column.")
_tablehead = base.UnicodeAttribute(
"tablehead",
default=None,
description="Terse phrase to put into table headers for this"
" column",
copyable=True)
_utype = base.UnicodeAttribute("utype",
default=None,
description="utype for this column",
copyable=True)
_required = base.BooleanAttribute(
"required",
default=False,
description="Record becomes invalid when this column is NULL",
copyable=True)
_displayHint = DisplayHintAttribute(
"displayHint",
description="Suggested presentation; the format is "
" <kw>=<value>{,<kw>=<value>}, where what is interpreted depends"
" on the output format. See, e.g., documentation on HTML renderers"
" and the formatter child of outputFields.",
copyable=True)
_verbLevel = base.IntAttribute(
"verbLevel",
default=20,
description="Minimal verbosity level at which to include this column",
copyable=True)
_values = base.StructAttribute("values",
default=None,
childFactory=Values,
description="Specification of legal values",
copyable=True)
_fixup = base.UnicodeAttribute(
"fixup",
description=
"A python expression the value of which will replace this column's"
" value on database reads. Write a ___ to access the original"
' value. You can use macros for the embedding table.'
' This is for, e.g., simple URL generation'
' (fixup="\'\\internallink{/this/svc}\'+___").'
' It will *only* kick in when tuples are deserialized from the'
" database, i.e., *not* for values taken from tables in memory.",
default=None,
copyable=True)
_note = base.UnicodeAttribute(
"note",
description="Reference to a note meta"
" on this table explaining more about this column",
default=None,
copyable=True)
_xtype = base.UnicodeAttribute("xtype",
description="VOTable xtype giving"
" the serialization form",
default=None,
copyable=True)
_stc = TableManagedAttribute(
"stc",
description="Internally used"
" STC information for this column (do not assign to unless instructed"
" to do so)",
default=None,
copyable=True)
_stcUtype = TableManagedAttribute(
"stcUtype",
description="Internally used"
" STC information for this column (do not assign to)",
default=None,
copyable=True)
_properties = base.PropertyAttribute(copyable=True)
_original = base.OriginalAttribute()
restrictedMode = False
def __repr__(self):
return "<Column %s>" % repr(self.name)
def setMetaParent(self, parent):
# columns should *not* take part in meta inheritance. The reason is
# that there are usually many columns to a table, and there's no
# way I can see that any piece of metadata should be repeated in
# all of them. On the other hand, for votlinks (no name an example),
# meta inheritance would have disastrous consequences.
# So, we bend the rules a bit.
raise base.StructureError(
"Columns may not have meta parents.",
hint="The rationale for this is explained in rscdef/column.py,"
" look for setMetaParent.")
def onParentComplete(self):
# we need to resolve note on construction since columns are routinely
# copied to other tables and meta info does not necessarily follow.
if isinstance(self.note, basestring):
try:
self.note = self.parent.getNote(self.note)
except base.NotFoundError: # non-existing notes silently ignored
self.note = None
def completeElement(self, ctx):
self.restrictedMode = getattr(ctx, "restricted", False)
if isinstance(self.name, utils.QuotedName):
self.key = self.name.name
if ')' in self.key:
# No '()' allowed in key for that breaks the %()s syntax (sigh!).
# Work around with the following quick hack that would break
# if people carefully chose proper names. Anyone using delim.
# ids in SQL deserves a good spanking anyway.
self.key = self.key.replace(')', "__").replace('(', "__")
else:
self.key = self.name
self._completeElementNext(ColumnBase, ctx)
def isEnumerated(self):
return self.values and self.values.options
def validate(self):
self._validateNext(ColumnBase)
if self.restrictedMode and self.fixup:
raise base.RestrictedElement("fixup")
def validateValue(self, value):
"""raises a ValidationError if value does not match the constraints
given here.
"""
if value is None:
if self.required:
raise base.ValidationError(
"Field %s is empty but non-optional" % self.name,
self.name)
return
# Only validate these if we're not a database column
if not isinstance(self, Column):
vals = self.values
if vals:
if vals.options:
if value and not vals.validateOptions(value):
raise base.ValidationError(
"Value %s not consistent with"
" legal values %s" % (value, vals.options),
self.name)
else:
if vals.min and value < vals.min:
raise base.ValidationError(
"%s too small (must be at least %s)" %
(value, vals.min), self.name)
if vals.max and value > vals.max:
raise base.ValidationError(
"%s too large (must be less than %s)" %
(value, vals.max), self.name)
def isIndexed(self):
"""returns a guess as to whether this column is part of an index.
This may return True, False, or None (unknown).
"""
if self.parent and hasattr(self.parent, "indexedColumns"):
# parent is something like a TableDef
if self.name in self.parent.indexedColumns:
return True
else:
return False
def isPrimary(self):
"""returns a guess as to whether this column is a primary key of the
embedding table.
This may return True, False, or None (unknown).
"""
if self.parent and hasattr(self.parent, "primary"):
# parent is something like a TableDef
if self.name in self.parent.primary:
return True
else:
return False
_indexedCleartext = {
True: "indexed",
False: "notIndexed",
None: "unknown",
}
def asInfoDict(self):
"""returns a dictionary of certain, "user-interesting" properties
of the data field, in a dict of strings.
"""
return {
"name": unicode(self.name),
"description": self.description or "N/A",
"tablehead": self.getLabel(),
"unit": self.unit or "N/A",
"ucd": self.ucd or "N/A",
"verbLevel": self.verbLevel,
"indexState": self._indexedCleartext[self.isIndexed()],
"note": self.note,
}
def getDDL(self):
"""returns an SQL fragment describing this column ready for
inclusion in a DDL statement.
"""
type = self.type
# we have one "artificial" type, and it shouldn't become more than
# one; so, a simple hack should do it.
if type.upper() == "UNICODE":
type = "TEXT"
# The "str" does magic for delimited identifiers, so it's important.
items = [str(self.name), type]
if self.required:
items.append("NOT NULL")
return " ".join(items)
def getDisplayHintAsString(self):
return self._displayHint.unparse(self.displayHint)
def getLabel(self):
"""returns a short label for this column.
The label is either the tablehead or, missing it, the capitalized
column name.
"""
if self.tablehead is not None:
return self.tablehead
return str(self.name).capitalize()
def _getVOTableType(self):
"""returns the VOTable type, arraysize and xtype for this
column-like thing.
"""
type, arraysize, xtype = base.sqltypeToVOTable(self.type)
if self.type == "date":
xtype = "dachs:DATE"
return type, arraysize, xtype
class Values(base.Structure):
"""Information on a column's values, in particular its domain.
This is quite like the values element in a VOTable. In particular,
to accomodate VOTable usage, we require nullLiteral to be a valid literal
for the parent's type.
Note that DaCHS does not validate for contraints from values on
table import. This is mainly because before ``gavo values`` has run,
values may not represent the new dataset in semiautomatic values.
With HTTP parameters, values validation does take place (but again,
that's mostly not too helpful because there are query languages
sitting in between most of the time).
Hence, the main utility of values is metadata declaration, both
in the form render (where they become placeholders) and in datalink
(where they are communicated as VOTable values).
"""
name_ = "values"
_min = base.UnicodeAttribute("min",
default=None,
description="Minimum acceptable"
" value as a datatype literal",
copyable=True)
_max = base.UnicodeAttribute("max",
default=None,
description="Maximum acceptable"
" value as a datatype literal",
copyable=True)
_options = base.StructListAttribute(
"options",
childFactory=Option,
description="List of acceptable values (if set)",
copyable=True)
_default = base.UnicodeAttribute(
"default",
default=None,
description="A default"
" value (currently only used for options).",
copyable=True)
_nullLiteral = base.UnicodeAttribute(
"nullLiteral",
default=None,
description=
"An appropriate value representing a NULL for this column in VOTables"
" and similar places. You usually should only set it for integer"
" types and chars. Note that rowmakers make no use of this nullLiteral,"
" i.e., you can and should choose null values independently of"
" your source. Again, for reals, floats and (mostly) text you probably"
" do not want to do this.",
copyable=True)
_multiOk = base.BooleanAttribute("multiOk",
False, "Deprecated, use"
" multiplicity=multiple instead.",
copyable=True)
_fromDB = base.ActionAttribute(
"fromdb",
"_evaluateFromDB",
description=
"A query fragment returning just one column to fill options from (will"
" add to options if some are given). Do not write SELECT or anything,"
" just the column name and the where clause.")
_original = base.OriginalAttribute()
validValues = None
@classmethod
def fromOptions(cls, labels):
"""returns Values with the elements of labels as valid options.
"""
return base.makeStruct(
cls, options=[base.makeStruct(Option, content_=l) for l in labels])
def makePythonVal(self, literal, sqltype):
return typesystems.sqltypeToPython(sqltype)(literal)
def _evaluateFromDB(self, ctx):
if not getattr(ctx, "doQueries", True):
return
try:
with base.getTableConn() as conn:
for row in conn.query(
self.parent.parent.expand("SELECT DISTINCT %s" %
(self.fromdb))):
self._options.feedObject(
self, base.makeStruct(Option, content_=row[0]))
except base.DBError: # Table probably doesn't exist yet, ignore.
base.ui.notifyWarning("Values fromdb '%s' failed, ignoring" %
self.fromdb)
def onParentComplete(self):
"""converts min, max, and options from string literals to python
objects.
"""
dataField = self.parent
# It would be nicer if we could handle this in properties for min etc, but
# unfortunately parent might not be complete then. The only
# way around that would have been delegations from Column, and that's
# not very attractive either.
if self.min:
self.min = self.makePythonVal(self.min, dataField.type)
if self.max:
self.max = self.makePythonVal(self.max, dataField.type)
if self.options:
dbt = dataField.type
for opt in self.options:
opt.content_ = self.makePythonVal(opt.content_, dbt)
self.validValues = set([o.content_ for o in self.options])
if self.nullLiteral:
try:
self.makePythonVal(self.nullLiteral, dataField.type)
except ValueError:
raise base.LiteralParseError(
"nullLiteral",
self.nullLiteral,
hint="If you want to *parse* whatever you gave into a NULL,"
" use the parseWithNull function in a rowmaker. The null"
" literal gives what value will be used for null values"
" when serializing to VOTables and the like.")
if self.default and isinstance(self.default, basestring):
type, arraysize, xtype = dataField._getVOTableType()
self.default = paramval.getVOTParser(type, arraysize,
xtype)(self.default)
def validateOptions(self, value):
"""returns false if value isn't either in options or doesn't consist of
items in options.
Various null values always validate here; non-null checking is done
by the column on its required attribute.
"""
if value == "None":
assert False, "Literal 'None' passed as a value to validateOptions"
if self.validValues is None:
return True
if isinstance(value, (list, tuple, set)):
for val in value:
if val and not val in self.validValues:
return False
else:
return value in self.validValues or value is None
return True
class SourceSpec(base.Structure):
"""A Specification of a data descriptor's inputs.
This will typcially be files taken from a file system. If so, DaCHS will,
in each directory, process the files in alphabetical order. No guarantees
are made as to the sequence directories are processed in.
Multiple patterns are processed in the order given in the RD.
"""
name_ = "sources"
_patterns = base.ListOfAtomsAttribute("patterns", description=
"Paths to the source files. You can use shell patterns here.",
itemAttD=base.UnicodeAttribute("pattern", description="Shell pattern"
" for source file(s), relative to resource directory."),
copyable=True)
_items = base.ListOfAtomsAttribute("items", description=
"String literals to pass to grammars. In contrast to patterns,"
" they are not interpreted as file names but passed to the"
" grammar verbatim. Normal grammars do not like this. It is"
" mainly intended for use with custom or null grammars.",
itemAttD=base.UnicodeAttribute("item",
description="Grammar-specific string"), copyable=True)
_recurse = base.BooleanAttribute("recurse", default=False,
description="Search for pattern(s) recursively in their directory"
" part(s)?", copyable=True)
_ignore = base.StructAttribute("ignoredSources", childFactory=
IgnoreSpec, description="Specification of sources that should not"
" be processed although they match patterns. Typically used"
" in update-type data descriptors.", copyable=True)
_file = base.DataContent(description="A single"
" file name (this is for convenience)", copyable="True")
_original = base.OriginalAttribute()
def __iter__(self):
return self.iterSources()
def completeElement(self, ctx):
if self.ignoredSources is base.Undefined:
self.ignoredSources = base.makeStruct(IgnoreSpec)
self._completeElementNext(SourceSpec, ctx)
def _expandDirParts(self, dirParts, ignoreDotDirs=True):
"""expands a list of directories into a list of them and all their
descendants.
It follows symbolic links but doesn't do any bookkeeping, so bad
things will happen if the directory graph contains cycles.
"""
res = []
for root in dirParts:
for root, dirs, files in os.walk(root):
if ignoreDotDirs:
if os.path.basename(root).startswith("."):
continue
dirs = [dir for dir in dirs if not dir.startswith(".")]
dirs = (os.path.join(root, dir) for dir in dirs)
res.extend(dir for dir in dirs if os.path.isdir(dir))
for child in files:
destName = os.path.join(root, child)
if os.path.islink(destName) and not os.path.isfile(destName):
res.extend(self._expandDirParts(destName))
return res
def iterSources(self, connection=None):
self.ignoredSources.prepare(connection)
for item in self.items:
if not self.ignoredSources.isIgnored(item):
yield item
baseDir = ""
if self.parent.rd:
baseDir = self.parent.rd.resdir
for pattern in self.patterns:
dirPart, baseName = os.path.split(pattern)
if self.parent.rd:
dirParts = [os.path.join(baseDir, dirPart)]
else:
dirParts = [dirPart]
if self.recurse:
dirParts = dirParts+self._expandDirParts(dirParts)
for dir in sorted(dirParts):
for name in sorted(glob.glob(os.path.join(dir, baseName))):
fullName = os.path.abspath(name)
if not self.ignoredSources.isIgnored(fullName):
yield fullName
if self.content_:
yield os.path.abspath(os.path.join(baseDir, self.content_))
def __nonzero__(self):
return (not not self.patterns) or (not not self.items
) or (not not self.content_)
class DataDescriptor(base.Structure, base.ComputedMetaMixin,
common.IVOMetaMixin, tabledef.PublishableDataMixin):
"""A description of how to process data from a given set of sources.
Data descriptors bring together a grammar, a source specification and
"makes", each giving a table and a rowmaker to feed the table from the
grammar output.
They are the "executable" parts of a resource descriptor. Their ids
are used as arguments to gavoimp for partial imports.
"""
name_ = "data"
resType = "data"
_rowmakers = base.StructListAttribute("rowmakers",
childFactory=rmkdef.RowmakerDef,
description="Embedded build rules (usually rowmakers are defined toplevel)",
copyable=True,
before="makes")
_tables = base.StructListAttribute("tables",
childFactory=tabledef.TableDef,
description="Embedded table definitions (usually, tables are defined"
" toplevel)",
copyable=True,
before="makes")
_grammar = base.MultiStructAttribute("grammar",
default=None,
childFactory=builtingrammars.getGrammar,
childNames=builtingrammars.GRAMMAR_REGISTRY.keys(),
description="Grammar used to parse this data set.",
copyable=True,
before="makes")
_sources = base.StructAttribute("sources",
default=None,
childFactory=SourceSpec,
description="Specification of sources that should be fed to the grammar.",
copyable=True,
before="grammar")
_dependents = base.ListOfAtomsAttribute("dependents",
itemAttD=base.UnicodeAttribute("recreateAfter"),
description="A data ID to recreate when this resource is"
" remade; use # syntax to reference in other RDs.")
_auto = base.BooleanAttribute("auto",
default=True,
description="Import this data set if not explicitly"
" mentioned on the command line?")
_updating = base.BooleanAttribute("updating",
default=False,
description="Keep existing tables on import? You usually want this"
" False unless you have some kind of sources management,"
" e.g., via a sources ignore specification.",
copyable=True)
_makes = base.StructListAttribute("makes",
childFactory=Make,
copyable=True,
description="Specification of a target table and the rowmaker"
" to feed them.")
_params = common.ColumnListAttribute("params",
childFactory=column.Param,
description='Param ("global columns") for this data (mostly for'
' VOTable serialization).',
copyable=True)
_properties = base.PropertyAttribute()
_rd = common.RDAttribute()
_original = base.OriginalAttribute()
metaModel = ("title(1), creationDate(1), description(1),"
"subject, referenceURL(1)")
def __repr__(self):
return "<data descriptor with id %s>"%self.id
def validate(self):
self._validateNext(DataDescriptor)
if self.registration and self.id is None:
raise base.StructureError("Published data needs an assigned id.")
def onElementComplete(self):
self._onElementCompleteNext(DataDescriptor)
for t in self.tables:
t.setMetaParent(self)
if self.registration:
self.registration.register()
# since we want to be able to create DDs dynamically , they must find their
# meta parent themselves. We do this while the DD is being adopted;
# the rules here are: if the parent is a meta mixin itself, it's the
# meta parent, if it has an rd attribute, use that, else give up.
# TODO: For DDs on cores, it would be *desirable* to come up
# with some magic that makes the current service their meta parent.
def _getParent(self):
return self.__parent
def _setParent(self, value):
self.__parent = value
if isinstance(value, base.MetaMixin):
self.setMetaParent(value)
elif hasattr(value, "rd"):
self.setMetaParent(value.rd)
parent = property(_getParent, _setParent)
def iterSources(self, connection=None):
if self.sources:
return self.sources.iterSources(connection)
else:
return iter([])
def __iter__(self):
for m in self.makes:
yield m.table
def iterTableDefs(self):
"""iterates over the definitions of all the tables built by this DD.
"""
for m in self.makes:
yield m.table
def getTableDefById(self, id):
for td in self.iterTableDefs():
if td.id==id:
return td
raise base.StructureError("No table name %s will be built"%id)
def getTableDefWithRole(self, role):
for m in self.makes:
if m.role==role:
return m.table
raise base.StructureError("No table def with role '%s'"%role)
def getPrimary(self):
"""returns the "primary" table definition in the data descriptor.
"primary" means the only table in a one-table dd, the table with the
role "primary" if there are more. If no matching table is found, a
StructureError is raised.
"""
if len(self.makes)==1:
return self.makes[0].table
else:
try:
return self.getTableDefWithRole("primary")
except base.StructureError: # raise more telling message
pass
raise base.StructureError("Ambiguous request for primary table")
def copyShallowly(self):
"""returns a shallow copy of self.
Sources are not copied.
"""
return DataDescriptor(self.parent, rowmakers=self.rowmakers[:],
tables=self.tables[:], grammar=self.grammar, makes=self.makes[:])
def getURL(self, rendName, absolute=True):
# there's no sensible URL for DDs; thus, let people browse
# the RD info. At least they should find links to any tables
# included here there.
basePath = "%sbrowse/%s"%(
base.getConfig("web", "nevowRoot"),
self.rd.sourceId)
if absolute:
return base.getConfig("web", "serverURL")+basePath
return basePath
class DBCore(TableBasedCore):
"""A core performing database queries on one table or view.
DBCores ask the service for the desired output schema and adapt their
output. The DBCore's output table, on the other hand, lists all fields
available from the queried table.
"""
name_ = "dbCore"
_sortKey = base.UnicodeAttribute("sortKey",
description="A pre-defined sort order (suppresses DB options widget)."
" The sort key accepts multiple columns, separated by commas.",
copyable=True)
_limit = base.IntAttribute("limit", description="A pre-defined"
" match limit (suppresses DB options widget).", copyable=True)
_distinct = base.BooleanAttribute("distinct", description="Add a"
" 'distinct' modifier to the query?", default=False, copyable=True)
_groupBy = base.UnicodeAttribute("groupBy", description=
"A group by clause. You shouldn't generally need this, and if"
" you use it, you must give an outputTable to your core.",
default=None)
def wantsTableWidget(self):
return self.sortKey is None and self.limit is None
def getQueryCols(self, service, queryMeta):
"""returns the fields we need in the output table.
The normal DbBased core just returns whatever the service wants.
Derived cores, e.g., for special protocols, could override this
to make sure they have some fields in the result they depend on.
"""
return service.getCurOutputFields(queryMeta)
def _runQuery(self, resultTableDef, fragment, pars, queryMeta,
**kwargs):
with base.getTableConn() as conn:
queriedTable = rsc.TableForDef(self.queriedTable, nometa=True,
create=False, connection=conn)
queriedTable.setTimeout(queryMeta["timeout"])
if fragment and pars:
resultTableDef.addMeta("info", repr(pars),
infoName="queryPars", infoValue=fragment)
iqArgs = {"limits": queryMeta.asSQL(), "distinct": self.distinct,
"groupBy": self.groupBy}
iqArgs.update(kwargs)
try:
try:
return self._makeTable(
queriedTable.iterQuery(resultTableDef, fragment, pars,
**iqArgs), resultTableDef, queryMeta)
except:
mapDBErrors(*sys.exc_info())
finally:
queriedTable.close()
def _makeResultTableDef(self, service, inputTable, queryMeta):
"""returns an OutputTableDef object for querying our table with queryMeta.
"""
return base.makeStruct(outputdef.OutputTableDef,
parent_=self.queriedTable.parent, id="result",
onDisk=False, columns=self.getQueryCols(service, queryMeta),
params=self.queriedTable.params)
def run(self, service, inputTable, queryMeta):
"""does the DB query and returns an InMemoryTable containing
the result.
"""
resultTableDef = self._makeResultTableDef(
service, inputTable, queryMeta)
resultTableDef.copyMetaFrom(self.queriedTable)
if not resultTableDef.columns:
raise base.ValidationError("No output columns with these settings."
"_OUTPUT")
sortKeys = None
if self.sortKey:
sortKeys = self.sortKey.split(",")
queryMeta.overrideDbOptions(limit=self.limit, sortKeys=sortKeys,
sortFallback=self.getProperty("defaultSortKey", None))
try:
fragment, pars = self._getSQLWhere(inputTable, queryMeta)
except base.LiteralParseError, ex:
raise base.ui.logOldExc(base.ValidationError(str(ex),
colName=ex.attName))
queryMeta["sqlQueryPars"] = pars
return self._runQuery(resultTableDef, fragment, pars, queryMeta)
class ContextGrammar(grammars.Grammar):
"""A grammar for web inputs.
These are almost exclusively in InputDDs. They hold InputKeys
defining what they take from the context.
For DBCores, the InputDDs are generally defined implicitely
via CondDescs. Thus, only for other cores will you ever need
to bother with ContextGrammars (unless you're going for special
effects).
The source tokens for context grammars are dictionaries; these
are either typed dictionaries from nevow, where the values
usually are atomic, or, preferably, the dictionaries of lists
from request.args.
ContextGrammars only yield rows if there's a rowKey defined.
In that case, an outer join of all other parameters is returned;
with rowKey defined, the input keys are obtained from the table's
columns.
In normal usage, they just yield a single parameter row,
corresponding to the source dictionary possibly completed with
defaults, where non-requried input keys get None defaults where not
given. Missing required parameters yield errors.
Since most VO protocols require case-insensitive matching of parameter
names, matching of input key names and the keys of the input dictionary
is attempted first literally, then disregarding case.
"""
name_ = "contextGrammar"
_inputTable = base.ReferenceAttribute(
"inputTable",
default=base.NotGiven,
description="The table that is to be built using this grammar",
copyable=True)
_inputKeys = rscdef.ColumnListAttribute(
"inputKeys",
childFactory=InputKey,
description="Input keys this context grammar should parse."
" These must not be given if there is an input table defined.")
_rowKey = base.UnicodeAttribute(
"rowKey",
default=base.NotGiven,
description="The name of a key that is used to generate"
" rows from the input",
copyable=True)
_rejectExtras = base.BooleanAttribute(
"rejectExtras",
default=False,
description="If true, the grammar will reject extra input parameters."
" Note that for form-based services, there *are* extra parameters"
" not declared in the services' input tables. Right now,"
" contextGrammar does not ignore those.",
copyable=True)
_original = base.OriginalAttribute("original")
rowIterator = ContextRowIterator
def onElementComplete(self):
if self.inputTable is not base.NotGiven:
if self.inputKeys != []:
raise base.StructureError(
"InputKeys and inputTable must not"
" both be given in a context grammar")
else:
if self.rowKey:
self.inputKeys = [
InputKey.fromColumn(c) for c in self.inputTable.columns
]
else:
self.inputKeys = self.inputTable.params
else:
columns = []
if self.rowKey:
columns = self.inputKeys
self.inputTable = MS(InputTable,
params=self.inputKeys,
columns=columns)
self.defaults = {}
for ik in self.iterInputKeys():
if not ik.required:
self.defaults[ik.name] = None
if ik.value is not None:
self.defaults[ik.name] = ik.value
self._onElementCompleteNext(ContextGrammar)
def iterInputKeys(self):
for ik in self.inputKeys:
yield ik
class EmbeddedGrammar(common.Grammar, base.RestrictionMixin):
"""A Grammar defined by a code application.
To define this grammar, write a ProcApp iterator leading to code yielding
row dictionaries. The grammar input is available as self.sourceToken;
for normal grammars within data elements, that would be a fully
qualified file name.
Grammars can also return one "parameter" dictionary per source (the
input to a make's parmaker). In an embedded grammar, you can define
a pargetter to do that. It works like the iterator, except that
it returns a single dictionary rather than yielding several of them.
This could look like this, when the grammar input is some iterable::
<embeddedGrammar>
<iterator>
<setup>
<code>
testData = "a"*1024
</code>
</setup>
<code>
for i in self.sourceToken:
yield {'index': i, 'data': testData}
</code>
</iterator>
</embeddedGrammar>
"""
name_ = "embeddedGrammar"
_iterator = base.StructAttribute(
"iterator",
default=base.Undefined,
childFactory=EmbeddedIterator,
description="Code yielding row dictionaries",
copyable=True)
_pargetter = base.StructAttribute(
"pargetter",
default=None,
childFactory=EmbeddedPargetter,
description="Code returning a parameter dictionary",
copyable=True)
_isDispatching = base.BooleanAttribute(
"isDispatching",
default=False,
description="Is this a dispatching grammar (i.e., does the row iterator"
" return pairs of role, row rather than only rows)?",
copyable=True)
_notify = base.BooleanAttribute(
"notify",
default=False,
description="Enable notification of begin/end of processing (as"
" for other grammars; embedded grammars often have odd source"
" tokens for which you don't want that).",
copyable=True)
def onElementComplete(self):
self._onElementCompleteNext(EmbeddedGrammar)
class RowIterator(common.RowIterator):
_iterRows = self.iterator.compile()
notify = self.notify
if self.pargetter:
RowIterator.getParameters = self.pargetter.compile()
self.rowIterator = RowIterator
class CondDesc(base.Structure):
"""A query specification for cores talking to the database.
CondDescs define inputs as a sequence of InputKeys (see `Element InputKey`_).
Internally, the values in the InputKeys can be translated to SQL.
"""
name_ = "condDesc"
_inputKeys = rscdef.ColumnListAttribute("inputKeys",
childFactory=inputdef.InputKey,
description="One or more InputKeys defining the condition's input.",
copyable=True)
_silent = base.BooleanAttribute("silent",
default=False,
description="Do not produce SQL from this CondDesc. This"
" can be used to convey meta information to the core. However,"
" in general, a service is a more appropriate place to deal with"
" such information, and thus you should prefer service InputKeys"
" to silent CondDescs.",
copyable=True)
_required = base.BooleanAttribute("required",
default=False,
description="Reject queries not filling the InputKeys of this CondDesc",
copyable=True)
_fixedSQL = base.UnicodeAttribute("fixedSQL",
default=None,
description="Always insert this SQL statement into the query. Deprecated.",
copyable=True)
_buildFrom = base.ReferenceAttribute("buildFrom",
description="A reference to a column or an InputKey to define"
" this CondDesc",
default=None)
_phraseMaker = base.StructAttribute("phraseMaker",
default=None,
description="Code to generate custom SQL from the input keys",
childFactory=PhraseMaker,
copyable=True)
_combining = base.BooleanAttribute("combining",
default=False,
description="Allow some input keys to be missing when others are given?"
" (you want this for pseudo-condDescs just collecting random input"
" keys)", # (and I wish I had a better idea)
copyable="True")
_group = base.StructAttribute("group",
default=None,
childFactory=rscdef.Group,
description="Group child input keys in the input table (primarily"
" interesting for web forms, where this grouping is shown graphically;"
" Set the style property to compact to have a one-line group there)")
_joiner = base.UnicodeAttribute("joiner",
default="OR",
description="When yielding multiple fragments, join them"
" using this operator (probably the only thing besides OR is"
" AND).",
copyable=True)
_original = base.OriginalAttribute()
def __init__(self, parent, **kwargs):
base.Structure.__init__(self, parent, **kwargs)
# copy parent's resolveName if present for buildFrom resolution
if hasattr(self.parent, "resolveName"):
self.resolveName = self.parent.resolveName
def __repr__(self):
return "<CondDesc %s>"%",".join(ik.name for ik in self.inputKeys)
@classmethod
def fromInputKey(cls, ik, **kwargs):
return base.makeStruct(CondDesc, inputKeys=[ik], **kwargs)
@classmethod
def fromColumn(cls, col, **kwargs):
return base.makeStruct(cls, buildFrom=col, **kwargs)
@property
def name(self):
"""returns some key for uniqueness of condDescs.
"""
# This is necessary for ColumnLists that are used
# for CondDescs as well. Ideally, we'd do this on an
# InputKeys basis and yield their names (because that's what
# formal counts on), but it's probably not worth the effort.
return "+".join([f.name for f in self.inputKeys])
def completeElement(self, ctx):
if self.buildFrom and not self.inputKeys:
# use the column as input key; special renderers may want
# to do type mapping, but the default is to have plain input
self.inputKeys = [inputdef.InputKey.fromColumn(self.buildFrom)]
self._completeElementNext(CondDesc, ctx)
def expand(self, *args, **kwargs):
"""hands macro expansion requests (from phraseMakers) upwards.
This is to the queried table if the parent has one (i.e., we're
part of a core), or to the RD if not (i.e., we're defined within
an rd).
"""
if hasattr(self.parent, "queriedTable"):
return self.parent.queriedTable.expand(*args, **kwargs)
else:
return self.parent.rd.expand(*args, **kwargs)
def _makePhraseDefault(self, ignored, inputKeys, inPars, outPars, core):
# the default phrase maker uses whatever the individual input keys
# come up with.
for ik in self.inputKeys:
yield base.getSQLForField(ik, inPars, outPars)
# We only want to compile the phraseMaker if actually necessary.
# condDescs may be defined within resource descriptors (e.g., in
# scs.rd), and they can't be compiled there (since macros may
# be missing); thus, we dispatch on the first call.
def _getPhraseMaker(self):
try:
return self.__compiledPhraseMaker
except AttributeError:
if self.phraseMaker is not None:
val = self.phraseMaker.compile()
else:
val = self._makePhraseDefault
self.__compiledPhraseMaker = val
return self.__compiledPhraseMaker
makePhrase = property(_getPhraseMaker)
def _isActive(self, inPars):
"""returns True if the dict inPars contains input to all our input keys.
"""
for f in self.inputKeys:
if f.name not in inPars:
return False
return True
def inputReceived(self, inPars, queryMeta):
"""returns True if all inputKeys can be filled from inPars.
As a side effect, inPars will receive defaults form the input keys
if there are any.
"""
if not self._isActive(inPars):
return False
keysFound, keysMissing = [], []
for f in self.inputKeys:
if inPars.get(f.name) is None:
keysMissing.append(f)
else:
if f.value!=inPars.get(f.name): # non-defaulted
keysFound.append(f)
if not keysMissing:
return True
# keys are missing. That's ok if none were found and we're not required
if not self.required and not keysFound:
return False
if self.required:
raise base.ValidationError("is mandatory but was not provided.",
colName=keysMissing[0].name)
# we're optional, but a value was given and others are missing
if not self.combining:
raise base.ValidationError("When you give a value for %s,"
" you must give value(s) for %s, too"%(keysFound[0].getLabel(),
", ".join(k.name for k in keysMissing)),
colName=keysMissing[0].name)
return True
def asSQL(self, inPars, sqlPars, queryMeta):
if self.silent or not self.inputReceived(inPars, queryMeta):
return ""
res = list(self.makePhrase(
self, self.inputKeys, inPars, sqlPars, self.parent))
sql = base.joinOperatorExpr(self.joiner, res)
if self.fixedSQL:
sql = base.joinOperatorExpr(self.joiner, [sql, self.fixedSQL])
return sql
def adaptForRenderer(self, renderer):
"""returns a changed version of self if renderer suggests such a
change.
This only happens if buildFrom is non-None. The method must
return a "defused" version that has buildFrom None (or self,
which will do because core.adaptForRenderer stops adapting if
the condDescs are stable).
The adaptors may also raise a Replace exception and return a
full CondDesc; this is done, e.g., for spoints for the form
renderer, since they need two input keys and a completely modified
phrase.
"""
if not self.buildFrom:
return self
adaptor = inputdef.getRendererAdaptor(renderer)
if adaptor is None:
return self
try:
newInputKeys = []
for ik in self.inputKeys:
newInputKeys.append(adaptor(ik))
if self.inputKeys==newInputKeys:
return self
else:
return self.change(inputKeys=newInputKeys, buildFrom=None)
except base.Replace, ex:
return ex.newOb
class Publication(base.Structure, base.ComputedMetaMixin):
"""A specification of how a service should be published.
This contains most of the metadata for what is an interface in
registry speak.
"""
name_ = "publish"
_rd = rscdef.RDAttribute()
_render = base.UnicodeAttribute(
"render",
default=base.Undefined,
description="The renderer the publication will point at.",
copyable=True)
_sets = base.StringSetAttribute(
"sets",
description="Comma-separated list of sets this service will be"
" published in. Predefined are: local=publish on front page,"
" ivo_managed=register with the VO registry. If you leave it"
" empty, 'local' publication is assumed.",
copyable="True")
_service = base.ReferenceAttribute(
"service",
default=base.NotGiven,
description="Reference for a service actually implementing the"
" capability corresponding to this publication. This is"
" mainly when there is a vs:WebBrowser service accompanying a VO"
" protocol service, and this other service should be published"
" in the same resource record. See also the operator's guide.",
copyable="True")
_auxiliary = base.BooleanAttribute(
"auxiliary",
default=False,
description="Auxiliary publications are for capabilities"
" not intended to be picked up for all-VO queries, typically"
" because they are already registered with other services."
" This is mostly used internally; you probably have no reason"
" to touch it.")
def completeElement(self, ctx):
if self.render is base.Undefined:
self.render = "form"
if not self.sets:
self.sets.add("local")
if self.service is base.NotGiven:
self.service = self.parent
self.setMetaParent(self.service)
self._completeElementNext(Publication, ctx)
def validate(self):
self._validateNext(Publication)
try:
renderers.getRenderer(self.render)
except KeyError:
raise base.StructureError("Unknown renderer: %s" % self.render)
def _meta_accessURL(self):
return self.service.getURL(self.render)
def _meta_urlUse(self):
return renderers.getRenderer(self.render).urlUse
def _meta_requestMethod(self):
return renderers.getRenderer(self.render).preferredMethod
def _meta_resultType(self):
return renderers.getRenderer(self.render).resultType
class ForeignKey(base.Structure):
"""A description of a foreign key relation between this table and another
one.
"""
name_ = "foreignKey"
_inTable = base.ReferenceAttribute(
"inTable",
default=base.Undefined,
description="Reference to the table the foreign key points to.",
copyable=True)
_source = base.UnicodeAttribute(
"source",
default=base.Undefined,
description="Comma-separated list of local columns corresponding"
" to the foreign key. No sanity checks are performed here.",
copyable=True)
_dest = base.UnicodeAttribute(
"dest",
default=base.NotGiven,
description="Comma-separated list of columns in the target table"
" belonging to its key. No checks for their existence, uniqueness,"
" etc. are done here. If not given, defaults to source.")
_metaOnly = base.BooleanAttribute(
"metaOnly",
default=False,
description="Do not tell the database to actually create the foreign"
" key, just declare it in the metadata. This is for when you want"
" to document a relationship but don't want the DB to actually"
" enforce this. This is typically a wise thing to do when you have, say"
" a gigarecord of flux/density pairs and only several thousand metadata"
" records -- you may want to update the latter without having"
" to tear down the former.")
def getDescription(self):
return "%s:%s -> %s:%s" % (self.parent.getQName(), ",".join(
self.source), self.destTableName, ".".join(self.dest))
def _parseList(self, raw):
if isinstance(raw, list):
# we're being copied
return raw
return [s.strip() for s in raw.split(",") if s.strip()]
def onElementComplete(self):
self.destTableName = self.inTable.getQName()
self.isADQLKey = self.inTable.adql and self.inTable.adql != 'hidden'
self.source = self._parseList(self.source)
if self.dest is base.NotGiven:
self.dest = self.source
else:
self.dest = self._parseList(self.dest)
self._onElementCompleteNext(ForeignKey)
def create(self, querier):
if self.metaOnly:
return
if not querier.foreignKeyExists(self.parent.getQName(),
self.destTableName, self.source,
self.dest):
return querier.query(
"ALTER TABLE %s ADD FOREIGN KEY (%s)"
" REFERENCES %s (%s)"
" ON DELETE CASCADE"
" DEFERRABLE INITIALLY DEFERRED" %
(self.parent.getQName(), ",".join(
self.source), self.destTableName, ",".join(self.dest)))
def delete(self, querier):
if self.metaOnly:
return
try:
constraintName = querier.getForeignKeyName(self.parent.getQName(),
self.destTableName,
self.source, self.dest)
except base.DBError: # key does not exist.
return
querier.query("ALTER TABLE %s DROP CONSTRAINT %s" %
(self.parent.getQName(), constraintName))
def getAnnotation(self, roleName, container):
"""returns a dm annotation for this foreign key.
"""
return dm.ForeignKeyAnnotation(roleName, self)
class Execute(base.Structure, base.ExpansionDelegator):
"""a container for calling code.
This is a cron-like functionality. The jobs are run in separate
threads, so they need to be thread-safe with respect to the
rest of DaCHS. DaCHS serializes calls, though, so that your
code should never run twice at the same time.
At least on CPython, you must make sure your code does not
block with the GIL held; this is still in the server process.
If you do daring things, fork off (note that you must not use
any database connections you may have after forking, which means
you can't safely use the RD passed in). See the docs on `Element job`_.
Then testing/debugging such code, use ``gavo admin execute rd#id``
to immediately run the jobs.
"""
name_ = "execute"
_title = base.UnicodeAttribute(
"title",
default=base.Undefined,
description="Some descriptive title for the job; this is used"
" in diagnostics.",
copyable=False,
)
_at = base.StringListAttribute(
"at",
description="One or more hour:minute pairs at which to run"
" the code each day. This conflicts with every. Optionally,"
" you can prefix each time by one of m<dom> or w<dow> for"
" jobs only to be exectued at some day of the month or week, both"
" counted from 1. So, 'm22 7:30, w3 15:02' would execute on"
" the 22nd of each month at 7:30 UTC and on every wednesday at 15:02.",
default=base.NotGiven,
copyable=True,
)
_every = base.IntAttribute(
"every",
default=base.NotGiven,
description="Run the job roughly every this many seconds."
" This conflicts with at. Note that the first execution of"
" such a job is after every/10 seconds, and that the timers"
" start anew at every server restart. So, if you restart"
" often, these jobs may run much more frequently or not at all"
" if the interval is large. If every is smaller than zero, the"
" job will be executed immediately when the RD is being loaded and is"
" then run every abs(every) seconds",
copyable=True,
)
_job = base.StructAttribute(
"job",
childFactory=CronJob,
default=base.Undefined,
description="The code to run.",
copyable=True,
)
_debug = base.BooleanAttribute(
"debug",
description="If true, on execution of external processes (span or"
" spawnPython), the output will be accumulated and mailed to"
" the administrator. Note that output of the actual cron job"
" itself is not caught (it might turn up in serverStderr)."
" You could use execDef.outputAccum.append(<stuff>) to have"
" information from within the code included.",
default=False)
_properties = base.PropertyAttribute()
_rd = common.RDAttribute()
def spawn(self, cliList):
"""spawns an external command, capturing the output and mailing it
to the admin if it failed.
Output is buffered and mailed, so it shouldn't be too large.
This does not raise an exception if it failed (in normal usage,
this would cause two mails to be sent). Instead, it returns the
returncode of the spawned process; if that's 0, you're ok. But
in general, you wouldn't want to check it.
"""
p = subprocess.Popen(cliList,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=True)
childOutput, _ = p.communicate()
if p.returncode:
cron.sendMailToAdmin(
"A process spawned by %s failed with %s" %
(self.title, p.returncode),
"Output of %s:\n\n%s" % (cliList, childOutput))
elif self.debug:
if childOutput:
self.outputAccum.append("\n\n%s -> %s\n" %
(cliList, p.returncode))
self.outputAccum.append(childOutput)
return p.returncode
def spawnPython(self, pythonFile):
"""spawns a new python interpreter executing pythonFile.
pythonFile may be resdir-relative.
"""
self.spawn(["python", os.path.join(self.rd.resdir, pythonFile)])
def _parseAt(self, atSpec, ctx):
"""returns a tuple ready for cron.repeatAt from atSpec.
see the at StringListAttribute for how it would look like; this
parses one element of that string list.
"""
mat = re.match(
r"(?P<dow>w\d\s+)?"
r"(?P<dom>m\d\d?\s+)?"
r"(?P<hr>\d+):(?P<min>\d+)", atSpec)
if not mat:
raise base.LiteralParseError("at", atSpec, pos=ctx.pos, hint=
"This is hour:minute optionally prefixed by either w<weekday> or"\
" m<day of month>, each counted from 1.")
hour, minute = int(mat.group("hr")), int(mat.group("min"))
if not (0 <= hour <= 23 and 0 <= minute <= 59):
raise base.LiteralParseError(
"at",
atSpec,
pos=ctx.pos,
hint=
"This must be hour:minute with 0<=hour<=23 or 0<=minute<=59")
dom = None
if mat.group("dom"):
dom = int(mat.group("dom")[1:])
if not 1 <= dom <= 28:
raise base.LiteralParseError(
"at",
atSpec,
pos=ctx.pos,
hint="day-of-month in at must be between 1 and 28.")
dow = None
if mat.group("dow"):
dow = int(mat.group("dow")[1:])
if not 1 <= dow <= 7:
raise base.LiteralParseError(
"at",
atSpec,
pos=ctx.pos,
hint="day-of-week in at must be between 1 and 7.")
return (dom, dow, hour, minute)
def completeElement(self, ctx):
self._completeElementNext(Execute, ctx)
if len([s for s in [self.at, self.every] if s is base.NotGiven]) != 1:
raise base.StructureError(
"Exactly one of at and every required"
" for Execute",
pos=ctx.pos)
if self.at is not base.NotGiven:
self.parsedAt = []
for literal in self.at:
self.parsedAt.append(self._parseAt(literal, ctx))
def onElementComplete(self):
self._onElementCompleteNext(Execute)
self.jobName = "%s#%s" % (self.rd.sourceId, self.title)
self.callable = _guardedFunctionFactory.makeGuardedThreaded(
self.job.compile(), self)
if self.at is not base.NotGiven:
cron.repeatAt(self.parsedAt, self.jobName, self.callable)
else:
cron.runEvery(self.every, self.jobName, self.callable)
class DBIndex(base.Structure):
"""A description of an index in the database.
In real databases, indices may be fairly complex things; still, the
most common usage here will be to just index a single column::
<index columns="my_col"/>
To index over functions, use the character content; parentheses are added
by DaCHS, so don't have them in the content. An explicit specification
of the index expression is also necessary to allow RE pattern matches using
indices in character columns (outside of the C locale). That would be::
<index columns="uri">uri text_pattern_ops</index>
(you still want to give columns so the metadata engine is aware of the
index). See section "Operator Classes and Operator Families" in
the Postgres documentation for details.
"""
name_ = "index"
_name = base.UnicodeAttribute(
"name",
default=base.Undefined,
description="Name of the index (defaults to something computed from"
" columns; the name of the parent table will be prepended in the DB)",
copyable=True)
_columns = base.StringListAttribute(
"columns",
description=
"Table columns taking part in the index (must be given even if there"
" is an expression building the index and mention all columns taking"
" part in the index generated by it",
copyable=True)
_cluster = base.BooleanAttribute(
"cluster",
default=False,
description="Cluster the table according to this index?",
copyable=True)
_code = base.DataContent(
copyable=True,
description="Raw SQL specifying an expression the table should be"
" indexed for. If not given, the expression will be generated from"
" columns (which is what you usually want).")
_method = base.UnicodeAttribute(
"method",
default=None,
description="The indexing method, like an index type. In the 8.x,"
" series of postgres, you need to set method=GIST for indices"
" over pgsphere columns; otherwise, you should not need to"
" worry about this.",
copyable=True)
def completeElement(self, ctx):
if self.content_ and getattr(ctx, "restricted", False):
raise base.RestrictedElement("index",
hint="Free-form SQL on indices"
" is not allowed in restricted mode")
self._completeElementNext(DBIndex, ctx)
if not self.columns:
raise base.StructureError("Index without columns is verboten.")
if self.name is base.Undefined:
self.name = "%s" % (re.sub("[^\w]+", "_", "_".join(self.columns)))
if not self.content_:
self.content_ = "%s" % ",".join(self.columns)
def iterCode(self):
destTableName = self.parent.getQName()
usingClause = ""
if self.method is not None:
usingClause = " USING %s" % self.method
yield self.parent.expand(
"CREATE INDEX %s ON %s%s (%s)" %
(self.dbname, destTableName, usingClause, self.content_))
if self.cluster:
yield self.parent.expand("CLUSTER %s ON %s" %
(self.dbname, destTableName))
def create(self, querier):
"""creates the index on the parent table if necessary.
querier is an object mixing in the DBMethodsMixin, usually the
DBTable object the index should be created on.
"""
if not querier.hasIndex(self.parent.getQName(), self.dbname):
if not self.parent.system:
base.ui.notifyIndexCreation(self.parent.expand(self.dbname))
for statement in self.iterCode():
querier.query(statement)
def drop(self, querier):
"""drops the index if it exists.
querier is an object mixing in the DBMethodsMixin, usually the
DBTable object the index possibly exists on.
"""
iName = self.parent.expand(self.dbname)
if querier.hasIndex(self.parent.getQName(), iName):
querier.query("DROP INDEX %s.%s" % (self.parent.rd.schema, iName))
@property
def dbname(self):
return "%s_%s" % (self.parent.id, self.name)
class FITSProdGrammar(Grammar):
r"""A grammar that returns FITS-headers as dictionaries.
This is the grammar you want when one FITS file corresponds to one
row in the destination table.
The keywords of the grammar record are the cards in the primary
header (or some other hdu using the same-named attribute). "-" in
keywords is replaced with an underscore for easier @-referencing.
You can use a mapKeys element to effect further name cosmetics.
This grammar should handle compressed FITS images transparently if
set qnd="False". This means that you will essentially get the readers
from the second extension for those even if you left hdu="0".
The original header is preserved as the value of the header\_ key. This
is mainly intended for use WCS use, as in ``pywcs.WCS(@header_)``.
If you have more complex structures in your FITS files, you can get access
to the pyfits HDU using the hdusField attribute. With
``hdusField="_H"``, you could say things like ``@_H[1].data[10][0]``
to get the first data item in the tenth row in the second HDU.
"""
name_ = "fitsProdGrammar"
_qnd = base.BooleanAttribute(
"qnd",
default=True,
description="Use a hack to read the FITS header more quickly. This only"
" works for the primary HDU",
copyable=True)
_hduIndex = base.IntAttribute(
"hdu",
default=0,
description="Take the header from this HDU. You must say qnd='False'"
" for this to take effect.",
copyable=True)
_mapKeys = base.StructAttribute(
"mapKeys",
childFactory=MapKeys,
default=None,
copyable=True,
description="Prescription for how to"
" map header keys to grammar dictionary keys")
_hdusAttr = base.UnicodeAttribute(
"hdusField",
default=None,
description="If set, the complete pyfits HDU list for the FITS"
" file is returned in this grammar field.",
copyable=True)
_maxHeaderBlocks = base.IntAttribute(
"maxHeaderBlocks",
default=40,
copyable=True,
description="Stop looking for"
" FITS END cards and raise an error after this many blocks."
" You may need to raise this for people dumping obscene amounts"
" of data or history into headers.")
rowIterator = FITSProdIterator
def onElementComplete(self):
if self.mapKeys is None:
self.mapKeys = base.makeStruct(MapKeys)
self._onElementCompleteNext(FITSProdGrammar)
class DirectGrammar(base.Structure, base.RestrictionMixin):
"""A user-defined external grammar.
See the separate document on user-defined code on more on direct grammars.
Also note the program gavomkboost that can help you generate core for
the C boosters used by direct grammars.
"""
name_ = "directGrammar"
_cbooster = rscdef.ResdirRelativeAttribute(
"cBooster",
default=base.Undefined,
description="resdir-relative path to the booster C source.",
copyable=True)
_gzippedInput = base.BooleanAttribute(
"gzippedInput",
default=False,
description="Pipe gzip before booster? (will not work for FITS)",
copyable=True)
_autoNull = base.UnicodeAttribute(
"autoNull",
default=None,
description="Use this string as general NULL value (when reading"
" from plain text).",
copyable=True)
_ignoreBadRecords = base.BooleanAttribute(
"ignoreBadRecords",
default=False,
description="Let booster ignore invalid records?",
copyable=True)
_recordSize = base.IntAttribute(
"recordSize",
default=4000,
description="For bin boosters, read this many bytes to make"
" up a record; for line-based boosters, this is the maximum"
" length of an input line.",
copyable=True)
_preFilter = base.UnicodeAttribute(
"preFilter",
default=None,
description="Pipe input through this program before handing it to"
" the booster; this string is shell-expanded (will not work for FITS).",
copyable=True)
_customFlags = base.UnicodeAttribute(
"customFlags",
default="",
description="Pass these flags to the C compiler when building the"
" booster.",
copyable=True)
_type = base.EnumeratedUnicodeAttribute(
"type",
default="col",
validValues=["col", "bin", "fits", "split"],
description="Make code for a booster parsing by column indices (col),"
" by splitting along separators (split), by reading fixed-length"
" binary records (bin), for from FITS binary tables (fits).",
copyable=True)
_splitChar = base.UnicodeAttribute(
"splitChar",
default="|",
description="For split boosters, use this as the separator.",
copyable=True)
_ext = base.IntAttribute(
"extension",
default=1,
description=
"For FITS table boosters, get the table from this extension.",
copyable=True)
_mapKeys = base.StructAttribute(
"mapKeys",
childFactory=common.MapKeys,
default=None,
copyable=True,
description="For a FITS booster, map DB table column names"
" to FITS column names (e.g., if the FITS table name flx is to"
" end up in the DB column flux, say flux:flx).")
_rd = rscdef.RDAttribute()
isDispatching = False
def validate(self):
self._validateNext(DirectGrammar)
if self.type == 'bin':
if not self.recordSize:
raise base.StructureError(
"DirectGrammars reading from binary need"
" a recordSize attribute")
if self.mapKeys is not None:
if self.type != "fits":
raise base.StructureError("mapKeys is only allowed for FITS"
" boosters.")
def onElementComplete(self):
if self.type == "fits":
if self.mapKeys:
self.keyMap = self.mapKeys.maps
else:
self.keyMap = {}
def getBooster(self):
return CBooster(self.cBooster,
self.parent,
gzippedInput=self.gzippedInput,
preFilter=self.preFilter,
autoNull=self.autoNull,
ignoreBadRecords=self.ignoreBadRecords,
customFlags=self.customFlags)
def parse(self, sourceToken, targetData=None):
booster = self.getBooster()
makes = self.parent.makes
if len(makes) != 1:
raise base.StructureError(
"Directgrammar only works for data having"
" exactly one table, but data '%s' has %d" %
(self.parent.id, len(makes)))
def copyIn(data):
data.tables.values()[0].copyIn(booster.getOutput(sourceToken))
if booster.getStatus():
raise base.SourceParseError("Booster returned error signature",
source=sourceToken)
return copyIn
class DataURL(base.Structure):
"""A source document for a regression test.
As string URLs, they specify where to get data from, but the additionally
let you specify uploads, authentication, headers and http methods,
while at the same time saving you manual escaping of parameters.
The bodies is the path to run the test against. This is
interpreted as relative to the RD if there's no leading slash,
relative to the server if there's a leading slash, and absolute
if there's a scheme.
The attributes are translated to parameters, except for a few
pre-defined names. If you actually need those as URL parameters,
should at us and we'll provide some way of escaping these.
We don't actually parse the URLs coming in here. GET parameters
are appended with a & if there's a ? in the existing URL, with a ?
if not. Again, shout if this is too dumb for you (but urlparse
really isn't all that robust either...)
"""
name_ = "url"
# httpURL will be set to the URL actually used in retrieveResource
# Only use this to report the source of the data for, e.g., failing
# tests.
httpURL = "(not retrieved)"
_base = base.DataContent(description="Base for URL generation; embedded"
" whitespace will be removed, so you're free to break those whereever"
" you like.",
copyable=True)
_httpMethod = base.UnicodeAttribute("httpMethod",
description="Request method; usually one of GET or POST",
default="GET")
_httpPost = common.ResdirRelativeAttribute("postPayload",
default=base.NotGiven,
description="Path to a file containing material that should go"
" with a POST request (conflicts with additional parameters).",
copyable=True)
_parset = base.EnumeratedUnicodeAttribute("parSet",
description="Preselect a default parameter set; form gives what"
" our framework adds to form queries.", default=base.NotGiven,
validValues=["form"],
copyable=True)
_httpHeaders = base.DictAttribute("httpHeader",
description="Additional HTTP headers to pass.",
copyable=True)
_httpAuthKey = base.UnicodeAttribute("httpAuthKey",
description="A key into ~/.gavo/test.creds to find a user/password"
" pair for this request.",
default=base.NotGiven,
copyable=True)
_httpUploads = base.StructListAttribute("uploads",
childFactory=Upload,
description='HTTP uploads to add to request (must have httpMethod="POST")',
copyable=True)
_httpHonorRedirects = base.BooleanAttribute("httpHonorRedirects",
default=False,
description="Follow 30x redirects instead of just using"
" status, headers, and payload of the initial request.",
copyable="True")
_rd = common.RDAttribute()
_open = DynamicOpenVocAttribute("open")
def getValue(self, serverURL):
"""returns a pair of full request URL and postable payload for this
test.
"""
urlBase = re.sub(r"\s+", "", self.content_)
if "://" in urlBase:
# we believe there's a scheme in there
pass
elif urlBase.startswith("/"):
urlBase = serverURL+urlBase
else:
urlBase = serverURL+"/"+self.parent.rd.sourceId+"/"+urlBase
if self.httpMethod=="POST":
return urlBase
else:
return self._addParams(urlBase, urllib.urlencode(self.getParams()))
def getParams(self):
"""returns the URL parameters as a sequence of kw, value pairs.
"""
params = getattr(self, "freeAttrs", [])
if self.parSet=="form":
params.extend([("__nevow_form__", "genForm"), ("submit", "Go"),
("_charset_", "UTF-8")])
return params
def retrieveResource(self, serverURL, timeout):
"""returns a triple of status, headers, and content for retrieving
this URL.
"""
self.httpURL, payload = self.getValue(serverURL), None
headers = {
"user-agent": "DaCHS regression tester"}
headers.update(self.httpHeader)
if self.httpMethod=="POST":
if self.postPayload:
with open(self.postPayload) as f:
payload = f.read()
elif self.uploads:
form = _FormData()
for key, value in self.getParams():
form.addParam(key, value)
for upload in self.uploads:
upload.addToForm(form)
boundary = "========== roughtest deadbeef"
form.set_param("boundary", boundary)
headers["Content-Type"] = form.get_content_type(
)+'; boundary="%s"'%boundary
payload = form.as_string()
else:
payload = urllib.urlencode(self.getParams())
headers["Content-Type"] = "application/x-www-form-urlencoded"
scheme, host, path, _, query, _ = urlparse.urlparse(str(self.httpURL))
assert scheme=="http"
if self.httpAuthKey is not base.NotGiven:
headers.update(getAuthFor(self.httpAuthKey))
status, respHeaders, content = doHTTPRequest(str(self.httpMethod),
host, path, query, payload, headers, timeout)
while self.httpHonorRedirects and status in [301, 302, 303]:
scheme, host, path, _, query, _ = urlparse.urlparse(
getHeaderValue(respHeaders, "location"))
status, respHeaders, content = doHTTPRequest("GET",
host, path, query, None, {}, timeout)
return status, respHeaders, content
def _addParams(self, urlBase, params):
"""a brief hack to add query parameters to GET-style URLs.
This is a workaround for not trusting urlparse and is fairly easy to
fool.
Params must already be fully encoded.
"""
if not params:
return urlBase
if "?" in urlBase:
return urlBase+"&"+params
else:
return urlBase+"?"+params
def validate(self):
if self.postPayload is not base.NotGiven:
if self.getParams():
raise base.StructureError("No parameters (or parSets) are"
" possible with postPayload")
if self.httpMethod!="POST":
raise base.StructureError("Only POST is allowed as httpMethod"
" together with postPayload")
if self.uploads:
if self.httpMethod!="POST":
raise base.StructureError("Only POST is allowed as httpMethod"
" together with upload")
self._validateNext(DataURL)
class InputKey(column.ParamBase):
"""A description of a piece of input.
Think of inputKeys as abstractions for input fields in forms, though
they are used for services not actually exposing HTML forms as well.
Some of the DDL-type attributes (e.g., references) only make sense here
if columns are being defined from the InputKey.
You can give a "defaultForForm" property on inputKeys to supply
a string literal default that will be pre-filled in the form
renderer and is friends but not for other renderers (like S*AP).
Properties evaluated:
* defaultForForm -- a value entered into form fields by default
(be stingy with those; while it's nice to not have to set things
presumably right for almost everyone, having to delete stuff
you don't want over and over is really annoying).
* adaptToRenderer -- a true boolean literal here causes the param
to be adapted for the renderer (e.g., float could become vizierexpr-float).
You'll usually not want this, because the expressions are
generally evaluated by the database, and the condDescs do the
adaptation themselves. This is mainly for rare situations like
file uploads in custom cores.
"""
name_ = "inputKey"
# XXX TODO: make widgetFactory and showItems properties.
_widgetFactory = base.UnicodeAttribute(
"widgetFactory",
default=None,
description="A python expression for a custom widget"
" factory for this input,"
" e.g., 'Hidden' or 'widgetFactory(TextArea, rows=15, cols=30)'",
copyable=True)
_showItems = base.IntAttribute(
"showItems",
default=3,
description="Number of items to show at one time on selection widgets.",
copyable=True)
_inputUnit = base.UnicodeAttribute(
"inputUnit",
default=None,
description="Override unit of the table column with this.",
copyable=True)
_std = base.BooleanAttribute(
"std",
default=False,
description="Is this input key part of a standard interface for"
" registry purposes?",
copyable=True)
_multiplicity = base.UnicodeAttribute(
"multiplicity",
default=None,
copyable=True,
description="Set"
" this to single to have an atomic value (chosen at random"
" if multiple input values are given),"
" forced-single to have an atomic value"
" and raise an exception if multiple values come in, or"
" multiple to receive lists. On the form renderer, this is"
" ignored, and the values are what nevow formal passes in."
" If not given, it is single unless there is a values element with"
" options, in which case it's multiple.")
# Don't validate meta for these -- while they are children
# of validated structures (services), they don't need any
# meta at all. This should go as soon as we have a sane
# inheritance hierarchy for tables.
metaModel = None
def completeElement(self, ctx):
self._completeElementNext(InputKey, ctx)
if self.restrictedMode and self.widgetFactory:
raise base.RestrictedElement("widgetFactory")
def onElementComplete(self):
self._onElementCompleteNext(InputKey)
# compute scaling if an input unit is given
self.scaling = None
if self.inputUnit:
self.scaling = base.computeConversionFactor(
self.inputUnit, self.unit)
if self.multiplicity is None:
self.multiplicity = "single"
if self.isEnumerated():
# these almost always want lists returned.
self.multiplicity = "multiple"
def onParentComplete(self):
if self.parent and hasattr(self.parent, "required"):
# children of condDescs inherit their requiredness
# (unless defaulted)
self.required = self.parent.required
# but if there's a default, never require an input
if self.value:
self.required = False
def validateValue(self, literal):
"""raises a ValidationError if literal cannot be deserialised into
an acceptable value for self.
"""
self._parse(literal)
def _getVOTableType(self):
"""returns the VOTable type for the param.
The reason this is overridden is that historically, we've been
cavalier about letting in multiple values for a single param
(as in enumerated values and such).
It's probably too late to fix this now, so for InputKeys with
multiplicity multiple we're allowing arrays, too.
"""
type, arraysize, xtype = column.ParamBase._getVOTableType(self)
if self.multiplicity == "multiple" and arraysize == '1':
arraysize = "*"
return type, arraysize, xtype
@classmethod
def fromColumn(cls, column, **kwargs):
"""returns an InputKey for query input to column.
"""
if isinstance(column, InputKey):
if kwargs:
return column.change(**kwargs)
else:
return column
instance = cls(None)
instance.feedObject("original", column)
if column.isEnumerated():
instance.feedObject("multiplicity", "multiple")
for k, v in kwargs.iteritems():
instance.feed(k, v)
if not "required" in kwargs:
instance.feedObject("required", False)
return instance.finishElement(None)
class TableDef(base.Structure, base.ComputedMetaMixin, common.PrivilegesMixin,
common.IVOMetaMixin, base.StandardMacroMixin,
PublishableDataMixin):
"""A definition of a table, both on-disk and internal.
Some attributes are ignored for in-memory tables, e.g., roles or adql.
Properties for tables:
* supportsModel -- a short name of a data model supported through this
table (for TAPRegExt dataModel); you can give multiple names separated
by commas.
* supportsModelURI -- a URI of a data model supported through this table.
You can give multiple URIs separated by blanks.
If you give multiple data model names or URIs, the sequences of names and
URIs must be identical (in particular, each name needs a URI).
"""
name_ = "table"
resType = "table"
# We don't want to force people to come up with an id for all their
# internal tables but want to avoid writing default-named tables to
# the db. Thus, the default is not a valid sql identifier.
_id = base.IdAttribute(
"id",
default=base.NotGiven,
description="Name of the table (must be SQL-legal for onDisk tables)")
_cols = common.ColumnListAttribute(
"columns",
childFactory=column.Column,
description="Columns making up this table.",
copyable=True)
_params = common.ColumnListAttribute(
"params",
childFactory=column.Param,
description='Param ("global columns") for this table.',
copyable=True)
_viewStatement = base.UnicodeAttribute(
"viewStatement",
default=None,
description="A single SQL statement to create a view. Setting this"
" makes this table a view. The statement will typically be something"
" like CREATE VIEW \\\\curtable AS (SELECT \\\\colNames FROM...).",
copyable=True)
# onDisk must not be copyable since queries might copy the tds and havoc
# would result if the queries were to end up on disk.
_onDisk = base.BooleanAttribute(
"onDisk",
default=False,
description="Table in the database rather than in memory?")
_temporary = base.BooleanAttribute(
"temporary",
default=False,
description="If this is an onDisk table, make it temporary?"
" This is mostly useful for custom cores and such.",
copyable=True)
_adql = ADQLVisibilityAttribute(
"adql",
default=False,
description="Should this table be available for ADQL queries? In"
" addition to True/False, this can also be 'hidden' for tables"
" readable from the TAP machinery but not published in the"
" metadata; this is useful for, e.g., tables contributing to a"
" published view. Warning: adql=hidden is incompatible with setting"
" readProfiles manually.")
_system = base.BooleanAttribute(
"system",
default=False,
description="Is this a system table? If it is, it will not be"
" dropped on normal imports, and accesses to it will not be logged.")
_forceUnique = base.BooleanAttribute(
"forceUnique",
default=False,
description="Enforce dupe policy for primary key (see dupePolicy)?")
_dupePolicy = base.EnumeratedUnicodeAttribute(
"dupePolicy",
default="check",
validValues=["check", "drop", "overwrite", "dropOld"],
description="Handle duplicate rows with identical primary keys manually"
" by raising an error if existing and new rows are not identical (check),"
" dropping the new one (drop), updating the old one (overwrite), or"
" dropping the old one and inserting the new one (dropOld)?")
_primary = ColumnTupleAttribute(
"primary",
default=(),
description=
"Comma separated names of columns making up the primary key.",
copyable=True)
_indices = base.StructListAttribute(
"indices",
childFactory=DBIndex,
description="Indices defined on this table",
copyable=True)
_foreignKeys = base.StructListAttribute(
"foreignKeys",
childFactory=ForeignKey,
description="Foreign keys used in this table",
copyable=False)
_groups = base.StructListAttribute(
"groups",
childFactory=group.Group,
description="Groups for columns and params of this table",
copyable=True)
# this actually induces an attribute annotations with the DM
# annotation instances
_annotations = base.StructListAttribute(
"dm",
childFactory=dm.DataModelRoles,
description="Annotations for data models.",
copyable=True)
_properties = base.PropertyAttribute()
# don't copy stc -- columns just keep the reference to the original
# stc on copy, and nothing should rely on column stc actually being
# defined in the parent tableDefs.
_stcs = base.StructListAttribute("stc",
description="STC-S definitions"
" of coordinate systems.",
childFactory=STCDef)
_rd = common.RDAttribute()
_mixins = mixins.MixinAttribute()
_original = base.OriginalAttribute()
_namePath = common.NamePathAttribute()
fixupFunction = None
metaModel = ("title(1), creationDate(1), description(1),"
"subject, referenceURL(1)")
@classmethod
def fromColumns(cls, columns, **kwargs):
"""returns a TableDef from a sequence of columns.
You can give additional constructor arguments. makeStruct is used
to build the instance, the mixin hack is applied.
Columns with identical names will be disambiguated.
"""
res = MS(cls,
columns=common.ColumnList(cls.disambiguateColumns(columns)),
**kwargs)
return res
def __iter__(self):
return iter(self.columns)
def __contains__(self, name):
try:
self.columns.getColumnByName(name)
except base.NotFoundError:
return False
return True
def __repr__(self):
try:
return "<Table definition of %s>" % self.getQName()
except base.Error:
return "<Non-RD table %s>" % self.id
def completeElement(self, ctx):
# we want a meta parent as soon as possible, and we always let it
# be our struct parent
if (not self.getMetaParent() and self.parent
and hasattr(self.parent, "_getMeta")):
self.setMetaParent(self.parent)
# Make room for DM annotations (these are currently filled by
# gavo.dm.dmrd.DataModelRoles, but we might reconsider this)
self.annotations = []
if self.viewStatement and getattr(ctx, "restricted", False):
raise base.RestrictedElement(
"table",
hint="tables with"
" view creation statements are not allowed in restricted mode")
if self.registration and self.id is base.NotGiven:
raise base.StructureError("Published tables need an assigned id.")
if not self.id:
self._id.feed(ctx, self, utils.intToFunnyWord(id(self)))
# allow iterables to be passed in for columns and convert them
# to a ColumnList here
if not isinstance(self.columns, common.ColumnList):
self.columns = common.ColumnList(self.columns)
self._resolveSTC()
self._completeElementNext(TableDef, ctx)
self.columns.withinId = self.params.tableName = "table " + self.id
def validate(self):
if self.id.upper() in adql.allReservedWords:
raise base.StructureError(
"Reserved word %s is not allowed as a table"
" name" % self.id)
self._validateNext(TableDef)
def onElementComplete(self):
if self.adql:
self.readProfiles = (self.readProfiles
| base.getConfig("db", "adqlProfiles"))
self.dictKeys = [c.key for c in self]
self.indexedColumns = set()
for index in self.indices:
for col in index.columns:
if "\\" in col:
try:
self.indexedColumns.add(self.expand(col))
except (base.Error,
ValueError): # cannot expand yet, ignore
pass
else:
self.indexedColumns.add(col)
if self.primary:
self.indexedColumns |= set(self.primary)
self._defineFixupFunction()
self._onElementCompleteNext(TableDef)
if self.registration:
self.registration.register()
def getElementForName(self, name):
"""returns the first of column and param having name name.
The function raises a NotFoundError if neiter column nor param with
name exists.
"""
try:
try:
return self.columns.getColumnByName(name)
except base.NotFoundError:
return self.params.getColumnByName(name)
except base.NotFoundError, ex:
ex.within = "table %s" % self.id
raise
