def _defineMacros(self, fillers, destination):
"""creates attributes macroExpansions and parentMacroPackage used by
execMacros.
Within mixins, you can use macros filled by mixin parameters or
expanded by the substrate. This information is local to a concrete
mixin application. Hence, applyTo calls this method, and the
attributes created are invalid for any subsequent or parallel applyTo
calls. Therefore, applyTo acquires the applicationLock before
calling this.
"""
self.parentMacroPackage = None
if hasattr(destination, "execMacro"):
self.parentMacroPackage = destination
self.macroExpansions = {}
for p in self.pars:
if p.key in fillers:
self.macroExpansions[p.key] = fillers.pop(p.key)
elif p.isDefaulted():
self.macroExpansions[p.key] = p.content_
else:
raise base.StructureError("Mixin parameter %s mandatory"%p.key)
if fillers:
raise base.StructureError("The attribute(s) %s is/are not allowed"
" on this mixin"%(",".join(fillers)))
def validate(self):
self._validateNext(DirectGrammar)
if self.type == 'bin':
if not self.recordSize:
raise base.StructureError(
"DirectGrammars reading from binary need"
" a recordSize attribute")
if self.mapKeys is not None:
if self.type != "fits":
raise base.StructureError("mapKeys is only allowed for FITS"
" boosters.")
def validate(self):
if self.postPayload is not base.NotGiven:
if self.getParams():
raise base.StructureError("No parameters (or parSets) are"
" possible with postPayload")
if self.httpMethod!="POST":
raise base.StructureError("Only POST is allowed as httpMethod"
" together with postPayload")
if self.uploads:
if self.httpMethod!="POST":
raise base.StructureError("Only POST is allowed as httpMethod"
" together with upload")
self._validateNext(DataURL)
def _ensureParsBound(self):
"""raises an error if non-defaulted pars of procDef are not filled
by the bindings.
"""
bindNames = set(b.key for b in self.bindings)
for p in self.getSetupPars():
if not p.isDefaulted():
if not p.key in bindNames:
raise base.StructureError("Parameter %s is not defaulted in"
" %s and thus must be bound."%(p.key, self.name))
if p.key in bindNames:
bindNames.remove(p.key)
if bindNames:
raise base.StructureError("May not bind non-existing parameter(s)"
" %s."%(", ".join(bindNames)))
def completeElement(self, ctx):
# we want a meta parent as soon as possible, and we always let it
# be our struct parent
if (not self.getMetaParent() and self.parent
and hasattr(self.parent, "_getMeta")):
self.setMetaParent(self.parent)
# Make room for DM annotations (these are currently filled by
# gavo.dm.dmrd.DataModelRoles, but we might reconsider this)
self.annotations = []
if self.viewStatement and getattr(ctx, "restricted", False):
raise base.RestrictedElement(
"table",
hint="tables with"
" view creation statements are not allowed in restricted mode")
if self.registration and self.id is base.NotGiven:
raise base.StructureError("Published tables need an assigned id.")
if not self.id:
self._id.feed(ctx, self, utils.intToFunnyWord(id(self)))
# allow iterables to be passed in for columns and convert them
# to a ColumnList here
if not isinstance(self.columns, common.ColumnList):
self.columns = common.ColumnList(self.columns)
self._resolveSTC()
self._completeElementNext(TableDef, ctx)
self.columns.withinId = self.params.tableName = "table " + self.id
def completeElement(self, ctx):
self._completeElementNext(STCDef, ctx)
try:
self.compiled = stc.parseQSTCS(self.content_)
except stc.STCSParseError, msg:
raise base.ui.logOldExc(
base.StructureError("Bad stc definition: %s" % str(msg)))
def makeAttributeAnnotation(container, attName, attValue):
"""returns a typed annotation for attValue within container.
When attValue is a literal, this is largely trivial. If it's a reference,
this figures out what it points to and creates an annotation of
the appropriate type (e.g., ColumnAnnotation, ParamAnnotation, etc).
container in current DaCHS should be a TableDef or something similar;
this function expects at least a getByName function and an rd attribute.
This is usually used as a callback from within sil.getAnnotation and
expects Atom and Reference instances as used there.
"""
if isinstance(attValue, sil.Atom):
return common.AtomicAnnotation(attName, attValue)
elif isinstance(attValue, sil.Reference):
# try name-resolving first (resolveId only does id resolving on
# unadorned strings)
try:
res = container.getByName(attValue)
except base.NotFoundError:
res = base.resolveId(container.rd, attValue, instance=container)
if not hasattr(res, "getAnnotation"):
raise base.StructureError("Element %s cannot be referenced"
" within a data model." % repr(res))
return res.getAnnotation(attName, container)
else:
assert False
def makeLinkFromFile(self,
localPath,
description,
semantics,
service=None,
contentType=None):
"""returns a LinkDef for a local file.
Arguments are as for LinkDef.fromFile, except you don't have
to pass in service if you're using the datalink service itself
to access the file; this method will try to find the service by
itself.
"""
if service is None:
try:
service = inspect.currentframe().f_back.f_locals["self"].parent
except (KeyError, AttributeError):
raise base.StructureError(
"Cannot infer service for datalink"
" file link. Pass an appropriate service manually.")
return LinkDef.fromFile(localPath,
description,
semantics,
service=service,
contentType=None)
def onElementComplete(self):
if self.inputTable is not base.NotGiven:
if self.inputKeys != []:
raise base.StructureError(
"InputKeys and inputTable must not"
" both be given in a context grammar")
else:
if self.rowKey:
self.inputKeys = [
InputKey.fromColumn(c) for c in self.inputTable.columns
]
else:
self.inputKeys = self.inputTable.params
else:
columns = []
if self.rowKey:
columns = self.inputKeys
self.inputTable = MS(InputTable,
params=self.inputKeys,
columns=columns)
self.defaults = {}
for ik in self.iterInputKeys():
if not ik.required:
self.defaults[ik.name] = None
if ik.value is not None:
self.defaults[ik.name] = ik.value
self._onElementCompleteNext(ContextGrammar)
def _computeMatches(self):
"""adds .fitsIndexForCol and .colForFITSIndex attributes.
These are matches based on the respective column names, where
we do a case-insensitive matching for now.
Nones mean that no corresponding column is present; for FITS columns,
this means they are ignored. For table columns, this means that
stand-in code is generated for filling out later.
"""
tableColumns = dict((col.name.lower(), col) for col in self.tableDef)
if len(tableColumns) != len(self.tableDef.columns):
raise base.StructureError(
"Table unsuitable for FITS boosting as"
" column names identical after case folding are present.",
hint="Use mapKeys to adapt FITS table names to resolve"
" the ambiguity")
self.colForFITSIndex = {}
for index, fitsCol in enumerate(self.fitsTable.columns):
columnName = self.grammar.keyMap.get(fitsCol.name,
fitsCol.name).lower()
self.colForFITSIndex[index] = tableColumns.get(columnName)
self.fitsIndexForColName = {}
for index, col in self.colForFITSIndex.iteritems():
if col is None:
continue
self.fitsIndexForColName[col.name.lower()] = index
def completeElement(self, ctx):
self._completeElementNext(Service, ctx)
if not self.allowed:
self.allowed.add("form")
if self.core is base.Undefined:
# undefined cores are only allowed with custom pages
# (Deprecated)
if self.customPage:
self.core = core.getCore("nullCore")(
self.rd).finishElement(None)
base.ui.notifyWarning(
"Custom page service %s without nullCore."
" This is deprecated, please fix" % self.id)
else:
raise base.StructureError(
"Services must have cores (add <nullCore/>"
" if you really do not want a core, e.g., with fixed renderers)."
)
# if there's only one renderer on this service, make it the default
if self.defaultRenderer is None and len(self.allowed) == 1:
self.defaultRenderer = list(self.allowed)[0]
# empty output tables are filled from the core
if self.outputTable is base.NotGiven:
self.outputTable = self.core.outputTable
# cache all kinds of things expensive to create and parse
self._coresCache = {}
self._inputDDCache = {}
self._loadedTemplates = {}
# Schedule the capabilities to be added when the parse is
# done (i.e., the RD is complete)
ctx.addExitFunc(lambda rd, ctx: self._addAutomaticCapabilities())
def validate(self):
"""checks that code content is a parseable python expression and that
the destination exists in the tableDef
"""
self._validateNext(MappedExpression)
if (self.content_ and self.source) or not (self.content_ or self.source):
raise base.StructureError("Map must have exactly one of source attribute"
" or element content")
if not utils.identifierPattern.match(self.key):
raise base.LiteralParseError("name", self.key,
hint="Var keys must be valid python"
" identifiers, and '%s' is not"%self.key)
if self.source:
if not utils.identifierPattern.match(self.source):
raise base.LiteralParseError("source", self.source,
hint="Map sources must be (python)"
" identifiers, and '%s' is not"%self.source)
if self.nullExpr is not base.NotGiven:
utils.ensureExpression(self.nullExpr)
if self.content_:
utils.ensureExpression(common.replaceProcDefAt(self.content_), self.name_)
if self.nullExcs is not base.NotGiven:
utils.ensureExpression(self.nullExcs, "%s.nullExcs"%(self.name_))
def _addNames(self, ctx, names):
# since autoCols is not copyable, we can require
# that _addNames only be called when there's a real parse context.
if ctx is None:
raise base.StructureError("outputTable autocols is"
" only available with a parse context")
for name in names:
self._addName(ctx, name)
def feed(self, ctx, instance, value):
if isinstance(value, _AttBox):
# synthetic event during object copying, accept
self.feedObject(instance, value.payload)
else:
# do not let people set that stuff directly
raise base.StructureError("Cannot set %s attributes from XML" %
self.name_)
def end_(self, ctx, name, value):
if self.curName: # end parsing parameter binding
self.curName = None
return self
else: # end of mixin application, run the mixin and hand control back to
# mixin parent
if "mixin name" not in self.fillers:
raise base.StructureError("Empty mixin children not allowed")
mixinRef = self.fillers.pop("mixin name")
self.parentAttr.feed(ctx, self.parent, mixinRef, fillers=self.fillers)
return self.parent
def getById(self, id, forceType=None):
try:
res = self.idmap[id]
except KeyError:
raise base.NotFoundError(id, "Element with id",
"RD %s" % (self.sourceId))
if forceType:
if not isinstance(res, forceType):
raise base.StructureError("Element with id '%s' is not a %s" %
(id, forceType.__name__))
return res
def setMetaParent(self, parent):
# columns should *not* take part in meta inheritance. The reason is
# that there are usually many columns to a table, and there's no
# way I can see that any piece of metadata should be repeated in
# all of them. On the other hand, for votlinks (no name an example),
# meta inheritance would have disastrous consequences.
# So, we bend the rules a bit.
raise base.StructureError(
"Columns may not have meta parents.",
hint="The rationale for this is explained in rscdef/column.py,"
" look for setMetaParent.")
def completeElement(self, ctx):
self._completeElementNext(Execute, ctx)
if len([s for s in [self.at, self.every] if s is base.NotGiven]) != 1:
raise base.StructureError(
"Exactly one of at and every required"
" for Execute",
pos=ctx.pos)
if self.at is not base.NotGiven:
self.parsedAt = []
for literal in self.at:
self.parsedAt.append(self._parseAt(literal, ctx))
def __init__(self, grammar, tableDef):
from gavo.utils import pyfits
_CodeGenerator.__init__(self, grammar, tableDef)
# now fetch the first source to figure out its schema
if self.grammar.parent.sources is None:
raise base.StructureError(
"Cannot make FITS bintable booster without"
" a sources element on the embedding data.")
self.forExtension = grammar.extension
try:
self.fitsTable = pyfits.open(
self.grammar.parent.sources.iterSources().next())[
self.forExtension]
except StopIteration:
raise base.StructureError(
"Buliding a FITS bintable booster requires"
" at least one matching source.")
self._computeMatches()
def completeElement(self, ctx):
if self.content_ and getattr(ctx, "restricted", False):
raise base.RestrictedElement("index",
hint="Free-form SQL on indices"
" is not allowed in restricted mode")
self._completeElementNext(DBIndex, ctx)
if not self.columns:
raise base.StructureError("Index without columns is verboten.")
if self.name is base.Undefined:
self.name = "%s" % (re.sub("[^\w]+", "_", "_".join(self.columns)))
if not self.content_:
self.content_ = "%s" % ",".join(self.columns)
def getGrammarAndTable(grammarId):
"""returns a pair of directGrammar and table being fed for a cross-rd
reference.
"""
grammar = rscdef.getReferencedElement(grammarId, forceType=DirectGrammar)
# to figure out the table built, use the parent's make
makes = grammar.parent.makes
if len(makes) != 1:
raise base.StructureError("Directgrammar only works for data having"
" exactly one table, but data '%s' has %d" %
(grammar.parent.id, len(makes)))
tableDef = makes[0].table
return grammar, tableDef
def getPrimary(self):
"""returns the "primary" table definition in the data descriptor.
"primary" means the only table in a one-table dd, the table with the
role "primary" if there are more. If no matching table is found, a
StructureError is raised.
"""
if len(self.makes)==1:
return self.makes[0].table
else:
try:
return self.getTableDefWithRole("primary")
except base.StructureError: # raise more telling message
pass
raise base.StructureError("Ambiguous request for primary table")
def parse(self, sourceToken, targetData=None):
booster = self.getBooster()
makes = self.parent.makes
if len(makes) != 1:
raise base.StructureError(
"Directgrammar only works for data having"
" exactly one table, but data '%s' has %d" %
(self.parent.id, len(makes)))
def copyIn(data):
data.tables.values()[0].copyIn(booster.getOutput(sourceToken))
if booster.getStatus():
raise base.SourceParseError("Booster returned error signature",
source=sourceToken)
return copyIn
def validate(self):
self._validateNext(Param)
if self.content_ is base.NotGiven:
self.set(None)
if self.required and self.value is None:
raise base.StructureError("Required value not given for param"
" %s" % self.name)
try:
# the value property will bomb on malformed literals
self.value
except ValueError, msg:
raise base.LiteralParseError(
self.name,
self.content_,
hint="Param content must be parseable by the DC default parsers."
" The value you passed caused the error: %s" % msg)
def onParentComplete(self):
"""checks that param and column names can be found in the parent table.
"""
# defer validation for sub-groups (parent group will cause validation)
if isinstance(self.parent, Group):
return
# forgo validation if the group doesn't have a table
if self.table is None:
return
try:
for col in self.iterColumns():
pass
for par in self.iterParams():
pass
except base.NotFoundError, msg:
raise base.StructureError(
"No param or field %s in found in table %s" %
(msg.what, self.table.id))
def EnumeratedWidget(ik):
"""is a widget factory for input keys over enumerated columns.
This probably contains a bit too much magic, but let's see. The current
rules are:
If values.multiOk is true, render a MultiSelectChoice, else
render a SelectChoice or a RadioChoice depending on how many
items there are.
If ik is not required, add an ANY key evaluating to None. For
MultiSelectChoices we don't need this since for them, you can
simply leave it all unselected.
If there is a default, it becomes the NoneOption.
"""
if not ik.isEnumerated():
raise base.StructureError("%s is not enumerated"%ik.name)
noneOption, options = _getDisplayOptions(ik)
moreArgs = {"noneOption": noneOption}
if ik.values.multiOk or ik.multiplicity=="multiple":
if ik.showItems==-1 or len(options)<4:
baseWidget = CheckboxMultiChoice
del moreArgs["noneOption"]
else:
baseWidget = MultiSelectChoice
moreArgs["size"] = ik.showItems
moreArgs["noneOption"] = None
else:
if len(options)<4:
baseWidget = RadioChoice
else:
baseWidget = SelectChoice
res = formal.widgetFactory(baseWidget, options=options,
**moreArgs)
return res
def validate(self):
if self.procDef and self.procDef.type and self.requiredType:
if self.procDef.type!=self.requiredType:
raise base.StructureError("The procDef %s has type %s, but"
" here %s procDefs are required."%(self.procDef.id,
self.procDef.type, self.requiredType))
if self.procDef:
if self.procDef.deprecated:
if self.getSourcePosition()!="<internally built>":
# for now, don't warn about these; they typically
# originate when copying/adapting cores and will just
# confuse operators
procId = "unnamed procApp"
if self.name:
procId = "procApp %s"%self.name
base.ui.notifyWarning("%s, %s: %s"%(
self.getSourcePosition(),
procId,
utils.fixIndentation(self.procDef.deprecated, "")))
self._validateNext(ProcApp)
self._ensureParsBound()
def parse(self, value):
if not utils.identifierPattern.match(value):
raise base.StructureError("'%s' is not a valid column name" %
value)
return value
def getTableDefWithRole(self, role):
for m in self.makes:
if m.role==role:
return m.table
raise base.StructureError("No table def with role '%s'"%role)
def getTableDefById(self, id):
for td in self.iterTableDefs():
if td.id==id:
return td
raise base.StructureError("No table name %s will be built"%id)
