class Script(base.Structure, base.RestrictionMixin): """A script, i.e., some executable item within a resource descriptor. The content of scripts is given by their type -- usually, they are either python scripts or SQL with special rules for breaking the script into individual statements (which are basically like python's). The special language AC_SQL is like SQL, but execution errors are ignored. This is not what you want for most data RDs (it's intended for housekeeping scripts). See `Scripting`_. """ name_ = "script" typeDesc_ = "Embedded executable code with a type definition" _lang = base.EnumeratedUnicodeAttribute( "lang", default=base.Undefined, description="Language of the script.", validValues=["SQL", "python", "AC_SQL"], copyable=True) _type = base.EnumeratedUnicodeAttribute( "type", default=base.Undefined, description="Point of time at which script is to run.", validValues=[ "preImport", "newSource", "preIndex", "postCreation", "beforeDrop", "sourceDone" ], copyable=True) _name = base.UnicodeAttribute( "name", default="anonymous", description="A human-consumable designation of the script.", copyable=True) _notify = base.BooleanAttribute( "notify", default=True, description="Send out a notification when running this" " script.", copyable=True) _content = base.DataContent(copyable=True, description="The script body.") _original = base.OriginalAttribute() def getSource(self): """returns the content with all macros expanded. """ return self.parent.getExpander().expand(self.content_)
class ContextGrammar(grammars.Grammar): """A grammar for web inputs. These are almost exclusively in InputDDs. They hold InputKeys defining what they take from the context. For DBCores, the InputDDs are generally defined implicitely via CondDescs. Thus, only for other cores will you ever need to bother with ContextGrammars (unless you're going for special effects). The source tokens for context grammars are dictionaries; these are either typed dictionaries from nevow, where the values usually are atomic, or, preferably, the dictionaries of lists from request.args. ContextGrammars only yield rows if there's a rowKey defined. In that case, an outer join of all other parameters is returned; with rowKey defined, the input keys are obtained from the table's columns. In normal usage, they just yield a single parameter row, corresponding to the source dictionary possibly completed with defaults, where non-requried input keys get None defaults where not given. Missing required parameters yield errors. Since most VO protocols require case-insensitive matching of parameter names, matching of input key names and the keys of the input dictionary is attempted first literally, then disregarding case. """ name_ = "contextGrammar" _inputTable = base.ReferenceAttribute( "inputTable", default=base.NotGiven, description="The table that is to be built using this grammar", copyable=True) _inputKeys = rscdef.ColumnListAttribute( "inputKeys", childFactory=InputKey, description="Input keys this context grammar should parse." " These must not be given if there is an input table defined.") _rowKey = base.UnicodeAttribute( "rowKey", default=base.NotGiven, description="The name of a key that is used to generate" " rows from the input", copyable=True) _rejectExtras = base.BooleanAttribute( "rejectExtras", default=False, description="If true, the grammar will reject extra input parameters." " Note that for form-based services, there *are* extra parameters" " not declared in the services' input tables. Right now," " contextGrammar does not ignore those.", copyable=True) _original = base.OriginalAttribute("original") rowIterator = ContextRowIterator def onElementComplete(self): if self.inputTable is not base.NotGiven: if self.inputKeys != []: raise base.StructureError( "InputKeys and inputTable must not" " both be given in a context grammar") else: if self.rowKey: self.inputKeys = [ InputKey.fromColumn(c) for c in self.inputTable.columns ] else: self.inputKeys = self.inputTable.params else: columns = [] if self.rowKey: columns = self.inputKeys self.inputTable = MS(InputTable, params=self.inputKeys, columns=columns) self.defaults = {} for ik in self.iterInputKeys(): if not ik.required: self.defaults[ik.name] = None if ik.value is not None: self.defaults[ik.name] = ik.value self._onElementCompleteNext(ContextGrammar) def iterInputKeys(self): for ik in self.inputKeys: yield ik
class RowmakerDef(base.Structure, RowmakerMacroMixin): """A definition of the mapping between grammar input and finished rows ready for shipout. Rowmakers consist of variables, procedures and mappings. They result in a python callable doing the mapping. RowmakerDefs double as macro packages for the expansion of various macros. The standard macros will need to be quoted, the rowmaker macros above yield python expressions. Within map and var bodies as well as late apply pars and apply bodies, you can refer to the grammar input as vars["name"] or, shorter @name. To add output keys, use map or, in apply bodies, add keys to the result dictionary. """ name_ = "rowmaker" _maps = base.StructListAttribute("maps", childFactory=MapRule, description="Mapping rules.", copyable=True) _vars = base.StructListAttribute("vars", childFactory=VarDef, description="Definitions of intermediate variables.", copyable=True) _apps = base.StructListAttribute("apps", childFactory=ApplyDef, description="Procedure applications.", copyable=True) _rd = common.RDAttribute() _idmaps = base.StringListAttribute("idmaps", description="List of" ' column names that are just "mapped through" (like map with key' " only); you can use shell patterns to select multiple colums at once.", copyable=True) _simplemaps = base.IdMapAttribute("simplemaps", description= "Abbreviated notation for <map source>; each pair is destination:source", copyable=True) _ignoreOn = base.StructAttribute("ignoreOn", default=None, childFactory=rowtriggers.IgnoreOn, description="Conditions on the" " input record coming from the grammar to cause the input" " record to be dropped by the rowmaker, i.e., for this specific" " table. If you need to drop a row for all tables being fed," " use a trigger on the grammar.", copyable=True) _original = base.OriginalAttribute() @classmethod def makeIdentityFromTable(cls, table, **kwargs): """returns a rowmaker that just maps input names to column names. """ if "id" not in kwargs: kwargs["id"] = "autogenerated rowmaker for table %s"%table.id return base.makeStruct(cls, idmaps=[c.key for c in table], **kwargs) @classmethod def makeTransparentFromTable(cls, table, **kwargs): """returns a rowmaker that maps input names to column names without touching them. This is for crazy cases in which the source actually provides pre-parsed data that any treatment would actually ruin. """ if "id" not in kwargs: kwargs["id"] = "autogenerated rowmaker for table %s"%table.id return base.makeStruct(cls, maps=[ base.makeStruct(MapRule, key=c.name, content_="vars[%s]"%repr(c.name)) for c in table], **kwargs) def completeElement(self, ctx): if self.simplemaps: for k,v in self.simplemaps.iteritems(): nullExcs = base.NotGiven if v.startswith("@"): v = v[1:] nullExcs = "KeyError," self.feedObject("maps", base.makeStruct(MapRule, key=k, source=v, nullExcs=nullExcs)) self._completeElementNext(RowmakerDef, ctx) def _getSourceFromColset(self, columns): """returns the source code for a mapper to a column set. """ lineMap, line = {}, 0 source = [] def appendToSource(srcLine, line, lineMarker): source.append(srcLine) line += 1 lineMap[line] = lineMarker line += source[-1].count("\n") return line if self.ignoreOn: line = appendToSource("if checkTrigger(vars):\n" " raise IgnoreThisRow(vars)", line, "Checking ignore") for v in self.vars: line = appendToSource(v.getCode(columns), line, "assigning "+v.key) for a in self.apps: line = appendToSource( "%s(vars, result, targetTable, _self)"%a.name, line, "executing "+a.name) for m in self.maps: line = appendToSource(m.getCode(columns), line, "building "+m.key) return "\n".join(source), lineMap def _getSource(self, tableDef): """returns the source code for a mapper to tableDef's columns. """ return self._getSourceFromColset(tableDef.columns) def _getGlobals(self, tableDef): globals = {} for a in self.apps: globals[a.name] = a.compile() if self.ignoreOn: globals["checkTrigger"] = self.ignoreOn globals["tableDef_"] = tableDef globals["rd_"] = tableDef.rd globals["curDD_"] = tableDef.parent return globals def _resolveIdmaps(self, columns): """adds mappings for self's idmap within column set. """ existingMaps = set(m.key for m in self.maps) baseNames = [c.key for c in columns] for colName in self.idmaps: matching = fnmatch.filter(baseNames, colName) if not matching: raise base.NotFoundError(colName, "columns matching", "unknown") for dest in matching: if dest not in existingMaps: self.maps.append(MapRule(self, key=dest).finishElement(None)) self.idmaps = [] def _checkTable(self, columns, id): """raises a LiteralParseError if we try to map to non-existing columns. """ for map in self.maps: try: columns.getColumnByName(map.key) except KeyError: raise base.ui.logOldExc(base.LiteralParseError(self.name_, map.key, "Cannot map to '%s' since it does not exist in %s"%( map.key, id))) def _buildForTable(self, tableDef): """returns a RowmakerDef with everything expanded and checked for tableDef. This may raise LiteralParseErrors if self's output is incompatible with tableDef. """ res = self.copyShallowly() try: res._resolveIdmaps(tableDef.columns) res._checkTable(tableDef.columns, tableDef.id) except base.NotFoundError, ex: ex.within = "table %s's columns"%tableDef.id raise return res
class ColumnBase(base.Structure, base.MetaMixin): """A base class for columns, parameters, output fields, etc. Actually, right now there's far too much cruft in here that should go into Column proper or still somewhere else. Hence: XXX TODO: Refactor. See also Column for a docstring that still applies to all we've in here. """ _name = ParamNameAttribute("name", default=base.Undefined, description="Name of the param", copyable=True, before="type") _type = TypeNameAttribute( "type", default="real", description="datatype for the column (SQL-like type system)", copyable=True, before="unit") _unit = base.UnicodeAttribute("unit", default="", description="Unit of the values", copyable=True, before="ucd", strip=True) _ucd = base.UnicodeAttribute("ucd", default="", description="UCD of the column", copyable=True, before="description") _description = base.NWUnicodeAttribute( "description", default="", copyable=True, description= "A short (one-line) description of the values in this column.") _tablehead = base.UnicodeAttribute( "tablehead", default=None, description="Terse phrase to put into table headers for this" " column", copyable=True) _utype = base.UnicodeAttribute("utype", default=None, description="utype for this column", copyable=True) _required = base.BooleanAttribute( "required", default=False, description="Record becomes invalid when this column is NULL", copyable=True) _displayHint = DisplayHintAttribute( "displayHint", description="Suggested presentation; the format is " " <kw>=<value>{,<kw>=<value>}, where what is interpreted depends" " on the output format. See, e.g., documentation on HTML renderers" " and the formatter child of outputFields.", copyable=True) _verbLevel = base.IntAttribute( "verbLevel", default=20, description="Minimal verbosity level at which to include this column", copyable=True) _values = base.StructAttribute("values", default=None, childFactory=Values, description="Specification of legal values", copyable=True) _fixup = base.UnicodeAttribute( "fixup", description= "A python expression the value of which will replace this column's" " value on database reads. Write a ___ to access the original" ' value. You can use macros for the embedding table.' ' This is for, e.g., simple URL generation' ' (fixup="\'\\internallink{/this/svc}\'+___").' ' It will *only* kick in when tuples are deserialized from the' " database, i.e., *not* for values taken from tables in memory.", default=None, copyable=True) _note = base.UnicodeAttribute( "note", description="Reference to a note meta" " on this table explaining more about this column", default=None, copyable=True) _xtype = base.UnicodeAttribute("xtype", description="VOTable xtype giving" " the serialization form", default=None, copyable=True) _stc = TableManagedAttribute( "stc", description="Internally used" " STC information for this column (do not assign to unless instructed" " to do so)", default=None, copyable=True) _stcUtype = TableManagedAttribute( "stcUtype", description="Internally used" " STC information for this column (do not assign to)", default=None, copyable=True) _properties = base.PropertyAttribute(copyable=True) _original = base.OriginalAttribute() restrictedMode = False def __repr__(self): return "<Column %s>" % repr(self.name) def setMetaParent(self, parent): # columns should *not* take part in meta inheritance. The reason is # that there are usually many columns to a table, and there's no # way I can see that any piece of metadata should be repeated in # all of them. On the other hand, for votlinks (no name an example), # meta inheritance would have disastrous consequences. # So, we bend the rules a bit. raise base.StructureError( "Columns may not have meta parents.", hint="The rationale for this is explained in rscdef/column.py," " look for setMetaParent.") def onParentComplete(self): # we need to resolve note on construction since columns are routinely # copied to other tables and meta info does not necessarily follow. if isinstance(self.note, basestring): try: self.note = self.parent.getNote(self.note) except base.NotFoundError: # non-existing notes silently ignored self.note = None def completeElement(self, ctx): self.restrictedMode = getattr(ctx, "restricted", False) if isinstance(self.name, utils.QuotedName): self.key = self.name.name if ')' in self.key: # No '()' allowed in key for that breaks the %()s syntax (sigh!). # Work around with the following quick hack that would break # if people carefully chose proper names. Anyone using delim. # ids in SQL deserves a good spanking anyway. self.key = self.key.replace(')', "__").replace('(', "__") else: self.key = self.name self._completeElementNext(ColumnBase, ctx) def isEnumerated(self): return self.values and self.values.options def validate(self): self._validateNext(ColumnBase) if self.restrictedMode and self.fixup: raise base.RestrictedElement("fixup") def validateValue(self, value): """raises a ValidationError if value does not match the constraints given here. """ if value is None: if self.required: raise base.ValidationError( "Field %s is empty but non-optional" % self.name, self.name) return # Only validate these if we're not a database column if not isinstance(self, Column): vals = self.values if vals: if vals.options: if value and not vals.validateOptions(value): raise base.ValidationError( "Value %s not consistent with" " legal values %s" % (value, vals.options), self.name) else: if vals.min and value < vals.min: raise base.ValidationError( "%s too small (must be at least %s)" % (value, vals.min), self.name) if vals.max and value > vals.max: raise base.ValidationError( "%s too large (must be less than %s)" % (value, vals.max), self.name) def isIndexed(self): """returns a guess as to whether this column is part of an index. This may return True, False, or None (unknown). """ if self.parent and hasattr(self.parent, "indexedColumns"): # parent is something like a TableDef if self.name in self.parent.indexedColumns: return True else: return False def isPrimary(self): """returns a guess as to whether this column is a primary key of the embedding table. This may return True, False, or None (unknown). """ if self.parent and hasattr(self.parent, "primary"): # parent is something like a TableDef if self.name in self.parent.primary: return True else: return False _indexedCleartext = { True: "indexed", False: "notIndexed", None: "unknown", } def asInfoDict(self): """returns a dictionary of certain, "user-interesting" properties of the data field, in a dict of strings. """ return { "name": unicode(self.name), "description": self.description or "N/A", "tablehead": self.getLabel(), "unit": self.unit or "N/A", "ucd": self.ucd or "N/A", "verbLevel": self.verbLevel, "indexState": self._indexedCleartext[self.isIndexed()], "note": self.note, } def getDDL(self): """returns an SQL fragment describing this column ready for inclusion in a DDL statement. """ type = self.type # we have one "artificial" type, and it shouldn't become more than # one; so, a simple hack should do it. if type.upper() == "UNICODE": type = "TEXT" # The "str" does magic for delimited identifiers, so it's important. items = [str(self.name), type] if self.required: items.append("NOT NULL") return " ".join(items) def getDisplayHintAsString(self): return self._displayHint.unparse(self.displayHint) def getLabel(self): """returns a short label for this column. The label is either the tablehead or, missing it, the capitalized column name. """ if self.tablehead is not None: return self.tablehead return str(self.name).capitalize() def _getVOTableType(self): """returns the VOTable type, arraysize and xtype for this column-like thing. """ type, arraysize, xtype = base.sqltypeToVOTable(self.type) if self.type == "date": xtype = "dachs:DATE" return type, arraysize, xtype
class Values(base.Structure): """Information on a column's values, in particular its domain. This is quite like the values element in a VOTable. In particular, to accomodate VOTable usage, we require nullLiteral to be a valid literal for the parent's type. Note that DaCHS does not validate for contraints from values on table import. This is mainly because before ``gavo values`` has run, values may not represent the new dataset in semiautomatic values. With HTTP parameters, values validation does take place (but again, that's mostly not too helpful because there are query languages sitting in between most of the time). Hence, the main utility of values is metadata declaration, both in the form render (where they become placeholders) and in datalink (where they are communicated as VOTable values). """ name_ = "values" _min = base.UnicodeAttribute("min", default=None, description="Minimum acceptable" " value as a datatype literal", copyable=True) _max = base.UnicodeAttribute("max", default=None, description="Maximum acceptable" " value as a datatype literal", copyable=True) _options = base.StructListAttribute( "options", childFactory=Option, description="List of acceptable values (if set)", copyable=True) _default = base.UnicodeAttribute( "default", default=None, description="A default" " value (currently only used for options).", copyable=True) _nullLiteral = base.UnicodeAttribute( "nullLiteral", default=None, description= "An appropriate value representing a NULL for this column in VOTables" " and similar places. You usually should only set it for integer" " types and chars. Note that rowmakers make no use of this nullLiteral," " i.e., you can and should choose null values independently of" " your source. Again, for reals, floats and (mostly) text you probably" " do not want to do this.", copyable=True) _multiOk = base.BooleanAttribute("multiOk", False, "Deprecated, use" " multiplicity=multiple instead.", copyable=True) _fromDB = base.ActionAttribute( "fromdb", "_evaluateFromDB", description= "A query fragment returning just one column to fill options from (will" " add to options if some are given). Do not write SELECT or anything," " just the column name and the where clause.") _original = base.OriginalAttribute() validValues = None @classmethod def fromOptions(cls, labels): """returns Values with the elements of labels as valid options. """ return base.makeStruct( cls, options=[base.makeStruct(Option, content_=l) for l in labels]) def makePythonVal(self, literal, sqltype): return typesystems.sqltypeToPython(sqltype)(literal) def _evaluateFromDB(self, ctx): if not getattr(ctx, "doQueries", True): return try: with base.getTableConn() as conn: for row in conn.query( self.parent.parent.expand("SELECT DISTINCT %s" % (self.fromdb))): self._options.feedObject( self, base.makeStruct(Option, content_=row[0])) except base.DBError: # Table probably doesn't exist yet, ignore. base.ui.notifyWarning("Values fromdb '%s' failed, ignoring" % self.fromdb) def onParentComplete(self): """converts min, max, and options from string literals to python objects. """ dataField = self.parent # It would be nicer if we could handle this in properties for min etc, but # unfortunately parent might not be complete then. The only # way around that would have been delegations from Column, and that's # not very attractive either. if self.min: self.min = self.makePythonVal(self.min, dataField.type) if self.max: self.max = self.makePythonVal(self.max, dataField.type) if self.options: dbt = dataField.type for opt in self.options: opt.content_ = self.makePythonVal(opt.content_, dbt) self.validValues = set([o.content_ for o in self.options]) if self.nullLiteral: try: self.makePythonVal(self.nullLiteral, dataField.type) except ValueError: raise base.LiteralParseError( "nullLiteral", self.nullLiteral, hint="If you want to *parse* whatever you gave into a NULL," " use the parseWithNull function in a rowmaker. The null" " literal gives what value will be used for null values" " when serializing to VOTables and the like.") if self.default and isinstance(self.default, basestring): type, arraysize, xtype = dataField._getVOTableType() self.default = paramval.getVOTParser(type, arraysize, xtype)(self.default) def validateOptions(self, value): """returns false if value isn't either in options or doesn't consist of items in options. Various null values always validate here; non-null checking is done by the column on its required attribute. """ if value == "None": assert False, "Literal 'None' passed as a value to validateOptions" if self.validValues is None: return True if isinstance(value, (list, tuple, set)): for val in value: if val and not val in self.validValues: return False else: return value in self.validValues or value is None return True
class SourceSpec(base.Structure): """A Specification of a data descriptor's inputs. This will typcially be files taken from a file system. If so, DaCHS will, in each directory, process the files in alphabetical order. No guarantees are made as to the sequence directories are processed in. Multiple patterns are processed in the order given in the RD. """ name_ = "sources" _patterns = base.ListOfAtomsAttribute("patterns", description= "Paths to the source files. You can use shell patterns here.", itemAttD=base.UnicodeAttribute("pattern", description="Shell pattern" " for source file(s), relative to resource directory."), copyable=True) _items = base.ListOfAtomsAttribute("items", description= "String literals to pass to grammars. In contrast to patterns," " they are not interpreted as file names but passed to the" " grammar verbatim. Normal grammars do not like this. It is" " mainly intended for use with custom or null grammars.", itemAttD=base.UnicodeAttribute("item", description="Grammar-specific string"), copyable=True) _recurse = base.BooleanAttribute("recurse", default=False, description="Search for pattern(s) recursively in their directory" " part(s)?", copyable=True) _ignore = base.StructAttribute("ignoredSources", childFactory= IgnoreSpec, description="Specification of sources that should not" " be processed although they match patterns. Typically used" " in update-type data descriptors.", copyable=True) _file = base.DataContent(description="A single" " file name (this is for convenience)", copyable="True") _original = base.OriginalAttribute() def __iter__(self): return self.iterSources() def completeElement(self, ctx): if self.ignoredSources is base.Undefined: self.ignoredSources = base.makeStruct(IgnoreSpec) self._completeElementNext(SourceSpec, ctx) def _expandDirParts(self, dirParts, ignoreDotDirs=True): """expands a list of directories into a list of them and all their descendants. It follows symbolic links but doesn't do any bookkeeping, so bad things will happen if the directory graph contains cycles. """ res = [] for root in dirParts: for root, dirs, files in os.walk(root): if ignoreDotDirs: if os.path.basename(root).startswith("."): continue dirs = [dir for dir in dirs if not dir.startswith(".")] dirs = (os.path.join(root, dir) for dir in dirs) res.extend(dir for dir in dirs if os.path.isdir(dir)) for child in files: destName = os.path.join(root, child) if os.path.islink(destName) and not os.path.isfile(destName): res.extend(self._expandDirParts(destName)) return res def iterSources(self, connection=None): self.ignoredSources.prepare(connection) for item in self.items: if not self.ignoredSources.isIgnored(item): yield item baseDir = "" if self.parent.rd: baseDir = self.parent.rd.resdir for pattern in self.patterns: dirPart, baseName = os.path.split(pattern) if self.parent.rd: dirParts = [os.path.join(baseDir, dirPart)] else: dirParts = [dirPart] if self.recurse: dirParts = dirParts+self._expandDirParts(dirParts) for dir in sorted(dirParts): for name in sorted(glob.glob(os.path.join(dir, baseName))): fullName = os.path.abspath(name) if not self.ignoredSources.isIgnored(fullName): yield fullName if self.content_: yield os.path.abspath(os.path.join(baseDir, self.content_)) def __nonzero__(self): return (not not self.patterns) or (not not self.items ) or (not not self.content_)
class DataDescriptor(base.Structure, base.ComputedMetaMixin, common.IVOMetaMixin, tabledef.PublishableDataMixin): """A description of how to process data from a given set of sources. Data descriptors bring together a grammar, a source specification and "makes", each giving a table and a rowmaker to feed the table from the grammar output. They are the "executable" parts of a resource descriptor. Their ids are used as arguments to gavoimp for partial imports. """ name_ = "data" resType = "data" _rowmakers = base.StructListAttribute("rowmakers", childFactory=rmkdef.RowmakerDef, description="Embedded build rules (usually rowmakers are defined toplevel)", copyable=True, before="makes") _tables = base.StructListAttribute("tables", childFactory=tabledef.TableDef, description="Embedded table definitions (usually, tables are defined" " toplevel)", copyable=True, before="makes") _grammar = base.MultiStructAttribute("grammar", default=None, childFactory=builtingrammars.getGrammar, childNames=builtingrammars.GRAMMAR_REGISTRY.keys(), description="Grammar used to parse this data set.", copyable=True, before="makes") _sources = base.StructAttribute("sources", default=None, childFactory=SourceSpec, description="Specification of sources that should be fed to the grammar.", copyable=True, before="grammar") _dependents = base.ListOfAtomsAttribute("dependents", itemAttD=base.UnicodeAttribute("recreateAfter"), description="A data ID to recreate when this resource is" " remade; use # syntax to reference in other RDs.") _auto = base.BooleanAttribute("auto", default=True, description="Import this data set if not explicitly" " mentioned on the command line?") _updating = base.BooleanAttribute("updating", default=False, description="Keep existing tables on import? You usually want this" " False unless you have some kind of sources management," " e.g., via a sources ignore specification.", copyable=True) _makes = base.StructListAttribute("makes", childFactory=Make, copyable=True, description="Specification of a target table and the rowmaker" " to feed them.") _params = common.ColumnListAttribute("params", childFactory=column.Param, description='Param ("global columns") for this data (mostly for' ' VOTable serialization).', copyable=True) _properties = base.PropertyAttribute() _rd = common.RDAttribute() _original = base.OriginalAttribute() metaModel = ("title(1), creationDate(1), description(1)," "subject, referenceURL(1)") def __repr__(self): return "<data descriptor with id %s>"%self.id def validate(self): self._validateNext(DataDescriptor) if self.registration and self.id is None: raise base.StructureError("Published data needs an assigned id.") def onElementComplete(self): self._onElementCompleteNext(DataDescriptor) for t in self.tables: t.setMetaParent(self) if self.registration: self.registration.register() # since we want to be able to create DDs dynamically , they must find their # meta parent themselves. We do this while the DD is being adopted; # the rules here are: if the parent is a meta mixin itself, it's the # meta parent, if it has an rd attribute, use that, else give up. # TODO: For DDs on cores, it would be *desirable* to come up # with some magic that makes the current service their meta parent. def _getParent(self): return self.__parent def _setParent(self, value): self.__parent = value if isinstance(value, base.MetaMixin): self.setMetaParent(value) elif hasattr(value, "rd"): self.setMetaParent(value.rd) parent = property(_getParent, _setParent) def iterSources(self, connection=None): if self.sources: return self.sources.iterSources(connection) else: return iter([]) def __iter__(self): for m in self.makes: yield m.table def iterTableDefs(self): """iterates over the definitions of all the tables built by this DD. """ for m in self.makes: yield m.table def getTableDefById(self, id): for td in self.iterTableDefs(): if td.id==id: return td raise base.StructureError("No table name %s will be built"%id) def getTableDefWithRole(self, role): for m in self.makes: if m.role==role: return m.table raise base.StructureError("No table def with role '%s'"%role) def getPrimary(self): """returns the "primary" table definition in the data descriptor. "primary" means the only table in a one-table dd, the table with the role "primary" if there are more. If no matching table is found, a StructureError is raised. """ if len(self.makes)==1: return self.makes[0].table else: try: return self.getTableDefWithRole("primary") except base.StructureError: # raise more telling message pass raise base.StructureError("Ambiguous request for primary table") def copyShallowly(self): """returns a shallow copy of self. Sources are not copied. """ return DataDescriptor(self.parent, rowmakers=self.rowmakers[:], tables=self.tables[:], grammar=self.grammar, makes=self.makes[:]) def getURL(self, rendName, absolute=True): # there's no sensible URL for DDs; thus, let people browse # the RD info. At least they should find links to any tables # included here there. basePath = "%sbrowse/%s"%( base.getConfig("web", "nevowRoot"), self.rd.sourceId) if absolute: return base.getConfig("web", "serverURL")+basePath return basePath
class MixinDef(activetags.ReplayBase): """A definition for a resource mixin. Resource mixins are resource descriptor fragments typically rooted in tables (though it's conceivable that other structures could grow mixin attributes as well). They are used to define and implement certain behaviours components of the DC software want to see: - products want to be added into their table, and certain fields are required within tables describing products - tables containing positions need some basic machinery to support scs. - siap needs quite a bunch of fields Mixins consist of events that are played back on the structure mixing in before anything else happens (much like original) and two procedure definitions, viz, processEarly and processLate. These can access the structure that has the mixin as substrate. processEarly is called as part of the substrate's completeElement method. processLate is executed just before the parser exits. This is the place to fix up anything that uses the table mixed in. Note, however, that you should be as conservative as possible here -- you should think of DC structures as immutable as long as possible. Programmatically, you can check if a certain table mixes in something by calling its mixesIn method. Recursive application of mixins, even to seperate objects, will deadlock. """ name_ = "mixinDef" _doc = base.UnicodeAttribute("doc", description="Documentation for" " this mixin", strip=False) _events = base.StructAttribute("events", childFactory=activetags.EmbeddedStream, description="Events to be played back into the structure mixing" " this in at mixin time.", copyable=True, default=base.NotGiven) _lateEvents = base.StructAttribute("lateEvents", childFactory=LateEvents, description="Events to be played back into the structure mixing" " this in at completion time.", copyable=True, default=base.NotGiven) _processEarly = base.StructAttribute("processEarly", default=None, childFactory=ProcessEarly, description="Code executed at element fixup.", copyable=True) _processLate = base.StructAttribute("processLate", default=None, childFactory=ProcessLate, description="Code executed resource fixup.", copyable=True) _pars = base.UniquedStructListAttribute("pars", childFactory=MixinPar, uniqueAttribute="key", description="Parameters available for this mixin.", copyable=True) _original = base.OriginalAttribute() def completeElement(self, ctx): # we want to double-expand macros in mixins. Thus, reset all # value/expanded events to plain values if self.events: self.events.unexpandMacros() if self.lateEvents: self.lateEvents.unexpandMacros() # This lock protects against multiple uses of applyTo. This is # necessary because during replay, we have macroExpansions and # macroParent reflect a concrete application's context. self.applicationLock = threading.Lock() self._completeElementNext(MixinDef, ctx) def _defineMacros(self, fillers, destination): """creates attributes macroExpansions and parentMacroPackage used by execMacros. Within mixins, you can use macros filled by mixin parameters or expanded by the substrate. This information is local to a concrete mixin application. Hence, applyTo calls this method, and the attributes created are invalid for any subsequent or parallel applyTo calls. Therefore, applyTo acquires the applicationLock before calling this. """ self.parentMacroPackage = None if hasattr(destination, "execMacro"): self.parentMacroPackage = destination self.macroExpansions = {} for p in self.pars: if p.key in fillers: self.macroExpansions[p.key] = fillers.pop(p.key) elif p.isDefaulted(): self.macroExpansions[p.key] = p.content_ else: raise base.StructureError("Mixin parameter %s mandatory"%p.key) if fillers: raise base.StructureError("The attribute(s) %s is/are not allowed" " on this mixin"%(",".join(fillers))) def execMacro(self, macName, args): if macName in self.macroExpansions: return self.macroExpansions[macName] try: if self.parentMacroPackage: return self.parentMacroPackage.execMacro(macName, args) except base.MacroError: raise base.MacroError( "No macro \\%s available in this mixin or substrate."%(macName), macName) def applyTo(self, destination, ctx, fillers={}): """replays the stored events on destination and arranges for processEarly and processLate to be run. """ with self.applicationLock: self._defineMacros(fillers.copy(), destination) if self.events: self.replay(self.events.events_, destination, ctx) if self.processEarly is not None: self.processEarly.compile(destination)(ctx, destination, self.macroExpansions) if self.processLate is not None: def procLate(rootStruct, parseContext): self.processLate.compile(destination)( destination, rootStruct, parseContext) ctx.addExitFunc(procLate) if self.lateEvents: origComplete = destination.completeElement def newComplete(ctx): with self.applicationLock: self._defineMacros(fillers.copy(), destination) self.replay(self.lateEvents.events_, destination, ctx) origComplete(ctx) destination.completeElement = newComplete def applyToFinished(self, destination): """applies the mixin to an object already parsed. Late callbacks will only be executed if destination has an rd attribute; if that is the case, this rd's idmap will be amended with anything the mixin comes up with. """ rd = None if hasattr(destination, "rd"): rd = destination.rd ctx = base.ParseContext() if rd is not None: ctx.idmap = destination.rd.idmap self.applyTo(destination, ctx) # we don't keep the application lock for this; applyToFinished # is more of a debugging thing, so we don't worry too much. if self.lateEvents: self.replay(self.lateEvents.events_, destination, ctx) if rd is not None: ctx.runExitFuncs(rd)
class Service(base.Structure, base.ComputedMetaMixin, base.StandardMacroMixin, rscdef.IVOMetaMixin): """A service definition. A service is a combination of a core and one or more renderers. They can be published, and they carry the metadata published into the VO. You can set the defaultSort property on the service to a name of an output column to preselect a sort order. Note again that this will slow down responses for all but the smallest tables unless there is an index on the corresponding column. Properties evaluated: * defaultSort -- a key to sort on by default with the form renderer. This differs from the dbCore's sortKey in that this does not suppress the widget itself, it just sets a default for its value. Don't use this unless you have to; the combination of sort and limit can have disastrous effects on the run time of queries. * votableRespectsOutputTable -- usually, VOTable output puts in all columns from the underlying database table with low enough verbLevel (essentially). When this property is "True" (case-sensitive), that's not done and only the service's output table is evaluated. [Note that column selection is such a mess it needs to be fixed before version 1.0 anyway] """ name_ = "service" _core = CoreAttribute() _templates = base.DictAttribute( "templates", description="Custom" ' nevow templates for this service; use key "form" to replace the Form' " renderer's standard template. Start the path with two slashes to" " access system templates.", itemAttD=rscdef.ResdirRelativeAttribute( "template", description="resdir-relative path to a nevow template" " used for the function given in key."), copyable=True) _publications = base.StructListAttribute( "publications", childFactory=Publication, description="Sets and renderers this service" " is published with.") _limitTo = base.UnicodeAttribute( "limitTo", default=None, description="Limit access to the group given; the empty default disables" " access control.", copyable="True") _customPage = rscdef.ResdirRelativeAttribute( "customPage", default=None, description="resdir-relative path to custom page code. It is used" " by the 'custom' renderer", copyable="True") _allowedRenderers = base.StringSetAttribute( "allowed", description="Names of renderers allowed on this service; leave emtpy" " to allow the form renderer only.", copyable=True) _customRF = base.StructListAttribute( "customRFs", description="Custom render functions for use in custom templates.", childFactory=CustomRF, copyable=True) _customDF = base.StructListAttribute( "customDFs", description="Custom data functions for use in custom templates.", childFactory=CustomDF, copyable=True) _inputData = base.StructAttribute( "inputDD", default=base.NotGiven, childFactory=inputdef.InputDescriptor, description="A data descriptor" " for obtaining the core's input, usually based on a contextGrammar." " For many cores (e.g., DBCores), you do not want to give this" " but rather want to let service figure this out from the core.", copyable=True) _outputTable = base.StructAttribute( "outputTable", default=base.NotGiven, childFactory=outputdef.OutputTableDef, copyable=True, description="The output fields of this service.") _serviceKeys = base.StructListAttribute( "serviceKeys", childFactory=inputdef.InputKey, description="Input widgets for" " processing by the service, e.g. output sets.", copyable=True) _defaultRenderer = base.UnicodeAttribute( "defaultRenderer", default=None, description="A name of a renderer used when" " none is provided in the URL (lets you have shorter URLs).") _rd = rscdef.RDAttribute() _props = base.PropertyAttribute() _original = base.OriginalAttribute() metaModel = ("title(1), creationDate(1), description(1)," "subject, referenceURL(1), shortName(!)") # formats that should query the same fields as HTML (the others behave # like VOTables and offer a "verbosity" widget in forms). htmlLikeFormats = ["HTML", "tar"] ####################### Housekeeping methods def __repr__(self): return "<Service at %x>" % id(self) def completeElement(self, ctx): self._completeElementNext(Service, ctx) if not self.allowed: self.allowed.add("form") if self.core is base.Undefined: # undefined cores are only allowed with custom pages # (Deprecated) if self.customPage: self.core = core.getCore("nullCore")( self.rd).finishElement(None) base.ui.notifyWarning( "Custom page service %s without nullCore." " This is deprecated, please fix" % self.id) else: raise base.StructureError( "Services must have cores (add <nullCore/>" " if you really do not want a core, e.g., with fixed renderers)." ) # if there's only one renderer on this service, make it the default if self.defaultRenderer is None and len(self.allowed) == 1: self.defaultRenderer = list(self.allowed)[0] # empty output tables are filled from the core if self.outputTable is base.NotGiven: self.outputTable = self.core.outputTable # cache all kinds of things expensive to create and parse self._coresCache = {} self._inputDDCache = {} self._loadedTemplates = {} # Schedule the capabilities to be added when the parse is # done (i.e., the RD is complete) ctx.addExitFunc(lambda rd, ctx: self._addAutomaticCapabilities()) def onElementComplete(self): self._onElementCompleteNext(Service) # Index custom render/data functions self.nevowRenderers = {} for customRF in self.customRFs: self.nevowRenderers[customRF.name] = customRF.func self.nevowDataFunctions = {} for customDF in self.customDFs: self.nevowDataFunctions[customDF.name] = customDF.func self._compileCustomPage() self._computeResourceType() def _compileCustomPage(self): if self.customPage: try: modNs, moddesc = utils.loadPythonModule(self.customPage) modNs.RD = self.rd getattr(modNs, "initModule", lambda: None)() page = modNs.MainPage except ImportError: raise base.ui.logOldExc( base.LiteralParseError( "customPage", self.customPage, hint= "This means that an exception was raised while DaCHS" " tried to import the renderer module. If DaCHS ran" " with --debug, the original traceback is available" " in the logs.")) self.customPageCode = page, (os.path.basename( self.customPage), ) + moddesc def getTemplate(self, key): """returns the nevow template for the function key on this service. """ if key not in self._loadedTemplates: from nevow import loaders tp = self.templates[key] if tp.startswith("//"): self._loadedTemplates[key] = common.loadSystemTemplate(tp[2:]) else: self._loadedTemplates[key] = loaders.xmlfile( os.path.join(self.rd.resdir, tp)) return self._loadedTemplates[key] def getUWS(self): """returns a user UWS instance for this service. This is a service for the UWSAsyncRenderer. """ if not hasattr(self, "uws"): from gavo.protocols import useruws self.uws = useruws.makeUWSForService(self) return self.uws ################### Registry and related methods. @property def isVOPublished(self, renderer=None): """is true if there is any ivo_managed publication on this service. If renderer is non-None, only publications with this renderer name count. """ for pub in self.publications: if "ivo_managed" in pub.sets: if renderer: if pub.render == renderer: return True else: return True return False def _computeResourceType(self): """sets the resType attribute. Services are resources, and the registry code wants to know what kind. This method ventures a guess. You can override this decision by setting the resType meta item. """ if (self.outputTable.columns or self.outputTable.verbLevel or "tap" in self.allowed): self.resType = "catalogService" else: # no output table defined, we're a plain service self.resType = "nonTabularService" def _addAutomaticCapabilities(self): """adds some publications that are automatic for certain types of services. For services with ivo_managed publications and with useful cores (this keeps out doc-like publications, which shouldn't have VOSI resources), artificial VOSI publications are added. If there is _example meta, an examples publication is added. If this service exposes a table (i.e., a DbCore with a queriedTable) and that table is adql-readable, also add an auxiliary TAP publication if going to the VO. This is being run as an exit function from the parse context as we want the RD to be complete at this point (e.g., _examples meta might come from it). This also lets us liberally resolve references anywhere. """ if not self.isVOPublished: return vosiSet = set(["ivo_managed"]) # All actual services get VOSI caps if not isinstance(self.core, core.getCore("nullCore")): self._publications.feedObject( self, base.makeStruct(Publication, render="availability", sets=vosiSet, parent_=self)) self._publications.feedObject( self, base.makeStruct(Publication, render="capabilities", sets=vosiSet, parent_=self)) self._publications.feedObject( self, base.makeStruct(Publication, render="tableMetadata", sets=vosiSet, parent_=self)) # things querying tables get a TAP relationship if # their table is adql-queriable if isinstance(self.core, core.getCore("dbCore")): if self.core.queriedTable.adql: tapService = base.resolveCrossId("//tap#run") self._publications.feedObject( self, base.makeStruct(Publication, render="tap", sets=vosiSet, auxiliary=True, service=tapService, parent_=self)) # and they need a servedBy, too. # According to the "discovering dependent" note, we don't # do the reverse relationship lest the TAP service # gets too related... self.addMeta("servedBy", base.getMetaText(tapService, "title"), ivoId=base.getMetaText(tapService, "identifier")) # things with examples meta get an examples capability try: self.getMeta("_example", raiseOnFail=True) self._publications.feedObject( self, base.makeStruct(Publication, render="examples", sets=utils.AllEncompassingSet(), parent_=self)) except base.NoMetaKey: pass def getPublicationsForSet(self, names): """returns publications for set names in names. names must be a set. """ additionals = [] # for ivo_managed, also return a datalink endpoints if they're # there; the specs imply that might be useful some day. if self.getProperty("datalink", None): dlSvc = self.rd.getById(self.getProperty("datalink")) if "dlget" in dlSvc.allowed: additionals.append( base.makeStruct(Publication, render="dlget", sets="ivo_managed", service=dlSvc)) if "dlasync" in dlSvc.allowed: additionals.append( base.makeStruct(Publication, render="dlasync", sets="ivo_managed", service=dlSvc)) if "dlmeta" in dlSvc.allowed: additionals.append( base.makeStruct(Publication, render="dlmeta", sets="ivo_managed", service=dlSvc)) return [pub for pub in self.publications if pub.sets & names] + additionals def getURL(self, rendName, absolute=True, **kwargs): """returns the full canonical access URL of this service together with renderer. rendName is the name of the intended renderer in the registry of renderers. With absolute, a fully qualified URL is being returned. Further keyword arguments are translated into URL parameters in the query part. """ basePath = "%s%s/%s" % (base.getConfig( "web", "nevowRoot"), self.rd.sourceId, self.id) if absolute: basePath = base.getConfig("web", "serverURL") + basePath res = renderers.getRenderer(rendName).makeAccessURL(basePath) if kwargs: res = res + "?" + urllib.urlencode(kwargs) return res # used by getBrowserURL; keep external higher than form as long as # we have mess like Potsdam CdC. _browserScores = { "form": 10, "external": 12, "fixed": 15, "custom": 3, "img.jpeg": 2, "static": 1 } def getBrowserURL(self, fq=True): """returns a published URL that's suitable for a web browser or None if no such URL can be guessed. If you pass fq=False, you will get a path rather than a URL. """ # There can be multiple candidates for browser URLs (like when a service # has both form, static, and external renderers). If so, we select # by plain scores. browseables = [] for rendName in self.allowed: if self.isBrowseableWith(rendName): browseables.append((self._browserScores.get(rendName, -1), rendName)) if browseables: return self.getURL(max(browseables)[1], absolute=fq) else: return None def isBrowseableWith(self, rendName): """returns true if rendering this service through rendName results in something pretty in a web browser. """ try: return bool(renderers.getRenderer(rendName).isBrowseable(self)) except base.NotFoundError: # renderer name not known return False def getTableSet(self): """returns a list of table definitions that have something to do with this service. This is for VOSI-type queries. Usually, that's just the core's queried table or an output table, except when there is a TAP renderer on the service. All this is a bit heuristic; but then again, there's no rigorous definition for what's to be in a tables endpoint either. """ tables = [] # output our own outputTable if it sounds reasonable; if so, # add the core's queried table, too, if it has one. if self.outputTable and self.outputTable.columns: tables.append(self.outputTable) tables.append(getattr(self.core, "queriedTable", None)) else: # if our outputTable is no good, just use the one of the core qt = getattr(self.core, "queriedTable", None) if qt is None: qt = getattr(self.core, "outputTable", None) if qt is not None: tables.append(qt) # XXX TODO: This stinks big time. It's because we got TAP factorization # wrong. Sync and async should be renderers, and there should # be a core that then could say this kind of thing. That's not # yet the case, so: if "tap" in self.allowed: # tap never has "native" tables, so start afresh tables = [] mth = base.caches.getMTH(None) for tableName in mth.getTAPTables(): try: tables.append(mth.getTableDefForTable(tableName)) except: base.ui.notifyError( "Failure trying to retrieve table definition" " for table %s. Please fix the corresponding RD." % tableName) return [t for t in tables if t is not None and t.rd is not None] def declareServes(self, data): """adds meta to self and data indicating that data is served by service. This is used by table/@adql and the publish element on data. """ if data.registration: self.addMeta("serviceFor", base.getMetaText(data, "title", default="Anonymous"), ivoId=base.getMetaText(data, "identifier")) data.addMeta("servedBy", base.getMetaText(self, "title"), ivoId=base.getMetaText(self, "identifier")) # Since this is always initiated by the data, the dependency # must show up in its RD to be properly added on publication # and to be removed when the data is removed. data.rd.addDependency(self.rd, data.rd) ########################## Output field selection (ouch!) def _getVOTableOutputFields(self, queryMeta): """returns a list of OutputFields suitable for a VOTable response described by queryMeta. This is what's given for HTML when the columns verbLevel is low enough and there's no displayHint of noxml present. In addition, more columns are added from outputTable's parent (which usually will be the database table itself) if their verbLevel is low enough. this may be suppressed by setting the votableRespectsOutputTable property to "True". """ verbLevel = queryMeta.get("verbosity", 20) fields = [ f for f in self.getHTMLOutputFields(queryMeta) if f.verbLevel <= verbLevel and f.displayHint.get("noxml") != "true" ] if (verbLevel != "HTML" and self.getProperty( "votableRespectsOutputTable", None) != "True"): htmlNames = set(f.name for f in fields) for field in self.outputTable.parentTable: if field.name in htmlNames: continue if (field.displayHint.get("type") == "suppress" or field.displayHint.get("noxml") == "true"): continue if field.verbLevel <= verbLevel: fields.append(field) return rscdef.ColumnList(fields) _allSet = set(["ALL"]) def getHTMLOutputFields(self, queryMeta, ignoreAdditionals=False, raiseOnUnknown=True): """returns a list of OutputFields suitable for an HTML response described by queryMeta. raiseOnUnknown is used by customwidgets to avoid exceptions because of bad additional fields during form construction (when they aren't properly caught). """ requireSet = queryMeta["columnSet"] res = rscdef.ColumnList() # add "normal" output fields if requireSet: res.extend([ f for f in self.outputTable if f.sets == self._allSet or requireSet & f.sets ]) else: res.extend([ f for f in self.outputTable if f.displayHint.get("type") != "suppress" ]) # add user-selected fields if not ignoreAdditionals and queryMeta["additionalFields"]: cofs = self.core.outputTable.columns try: for fieldName in queryMeta["additionalFields"]: col = cofs.getColumnByName(fieldName) if isinstance(col, outputdef.OutputField): res.append(col) else: res.append(outputdef.OutputField.fromColumn(col)) except base.NotFoundError, msg: if raiseOnUnknown: raise base.ValidationError( "The additional field %s you requested" " does not exist" % repr(msg.lookedFor), colName="_OUTPUT") return res
class Core(base.Structure): """A definition of the "active" part of a service. Cores receive their input in tables the structure of which is defined by their inputTable attribute. The abstract core element will never occur in resource descriptors. See `Cores Available`_ for concrete cores. Use the names of the concrete cores in RDs. You can specify an input table in an inputTableXML and an output table in an outputTableXML class attribute. """ name_ = "core" inputTableXML = None outputTableXML = None # the cached prototype of the output table, filled in by # _OutputTableFactory _ot_prototype = None _rd = rscdef.RDAttribute() _inputTable = base.StructAttribute( "inputTable", default=base.NotGiven, childFactory=inputdef.InputTable, description="Description of the input data", copyable=True) _outputTable = base.StructAttribute( "outputTable", default=base.NotGiven, childFactory=_OutputTableFactory(), description="Table describing what fields are available from this core", copyable=True) _original = base.OriginalAttribute() _properties = base.PropertyAttribute() def __init__(self, parent, **kwargs): if self.inputTableXML is not None: if "inputTable" not in kwargs: kwargs["inputTable"] = base.parseFromString( inputdef.InputTable, self.inputTableXML) base.Structure.__init__(self, parent, **kwargs) def __repr__(self): return "<%s at %s>" % (self.__class__.__name__, id(self)) def __str__(self): return repr(self) def completeElement(self, ctx): self._completeElementNext(Core, ctx) if self.inputTable is base.NotGiven: self.inputTable = base.makeStruct(inputdef.InputTable) if self.outputTable is base.NotGiven: self.outputTable = self._outputTable.childFactory(self) def adaptForRenderer(self, renderer): """returns a core object tailored for renderer. """ newIT = self.inputTable.adaptForRenderer(renderer) if newIT is self.inputTable: return self else: return self.change(inputTable=newIT) def run(self, service, inputData, queryMeta): raise NotImplementedError("%s cores are missing the run method" % self.__class__.__name__) def makeUserDoc(self): return ("Polymorphous core element. May contain any of the cores" " mentioned in `Cores Available`_ .")
class ProcSetup(base.Structure): """Prescriptions for setting up a namespace for a procedure application. You can add names to this namespace you using par(ameter)s. If a parameter has no default and an procedure application does not provide them, an error is raised. You can also add names by providing a code attribute containing a python function body in code. Within, the parameters are available. The procedure application's parent can be accessed as parent. All names you define in the code are available as globals to the procedure body. Caution: Macros are expanded within the code; this means you need double backslashes if you want a single backslash in python code. """ name_ = "setup" _code = base.ListOfAtomsAttribute("codeFrags", description="Python function bodies setting globals for the function" " application. Macros are expanded in the context" " of the procedure's parent.", itemAttD=base.UnicodeAttribute("code", description="Python function" " bodies setting globals for the function application. Macros" " are expanded in the context of the procedure's parent.", copyable=True), copyable=True) _pars = base.StructListAttribute("pars", ProcPar, description="Names to add to the procedure's global namespace.", copyable=True) _original = base.OriginalAttribute() def _getParSettingCode(self, useLate, indent, bindings): """returns code that sets our parameters. If useLate is true, generate for late bindings. Indent the code by indent. Bindings is is a dictionary overriding the defaults or setting parameter values. """ parCode = [] for p in self.pars: if p.late==useLate: val = bindings.get(p.key, base.NotGiven) if val is base.NotGiven: val = p.content_ parCode.append("%s%s = %s"%(indent, p.key, val)) return "\n".join(parCode) def getParCode(self, bindings): """returns code doing setup bindings un-indented. """ return self._getParSettingCode(False, "", bindings) def getLateCode(self, bindings): """returns code doing late (in-function) bindings indented with two spaces. """ return self._getParSettingCode(True, " ", bindings) def getBodyCode(self): """returns the body code un-indented. """ collectedCode = [] for frag in self.codeFrags: collectedCode.append( utils.fixIndentation(frag, "", governingLine=1)) return "\n".join(collectedCode)
class ProcDef(base.Structure, base.RestrictionMixin): """An embedded procedure. Embedded procedures are python code fragments with some interface defined by their type. They can occur at various places (which is called procedure application generically), e.g., as row generators in grammars, as applys in rowmakers, or as SQL phrase makers in condDescs. They consist of the actual actual code and, optionally, definitions like the namespace setup, configuration parameters, or a documentation. The procedure applications compile into python functions with special global namespaces. The signatures of the functions are determined by the type attribute. ProcDefs are referred to by procedure applications using their id. """ name_ = "procDef" _code = base.UnicodeAttribute("code", default=base.NotGiven, copyable=True, description="A python function body.") _setup = base.StructListAttribute("setups", ProcSetup, description="Setup of the namespace the function will run in", copyable=True) _doc = base.UnicodeAttribute("doc", default="", description= "Human-readable docs for this proc (may be interpreted as restructured" " text).", copyable=True) _type = base.EnumeratedUnicodeAttribute("type", default=None, description= "The type of the procedure definition. The procedure applications" " will in general require certain types of definitions.", validValues=["t_t", "apply", "rowfilter", "sourceFields", "mixinProc", "phraseMaker", "descriptorGenerator", "dataFunction", "dataFormatter", "metaMaker", "regTest", "iterator", "pargetter"], copyable=True, strip=True) _deprecated = base.UnicodeAttribute("deprecated", default=None, copyable=True, description="A deprecation message. This will" " be shown if this procDef is being compiled.") _original = base.OriginalAttribute() def getCode(self): """returns the body code indented with two spaces. """ if self.code is base.NotGiven: return "" else: return utils.fixIndentation(self.code, " ", governingLine=1) @utils.memoized def getSetupPars(self): """returns all parameters used by setup items, where lexically later items override earlier items of the same name. """ return unionByKey(*[s.pars for s in self.setups]) def getLateSetupCode(self, boundNames): return "\n".join(s.getLateCode(boundNames) for s in self.setups) def getParSetupCode(self, boundNames): return "\n".join(s.getParCode(boundNames) for s in self.setups) def getBodySetupCode(self, boundNames): return "\n".join(s.getBodyCode() for s in self.setups)
class TableDef(base.Structure, base.ComputedMetaMixin, common.PrivilegesMixin, common.IVOMetaMixin, base.StandardMacroMixin, PublishableDataMixin): """A definition of a table, both on-disk and internal. Some attributes are ignored for in-memory tables, e.g., roles or adql. Properties for tables: * supportsModel -- a short name of a data model supported through this table (for TAPRegExt dataModel); you can give multiple names separated by commas. * supportsModelURI -- a URI of a data model supported through this table. You can give multiple URIs separated by blanks. If you give multiple data model names or URIs, the sequences of names and URIs must be identical (in particular, each name needs a URI). """ name_ = "table" resType = "table" # We don't want to force people to come up with an id for all their # internal tables but want to avoid writing default-named tables to # the db. Thus, the default is not a valid sql identifier. _id = base.IdAttribute( "id", default=base.NotGiven, description="Name of the table (must be SQL-legal for onDisk tables)") _cols = common.ColumnListAttribute( "columns", childFactory=column.Column, description="Columns making up this table.", copyable=True) _params = common.ColumnListAttribute( "params", childFactory=column.Param, description='Param ("global columns") for this table.', copyable=True) _viewStatement = base.UnicodeAttribute( "viewStatement", default=None, description="A single SQL statement to create a view. Setting this" " makes this table a view. The statement will typically be something" " like CREATE VIEW \\\\curtable AS (SELECT \\\\colNames FROM...).", copyable=True) # onDisk must not be copyable since queries might copy the tds and havoc # would result if the queries were to end up on disk. _onDisk = base.BooleanAttribute( "onDisk", default=False, description="Table in the database rather than in memory?") _temporary = base.BooleanAttribute( "temporary", default=False, description="If this is an onDisk table, make it temporary?" " This is mostly useful for custom cores and such.", copyable=True) _adql = ADQLVisibilityAttribute( "adql", default=False, description="Should this table be available for ADQL queries? In" " addition to True/False, this can also be 'hidden' for tables" " readable from the TAP machinery but not published in the" " metadata; this is useful for, e.g., tables contributing to a" " published view. Warning: adql=hidden is incompatible with setting" " readProfiles manually.") _system = base.BooleanAttribute( "system", default=False, description="Is this a system table? If it is, it will not be" " dropped on normal imports, and accesses to it will not be logged.") _forceUnique = base.BooleanAttribute( "forceUnique", default=False, description="Enforce dupe policy for primary key (see dupePolicy)?") _dupePolicy = base.EnumeratedUnicodeAttribute( "dupePolicy", default="check", validValues=["check", "drop", "overwrite", "dropOld"], description="Handle duplicate rows with identical primary keys manually" " by raising an error if existing and new rows are not identical (check)," " dropping the new one (drop), updating the old one (overwrite), or" " dropping the old one and inserting the new one (dropOld)?") _primary = ColumnTupleAttribute( "primary", default=(), description= "Comma separated names of columns making up the primary key.", copyable=True) _indices = base.StructListAttribute( "indices", childFactory=DBIndex, description="Indices defined on this table", copyable=True) _foreignKeys = base.StructListAttribute( "foreignKeys", childFactory=ForeignKey, description="Foreign keys used in this table", copyable=False) _groups = base.StructListAttribute( "groups", childFactory=group.Group, description="Groups for columns and params of this table", copyable=True) # this actually induces an attribute annotations with the DM # annotation instances _annotations = base.StructListAttribute( "dm", childFactory=dm.DataModelRoles, description="Annotations for data models.", copyable=True) _properties = base.PropertyAttribute() # don't copy stc -- columns just keep the reference to the original # stc on copy, and nothing should rely on column stc actually being # defined in the parent tableDefs. _stcs = base.StructListAttribute("stc", description="STC-S definitions" " of coordinate systems.", childFactory=STCDef) _rd = common.RDAttribute() _mixins = mixins.MixinAttribute() _original = base.OriginalAttribute() _namePath = common.NamePathAttribute() fixupFunction = None metaModel = ("title(1), creationDate(1), description(1)," "subject, referenceURL(1)") @classmethod def fromColumns(cls, columns, **kwargs): """returns a TableDef from a sequence of columns. You can give additional constructor arguments. makeStruct is used to build the instance, the mixin hack is applied. Columns with identical names will be disambiguated. """ res = MS(cls, columns=common.ColumnList(cls.disambiguateColumns(columns)), **kwargs) return res def __iter__(self): return iter(self.columns) def __contains__(self, name): try: self.columns.getColumnByName(name) except base.NotFoundError: return False return True def __repr__(self): try: return "<Table definition of %s>" % self.getQName() except base.Error: return "<Non-RD table %s>" % self.id def completeElement(self, ctx): # we want a meta parent as soon as possible, and we always let it # be our struct parent if (not self.getMetaParent() and self.parent and hasattr(self.parent, "_getMeta")): self.setMetaParent(self.parent) # Make room for DM annotations (these are currently filled by # gavo.dm.dmrd.DataModelRoles, but we might reconsider this) self.annotations = [] if self.viewStatement and getattr(ctx, "restricted", False): raise base.RestrictedElement( "table", hint="tables with" " view creation statements are not allowed in restricted mode") if self.registration and self.id is base.NotGiven: raise base.StructureError("Published tables need an assigned id.") if not self.id: self._id.feed(ctx, self, utils.intToFunnyWord(id(self))) # allow iterables to be passed in for columns and convert them # to a ColumnList here if not isinstance(self.columns, common.ColumnList): self.columns = common.ColumnList(self.columns) self._resolveSTC() self._completeElementNext(TableDef, ctx) self.columns.withinId = self.params.tableName = "table " + self.id def validate(self): if self.id.upper() in adql.allReservedWords: raise base.StructureError( "Reserved word %s is not allowed as a table" " name" % self.id) self._validateNext(TableDef) def onElementComplete(self): if self.adql: self.readProfiles = (self.readProfiles | base.getConfig("db", "adqlProfiles")) self.dictKeys = [c.key for c in self] self.indexedColumns = set() for index in self.indices: for col in index.columns: if "\\" in col: try: self.indexedColumns.add(self.expand(col)) except (base.Error, ValueError): # cannot expand yet, ignore pass else: self.indexedColumns.add(col) if self.primary: self.indexedColumns |= set(self.primary) self._defineFixupFunction() self._onElementCompleteNext(TableDef) if self.registration: self.registration.register() def getElementForName(self, name): """returns the first of column and param having name name. The function raises a NotFoundError if neiter column nor param with name exists. """ try: try: return self.columns.getColumnByName(name) except base.NotFoundError: return self.params.getColumnByName(name) except base.NotFoundError, ex: ex.within = "table %s" % self.id raise
class CondDesc(base.Structure): """A query specification for cores talking to the database. CondDescs define inputs as a sequence of InputKeys (see `Element InputKey`_). Internally, the values in the InputKeys can be translated to SQL. """ name_ = "condDesc" _inputKeys = rscdef.ColumnListAttribute("inputKeys", childFactory=inputdef.InputKey, description="One or more InputKeys defining the condition's input.", copyable=True) _silent = base.BooleanAttribute("silent", default=False, description="Do not produce SQL from this CondDesc. This" " can be used to convey meta information to the core. However," " in general, a service is a more appropriate place to deal with" " such information, and thus you should prefer service InputKeys" " to silent CondDescs.", copyable=True) _required = base.BooleanAttribute("required", default=False, description="Reject queries not filling the InputKeys of this CondDesc", copyable=True) _fixedSQL = base.UnicodeAttribute("fixedSQL", default=None, description="Always insert this SQL statement into the query. Deprecated.", copyable=True) _buildFrom = base.ReferenceAttribute("buildFrom", description="A reference to a column or an InputKey to define" " this CondDesc", default=None) _phraseMaker = base.StructAttribute("phraseMaker", default=None, description="Code to generate custom SQL from the input keys", childFactory=PhraseMaker, copyable=True) _combining = base.BooleanAttribute("combining", default=False, description="Allow some input keys to be missing when others are given?" " (you want this for pseudo-condDescs just collecting random input" " keys)", # (and I wish I had a better idea) copyable="True") _group = base.StructAttribute("group", default=None, childFactory=rscdef.Group, description="Group child input keys in the input table (primarily" " interesting for web forms, where this grouping is shown graphically;" " Set the style property to compact to have a one-line group there)") _joiner = base.UnicodeAttribute("joiner", default="OR", description="When yielding multiple fragments, join them" " using this operator (probably the only thing besides OR is" " AND).", copyable=True) _original = base.OriginalAttribute() def __init__(self, parent, **kwargs): base.Structure.__init__(self, parent, **kwargs) # copy parent's resolveName if present for buildFrom resolution if hasattr(self.parent, "resolveName"): self.resolveName = self.parent.resolveName def __repr__(self): return "<CondDesc %s>"%",".join(ik.name for ik in self.inputKeys) @classmethod def fromInputKey(cls, ik, **kwargs): return base.makeStruct(CondDesc, inputKeys=[ik], **kwargs) @classmethod def fromColumn(cls, col, **kwargs): return base.makeStruct(cls, buildFrom=col, **kwargs) @property def name(self): """returns some key for uniqueness of condDescs. """ # This is necessary for ColumnLists that are used # for CondDescs as well. Ideally, we'd do this on an # InputKeys basis and yield their names (because that's what # formal counts on), but it's probably not worth the effort. return "+".join([f.name for f in self.inputKeys]) def completeElement(self, ctx): if self.buildFrom and not self.inputKeys: # use the column as input key; special renderers may want # to do type mapping, but the default is to have plain input self.inputKeys = [inputdef.InputKey.fromColumn(self.buildFrom)] self._completeElementNext(CondDesc, ctx) def expand(self, *args, **kwargs): """hands macro expansion requests (from phraseMakers) upwards. This is to the queried table if the parent has one (i.e., we're part of a core), or to the RD if not (i.e., we're defined within an rd). """ if hasattr(self.parent, "queriedTable"): return self.parent.queriedTable.expand(*args, **kwargs) else: return self.parent.rd.expand(*args, **kwargs) def _makePhraseDefault(self, ignored, inputKeys, inPars, outPars, core): # the default phrase maker uses whatever the individual input keys # come up with. for ik in self.inputKeys: yield base.getSQLForField(ik, inPars, outPars) # We only want to compile the phraseMaker if actually necessary. # condDescs may be defined within resource descriptors (e.g., in # scs.rd), and they can't be compiled there (since macros may # be missing); thus, we dispatch on the first call. def _getPhraseMaker(self): try: return self.__compiledPhraseMaker except AttributeError: if self.phraseMaker is not None: val = self.phraseMaker.compile() else: val = self._makePhraseDefault self.__compiledPhraseMaker = val return self.__compiledPhraseMaker makePhrase = property(_getPhraseMaker) def _isActive(self, inPars): """returns True if the dict inPars contains input to all our input keys. """ for f in self.inputKeys: if f.name not in inPars: return False return True def inputReceived(self, inPars, queryMeta): """returns True if all inputKeys can be filled from inPars. As a side effect, inPars will receive defaults form the input keys if there are any. """ if not self._isActive(inPars): return False keysFound, keysMissing = [], [] for f in self.inputKeys: if inPars.get(f.name) is None: keysMissing.append(f) else: if f.value!=inPars.get(f.name): # non-defaulted keysFound.append(f) if not keysMissing: return True # keys are missing. That's ok if none were found and we're not required if not self.required and not keysFound: return False if self.required: raise base.ValidationError("is mandatory but was not provided.", colName=keysMissing[0].name) # we're optional, but a value was given and others are missing if not self.combining: raise base.ValidationError("When you give a value for %s," " you must give value(s) for %s, too"%(keysFound[0].getLabel(), ", ".join(k.name for k in keysMissing)), colName=keysMissing[0].name) return True def asSQL(self, inPars, sqlPars, queryMeta): if self.silent or not self.inputReceived(inPars, queryMeta): return "" res = list(self.makePhrase( self, self.inputKeys, inPars, sqlPars, self.parent)) sql = base.joinOperatorExpr(self.joiner, res) if self.fixedSQL: sql = base.joinOperatorExpr(self.joiner, [sql, self.fixedSQL]) return sql def adaptForRenderer(self, renderer): """returns a changed version of self if renderer suggests such a change. This only happens if buildFrom is non-None. The method must return a "defused" version that has buildFrom None (or self, which will do because core.adaptForRenderer stops adapting if the condDescs are stable). The adaptors may also raise a Replace exception and return a full CondDesc; this is done, e.g., for spoints for the form renderer, since they need two input keys and a completely modified phrase. """ if not self.buildFrom: return self adaptor = inputdef.getRendererAdaptor(renderer) if adaptor is None: return self try: newInputKeys = [] for ik in self.inputKeys: newInputKeys.append(adaptor(ik)) if self.inputKeys==newInputKeys: return self else: return self.change(inputKeys=newInputKeys, buildFrom=None) except base.Replace, ex: return ex.newOb