class ColumnBase(base.Structure, base.MetaMixin):
"""A base class for columns, parameters, output fields, etc.
Actually, right now there's far too much cruft in here that
should go into Column proper or still somewhere else. Hence:
XXX TODO: Refactor.
See also Column for a docstring that still applies to all we've in
here.
"""
_name = ParamNameAttribute("name",
default=base.Undefined,
description="Name of the param",
copyable=True,
before="type")
_type = TypeNameAttribute(
"type",
default="real",
description="datatype for the column (SQL-like type system)",
copyable=True,
before="unit")
_unit = base.UnicodeAttribute("unit",
default="",
description="Unit of the values",
copyable=True,
before="ucd",
strip=True)
_ucd = base.UnicodeAttribute("ucd",
default="",
description="UCD of the column",
copyable=True,
before="description")
_description = base.NWUnicodeAttribute(
"description",
default="",
copyable=True,
description=
"A short (one-line) description of the values in this column.")
_tablehead = base.UnicodeAttribute(
"tablehead",
default=None,
description="Terse phrase to put into table headers for this"
" column",
copyable=True)
_utype = base.UnicodeAttribute("utype",
default=None,
description="utype for this column",
copyable=True)
_required = base.BooleanAttribute(
"required",
default=False,
description="Record becomes invalid when this column is NULL",
copyable=True)
_displayHint = DisplayHintAttribute(
"displayHint",
description="Suggested presentation; the format is "
" <kw>=<value>{,<kw>=<value>}, where what is interpreted depends"
" on the output format. See, e.g., documentation on HTML renderers"
" and the formatter child of outputFields.",
copyable=True)
_verbLevel = base.IntAttribute(
"verbLevel",
default=20,
description="Minimal verbosity level at which to include this column",
copyable=True)
_values = base.StructAttribute("values",
default=None,
childFactory=Values,
description="Specification of legal values",
copyable=True)
_fixup = base.UnicodeAttribute(
"fixup",
description=
"A python expression the value of which will replace this column's"
" value on database reads. Write a ___ to access the original"
' value. You can use macros for the embedding table.'
' This is for, e.g., simple URL generation'
' (fixup="\'\\internallink{/this/svc}\'+___").'
' It will *only* kick in when tuples are deserialized from the'
" database, i.e., *not* for values taken from tables in memory.",
default=None,
copyable=True)
_note = base.UnicodeAttribute(
"note",
description="Reference to a note meta"
" on this table explaining more about this column",
default=None,
copyable=True)
_xtype = base.UnicodeAttribute("xtype",
description="VOTable xtype giving"
" the serialization form",
default=None,
copyable=True)
_stc = TableManagedAttribute(
"stc",
description="Internally used"
" STC information for this column (do not assign to unless instructed"
" to do so)",
default=None,
copyable=True)
_stcUtype = TableManagedAttribute(
"stcUtype",
description="Internally used"
" STC information for this column (do not assign to)",
default=None,
copyable=True)
_properties = base.PropertyAttribute(copyable=True)
_original = base.OriginalAttribute()
restrictedMode = False
def __repr__(self):
return "<Column %s>" % repr(self.name)
def setMetaParent(self, parent):
# columns should *not* take part in meta inheritance. The reason is
# that there are usually many columns to a table, and there's no
# way I can see that any piece of metadata should be repeated in
# all of them. On the other hand, for votlinks (no name an example),
# meta inheritance would have disastrous consequences.
# So, we bend the rules a bit.
raise base.StructureError(
"Columns may not have meta parents.",
hint="The rationale for this is explained in rscdef/column.py,"
" look for setMetaParent.")
def onParentComplete(self):
# we need to resolve note on construction since columns are routinely
# copied to other tables and meta info does not necessarily follow.
if isinstance(self.note, basestring):
try:
self.note = self.parent.getNote(self.note)
except base.NotFoundError: # non-existing notes silently ignored
self.note = None
def completeElement(self, ctx):
self.restrictedMode = getattr(ctx, "restricted", False)
if isinstance(self.name, utils.QuotedName):
self.key = self.name.name
if ')' in self.key:
# No '()' allowed in key for that breaks the %()s syntax (sigh!).
# Work around with the following quick hack that would break
# if people carefully chose proper names. Anyone using delim.
# ids in SQL deserves a good spanking anyway.
self.key = self.key.replace(')', "__").replace('(', "__")
else:
self.key = self.name
self._completeElementNext(ColumnBase, ctx)
def isEnumerated(self):
return self.values and self.values.options
def validate(self):
self._validateNext(ColumnBase)
if self.restrictedMode and self.fixup:
raise base.RestrictedElement("fixup")
def validateValue(self, value):
"""raises a ValidationError if value does not match the constraints
given here.
"""
if value is None:
if self.required:
raise base.ValidationError(
"Field %s is empty but non-optional" % self.name,
self.name)
return
# Only validate these if we're not a database column
if not isinstance(self, Column):
vals = self.values
if vals:
if vals.options:
if value and not vals.validateOptions(value):
raise base.ValidationError(
"Value %s not consistent with"
" legal values %s" % (value, vals.options),
self.name)
else:
if vals.min and value < vals.min:
raise base.ValidationError(
"%s too small (must be at least %s)" %
(value, vals.min), self.name)
if vals.max and value > vals.max:
raise base.ValidationError(
"%s too large (must be less than %s)" %
(value, vals.max), self.name)
def isIndexed(self):
"""returns a guess as to whether this column is part of an index.
This may return True, False, or None (unknown).
"""
if self.parent and hasattr(self.parent, "indexedColumns"):
# parent is something like a TableDef
if self.name in self.parent.indexedColumns:
return True
else:
return False
def isPrimary(self):
"""returns a guess as to whether this column is a primary key of the
embedding table.
This may return True, False, or None (unknown).
"""
if self.parent and hasattr(self.parent, "primary"):
# parent is something like a TableDef
if self.name in self.parent.primary:
return True
else:
return False
_indexedCleartext = {
True: "indexed",
False: "notIndexed",
None: "unknown",
}
def asInfoDict(self):
"""returns a dictionary of certain, "user-interesting" properties
of the data field, in a dict of strings.
"""
return {
"name": unicode(self.name),
"description": self.description or "N/A",
"tablehead": self.getLabel(),
"unit": self.unit or "N/A",
"ucd": self.ucd or "N/A",
"verbLevel": self.verbLevel,
"indexState": self._indexedCleartext[self.isIndexed()],
"note": self.note,
}
def getDDL(self):
"""returns an SQL fragment describing this column ready for
inclusion in a DDL statement.
"""
type = self.type
# we have one "artificial" type, and it shouldn't become more than
# one; so, a simple hack should do it.
if type.upper() == "UNICODE":
type = "TEXT"
# The "str" does magic for delimited identifiers, so it's important.
items = [str(self.name), type]
if self.required:
items.append("NOT NULL")
return " ".join(items)
def getDisplayHintAsString(self):
return self._displayHint.unparse(self.displayHint)
def getLabel(self):
"""returns a short label for this column.
The label is either the tablehead or, missing it, the capitalized
column name.
"""
if self.tablehead is not None:
return self.tablehead
return str(self.name).capitalize()
def _getVOTableType(self):
"""returns the VOTable type, arraysize and xtype for this
column-like thing.
"""
type, arraysize, xtype = base.sqltypeToVOTable(self.type)
if self.type == "date":
xtype = "dachs:DATE"
return type, arraysize, xtype
class Service(base.Structure, base.ComputedMetaMixin, base.StandardMacroMixin,
rscdef.IVOMetaMixin):
"""A service definition.
A service is a combination of a core and one or more renderers. They
can be published, and they carry the metadata published into the VO.
You can set the defaultSort property on the service to a name of an
output column to preselect a sort order. Note again that this will
slow down responses for all but the smallest tables unless there is
an index on the corresponding column.
Properties evaluated:
* defaultSort -- a key to sort on by default with the form renderer.
This differs from the dbCore's sortKey in that this does not suppress the
widget itself, it just sets a default for its value. Don't use this unless
you have to; the combination of sort and limit can have disastrous effects
on the run time of queries.
* votableRespectsOutputTable -- usually, VOTable output puts in
all columns from the underlying database table with low enough
verbLevel (essentially). When this property is "True" (case-sensitive),
that's not done and only the service's output table is evaluated.
[Note that column selection is such a mess it needs to be fixed
before version 1.0 anyway]
"""
name_ = "service"
_core = CoreAttribute()
_templates = base.DictAttribute(
"templates",
description="Custom"
' nevow templates for this service; use key "form" to replace the Form'
" renderer's standard template. Start the path with two slashes to"
" access system templates.",
itemAttD=rscdef.ResdirRelativeAttribute(
"template",
description="resdir-relative path to a nevow template"
" used for the function given in key."),
copyable=True)
_publications = base.StructListAttribute(
"publications",
childFactory=Publication,
description="Sets and renderers this service"
" is published with.")
_limitTo = base.UnicodeAttribute(
"limitTo",
default=None,
description="Limit access to the group given; the empty default disables"
" access control.",
copyable="True")
_customPage = rscdef.ResdirRelativeAttribute(
"customPage",
default=None,
description="resdir-relative path to custom page code. It is used"
" by the 'custom' renderer",
copyable="True")
_allowedRenderers = base.StringSetAttribute(
"allowed",
description="Names of renderers allowed on this service; leave emtpy"
" to allow the form renderer only.",
copyable=True)
_customRF = base.StructListAttribute(
"customRFs",
description="Custom render functions for use in custom templates.",
childFactory=CustomRF,
copyable=True)
_customDF = base.StructListAttribute(
"customDFs",
description="Custom data functions for use in custom templates.",
childFactory=CustomDF,
copyable=True)
_inputData = base.StructAttribute(
"inputDD",
default=base.NotGiven,
childFactory=inputdef.InputDescriptor,
description="A data descriptor"
" for obtaining the core's input, usually based on a contextGrammar."
" For many cores (e.g., DBCores), you do not want to give this"
" but rather want to let service figure this out from the core.",
copyable=True)
_outputTable = base.StructAttribute(
"outputTable",
default=base.NotGiven,
childFactory=outputdef.OutputTableDef,
copyable=True,
description="The output fields of this service.")
_serviceKeys = base.StructListAttribute(
"serviceKeys",
childFactory=inputdef.InputKey,
description="Input widgets for"
" processing by the service, e.g. output sets.",
copyable=True)
_defaultRenderer = base.UnicodeAttribute(
"defaultRenderer",
default=None,
description="A name of a renderer used when"
" none is provided in the URL (lets you have shorter URLs).")
_rd = rscdef.RDAttribute()
_props = base.PropertyAttribute()
_original = base.OriginalAttribute()
metaModel = ("title(1), creationDate(1), description(1),"
"subject, referenceURL(1), shortName(!)")
# formats that should query the same fields as HTML (the others behave
# like VOTables and offer a "verbosity" widget in forms).
htmlLikeFormats = ["HTML", "tar"]
####################### Housekeeping methods
def __repr__(self):
return "<Service at %x>" % id(self)
def completeElement(self, ctx):
self._completeElementNext(Service, ctx)
if not self.allowed:
self.allowed.add("form")
if self.core is base.Undefined:
# undefined cores are only allowed with custom pages
# (Deprecated)
if self.customPage:
self.core = core.getCore("nullCore")(
self.rd).finishElement(None)
base.ui.notifyWarning(
"Custom page service %s without nullCore."
" This is deprecated, please fix" % self.id)
else:
raise base.StructureError(
"Services must have cores (add <nullCore/>"
" if you really do not want a core, e.g., with fixed renderers)."
)
# if there's only one renderer on this service, make it the default
if self.defaultRenderer is None and len(self.allowed) == 1:
self.defaultRenderer = list(self.allowed)[0]
# empty output tables are filled from the core
if self.outputTable is base.NotGiven:
self.outputTable = self.core.outputTable
# cache all kinds of things expensive to create and parse
self._coresCache = {}
self._inputDDCache = {}
self._loadedTemplates = {}
# Schedule the capabilities to be added when the parse is
# done (i.e., the RD is complete)
ctx.addExitFunc(lambda rd, ctx: self._addAutomaticCapabilities())
def onElementComplete(self):
self._onElementCompleteNext(Service)
# Index custom render/data functions
self.nevowRenderers = {}
for customRF in self.customRFs:
self.nevowRenderers[customRF.name] = customRF.func
self.nevowDataFunctions = {}
for customDF in self.customDFs:
self.nevowDataFunctions[customDF.name] = customDF.func
self._compileCustomPage()
self._computeResourceType()
def _compileCustomPage(self):
if self.customPage:
try:
modNs, moddesc = utils.loadPythonModule(self.customPage)
modNs.RD = self.rd
getattr(modNs, "initModule", lambda: None)()
page = modNs.MainPage
except ImportError:
raise base.ui.logOldExc(
base.LiteralParseError(
"customPage",
self.customPage,
hint=
"This means that an exception was raised while DaCHS"
" tried to import the renderer module. If DaCHS ran"
" with --debug, the original traceback is available"
" in the logs."))
self.customPageCode = page, (os.path.basename(
self.customPage), ) + moddesc
def getTemplate(self, key):
"""returns the nevow template for the function key on this service.
"""
if key not in self._loadedTemplates:
from nevow import loaders
tp = self.templates[key]
if tp.startswith("//"):
self._loadedTemplates[key] = common.loadSystemTemplate(tp[2:])
else:
self._loadedTemplates[key] = loaders.xmlfile(
os.path.join(self.rd.resdir, tp))
return self._loadedTemplates[key]
def getUWS(self):
"""returns a user UWS instance for this service.
This is a service for the UWSAsyncRenderer.
"""
if not hasattr(self, "uws"):
from gavo.protocols import useruws
self.uws = useruws.makeUWSForService(self)
return self.uws
################### Registry and related methods.
@property
def isVOPublished(self, renderer=None):
"""is true if there is any ivo_managed publication on this
service.
If renderer is non-None, only publications with this renderer name
count.
"""
for pub in self.publications:
if "ivo_managed" in pub.sets:
if renderer:
if pub.render == renderer:
return True
else:
return True
return False
def _computeResourceType(self):
"""sets the resType attribute.
Services are resources, and the registry code wants to know what kind.
This method ventures a guess. You can override this decision by setting
the resType meta item.
"""
if (self.outputTable.columns or self.outputTable.verbLevel
or "tap" in self.allowed):
self.resType = "catalogService"
else: # no output table defined, we're a plain service
self.resType = "nonTabularService"
def _addAutomaticCapabilities(self):
"""adds some publications that are automatic for certain types
of services.
For services with ivo_managed publications and with useful cores
(this keeps out doc-like publications, which shouldn't have VOSI
resources), artificial VOSI publications are added.
If there is _example meta, an examples publication is added.
If this service exposes a table (i.e., a DbCore with a queriedTable)
and that table is adql-readable, also add an auxiliary TAP publication
if going to the VO.
This is being run as an exit function from the parse context as
we want the RD to be complete at this point (e.g., _examples
meta might come from it). This also lets us liberally resolve
references anywhere.
"""
if not self.isVOPublished:
return
vosiSet = set(["ivo_managed"])
# All actual services get VOSI caps
if not isinstance(self.core, core.getCore("nullCore")):
self._publications.feedObject(
self,
base.makeStruct(Publication,
render="availability",
sets=vosiSet,
parent_=self))
self._publications.feedObject(
self,
base.makeStruct(Publication,
render="capabilities",
sets=vosiSet,
parent_=self))
self._publications.feedObject(
self,
base.makeStruct(Publication,
render="tableMetadata",
sets=vosiSet,
parent_=self))
# things querying tables get a TAP relationship if
# their table is adql-queriable
if isinstance(self.core, core.getCore("dbCore")):
if self.core.queriedTable.adql:
tapService = base.resolveCrossId("//tap#run")
self._publications.feedObject(
self,
base.makeStruct(Publication,
render="tap",
sets=vosiSet,
auxiliary=True,
service=tapService,
parent_=self))
# and they need a servedBy, too.
# According to the "discovering dependent" note, we don't
# do the reverse relationship lest the TAP service
# gets too related...
self.addMeta("servedBy",
base.getMetaText(tapService, "title"),
ivoId=base.getMetaText(tapService, "identifier"))
# things with examples meta get an examples capability
try:
self.getMeta("_example", raiseOnFail=True)
self._publications.feedObject(
self,
base.makeStruct(Publication,
render="examples",
sets=utils.AllEncompassingSet(),
parent_=self))
except base.NoMetaKey:
pass
def getPublicationsForSet(self, names):
"""returns publications for set names in names.
names must be a set.
"""
additionals = []
# for ivo_managed, also return a datalink endpoints if they're
# there; the specs imply that might be useful some day.
if self.getProperty("datalink", None):
dlSvc = self.rd.getById(self.getProperty("datalink"))
if "dlget" in dlSvc.allowed:
additionals.append(
base.makeStruct(Publication,
render="dlget",
sets="ivo_managed",
service=dlSvc))
if "dlasync" in dlSvc.allowed:
additionals.append(
base.makeStruct(Publication,
render="dlasync",
sets="ivo_managed",
service=dlSvc))
if "dlmeta" in dlSvc.allowed:
additionals.append(
base.makeStruct(Publication,
render="dlmeta",
sets="ivo_managed",
service=dlSvc))
return [pub
for pub in self.publications if pub.sets & names] + additionals
def getURL(self, rendName, absolute=True, **kwargs):
"""returns the full canonical access URL of this service together
with renderer.
rendName is the name of the intended renderer in the registry
of renderers.
With absolute, a fully qualified URL is being returned.
Further keyword arguments are translated into URL parameters in the
query part.
"""
basePath = "%s%s/%s" % (base.getConfig(
"web", "nevowRoot"), self.rd.sourceId, self.id)
if absolute:
basePath = base.getConfig("web", "serverURL") + basePath
res = renderers.getRenderer(rendName).makeAccessURL(basePath)
if kwargs:
res = res + "?" + urllib.urlencode(kwargs)
return res
# used by getBrowserURL; keep external higher than form as long as
# we have mess like Potsdam CdC.
_browserScores = {
"form": 10,
"external": 12,
"fixed": 15,
"custom": 3,
"img.jpeg": 2,
"static": 1
}
def getBrowserURL(self, fq=True):
"""returns a published URL that's suitable for a web browser or None if
no such URL can be guessed.
If you pass fq=False, you will get a path rather than a URL.
"""
# There can be multiple candidates for browser URLs (like when a service
# has both form, static, and external renderers). If so, we select
# by plain scores.
browseables = []
for rendName in self.allowed:
if self.isBrowseableWith(rendName):
browseables.append((self._browserScores.get(rendName,
-1), rendName))
if browseables:
return self.getURL(max(browseables)[1], absolute=fq)
else:
return None
def isBrowseableWith(self, rendName):
"""returns true if rendering this service through rendName results
in something pretty in a web browser.
"""
try:
return bool(renderers.getRenderer(rendName).isBrowseable(self))
except base.NotFoundError: # renderer name not known
return False
def getTableSet(self):
"""returns a list of table definitions that have something to do with
this service.
This is for VOSI-type queries. Usually, that's just the core's
queried table or an output table, except when there is a TAP renderer on
the service.
All this is a bit heuristic; but then again, there's no rigorous
definition for what's to be in a tables endpoint either.
"""
tables = []
# output our own outputTable if it sounds reasonable; if so,
# add the core's queried table, too, if it has one.
if self.outputTable and self.outputTable.columns:
tables.append(self.outputTable)
tables.append(getattr(self.core, "queriedTable", None))
else:
# if our outputTable is no good, just use the one of the core
qt = getattr(self.core, "queriedTable", None)
if qt is None:
qt = getattr(self.core, "outputTable", None)
if qt is not None:
tables.append(qt)
# XXX TODO: This stinks big time. It's because we got TAP factorization
# wrong. Sync and async should be renderers, and there should
# be a core that then could say this kind of thing. That's not
# yet the case, so:
if "tap" in self.allowed:
# tap never has "native" tables, so start afresh
tables = []
mth = base.caches.getMTH(None)
for tableName in mth.getTAPTables():
try:
tables.append(mth.getTableDefForTable(tableName))
except:
base.ui.notifyError(
"Failure trying to retrieve table definition"
" for table %s. Please fix the corresponding RD." %
tableName)
return [t for t in tables if t is not None and t.rd is not None]
def declareServes(self, data):
"""adds meta to self and data indicating that data is served by
service.
This is used by table/@adql and the publish element on data.
"""
if data.registration:
self.addMeta("serviceFor",
base.getMetaText(data, "title", default="Anonymous"),
ivoId=base.getMetaText(data, "identifier"))
data.addMeta("servedBy",
base.getMetaText(self, "title"),
ivoId=base.getMetaText(self, "identifier"))
# Since this is always initiated by the data, the dependency
# must show up in its RD to be properly added on publication
# and to be removed when the data is removed.
data.rd.addDependency(self.rd, data.rd)
########################## Output field selection (ouch!)
def _getVOTableOutputFields(self, queryMeta):
"""returns a list of OutputFields suitable for a VOTable response
described by queryMeta.
This is what's given for HTML when the columns verbLevel is low
enough and there's no displayHint of noxml present.
In addition, more columns are added from outputTable's parent (which
usually will be the database table itself) if their verbLevel is low
enough. this may be suppressed by setting the
votableRespectsOutputTable property to "True".
"""
verbLevel = queryMeta.get("verbosity", 20)
fields = [
f for f in self.getHTMLOutputFields(queryMeta) if
f.verbLevel <= verbLevel and f.displayHint.get("noxml") != "true"
]
if (verbLevel != "HTML" and self.getProperty(
"votableRespectsOutputTable", None) != "True"):
htmlNames = set(f.name for f in fields)
for field in self.outputTable.parentTable:
if field.name in htmlNames:
continue
if (field.displayHint.get("type") == "suppress"
or field.displayHint.get("noxml") == "true"):
continue
if field.verbLevel <= verbLevel:
fields.append(field)
return rscdef.ColumnList(fields)
_allSet = set(["ALL"])
def getHTMLOutputFields(self,
queryMeta,
ignoreAdditionals=False,
raiseOnUnknown=True):
"""returns a list of OutputFields suitable for an HTML response described
by queryMeta.
raiseOnUnknown is used by customwidgets to avoid exceptions because of
bad additional fields during form construction (when they aren't
properly caught).
"""
requireSet = queryMeta["columnSet"]
res = rscdef.ColumnList()
# add "normal" output fields
if requireSet:
res.extend([
f for f in self.outputTable
if f.sets == self._allSet or requireSet & f.sets
])
else:
res.extend([
f for f in self.outputTable
if f.displayHint.get("type") != "suppress"
])
# add user-selected fields
if not ignoreAdditionals and queryMeta["additionalFields"]:
cofs = self.core.outputTable.columns
try:
for fieldName in queryMeta["additionalFields"]:
col = cofs.getColumnByName(fieldName)
if isinstance(col, outputdef.OutputField):
res.append(col)
else:
res.append(outputdef.OutputField.fromColumn(col))
except base.NotFoundError, msg:
if raiseOnUnknown:
raise base.ValidationError(
"The additional field %s you requested"
" does not exist" % repr(msg.lookedFor),
colName="_OUTPUT")
return res
class DataDescriptor(base.Structure, base.ComputedMetaMixin,
common.IVOMetaMixin, tabledef.PublishableDataMixin):
"""A description of how to process data from a given set of sources.
Data descriptors bring together a grammar, a source specification and
"makes", each giving a table and a rowmaker to feed the table from the
grammar output.
They are the "executable" parts of a resource descriptor. Their ids
are used as arguments to gavoimp for partial imports.
"""
name_ = "data"
resType = "data"
_rowmakers = base.StructListAttribute("rowmakers",
childFactory=rmkdef.RowmakerDef,
description="Embedded build rules (usually rowmakers are defined toplevel)",
copyable=True,
before="makes")
_tables = base.StructListAttribute("tables",
childFactory=tabledef.TableDef,
description="Embedded table definitions (usually, tables are defined"
" toplevel)",
copyable=True,
before="makes")
_grammar = base.MultiStructAttribute("grammar",
default=None,
childFactory=builtingrammars.getGrammar,
childNames=builtingrammars.GRAMMAR_REGISTRY.keys(),
description="Grammar used to parse this data set.",
copyable=True,
before="makes")
_sources = base.StructAttribute("sources",
default=None,
childFactory=SourceSpec,
description="Specification of sources that should be fed to the grammar.",
copyable=True,
before="grammar")
_dependents = base.ListOfAtomsAttribute("dependents",
itemAttD=base.UnicodeAttribute("recreateAfter"),
description="A data ID to recreate when this resource is"
" remade; use # syntax to reference in other RDs.")
_auto = base.BooleanAttribute("auto",
default=True,
description="Import this data set if not explicitly"
" mentioned on the command line?")
_updating = base.BooleanAttribute("updating",
default=False,
description="Keep existing tables on import? You usually want this"
" False unless you have some kind of sources management,"
" e.g., via a sources ignore specification.",
copyable=True)
_makes = base.StructListAttribute("makes",
childFactory=Make,
copyable=True,
description="Specification of a target table and the rowmaker"
" to feed them.")
_params = common.ColumnListAttribute("params",
childFactory=column.Param,
description='Param ("global columns") for this data (mostly for'
' VOTable serialization).',
copyable=True)
_properties = base.PropertyAttribute()
_rd = common.RDAttribute()
_original = base.OriginalAttribute()
metaModel = ("title(1), creationDate(1), description(1),"
"subject, referenceURL(1)")
def __repr__(self):
return "<data descriptor with id %s>"%self.id
def validate(self):
self._validateNext(DataDescriptor)
if self.registration and self.id is None:
raise base.StructureError("Published data needs an assigned id.")
def onElementComplete(self):
self._onElementCompleteNext(DataDescriptor)
for t in self.tables:
t.setMetaParent(self)
if self.registration:
self.registration.register()
# since we want to be able to create DDs dynamically , they must find their
# meta parent themselves. We do this while the DD is being adopted;
# the rules here are: if the parent is a meta mixin itself, it's the
# meta parent, if it has an rd attribute, use that, else give up.
# TODO: For DDs on cores, it would be *desirable* to come up
# with some magic that makes the current service their meta parent.
def _getParent(self):
return self.__parent
def _setParent(self, value):
self.__parent = value
if isinstance(value, base.MetaMixin):
self.setMetaParent(value)
elif hasattr(value, "rd"):
self.setMetaParent(value.rd)
parent = property(_getParent, _setParent)
def iterSources(self, connection=None):
if self.sources:
return self.sources.iterSources(connection)
else:
return iter([])
def __iter__(self):
for m in self.makes:
yield m.table
def iterTableDefs(self):
"""iterates over the definitions of all the tables built by this DD.
"""
for m in self.makes:
yield m.table
def getTableDefById(self, id):
for td in self.iterTableDefs():
if td.id==id:
return td
raise base.StructureError("No table name %s will be built"%id)
def getTableDefWithRole(self, role):
for m in self.makes:
if m.role==role:
return m.table
raise base.StructureError("No table def with role '%s'"%role)
def getPrimary(self):
"""returns the "primary" table definition in the data descriptor.
"primary" means the only table in a one-table dd, the table with the
role "primary" if there are more. If no matching table is found, a
StructureError is raised.
"""
if len(self.makes)==1:
return self.makes[0].table
else:
try:
return self.getTableDefWithRole("primary")
except base.StructureError: # raise more telling message
pass
raise base.StructureError("Ambiguous request for primary table")
def copyShallowly(self):
"""returns a shallow copy of self.
Sources are not copied.
"""
return DataDescriptor(self.parent, rowmakers=self.rowmakers[:],
tables=self.tables[:], grammar=self.grammar, makes=self.makes[:])
def getURL(self, rendName, absolute=True):
# there's no sensible URL for DDs; thus, let people browse
# the RD info. At least they should find links to any tables
# included here there.
basePath = "%sbrowse/%s"%(
base.getConfig("web", "nevowRoot"),
self.rd.sourceId)
if absolute:
return base.getConfig("web", "serverURL")+basePath
return basePath
class Group(base.Structure):
"""A group is a collection of columns, parameters and other groups
with a dash of metadata.
Within a group, you can refer to columns or params of the enclosing table
by their names. Nothing outside of the enclosing table can be
part of a group.
Rather than referring to params, you can also embed them into a group;
they will then *not* be present in the embedding table.
Groups may contain groups.
One application for this is grouping input keys for the form renderer.
For such groups, you probably want to give the label property (and
possibly cssClass).
"""
name_ = "group"
_name = column.ParamNameAttribute(
"name",
default=None,
description="Name of the column (must be SQL-valid for onDisk tables)",
copyable=True)
_ucd = base.UnicodeAttribute("ucd",
default=None,
description="The UCD of the group",
copyable=True)
_description = base.NWUnicodeAttribute(
"description",
default=None,
copyable=True,
description="A short (one-line) description of the group")
_utype = base.UnicodeAttribute("utype",
default=None,
description="A utype for the group",
copyable=True)
_columnRefs = base.StructListAttribute(
"columnRefs",
description="References to table columns belonging to this group",
childFactory=ColumnReference,
copyable=True)
_paramRefs = base.StructListAttribute(
"paramRefs",
description="Names of table parameters belonging to this group",
childFactory=ParameterReference,
copyable=True)
_params = common.ColumnListAttribute(
"params",
childFactory=column.Param,
description="Immediate param elements for this group (use paramref"
" to reference params defined in the parent table)",
copyable=True)
_groups = base.StructListAttribute(
"groups",
childFactory=attrdef.Recursive,
description="Sub-groups of this group (names are still referenced"
" from the enclosing table)",
copyable=True,
xmlName="group")
_props = base.PropertyAttribute(copyable=True)
@property
def table(self):
"""the table definition this group lives in.
For nested groups, this still is the ancestor table.
"""
try:
# (re) compute the table we belong to if there's no table cache
# or determination has failed so far.
if self.__tableCache is None:
raise AttributeError
except AttributeError:
# find something that has columns (presumably a table def) in our
# ancestors. I don't want to check for a TableDef instance
# since I don't want to import rscdef.table here (circular import)
# and things with column and params would work as well.
anc = self.parent
while anc:
if hasattr(anc, "columns"):
self.__tableCache = anc
break
anc = anc.parent
else:
self.__tableCache = None
return self.__tableCache
def onParentComplete(self):
"""checks that param and column names can be found in the parent table.
"""
# defer validation for sub-groups (parent group will cause validation)
if isinstance(self.parent, Group):
return
# forgo validation if the group doesn't have a table
if self.table is None:
return
try:
for col in self.iterColumns():
pass
for par in self.iterParams():
pass
except base.NotFoundError, msg:
raise base.StructureError(
"No param or field %s in found in table %s" %
(msg.what, self.table.id))
for group in self.groups:
group.onParentComplete()
class Core(base.Structure):
"""A definition of the "active" part of a service.
Cores receive their input in tables the structure of which is
defined by their inputTable attribute.
The abstract core element will never occur in resource descriptors. See
`Cores Available`_ for concrete cores. Use the names of the concrete
cores in RDs.
You can specify an input table in an inputTableXML and an output table
in an outputTableXML class attribute.
"""
name_ = "core"
inputTableXML = None
outputTableXML = None
# the cached prototype of the output table, filled in by
# _OutputTableFactory
_ot_prototype = None
_rd = rscdef.RDAttribute()
_inputTable = base.StructAttribute(
"inputTable",
default=base.NotGiven,
childFactory=inputdef.InputTable,
description="Description of the input data",
copyable=True)
_outputTable = base.StructAttribute(
"outputTable",
default=base.NotGiven,
childFactory=_OutputTableFactory(),
description="Table describing what fields are available from this core",
copyable=True)
_original = base.OriginalAttribute()
_properties = base.PropertyAttribute()
def __init__(self, parent, **kwargs):
if self.inputTableXML is not None:
if "inputTable" not in kwargs:
kwargs["inputTable"] = base.parseFromString(
inputdef.InputTable, self.inputTableXML)
base.Structure.__init__(self, parent, **kwargs)
def __repr__(self):
return "<%s at %s>" % (self.__class__.__name__, id(self))
def __str__(self):
return repr(self)
def completeElement(self, ctx):
self._completeElementNext(Core, ctx)
if self.inputTable is base.NotGiven:
self.inputTable = base.makeStruct(inputdef.InputTable)
if self.outputTable is base.NotGiven:
self.outputTable = self._outputTable.childFactory(self)
def adaptForRenderer(self, renderer):
"""returns a core object tailored for renderer.
"""
newIT = self.inputTable.adaptForRenderer(renderer)
if newIT is self.inputTable:
return self
else:
return self.change(inputTable=newIT)
def run(self, service, inputData, queryMeta):
raise NotImplementedError("%s cores are missing the run method" %
self.__class__.__name__)
def makeUserDoc(self):
return ("Polymorphous core element. May contain any of the cores"
" mentioned in `Cores Available`_ .")
class Execute(base.Structure, base.ExpansionDelegator):
"""a container for calling code.
This is a cron-like functionality. The jobs are run in separate
threads, so they need to be thread-safe with respect to the
rest of DaCHS. DaCHS serializes calls, though, so that your
code should never run twice at the same time.
At least on CPython, you must make sure your code does not
block with the GIL held; this is still in the server process.
If you do daring things, fork off (note that you must not use
any database connections you may have after forking, which means
you can't safely use the RD passed in). See the docs on `Element job`_.
Then testing/debugging such code, use ``gavo admin execute rd#id``
to immediately run the jobs.
"""
name_ = "execute"
_title = base.UnicodeAttribute(
"title",
default=base.Undefined,
description="Some descriptive title for the job; this is used"
" in diagnostics.",
copyable=False,
)
_at = base.StringListAttribute(
"at",
description="One or more hour:minute pairs at which to run"
" the code each day. This conflicts with every. Optionally,"
" you can prefix each time by one of m<dom> or w<dow> for"
" jobs only to be exectued at some day of the month or week, both"
" counted from 1. So, 'm22 7:30, w3 15:02' would execute on"
" the 22nd of each month at 7:30 UTC and on every wednesday at 15:02.",
default=base.NotGiven,
copyable=True,
)
_every = base.IntAttribute(
"every",
default=base.NotGiven,
description="Run the job roughly every this many seconds."
" This conflicts with at. Note that the first execution of"
" such a job is after every/10 seconds, and that the timers"
" start anew at every server restart. So, if you restart"
" often, these jobs may run much more frequently or not at all"
" if the interval is large. If every is smaller than zero, the"
" job will be executed immediately when the RD is being loaded and is"
" then run every abs(every) seconds",
copyable=True,
)
_job = base.StructAttribute(
"job",
childFactory=CronJob,
default=base.Undefined,
description="The code to run.",
copyable=True,
)
_debug = base.BooleanAttribute(
"debug",
description="If true, on execution of external processes (span or"
" spawnPython), the output will be accumulated and mailed to"
" the administrator. Note that output of the actual cron job"
" itself is not caught (it might turn up in serverStderr)."
" You could use execDef.outputAccum.append(<stuff>) to have"
" information from within the code included.",
default=False)
_properties = base.PropertyAttribute()
_rd = common.RDAttribute()
def spawn(self, cliList):
"""spawns an external command, capturing the output and mailing it
to the admin if it failed.
Output is buffered and mailed, so it shouldn't be too large.
This does not raise an exception if it failed (in normal usage,
this would cause two mails to be sent). Instead, it returns the
returncode of the spawned process; if that's 0, you're ok. But
in general, you wouldn't want to check it.
"""
p = subprocess.Popen(cliList,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=True)
childOutput, _ = p.communicate()
if p.returncode:
cron.sendMailToAdmin(
"A process spawned by %s failed with %s" %
(self.title, p.returncode),
"Output of %s:\n\n%s" % (cliList, childOutput))
elif self.debug:
if childOutput:
self.outputAccum.append("\n\n%s -> %s\n" %
(cliList, p.returncode))
self.outputAccum.append(childOutput)
return p.returncode
def spawnPython(self, pythonFile):
"""spawns a new python interpreter executing pythonFile.
pythonFile may be resdir-relative.
"""
self.spawn(["python", os.path.join(self.rd.resdir, pythonFile)])
def _parseAt(self, atSpec, ctx):
"""returns a tuple ready for cron.repeatAt from atSpec.
see the at StringListAttribute for how it would look like; this
parses one element of that string list.
"""
mat = re.match(
r"(?P<dow>w\d\s+)?"
r"(?P<dom>m\d\d?\s+)?"
r"(?P<hr>\d+):(?P<min>\d+)", atSpec)
if not mat:
raise base.LiteralParseError("at", atSpec, pos=ctx.pos, hint=
"This is hour:minute optionally prefixed by either w<weekday> or"\
" m<day of month>, each counted from 1.")
hour, minute = int(mat.group("hr")), int(mat.group("min"))
if not (0 <= hour <= 23 and 0 <= minute <= 59):
raise base.LiteralParseError(
"at",
atSpec,
pos=ctx.pos,
hint=
"This must be hour:minute with 0<=hour<=23 or 0<=minute<=59")
dom = None
if mat.group("dom"):
dom = int(mat.group("dom")[1:])
if not 1 <= dom <= 28:
raise base.LiteralParseError(
"at",
atSpec,
pos=ctx.pos,
hint="day-of-month in at must be between 1 and 28.")
dow = None
if mat.group("dow"):
dow = int(mat.group("dow")[1:])
if not 1 <= dow <= 7:
raise base.LiteralParseError(
"at",
atSpec,
pos=ctx.pos,
hint="day-of-week in at must be between 1 and 7.")
return (dom, dow, hour, minute)
def completeElement(self, ctx):
self._completeElementNext(Execute, ctx)
if len([s for s in [self.at, self.every] if s is base.NotGiven]) != 1:
raise base.StructureError(
"Exactly one of at and every required"
" for Execute",
pos=ctx.pos)
if self.at is not base.NotGiven:
self.parsedAt = []
for literal in self.at:
self.parsedAt.append(self._parseAt(literal, ctx))
def onElementComplete(self):
self._onElementCompleteNext(Execute)
self.jobName = "%s#%s" % (self.rd.sourceId, self.title)
self.callable = _guardedFunctionFactory.makeGuardedThreaded(
self.job.compile(), self)
if self.at is not base.NotGiven:
cron.repeatAt(self.parsedAt, self.jobName, self.callable)
else:
cron.runEvery(self.every, self.jobName, self.callable)
class TableDef(base.Structure, base.ComputedMetaMixin, common.PrivilegesMixin,
common.IVOMetaMixin, base.StandardMacroMixin,
PublishableDataMixin):
"""A definition of a table, both on-disk and internal.
Some attributes are ignored for in-memory tables, e.g., roles or adql.
Properties for tables:
* supportsModel -- a short name of a data model supported through this
table (for TAPRegExt dataModel); you can give multiple names separated
by commas.
* supportsModelURI -- a URI of a data model supported through this table.
You can give multiple URIs separated by blanks.
If you give multiple data model names or URIs, the sequences of names and
URIs must be identical (in particular, each name needs a URI).
"""
name_ = "table"
resType = "table"
# We don't want to force people to come up with an id for all their
# internal tables but want to avoid writing default-named tables to
# the db. Thus, the default is not a valid sql identifier.
_id = base.IdAttribute(
"id",
default=base.NotGiven,
description="Name of the table (must be SQL-legal for onDisk tables)")
_cols = common.ColumnListAttribute(
"columns",
childFactory=column.Column,
description="Columns making up this table.",
copyable=True)
_params = common.ColumnListAttribute(
"params",
childFactory=column.Param,
description='Param ("global columns") for this table.',
copyable=True)
_viewStatement = base.UnicodeAttribute(
"viewStatement",
default=None,
description="A single SQL statement to create a view. Setting this"
" makes this table a view. The statement will typically be something"
" like CREATE VIEW \\\\curtable AS (SELECT \\\\colNames FROM...).",
copyable=True)
# onDisk must not be copyable since queries might copy the tds and havoc
# would result if the queries were to end up on disk.
_onDisk = base.BooleanAttribute(
"onDisk",
default=False,
description="Table in the database rather than in memory?")
_temporary = base.BooleanAttribute(
"temporary",
default=False,
description="If this is an onDisk table, make it temporary?"
" This is mostly useful for custom cores and such.",
copyable=True)
_adql = ADQLVisibilityAttribute(
"adql",
default=False,
description="Should this table be available for ADQL queries? In"
" addition to True/False, this can also be 'hidden' for tables"
" readable from the TAP machinery but not published in the"
" metadata; this is useful for, e.g., tables contributing to a"
" published view. Warning: adql=hidden is incompatible with setting"
" readProfiles manually.")
_system = base.BooleanAttribute(
"system",
default=False,
description="Is this a system table? If it is, it will not be"
" dropped on normal imports, and accesses to it will not be logged.")
_forceUnique = base.BooleanAttribute(
"forceUnique",
default=False,
description="Enforce dupe policy for primary key (see dupePolicy)?")
_dupePolicy = base.EnumeratedUnicodeAttribute(
"dupePolicy",
default="check",
validValues=["check", "drop", "overwrite", "dropOld"],
description="Handle duplicate rows with identical primary keys manually"
" by raising an error if existing and new rows are not identical (check),"
" dropping the new one (drop), updating the old one (overwrite), or"
" dropping the old one and inserting the new one (dropOld)?")
_primary = ColumnTupleAttribute(
"primary",
default=(),
description=
"Comma separated names of columns making up the primary key.",
copyable=True)
_indices = base.StructListAttribute(
"indices",
childFactory=DBIndex,
description="Indices defined on this table",
copyable=True)
_foreignKeys = base.StructListAttribute(
"foreignKeys",
childFactory=ForeignKey,
description="Foreign keys used in this table",
copyable=False)
_groups = base.StructListAttribute(
"groups",
childFactory=group.Group,
description="Groups for columns and params of this table",
copyable=True)
# this actually induces an attribute annotations with the DM
# annotation instances
_annotations = base.StructListAttribute(
"dm",
childFactory=dm.DataModelRoles,
description="Annotations for data models.",
copyable=True)
_properties = base.PropertyAttribute()
# don't copy stc -- columns just keep the reference to the original
# stc on copy, and nothing should rely on column stc actually being
# defined in the parent tableDefs.
_stcs = base.StructListAttribute("stc",
description="STC-S definitions"
" of coordinate systems.",
childFactory=STCDef)
_rd = common.RDAttribute()
_mixins = mixins.MixinAttribute()
_original = base.OriginalAttribute()
_namePath = common.NamePathAttribute()
fixupFunction = None
metaModel = ("title(1), creationDate(1), description(1),"
"subject, referenceURL(1)")
@classmethod
def fromColumns(cls, columns, **kwargs):
"""returns a TableDef from a sequence of columns.
You can give additional constructor arguments. makeStruct is used
to build the instance, the mixin hack is applied.
Columns with identical names will be disambiguated.
"""
res = MS(cls,
columns=common.ColumnList(cls.disambiguateColumns(columns)),
**kwargs)
return res
def __iter__(self):
return iter(self.columns)
def __contains__(self, name):
try:
self.columns.getColumnByName(name)
except base.NotFoundError:
return False
return True
def __repr__(self):
try:
return "<Table definition of %s>" % self.getQName()
except base.Error:
return "<Non-RD table %s>" % self.id
def completeElement(self, ctx):
# we want a meta parent as soon as possible, and we always let it
# be our struct parent
if (not self.getMetaParent() and self.parent
and hasattr(self.parent, "_getMeta")):
self.setMetaParent(self.parent)
# Make room for DM annotations (these are currently filled by
# gavo.dm.dmrd.DataModelRoles, but we might reconsider this)
self.annotations = []
if self.viewStatement and getattr(ctx, "restricted", False):
raise base.RestrictedElement(
"table",
hint="tables with"
" view creation statements are not allowed in restricted mode")
if self.registration and self.id is base.NotGiven:
raise base.StructureError("Published tables need an assigned id.")
if not self.id:
self._id.feed(ctx, self, utils.intToFunnyWord(id(self)))
# allow iterables to be passed in for columns and convert them
# to a ColumnList here
if not isinstance(self.columns, common.ColumnList):
self.columns = common.ColumnList(self.columns)
self._resolveSTC()
self._completeElementNext(TableDef, ctx)
self.columns.withinId = self.params.tableName = "table " + self.id
def validate(self):
if self.id.upper() in adql.allReservedWords:
raise base.StructureError(
"Reserved word %s is not allowed as a table"
" name" % self.id)
self._validateNext(TableDef)
def onElementComplete(self):
if self.adql:
self.readProfiles = (self.readProfiles
| base.getConfig("db", "adqlProfiles"))
self.dictKeys = [c.key for c in self]
self.indexedColumns = set()
for index in self.indices:
for col in index.columns:
if "\\" in col:
try:
self.indexedColumns.add(self.expand(col))
except (base.Error,
ValueError): # cannot expand yet, ignore
pass
else:
self.indexedColumns.add(col)
if self.primary:
self.indexedColumns |= set(self.primary)
self._defineFixupFunction()
self._onElementCompleteNext(TableDef)
if self.registration:
self.registration.register()
def getElementForName(self, name):
"""returns the first of column and param having name name.
The function raises a NotFoundError if neiter column nor param with
name exists.
"""
try:
try:
return self.columns.getColumnByName(name)
except base.NotFoundError:
return self.params.getColumnByName(name)
except base.NotFoundError, ex:
ex.within = "table %s" % self.id
raise
class RD(base.Structure, base.ComputedMetaMixin, scripting.ScriptingMixin,
base.StandardMacroMixin, common.PrivilegesMixin,
registry.DateUpdatedMixin):
"""A resource descriptor (RD); the root for all elements described here.
RDs collect all information about how to parse a particular source (like a
collection of FITS images, a catalogue, or whatever), about the database
tables the data ends up in, and the services used to access them.
"""
name_ = "resource"
# this is set somewhere below once parsing has proceeded far enough
# such that caching the RD make sense
cacheable = False
_resdir = base.FunctionRelativePathAttribute(
"resdir",
default=None,
baseFunction=lambda instance: base.getConfig("inputsDir"),
description="Base directory for source files and everything else"
" belonging to the resource.",
copyable=True)
_schema = base.UnicodeAttribute(
"schema",
default=base.Undefined,
description="Database schema for tables defined here. Follow the rule"
" 'one schema, one RD' if at all possible. If two RDs share the same"
" schema, the must generate exactly the same permissions for that"
" schema; this means, in particular, that if one has an ADQL-published"
" table, so must the other. In a nutshell: one schema, one RD.",
copyable=True,
callbacks=["_inferResdir"])
_dds = base.StructListAttribute(
"dds",
childFactory=rscdef.DataDescriptor,
description="Descriptors for the data generated and/or published"
" within this resource.",
copyable=True,
before="outputTables")
_tables = base.StructListAttribute(
"tables",
childFactory=rscdef.TableDef,
description="A table used or created by this resource",
copyable=True,
before="dds")
_outputTables = base.StructListAttribute(
"outputTables",
childFactory=svcs.OutputTableDef,
description="Canned output tables for later reference.",
copyable=True)
_rowmakers = base.StructListAttribute(
"rowmakers",
childFactory=rscdef.RowmakerDef,
description="Transformations for going from grammars to tables."
" If specified in the RD, they must be referenced from make"
" elements to become active.",
copyable=True,
before="dds")
_procDefs = base.StructListAttribute(
"procDefs",
childFactory=rscdef.ProcDef,
description="Procedure definintions (rowgens, rowmaker applys)",
copyable=True,
before="rowmakers")
_condDescs = base.StructListAttribute(
"condDescs",
childFactory=svcs.CondDesc,
description="Global condition descriptors for later reference",
copyable=True,
before="cores")
_resRecs = base.StructListAttribute(
"resRecs",
childFactory=registry.ResRec,
description="Non-service resources for the IVOA registry. They will"
" be published when gavo publish is run on the RD.")
_services = base.StructListAttribute(
"services",
childFactory=svcs.Service,
description="Services exposing data from this resource.",
copyable=True)
_macDefs = base.MacDefAttribute(
before="tables",
description="User-defined macros available on this RD")
_mixinDefs = base.StructListAttribute(
"mixdefs",
childFactory=rscdef.MixinDef,
description="Mixin definitions (usually not for users)")
_require = base.ActionAttribute(
"require",
methodName="importModule",
description="Import the named gavo module (for when you need something"
" registred)")
_cores = base.MultiStructListAttribute(
"cores",
childFactory=svcs.getCore,
childNames=svcs.CORE_REGISTRY.keys(),
description="Cores available in this resource.",
copyable=True,
before="services")
_jobs = base.StructListAttribute(
"jobs",
childFactory=executing.Execute,
description="Jobs to be run while this RD is active.")
_tests = base.StructListAttribute(
"tests",
childFactory=regtest.RegTestSuite,
description="Suites of regression tests connected to this RD.")
# These replace themselves with expanded tables
_viewDefs = base.StructAttribute(
"simpleView",
childFactory=rscdef.SimpleView,
description="Definitions of views created from natural joins",
default=None)
_properties = base.PropertyAttribute()
def __init__(self, srcId, **kwargs):
# RDs never have parents, so contrary to all other structures they
# are constructed with with a srcId instead of a parent. You
# *can* have that None, but such RDs cannot be used to create
# non-temporary tables, services, etc, since the srcId is used
# in the construction of identifiers and such.
self.sourceId = srcId
base.Structure.__init__(self, None, **kwargs)
# The rd attribute is a weakref on self. Always. So, this is the class
# that roots common.RDAttributes
self.rd = weakref.proxy(self)
# real dateUpdated is set by getRD, this is just for RDs created
# on the fly.
self.dateUpdated = datetime.datetime.utcnow()
# if an RD is parsed from a disk file, this gets set to its path
# by getRD below
self.srcPath = None
# this is for modified-since and friends.
self.loadedAt = time.time()
# keep track of RDs depending on us for the registry code
# (only read this)
self.rdDependencies = set()
def __iter__(self):
return iter(self.dds)
def __repr__(self):
return "<resource descriptor for %s>" % self.sourceId
def validate(self):
if not utils.identifierPattern.match(self.schema):
raise base.StructureError("DaCHS schema attributes must be valid"
" python identifiers")
def isDirty(self):
"""returns true if the RD on disk has a timestamp newer than
loadedAt.
"""
if isinstance(self.srcPath, PkgResourcePath):
# stuff from the resource package should not change underneath us.
return False
try:
if self.srcPath is not None:
return os.path.getmtime(self.srcPath) > self.loadedAt
except os.error:
# this will ususally mean the file went away
return True
return False
def importModule(self, ctx):
# this is a callback for the require attribute
utils.loadInternalObject(self.require, "__doc__")
def onElementComplete(self):
for table in self.tables:
self.readProfiles = self.readProfiles | table.readProfiles
table.setMetaParent(self)
self.serviceIndex = {}
for svc in self.services:
self.serviceIndex[svc.id] = svc
svc.setMetaParent(self)
for dd in self.dds:
dd.setMetaParent(self)
if self.resdir and not os.path.isdir(self.resdir):
base.ui.notifyWarning(
"RD %s: resource directory '%s' does not exist" %
(self.sourceId, self.resdir))
self._onElementCompleteNext(RD)
def _inferResdir(self, value):
if self.resdir is None:
self._resdir.feedObject(self, value)
def iterDDs(self):
return iter(self.dds)
def getService(self, id):
return self.serviceIndex.get(id, None)
def getTableDefById(self, id):
return self.getById(id, rscdef.TableDef)
def getDataDescById(self, id):
return self.getById(id, rscdef.DataDescriptor)
def getById(self, id, forceType=None):
try:
res = self.idmap[id]
except KeyError:
raise base.NotFoundError(id, "Element with id",
"RD %s" % (self.sourceId))
if forceType:
if not isinstance(res, forceType):
raise base.StructureError("Element with id '%s' is not a %s" %
(id, forceType.__name__))
return res
def getAbsPath(self, relPath):
"""returns the absolute path for a resdir-relative relPath.
"""
return os.path.join(self.resdir, relPath)
def openRes(self, relPath, mode="r"):
"""returns a file object for relPath within self's resdir.
Deprecated. This is going to go away, use getAbsPath and a context
manager.
"""
return open(self.getAbsPath(relPath), mode)
def getTimestampPath(self):
"""returns a path to a file that's accessed by Resource each time
a bit of the described resource is written to the db.
"""
return os.path.join(base.getConfig("stateDir"),
"updated_" + self.sourceId.replace("/", "+"))
def touchTimestamp(self):
"""updates the timestamp on the rd's state file.
"""
fn = self.getTimestampPath()
try:
try:
os.unlink(fn)
except os.error:
pass
f = open(fn, "w")
f.close()
os.chmod(fn, 0664)
try:
os.chown(fn, -1, grp.getgrnam(base.getConfig("GavoGroup")[2]))
except (KeyError, os.error):
pass
except (os.error, IOError):
base.ui.notifyWarning("Could not update timestamp on RD %s" %
self.sourceId)
def _computeIdmap(self):
res = {}
for child in self.iterChildren():
if hasattr(child, "id"):
res[child.id] = child
return res
def addDependency(self, rd, prereq):
"""declares that rd needs the RD prereq to properly work.
This is used in the generation of resource records to ensure that, e.g.
registred data have added their served-bys to the service resources.
"""
if rd.sourceId != prereq.sourceId:
self.rdDependencies.add((rd.sourceId, prereq.sourceId))
def copy(self, parent):
base.ui.notifyWarning("Copying an RD -- this may not be a good idea")
new = base.Structure.copy(self, parent)
new.idmap = new._computeIdmap()
new.sourceId = self.sourceId
return new
def invalidate(self):
"""make the RD fail on every attribute read.
See rscdesc._loadRDIntoCache for why we want this.
"""
errMsg = ("Loading of %s failed in another thread; this RD cannot"
" be used here") % self.sourceId
class BrokenClass(object):
"""A class that reacts to all attribute requests with a some exception.
"""
def __getattribute__(self, attributeName):
if attributeName == "__class__":
return BrokenClass
raise base.ReportableError(errMsg)
self.__class__ = BrokenClass
def macro_RSTccby(self, stuffDesignation):
"""expands to a declaration that stuffDesignation is available under
CC-BY.
This only works in reStructured text (though it's still almost
readable as source).
"""
return ("%s is licensed under the `Creative Commons Attribution 3.0"
" License <http://creativecommons.org/licenses/by/3.0/>`_\n\n"
".. image:: /static/img/ccby.png\n\n") % stuffDesignation
def macro_RSTcc0(self, stuffDesignation):
"""expands to a declaration that stuffDesignation is available under
CC-0.
This only works in reStructured text (though it's still almost
readable as source).
"""
return (
"To the extent possible under law, the publisher has"
" waived all copyright and related or neighboring rights to %s."
" For details, see the `Creative Commons CC0 1.0"
" Public Domain dedication"
" <http://creativecommons.org/publicdomain/zero/1.0/>`_. Of course,"
" you should still give proper credit when using this data as"
" required by good scientific practice.\n\n"
".. image:: /static/img/cc0.png\n\n") % stuffDesignation