class TableBasedCore(core.Core):
"""A core knowing a DB table it operates on and allowing the definition
of condDescs.
"""
_queriedTable = base.ReferenceAttribute("queriedTable",
default=base.Undefined, description="A reference to the table"
" this core queries.", copyable=True, callbacks=["_fixNamePath"])
_condDescs = base.StructListAttribute("condDescs", childFactory=CondDesc,
description="Descriptions of the SQL and input generating entities"
" for this core; if not given, they will be generated from the"
" table columns.", copyable=True)
_namePath = rscdef.NamePathAttribute(description="Id of an element"
" that will be used to located names in id references. Defaults"
" to the queriedTable's id.")
def _buildInputTableFromCondDescs(self):
groups, iks = [], []
for cd in self.condDescs:
for ik in cd.inputKeys:
iks.append(ik)
if cd.group:
groups.append(cd.group.change(
paramRefs=[MS(rscdef.ParameterReference, dest=ik.name)
for ik in cd.inputKeys],
parent_=None))
self.inputTable = MS(inputdef.InputTable,
params=iks,
groups=groups)
def completeElement(self, ctx):
# if no condDescs have been given, make them up from the table columns.
if not self.condDescs and self.queriedTable:
self.condDescs = [self.adopt(CondDesc.fromColumn(c))
for c in self.queriedTable]
# if an inputTable is given, trust it fits the condDescs, else
# build the input table
if self.inputTable is base.NotGiven:
self._buildInputTableFromCondDescs()
# if no outputTable has been given, make it up from the columns
# of the queried table unless a prototype is defined (which is
# handled by core itself).
if (self.outputTableXML is None
and self.outputTable is base.NotGiven
and self.queriedTable):
self.outputTable = outputdef.OutputTableDef.fromTableDef(
self.queriedTable, ctx)
self._completeElementNext(TableBasedCore, ctx)
def _fixNamePath(self, qTable):
# callback from queriedTable to make it the default for namePath as well
if self.namePath is None:
self.namePath = qTable.getFullId()
def _getSQLWhere(self, inputTable, queryMeta):
"""returns a where fragment and the appropriate parameters
for the query defined by inputTable and queryMeta.
"""
sqlPars = {}
inputPars = dict((p.name, p.value) for p in inputTable.iterParams())
return base.joinOperatorExpr("AND",
[cd.asSQL(inputPars, sqlPars, queryMeta)
for cd in self.condDescs]), sqlPars
def _makeTable(self, rowIter, resultTableDef, queryMeta):
"""returns a table from the row iterator rowIter, updating queryMeta
as necessary.
"""
rows = list(rowIter)
isOverflowed = len(rows)>queryMeta.get("dbLimit", 1e10)
if isOverflowed:
del rows[-1]
queryMeta["Matched"] = len(rows)
res = rsc.TableForDef(resultTableDef, rows=rows)
if isOverflowed:
queryMeta["Overflow"] = True
res.addMeta("_warning", "The query limit was reached. Increase it"
" to retrieve more matches. Note that unsorted truncated queries"
" are not reproducible (i.e., might return a different result set"
" at a later time).")
res.addMeta("_queryStatus", "Overflowed")
else:
res.addMeta("_queryStatus", "Ok")
return res
def adaptForRenderer(self, renderer):
"""returns a core tailored to renderer renderers.
This mainly means asking the condDescs to build themselves for
a certain renderer. If no polymorphuous condDescs are there,
self is returned.
"""
newCondDescs = []
for cd in self.condDescs:
newCondDescs.append(cd.adaptForRenderer(renderer))
if newCondDescs!=self.condDescs:
return self.change(condDescs=newCondDescs, inputTable=base.NotGiven
).adaptForRenderer(renderer)
else:
return core.Core.adaptForRenderer(self, renderer)
class RD(base.Structure, base.ComputedMetaMixin, scripting.ScriptingMixin,
base.StandardMacroMixin, common.PrivilegesMixin,
registry.DateUpdatedMixin):
"""A resource descriptor (RD); the root for all elements described here.
RDs collect all information about how to parse a particular source (like a
collection of FITS images, a catalogue, or whatever), about the database
tables the data ends up in, and the services used to access them.
"""
name_ = "resource"
# this is set somewhere below once parsing has proceeded far enough
# such that caching the RD make sense
cacheable = False
_resdir = base.FunctionRelativePathAttribute(
"resdir",
default=None,
baseFunction=lambda instance: base.getConfig("inputsDir"),
description="Base directory for source files and everything else"
" belonging to the resource.",
copyable=True)
_schema = base.UnicodeAttribute(
"schema",
default=base.Undefined,
description="Database schema for tables defined here. Follow the rule"
" 'one schema, one RD' if at all possible. If two RDs share the same"
" schema, the must generate exactly the same permissions for that"
" schema; this means, in particular, that if one has an ADQL-published"
" table, so must the other. In a nutshell: one schema, one RD.",
copyable=True,
callbacks=["_inferResdir"])
_dds = base.StructListAttribute(
"dds",
childFactory=rscdef.DataDescriptor,
description="Descriptors for the data generated and/or published"
" within this resource.",
copyable=True,
before="outputTables")
_tables = base.StructListAttribute(
"tables",
childFactory=rscdef.TableDef,
description="A table used or created by this resource",
copyable=True,
before="dds")
_outputTables = base.StructListAttribute(
"outputTables",
childFactory=svcs.OutputTableDef,
description="Canned output tables for later reference.",
copyable=True)
_rowmakers = base.StructListAttribute(
"rowmakers",
childFactory=rscdef.RowmakerDef,
description="Transformations for going from grammars to tables."
" If specified in the RD, they must be referenced from make"
" elements to become active.",
copyable=True,
before="dds")
_procDefs = base.StructListAttribute(
"procDefs",
childFactory=rscdef.ProcDef,
description="Procedure definintions (rowgens, rowmaker applys)",
copyable=True,
before="rowmakers")
_condDescs = base.StructListAttribute(
"condDescs",
childFactory=svcs.CondDesc,
description="Global condition descriptors for later reference",
copyable=True,
before="cores")
_resRecs = base.StructListAttribute(
"resRecs",
childFactory=registry.ResRec,
description="Non-service resources for the IVOA registry. They will"
" be published when gavo publish is run on the RD.")
_services = base.StructListAttribute(
"services",
childFactory=svcs.Service,
description="Services exposing data from this resource.",
copyable=True)
_macDefs = base.MacDefAttribute(
before="tables",
description="User-defined macros available on this RD")
_mixinDefs = base.StructListAttribute(
"mixdefs",
childFactory=rscdef.MixinDef,
description="Mixin definitions (usually not for users)")
_require = base.ActionAttribute(
"require",
methodName="importModule",
description="Import the named gavo module (for when you need something"
" registred)")
_cores = base.MultiStructListAttribute(
"cores",
childFactory=svcs.getCore,
childNames=svcs.CORE_REGISTRY.keys(),
description="Cores available in this resource.",
copyable=True,
before="services")
_jobs = base.StructListAttribute(
"jobs",
childFactory=executing.Execute,
description="Jobs to be run while this RD is active.")
_tests = base.StructListAttribute(
"tests",
childFactory=regtest.RegTestSuite,
description="Suites of regression tests connected to this RD.")
# These replace themselves with expanded tables
_viewDefs = base.StructAttribute(
"simpleView",
childFactory=rscdef.SimpleView,
description="Definitions of views created from natural joins",
default=None)
_properties = base.PropertyAttribute()
def __init__(self, srcId, **kwargs):
# RDs never have parents, so contrary to all other structures they
# are constructed with with a srcId instead of a parent. You
# *can* have that None, but such RDs cannot be used to create
# non-temporary tables, services, etc, since the srcId is used
# in the construction of identifiers and such.
self.sourceId = srcId
base.Structure.__init__(self, None, **kwargs)
# The rd attribute is a weakref on self. Always. So, this is the class
# that roots common.RDAttributes
self.rd = weakref.proxy(self)
# real dateUpdated is set by getRD, this is just for RDs created
# on the fly.
self.dateUpdated = datetime.datetime.utcnow()
# if an RD is parsed from a disk file, this gets set to its path
# by getRD below
self.srcPath = None
# this is for modified-since and friends.
self.loadedAt = time.time()
# keep track of RDs depending on us for the registry code
# (only read this)
self.rdDependencies = set()
def __iter__(self):
return iter(self.dds)
def __repr__(self):
return "<resource descriptor for %s>" % self.sourceId
def validate(self):
if not utils.identifierPattern.match(self.schema):
raise base.StructureError("DaCHS schema attributes must be valid"
" python identifiers")
def isDirty(self):
"""returns true if the RD on disk has a timestamp newer than
loadedAt.
"""
if isinstance(self.srcPath, PkgResourcePath):
# stuff from the resource package should not change underneath us.
return False
try:
if self.srcPath is not None:
return os.path.getmtime(self.srcPath) > self.loadedAt
except os.error:
# this will ususally mean the file went away
return True
return False
def importModule(self, ctx):
# this is a callback for the require attribute
utils.loadInternalObject(self.require, "__doc__")
def onElementComplete(self):
for table in self.tables:
self.readProfiles = self.readProfiles | table.readProfiles
table.setMetaParent(self)
self.serviceIndex = {}
for svc in self.services:
self.serviceIndex[svc.id] = svc
svc.setMetaParent(self)
for dd in self.dds:
dd.setMetaParent(self)
if self.resdir and not os.path.isdir(self.resdir):
base.ui.notifyWarning(
"RD %s: resource directory '%s' does not exist" %
(self.sourceId, self.resdir))
self._onElementCompleteNext(RD)
def _inferResdir(self, value):
if self.resdir is None:
self._resdir.feedObject(self, value)
def iterDDs(self):
return iter(self.dds)
def getService(self, id):
return self.serviceIndex.get(id, None)
def getTableDefById(self, id):
return self.getById(id, rscdef.TableDef)
def getDataDescById(self, id):
return self.getById(id, rscdef.DataDescriptor)
def getById(self, id, forceType=None):
try:
res = self.idmap[id]
except KeyError:
raise base.NotFoundError(id, "Element with id",
"RD %s" % (self.sourceId))
if forceType:
if not isinstance(res, forceType):
raise base.StructureError("Element with id '%s' is not a %s" %
(id, forceType.__name__))
return res
def getAbsPath(self, relPath):
"""returns the absolute path for a resdir-relative relPath.
"""
return os.path.join(self.resdir, relPath)
def openRes(self, relPath, mode="r"):
"""returns a file object for relPath within self's resdir.
Deprecated. This is going to go away, use getAbsPath and a context
manager.
"""
return open(self.getAbsPath(relPath), mode)
def getTimestampPath(self):
"""returns a path to a file that's accessed by Resource each time
a bit of the described resource is written to the db.
"""
return os.path.join(base.getConfig("stateDir"),
"updated_" + self.sourceId.replace("/", "+"))
def touchTimestamp(self):
"""updates the timestamp on the rd's state file.
"""
fn = self.getTimestampPath()
try:
try:
os.unlink(fn)
except os.error:
pass
f = open(fn, "w")
f.close()
os.chmod(fn, 0664)
try:
os.chown(fn, -1, grp.getgrnam(base.getConfig("GavoGroup")[2]))
except (KeyError, os.error):
pass
except (os.error, IOError):
base.ui.notifyWarning("Could not update timestamp on RD %s" %
self.sourceId)
def _computeIdmap(self):
res = {}
for child in self.iterChildren():
if hasattr(child, "id"):
res[child.id] = child
return res
def addDependency(self, rd, prereq):
"""declares that rd needs the RD prereq to properly work.
This is used in the generation of resource records to ensure that, e.g.
registred data have added their served-bys to the service resources.
"""
if rd.sourceId != prereq.sourceId:
self.rdDependencies.add((rd.sourceId, prereq.sourceId))
def copy(self, parent):
base.ui.notifyWarning("Copying an RD -- this may not be a good idea")
new = base.Structure.copy(self, parent)
new.idmap = new._computeIdmap()
new.sourceId = self.sourceId
return new
def invalidate(self):
"""make the RD fail on every attribute read.
See rscdesc._loadRDIntoCache for why we want this.
"""
errMsg = ("Loading of %s failed in another thread; this RD cannot"
" be used here") % self.sourceId
class BrokenClass(object):
"""A class that reacts to all attribute requests with a some exception.
"""
def __getattribute__(self, attributeName):
if attributeName == "__class__":
return BrokenClass
raise base.ReportableError(errMsg)
self.__class__ = BrokenClass
def macro_RSTccby(self, stuffDesignation):
"""expands to a declaration that stuffDesignation is available under
CC-BY.
This only works in reStructured text (though it's still almost
readable as source).
"""
return ("%s is licensed under the `Creative Commons Attribution 3.0"
" License <http://creativecommons.org/licenses/by/3.0/>`_\n\n"
".. image:: /static/img/ccby.png\n\n") % stuffDesignation
def macro_RSTcc0(self, stuffDesignation):
"""expands to a declaration that stuffDesignation is available under
CC-0.
This only works in reStructured text (though it's still almost
readable as source).
"""
return (
"To the extent possible under law, the publisher has"
" waived all copyright and related or neighboring rights to %s."
" For details, see the `Creative Commons CC0 1.0"
" Public Domain dedication"
" <http://creativecommons.org/publicdomain/zero/1.0/>`_. Of course,"
" you should still give proper credit when using this data as"
" required by good scientific practice.\n\n"
".. image:: /static/img/cc0.png\n\n") % stuffDesignation