class ResRec(rscdef.IVOMetaMixin, NonServiceResource):
"""A resource for pure registration purposes.
A Resource does nothing; it is for registration of Authorities,
Organizations, Instruments, or whatever. Thus, they consist
of metadata only (resources that do something are services; they
carry their own metadata and care for their registration themselves.).
All resources must either have an id (which is used in the construction of
their IVOID), or you must give an identifier meta item.
You must further set the following meta items:
- resType specifying the kind of resource record. You should not
use this element to build resource records for services or tables
(use the normal elements, even if the actual resrouces are external
to DaCHS). resType can be registry, organization, authority,
deleted, or anything else for which registry.builders has a
handling class.
- title
- subject(s)
- description
- referenceURL
- creationDate
Additional meta keys (e.g., accessURL for a registry) may be required
depending on resType. See the registry session in the operator's guide.
"""
name_ = "resRec"
_rd = rscdef.RDAttribute()
class DirectGrammar(base.Structure, base.RestrictionMixin):
"""A user-defined external grammar.
See the separate document on user-defined code on more on direct grammars.
Also note the program gavomkboost that can help you generate core for
the C boosters used by direct grammars.
"""
name_ = "directGrammar"
_cbooster = rscdef.ResdirRelativeAttribute(
"cBooster",
default=base.Undefined,
description="resdir-relative path to the booster C source.",
copyable=True)
_gzippedInput = base.BooleanAttribute(
"gzippedInput",
default=False,
description="Pipe gzip before booster? (will not work for FITS)",
copyable=True)
_autoNull = base.UnicodeAttribute(
"autoNull",
default=None,
description="Use this string as general NULL value (when reading"
" from plain text).",
copyable=True)
_ignoreBadRecords = base.BooleanAttribute(
"ignoreBadRecords",
default=False,
description="Let booster ignore invalid records?",
copyable=True)
_recordSize = base.IntAttribute(
"recordSize",
default=4000,
description="For bin boosters, read this many bytes to make"
" up a record; for line-based boosters, this is the maximum"
" length of an input line.",
copyable=True)
_preFilter = base.UnicodeAttribute(
"preFilter",
default=None,
description="Pipe input through this program before handing it to"
" the booster; this string is shell-expanded (will not work for FITS).",
copyable=True)
_customFlags = base.UnicodeAttribute(
"customFlags",
default="",
description="Pass these flags to the C compiler when building the"
" booster.",
copyable=True)
_type = base.EnumeratedUnicodeAttribute(
"type",
default="col",
validValues=["col", "bin", "fits", "split"],
description="Make code for a booster parsing by column indices (col),"
" by splitting along separators (split), by reading fixed-length"
" binary records (bin), for from FITS binary tables (fits).",
copyable=True)
_splitChar = base.UnicodeAttribute(
"splitChar",
default="|",
description="For split boosters, use this as the separator.",
copyable=True)
_ext = base.IntAttribute(
"extension",
default=1,
description=
"For FITS table boosters, get the table from this extension.",
copyable=True)
_mapKeys = base.StructAttribute(
"mapKeys",
childFactory=common.MapKeys,
default=None,
copyable=True,
description="For a FITS booster, map DB table column names"
" to FITS column names (e.g., if the FITS table name flx is to"
" end up in the DB column flux, say flux:flx).")
_rd = rscdef.RDAttribute()
isDispatching = False
def validate(self):
self._validateNext(DirectGrammar)
if self.type == 'bin':
if not self.recordSize:
raise base.StructureError(
"DirectGrammars reading from binary need"
" a recordSize attribute")
if self.mapKeys is not None:
if self.type != "fits":
raise base.StructureError("mapKeys is only allowed for FITS"
" boosters.")
def onElementComplete(self):
if self.type == "fits":
if self.mapKeys:
self.keyMap = self.mapKeys.maps
else:
self.keyMap = {}
def getBooster(self):
return CBooster(self.cBooster,
self.parent,
gzippedInput=self.gzippedInput,
preFilter=self.preFilter,
autoNull=self.autoNull,
ignoreBadRecords=self.ignoreBadRecords,
customFlags=self.customFlags)
def parse(self, sourceToken, targetData=None):
booster = self.getBooster()
makes = self.parent.makes
if len(makes) != 1:
raise base.StructureError(
"Directgrammar only works for data having"
" exactly one table, but data '%s' has %d" %
(self.parent.id, len(makes)))
def copyIn(data):
data.tables.values()[0].copyIn(booster.getOutput(sourceToken))
if booster.getStatus():
raise base.SourceParseError("Booster returned error signature",
source=sourceToken)
return copyIn
class Publication(base.Structure, base.ComputedMetaMixin):
"""A specification of how a service should be published.
This contains most of the metadata for what is an interface in
registry speak.
"""
name_ = "publish"
_rd = rscdef.RDAttribute()
_render = base.UnicodeAttribute(
"render",
default=base.Undefined,
description="The renderer the publication will point at.",
copyable=True)
_sets = base.StringSetAttribute(
"sets",
description="Comma-separated list of sets this service will be"
" published in. Predefined are: local=publish on front page,"
" ivo_managed=register with the VO registry. If you leave it"
" empty, 'local' publication is assumed.",
copyable="True")
_service = base.ReferenceAttribute(
"service",
default=base.NotGiven,
description="Reference for a service actually implementing the"
" capability corresponding to this publication. This is"
" mainly when there is a vs:WebBrowser service accompanying a VO"
" protocol service, and this other service should be published"
" in the same resource record. See also the operator's guide.",
copyable="True")
_auxiliary = base.BooleanAttribute(
"auxiliary",
default=False,
description="Auxiliary publications are for capabilities"
" not intended to be picked up for all-VO queries, typically"
" because they are already registered with other services."
" This is mostly used internally; you probably have no reason"
" to touch it.")
def completeElement(self, ctx):
if self.render is base.Undefined:
self.render = "form"
if not self.sets:
self.sets.add("local")
if self.service is base.NotGiven:
self.service = self.parent
self.setMetaParent(self.service)
self._completeElementNext(Publication, ctx)
def validate(self):
self._validateNext(Publication)
try:
renderers.getRenderer(self.render)
except KeyError:
raise base.StructureError("Unknown renderer: %s" % self.render)
def _meta_accessURL(self):
return self.service.getURL(self.render)
def _meta_urlUse(self):
return renderers.getRenderer(self.render).urlUse
def _meta_requestMethod(self):
return renderers.getRenderer(self.render).preferredMethod
def _meta_resultType(self):
return renderers.getRenderer(self.render).resultType
class Service(base.Structure, base.ComputedMetaMixin, base.StandardMacroMixin,
rscdef.IVOMetaMixin):
"""A service definition.
A service is a combination of a core and one or more renderers. They
can be published, and they carry the metadata published into the VO.
You can set the defaultSort property on the service to a name of an
output column to preselect a sort order. Note again that this will
slow down responses for all but the smallest tables unless there is
an index on the corresponding column.
Properties evaluated:
* defaultSort -- a key to sort on by default with the form renderer.
This differs from the dbCore's sortKey in that this does not suppress the
widget itself, it just sets a default for its value. Don't use this unless
you have to; the combination of sort and limit can have disastrous effects
on the run time of queries.
* votableRespectsOutputTable -- usually, VOTable output puts in
all columns from the underlying database table with low enough
verbLevel (essentially). When this property is "True" (case-sensitive),
that's not done and only the service's output table is evaluated.
[Note that column selection is such a mess it needs to be fixed
before version 1.0 anyway]
"""
name_ = "service"
_core = CoreAttribute()
_templates = base.DictAttribute(
"templates",
description="Custom"
' nevow templates for this service; use key "form" to replace the Form'
" renderer's standard template. Start the path with two slashes to"
" access system templates.",
itemAttD=rscdef.ResdirRelativeAttribute(
"template",
description="resdir-relative path to a nevow template"
" used for the function given in key."),
copyable=True)
_publications = base.StructListAttribute(
"publications",
childFactory=Publication,
description="Sets and renderers this service"
" is published with.")
_limitTo = base.UnicodeAttribute(
"limitTo",
default=None,
description="Limit access to the group given; the empty default disables"
" access control.",
copyable="True")
_customPage = rscdef.ResdirRelativeAttribute(
"customPage",
default=None,
description="resdir-relative path to custom page code. It is used"
" by the 'custom' renderer",
copyable="True")
_allowedRenderers = base.StringSetAttribute(
"allowed",
description="Names of renderers allowed on this service; leave emtpy"
" to allow the form renderer only.",
copyable=True)
_customRF = base.StructListAttribute(
"customRFs",
description="Custom render functions for use in custom templates.",
childFactory=CustomRF,
copyable=True)
_customDF = base.StructListAttribute(
"customDFs",
description="Custom data functions for use in custom templates.",
childFactory=CustomDF,
copyable=True)
_inputData = base.StructAttribute(
"inputDD",
default=base.NotGiven,
childFactory=inputdef.InputDescriptor,
description="A data descriptor"
" for obtaining the core's input, usually based on a contextGrammar."
" For many cores (e.g., DBCores), you do not want to give this"
" but rather want to let service figure this out from the core.",
copyable=True)
_outputTable = base.StructAttribute(
"outputTable",
default=base.NotGiven,
childFactory=outputdef.OutputTableDef,
copyable=True,
description="The output fields of this service.")
_serviceKeys = base.StructListAttribute(
"serviceKeys",
childFactory=inputdef.InputKey,
description="Input widgets for"
" processing by the service, e.g. output sets.",
copyable=True)
_defaultRenderer = base.UnicodeAttribute(
"defaultRenderer",
default=None,
description="A name of a renderer used when"
" none is provided in the URL (lets you have shorter URLs).")
_rd = rscdef.RDAttribute()
_props = base.PropertyAttribute()
_original = base.OriginalAttribute()
metaModel = ("title(1), creationDate(1), description(1),"
"subject, referenceURL(1), shortName(!)")
# formats that should query the same fields as HTML (the others behave
# like VOTables and offer a "verbosity" widget in forms).
htmlLikeFormats = ["HTML", "tar"]
####################### Housekeeping methods
def __repr__(self):
return "<Service at %x>" % id(self)
def completeElement(self, ctx):
self._completeElementNext(Service, ctx)
if not self.allowed:
self.allowed.add("form")
if self.core is base.Undefined:
# undefined cores are only allowed with custom pages
# (Deprecated)
if self.customPage:
self.core = core.getCore("nullCore")(
self.rd).finishElement(None)
base.ui.notifyWarning(
"Custom page service %s without nullCore."
" This is deprecated, please fix" % self.id)
else:
raise base.StructureError(
"Services must have cores (add <nullCore/>"
" if you really do not want a core, e.g., with fixed renderers)."
)
# if there's only one renderer on this service, make it the default
if self.defaultRenderer is None and len(self.allowed) == 1:
self.defaultRenderer = list(self.allowed)[0]
# empty output tables are filled from the core
if self.outputTable is base.NotGiven:
self.outputTable = self.core.outputTable
# cache all kinds of things expensive to create and parse
self._coresCache = {}
self._inputDDCache = {}
self._loadedTemplates = {}
# Schedule the capabilities to be added when the parse is
# done (i.e., the RD is complete)
ctx.addExitFunc(lambda rd, ctx: self._addAutomaticCapabilities())
def onElementComplete(self):
self._onElementCompleteNext(Service)
# Index custom render/data functions
self.nevowRenderers = {}
for customRF in self.customRFs:
self.nevowRenderers[customRF.name] = customRF.func
self.nevowDataFunctions = {}
for customDF in self.customDFs:
self.nevowDataFunctions[customDF.name] = customDF.func
self._compileCustomPage()
self._computeResourceType()
def _compileCustomPage(self):
if self.customPage:
try:
modNs, moddesc = utils.loadPythonModule(self.customPage)
modNs.RD = self.rd
getattr(modNs, "initModule", lambda: None)()
page = modNs.MainPage
except ImportError:
raise base.ui.logOldExc(
base.LiteralParseError(
"customPage",
self.customPage,
hint=
"This means that an exception was raised while DaCHS"
" tried to import the renderer module. If DaCHS ran"
" with --debug, the original traceback is available"
" in the logs."))
self.customPageCode = page, (os.path.basename(
self.customPage), ) + moddesc
def getTemplate(self, key):
"""returns the nevow template for the function key on this service.
"""
if key not in self._loadedTemplates:
from nevow import loaders
tp = self.templates[key]
if tp.startswith("//"):
self._loadedTemplates[key] = common.loadSystemTemplate(tp[2:])
else:
self._loadedTemplates[key] = loaders.xmlfile(
os.path.join(self.rd.resdir, tp))
return self._loadedTemplates[key]
def getUWS(self):
"""returns a user UWS instance for this service.
This is a service for the UWSAsyncRenderer.
"""
if not hasattr(self, "uws"):
from gavo.protocols import useruws
self.uws = useruws.makeUWSForService(self)
return self.uws
################### Registry and related methods.
@property
def isVOPublished(self, renderer=None):
"""is true if there is any ivo_managed publication on this
service.
If renderer is non-None, only publications with this renderer name
count.
"""
for pub in self.publications:
if "ivo_managed" in pub.sets:
if renderer:
if pub.render == renderer:
return True
else:
return True
return False
def _computeResourceType(self):
"""sets the resType attribute.
Services are resources, and the registry code wants to know what kind.
This method ventures a guess. You can override this decision by setting
the resType meta item.
"""
if (self.outputTable.columns or self.outputTable.verbLevel
or "tap" in self.allowed):
self.resType = "catalogService"
else: # no output table defined, we're a plain service
self.resType = "nonTabularService"
def _addAutomaticCapabilities(self):
"""adds some publications that are automatic for certain types
of services.
For services with ivo_managed publications and with useful cores
(this keeps out doc-like publications, which shouldn't have VOSI
resources), artificial VOSI publications are added.
If there is _example meta, an examples publication is added.
If this service exposes a table (i.e., a DbCore with a queriedTable)
and that table is adql-readable, also add an auxiliary TAP publication
if going to the VO.
This is being run as an exit function from the parse context as
we want the RD to be complete at this point (e.g., _examples
meta might come from it). This also lets us liberally resolve
references anywhere.
"""
if not self.isVOPublished:
return
vosiSet = set(["ivo_managed"])
# All actual services get VOSI caps
if not isinstance(self.core, core.getCore("nullCore")):
self._publications.feedObject(
self,
base.makeStruct(Publication,
render="availability",
sets=vosiSet,
parent_=self))
self._publications.feedObject(
self,
base.makeStruct(Publication,
render="capabilities",
sets=vosiSet,
parent_=self))
self._publications.feedObject(
self,
base.makeStruct(Publication,
render="tableMetadata",
sets=vosiSet,
parent_=self))
# things querying tables get a TAP relationship if
# their table is adql-queriable
if isinstance(self.core, core.getCore("dbCore")):
if self.core.queriedTable.adql:
tapService = base.resolveCrossId("//tap#run")
self._publications.feedObject(
self,
base.makeStruct(Publication,
render="tap",
sets=vosiSet,
auxiliary=True,
service=tapService,
parent_=self))
# and they need a servedBy, too.
# According to the "discovering dependent" note, we don't
# do the reverse relationship lest the TAP service
# gets too related...
self.addMeta("servedBy",
base.getMetaText(tapService, "title"),
ivoId=base.getMetaText(tapService, "identifier"))
# things with examples meta get an examples capability
try:
self.getMeta("_example", raiseOnFail=True)
self._publications.feedObject(
self,
base.makeStruct(Publication,
render="examples",
sets=utils.AllEncompassingSet(),
parent_=self))
except base.NoMetaKey:
pass
def getPublicationsForSet(self, names):
"""returns publications for set names in names.
names must be a set.
"""
additionals = []
# for ivo_managed, also return a datalink endpoints if they're
# there; the specs imply that might be useful some day.
if self.getProperty("datalink", None):
dlSvc = self.rd.getById(self.getProperty("datalink"))
if "dlget" in dlSvc.allowed:
additionals.append(
base.makeStruct(Publication,
render="dlget",
sets="ivo_managed",
service=dlSvc))
if "dlasync" in dlSvc.allowed:
additionals.append(
base.makeStruct(Publication,
render="dlasync",
sets="ivo_managed",
service=dlSvc))
if "dlmeta" in dlSvc.allowed:
additionals.append(
base.makeStruct(Publication,
render="dlmeta",
sets="ivo_managed",
service=dlSvc))
return [pub
for pub in self.publications if pub.sets & names] + additionals
def getURL(self, rendName, absolute=True, **kwargs):
"""returns the full canonical access URL of this service together
with renderer.
rendName is the name of the intended renderer in the registry
of renderers.
With absolute, a fully qualified URL is being returned.
Further keyword arguments are translated into URL parameters in the
query part.
"""
basePath = "%s%s/%s" % (base.getConfig(
"web", "nevowRoot"), self.rd.sourceId, self.id)
if absolute:
basePath = base.getConfig("web", "serverURL") + basePath
res = renderers.getRenderer(rendName).makeAccessURL(basePath)
if kwargs:
res = res + "?" + urllib.urlencode(kwargs)
return res
# used by getBrowserURL; keep external higher than form as long as
# we have mess like Potsdam CdC.
_browserScores = {
"form": 10,
"external": 12,
"fixed": 15,
"custom": 3,
"img.jpeg": 2,
"static": 1
}
def getBrowserURL(self, fq=True):
"""returns a published URL that's suitable for a web browser or None if
no such URL can be guessed.
If you pass fq=False, you will get a path rather than a URL.
"""
# There can be multiple candidates for browser URLs (like when a service
# has both form, static, and external renderers). If so, we select
# by plain scores.
browseables = []
for rendName in self.allowed:
if self.isBrowseableWith(rendName):
browseables.append((self._browserScores.get(rendName,
-1), rendName))
if browseables:
return self.getURL(max(browseables)[1], absolute=fq)
else:
return None
def isBrowseableWith(self, rendName):
"""returns true if rendering this service through rendName results
in something pretty in a web browser.
"""
try:
return bool(renderers.getRenderer(rendName).isBrowseable(self))
except base.NotFoundError: # renderer name not known
return False
def getTableSet(self):
"""returns a list of table definitions that have something to do with
this service.
This is for VOSI-type queries. Usually, that's just the core's
queried table or an output table, except when there is a TAP renderer on
the service.
All this is a bit heuristic; but then again, there's no rigorous
definition for what's to be in a tables endpoint either.
"""
tables = []
# output our own outputTable if it sounds reasonable; if so,
# add the core's queried table, too, if it has one.
if self.outputTable and self.outputTable.columns:
tables.append(self.outputTable)
tables.append(getattr(self.core, "queriedTable", None))
else:
# if our outputTable is no good, just use the one of the core
qt = getattr(self.core, "queriedTable", None)
if qt is None:
qt = getattr(self.core, "outputTable", None)
if qt is not None:
tables.append(qt)
# XXX TODO: This stinks big time. It's because we got TAP factorization
# wrong. Sync and async should be renderers, and there should
# be a core that then could say this kind of thing. That's not
# yet the case, so:
if "tap" in self.allowed:
# tap never has "native" tables, so start afresh
tables = []
mth = base.caches.getMTH(None)
for tableName in mth.getTAPTables():
try:
tables.append(mth.getTableDefForTable(tableName))
except:
base.ui.notifyError(
"Failure trying to retrieve table definition"
" for table %s. Please fix the corresponding RD." %
tableName)
return [t for t in tables if t is not None and t.rd is not None]
def declareServes(self, data):
"""adds meta to self and data indicating that data is served by
service.
This is used by table/@adql and the publish element on data.
"""
if data.registration:
self.addMeta("serviceFor",
base.getMetaText(data, "title", default="Anonymous"),
ivoId=base.getMetaText(data, "identifier"))
data.addMeta("servedBy",
base.getMetaText(self, "title"),
ivoId=base.getMetaText(self, "identifier"))
# Since this is always initiated by the data, the dependency
# must show up in its RD to be properly added on publication
# and to be removed when the data is removed.
data.rd.addDependency(self.rd, data.rd)
########################## Output field selection (ouch!)
def _getVOTableOutputFields(self, queryMeta):
"""returns a list of OutputFields suitable for a VOTable response
described by queryMeta.
This is what's given for HTML when the columns verbLevel is low
enough and there's no displayHint of noxml present.
In addition, more columns are added from outputTable's parent (which
usually will be the database table itself) if their verbLevel is low
enough. this may be suppressed by setting the
votableRespectsOutputTable property to "True".
"""
verbLevel = queryMeta.get("verbosity", 20)
fields = [
f for f in self.getHTMLOutputFields(queryMeta) if
f.verbLevel <= verbLevel and f.displayHint.get("noxml") != "true"
]
if (verbLevel != "HTML" and self.getProperty(
"votableRespectsOutputTable", None) != "True"):
htmlNames = set(f.name for f in fields)
for field in self.outputTable.parentTable:
if field.name in htmlNames:
continue
if (field.displayHint.get("type") == "suppress"
or field.displayHint.get("noxml") == "true"):
continue
if field.verbLevel <= verbLevel:
fields.append(field)
return rscdef.ColumnList(fields)
_allSet = set(["ALL"])
def getHTMLOutputFields(self,
queryMeta,
ignoreAdditionals=False,
raiseOnUnknown=True):
"""returns a list of OutputFields suitable for an HTML response described
by queryMeta.
raiseOnUnknown is used by customwidgets to avoid exceptions because of
bad additional fields during form construction (when they aren't
properly caught).
"""
requireSet = queryMeta["columnSet"]
res = rscdef.ColumnList()
# add "normal" output fields
if requireSet:
res.extend([
f for f in self.outputTable
if f.sets == self._allSet or requireSet & f.sets
])
else:
res.extend([
f for f in self.outputTable
if f.displayHint.get("type") != "suppress"
])
# add user-selected fields
if not ignoreAdditionals and queryMeta["additionalFields"]:
cofs = self.core.outputTable.columns
try:
for fieldName in queryMeta["additionalFields"]:
col = cofs.getColumnByName(fieldName)
if isinstance(col, outputdef.OutputField):
res.append(col)
else:
res.append(outputdef.OutputField.fromColumn(col))
except base.NotFoundError, msg:
if raiseOnUnknown:
raise base.ValidationError(
"The additional field %s you requested"
" does not exist" % repr(msg.lookedFor),
colName="_OUTPUT")
return res
class Core(base.Structure):
"""A definition of the "active" part of a service.
Cores receive their input in tables the structure of which is
defined by their inputTable attribute.
The abstract core element will never occur in resource descriptors. See
`Cores Available`_ for concrete cores. Use the names of the concrete
cores in RDs.
You can specify an input table in an inputTableXML and an output table
in an outputTableXML class attribute.
"""
name_ = "core"
inputTableXML = None
outputTableXML = None
# the cached prototype of the output table, filled in by
# _OutputTableFactory
_ot_prototype = None
_rd = rscdef.RDAttribute()
_inputTable = base.StructAttribute(
"inputTable",
default=base.NotGiven,
childFactory=inputdef.InputTable,
description="Description of the input data",
copyable=True)
_outputTable = base.StructAttribute(
"outputTable",
default=base.NotGiven,
childFactory=_OutputTableFactory(),
description="Table describing what fields are available from this core",
copyable=True)
_original = base.OriginalAttribute()
_properties = base.PropertyAttribute()
def __init__(self, parent, **kwargs):
if self.inputTableXML is not None:
if "inputTable" not in kwargs:
kwargs["inputTable"] = base.parseFromString(
inputdef.InputTable, self.inputTableXML)
base.Structure.__init__(self, parent, **kwargs)
def __repr__(self):
return "<%s at %s>" % (self.__class__.__name__, id(self))
def __str__(self):
return repr(self)
def completeElement(self, ctx):
self._completeElementNext(Core, ctx)
if self.inputTable is base.NotGiven:
self.inputTable = base.makeStruct(inputdef.InputTable)
if self.outputTable is base.NotGiven:
self.outputTable = self._outputTable.childFactory(self)
def adaptForRenderer(self, renderer):
"""returns a core object tailored for renderer.
"""
newIT = self.inputTable.adaptForRenderer(renderer)
if newIT is self.inputTable:
return self
else:
return self.change(inputTable=newIT)
def run(self, service, inputData, queryMeta):
raise NotImplementedError("%s cores are missing the run method" %
self.__class__.__name__)
def makeUserDoc(self):
return ("Polymorphous core element. May contain any of the cores"
" mentioned in `Cores Available`_ .")