def getPublicationsForSet(self, names):
"""returns publications for set names in names.
names must be a set.
"""
additionals = []
# for ivo_managed, also return a datalink endpoints if they're
# there; the specs imply that might be useful some day.
if self.getProperty("datalink", None):
dlSvc = self.rd.getById(self.getProperty("datalink"))
if "dlget" in dlSvc.allowed:
additionals.append(
base.makeStruct(Publication,
render="dlget",
sets="ivo_managed",
service=dlSvc))
if "dlasync" in dlSvc.allowed:
additionals.append(
base.makeStruct(Publication,
render="dlasync",
sets="ivo_managed",
service=dlSvc))
if "dlmeta" in dlSvc.allowed:
additionals.append(
base.makeStruct(Publication,
render="dlmeta",
sets="ivo_managed",
service=dlSvc))
return [pub
for pub in self.publications if pub.sets & names] + additionals
def _makeMetadata(self, service):
metaTD = self.outputTable.change(id="results")
for param in metaTD.params:
param.name = "OUTPUT:" + param.name
dd = base.makeStruct(rscdef.DataDescriptor,
parent_=self.rd,
makes=[
base.makeStruct(rscdef.Make,
table=metaTD,
rowmaker=base.makeStruct(
rscdef.RowmakerDef))
])
dd.setMetaParent(service)
for inP in self.inputTable.params:
dd.feedObject("param", inP.change(name="INPUT:" + inP.name))
dd.setMeta("_type", "meta")
dd.addMeta("info", "", infoName="QUERY_STATUS", infoValue="OK")
dd.addMeta("info",
"SSAP",
infoName="SERVICE_PROTOCOL",
infoValue="1.04")
data = rsc.makeData(dd)
return base.votableType, votablewrite.getAsVOTable(data)
def _getValuesFromField(votField):
"""returns None or an rscdef.Values instance for whatever is given
in votField.
"""
valArgs = {}
for valSpec in votField.iterChildrenOfType(V.VALUES):
if valSpec.null is not None:
valArgs["nullLiteral"] = valSpec.null
for minSpec in valSpec.iterChildrenOfType(V.MIN):
valArgs["min"] = minSpec.value
for maxSpec in valSpec.iterChildrenOfType(V.MAX):
valArgs["max"] = maxSpec.value
options = []
for optSpec in valSpec.iterChildrenOfType(V.OPTION):
# We don't support nested options in rscdef.
consArgs = {"content_": optSpec.value}
if optSpec.name:
consArgs["title"] = optSpec.name
options.append(base.makeStruct(rscdef.Option, **consArgs))
if options:
valArgs["options"] = options
if valArgs:
return base.makeStruct(rscdef.Values, **valArgs)
def makeTransparentFromTable(cls, table, **kwargs): """returns a rowmaker that maps input names to column names without touching them. This is for crazy cases in which the source actually provides pre-parsed data that any treatment would actually ruin. """ if "id" not in kwargs: kwargs["id"] = "autogenerated rowmaker for table %s"%table.id return base.makeStruct(cls, maps=[ base.makeStruct(MapRule, key=c.name, content_="vars[%s]"%repr(c.name)) for c in table], **kwargs)
def _getDisplayOptions(ik): """helps EnumeratedWidget figure out the None option and the options for selection. """ noneOption = None options = [] default = ik.values.default if ik.value: default = ik.value if default is not None: if ik.required: # default given and field required: There's no noneOption but a # selected default (this shouldn't happen when values.default is gone) options = ik.values.options else: # default given and becomes the noneOption for o in ik.values.options: if o.content_==ik.values.default: noneOption = o else: options.append(o) else: # no default given, make up ANY option as noneOption unless # ik is required. options.extend(ik.values.options) noneOption = None if not ik.required and not ik.values.multiOk or ik.multiplicity=="multiple": noneOption = base.makeStruct(rscdef.Option, title="ANY", content_="__DaCHS__ANY__") return noneOption, options
def onElementComplete(self):
self.recSplitter = re.compile("[%s]"%self.pairSeparators)
self.pairSplitter = re.compile("([^%s]+)[%s](.*)"%(
self.kvSeparators, self.kvSeparators))
if self.mapKeys is None:
self.mapKeys = base.makeStruct(MapKeys)
self._onElementCompleteNext(KeyValueGrammar)
def _makeResultTableDef(self, service, inputTable, queryMeta): """returns an OutputTableDef object for querying our table with queryMeta. """ return base.makeStruct(outputdef.OutputTableDef, parent_=self.queriedTable.parent, id="result", onDisk=False, columns=self.getQueryCols(service, queryMeta), params=self.queriedTable.params)
def _getTableDescForOutput(parsedTree):
"""returns a sequence of Column instances describing the output of the
parsed and annotated ADQL query parsedTree.
"""
ctx = TDContext()
columns = [
_makeColumnFromFieldInfo(ctx, *fi) for fi in parsedTree.fieldInfos.seq
]
resTable = base.makeStruct(rscdef.TableDef,
columns=columns,
id=parsedTree.suggestAName())
# if this is a simple one-table query, take the metadata and params
# from that table.
fromNames = [
t.qName for t in parsedTree.fromClause.getAllTables()
if hasattr(t, "qName")
]
if len(fromNames) == 1:
try:
srcTable = base.caches.getMTH(None).getTableDefForTable(
fromNames[0])
params = srcTable.params
if params:
resTable = resTable.change(params=params)
resTable.copyMetaFrom(srcTable)
resTable.id = srcTable.id
except base.NotFoundError:
# Single source is not one of our tables, hence no metadata
pass
return resTable
def makeTDForColumns(name, cols, **moreKWs):
"""returns a TableDef object named names and having the columns cols.
cols is some sequence of Column objects. You can give arbitrary
table attributes in keyword arguments.
"""
kws = {"id": name, "columns": common.ColumnList(cols)}
kws.update(moreKWs)
return base.makeStruct(TableDef, **kws)
def fromColumns(cls, colSpec, query, connection, **kwargs):
"""returns a QueryTable object for query, where the result table is
inferred from colSpec.
colSpec is a sequence consisting of either dictionaries with constructor
arguments to rscdef.Column or complete objects suitable as rscdef.Column
objects; futher kwargs are passed on the the QueryTable's constructor.
"""
columns = []
for c in colSpec:
if isinstance(c, dict):
columns.append(base.makeStruct(rscdef.Column, **c))
else:
columns.append(c)
return cls(base.makeStruct(rscdef.TableDef, columns=columns),
query,
connection=connection,
**kwargs)
def completeElement(self, ctx):
if self.simplemaps:
for k,v in self.simplemaps.iteritems():
nullExcs = base.NotGiven
if v.startswith("@"):
v = v[1:]
nullExcs = "KeyError,"
self.feedObject("maps", base.makeStruct(MapRule,
key=k, source=v, nullExcs=nullExcs))
self._completeElementNext(RowmakerDef, ctx)
def getSetsForResource(restup):
"""returns the list of set names the resource described by restup belongs to.
"""
tableDef = common.getServicesRD().getById("sets")
table = rsc.TableForDef(tableDef)
destTableDef = base.makeStruct(rscdef.TableDef,
columns=[tableDef.getColumnByName("setName")])
return set(str(r["setName"])
for r in table.iterQuery(destTableDef,
"sourceRD=%(sourceRD)s AND resId=%(resId)s", restup))
def _makeResultTableDef(self, service, inputTable, queryMeta):
destPos = self._guessDestPos(inputTable)
outCols = self._fixupQueryColumns(
destPos, self.getQueryCols(service, queryMeta))
return base.makeStruct(outputdef.OutputTableDef,
parent_=self.queriedTable.parent,
id="result",
onDisk=False,
columns=outCols,
params=self.queriedTable.params)
def _evaluateFromDB(self, ctx):
if not getattr(ctx, "doQueries", True):
return
try:
with base.getTableConn() as conn:
for row in conn.query(
self.parent.parent.expand("SELECT DISTINCT %s" %
(self.fromdb))):
self._options.feedObject(
self, base.makeStruct(Option, content_=row[0]))
except base.DBError: # Table probably doesn't exist yet, ignore.
base.ui.notifyWarning("Values fromdb '%s' failed, ignoring" %
self.fromdb)
def _getTree(self, request):
root = CAP.capabilities
for renderName in ["availability", "capabilities", "tableMetadata"
]+list(self.service.allowed):
try:
cap = capabilities.getCapabilityElement(base.makeStruct(
svcs.Publication, render=renderName, sets=["vosi"],
parent_=self.service))
root = root[cap]
except Exception:
base.ui.notifyWarning("Error while creating VOSI capability"
" for %s"%(self.service.getURL(renderName, absolute=False)))
return root
def _run_getTargetNames(self, service, inputTable, queryMeta):
with base.getTableConn() as conn:
table = rsc.TableForDef(self.queriedTable,
create=False,
connection=conn)
destTD = base.makeStruct(
outputdef.OutputTableDef,
parent_=self.queriedTable.parent,
id="result",
onDisk=False,
columns=[self.queriedTable.getColumnByName("ssa_targname")])
res = rsc.TableForDef(destTD,
rows=table.iterQuery(destTD,
"",
distinct=True))
res.noPostprocess = True
return res
def getResobFromRestup(restup):
"""returns a resob for a res tuple.
restup at least has to contain the sourceRD and resId fields.
The item that is being returned is either a service or a
NonServiceResource (including DeletedResource). All of these have
a getMeta method and should be able to return the standard DC
metadata.
"""
if restup["deleted"]:
return base.makeStruct(nonservice.DeletedResource, resTuple=restup)
sourceRD, resId = restup["sourceRD"], restup["resId"]
try:
return base.caches.getRD(sourceRD).getById(resId)
except KeyError:
raise base.ui.logOldExc(
base.NotFoundError(resId,
what="service",
within="RD %s" % sourceRD,
hint="This usually happens when you"
" forgot to run gavopublish %s" % sourceRD))
def getQuery(self,
resultTableDef,
fragment,
pars=None,
distinct=False,
limits=None,
groupBy=None):
"""returns a result table definition, query string and a parameters
dictionary for a query against this table.
See iterQuery for the meaning of the arguments.
"""
if pars is None:
pars = {}
if not isinstance(resultTableDef, rscdef.TableDef):
resultTableDef = base.makeStruct(rscdef.TableDef,
id="iterQuery",
columns=resultTableDef)
query = ["SELECT "]
if distinct:
query.append("DISTINCT ")
query.append(self.getSelectClause(resultTableDef) + " ")
query.append("FROM %s " % self.tableName)
if fragment and fragment.strip():
query.append("WHERE %s " % fragment)
if groupBy:
query.append("GROUP BY %s " % groupBy)
if limits:
query.append(limits[0] + " ")
pars.update(limits[1])
if self.exclusive:
query.append("FOR UPDATE ")
return resultTableDef, "".join(query), pars
def onElementComplete(self):
if self.mapKeys is None:
self.mapKeys = base.makeStruct(MapKeys)
self._onElementCompleteNext(FITSProdGrammar)
def completeElement(self, ctx):
self._completeElementNext(Core, ctx)
if self.inputTable is base.NotGiven:
self.inputTable = base.makeStruct(inputdef.InputTable)
if self.outputTable is base.NotGiven:
self.outputTable = self._outputTable.childFactory(self)
def copyShallowly(self): return base.makeStruct(self.__class__, maps=self.maps[:], vars=self.vars[:], idmaps=self.idmaps[:], apps=self.apps[:], ignoreOn=self.ignoreOn)
def makeIdentityFromTable(cls, table, **kwargs): """returns a rowmaker that just maps input names to column names. """ if "id" not in kwargs: kwargs["id"] = "autogenerated rowmaker for table %s"%table.id return base.makeStruct(cls, idmaps=[c.key for c in table], **kwargs)
def onElementComplete(self): if self.mapKeys is None: self.mapKeys = base.makeStruct(MapKeys) self._onElementCompleteNext(CDFHeaderGrammar)
def _addAutomaticCapabilities(self):
"""adds some publications that are automatic for certain types
of services.
For services with ivo_managed publications and with useful cores
(this keeps out doc-like publications, which shouldn't have VOSI
resources), artificial VOSI publications are added.
If there is _example meta, an examples publication is added.
If this service exposes a table (i.e., a DbCore with a queriedTable)
and that table is adql-readable, also add an auxiliary TAP publication
if going to the VO.
This is being run as an exit function from the parse context as
we want the RD to be complete at this point (e.g., _examples
meta might come from it). This also lets us liberally resolve
references anywhere.
"""
if not self.isVOPublished:
return
vosiSet = set(["ivo_managed"])
# All actual services get VOSI caps
if not isinstance(self.core, core.getCore("nullCore")):
self._publications.feedObject(
self,
base.makeStruct(Publication,
render="availability",
sets=vosiSet,
parent_=self))
self._publications.feedObject(
self,
base.makeStruct(Publication,
render="capabilities",
sets=vosiSet,
parent_=self))
self._publications.feedObject(
self,
base.makeStruct(Publication,
render="tableMetadata",
sets=vosiSet,
parent_=self))
# things querying tables get a TAP relationship if
# their table is adql-queriable
if isinstance(self.core, core.getCore("dbCore")):
if self.core.queriedTable.adql:
tapService = base.resolveCrossId("//tap#run")
self._publications.feedObject(
self,
base.makeStruct(Publication,
render="tap",
sets=vosiSet,
auxiliary=True,
service=tapService,
parent_=self))
# and they need a servedBy, too.
# According to the "discovering dependent" note, we don't
# do the reverse relationship lest the TAP service
# gets too related...
self.addMeta("servedBy",
base.getMetaText(tapService, "title"),
ivoId=base.getMetaText(tapService, "identifier"))
# things with examples meta get an examples capability
try:
self.getMeta("_example", raiseOnFail=True)
self._publications.feedObject(
self,
base.makeStruct(Publication,
render="examples",
sets=utils.AllEncompassingSet(),
parent_=self))
except base.NoMetaKey:
pass
res = self.copyShallowly() try: res._resolveIdmaps(tableDef.params) res._checkTable(tableDef.params, tableDef.id) except base.NotFoundError, ex: ex.within = "table %s's params"%tableDef.id raise return res def _getSource(self, tableDef): """returns the source code for a mapper to tableDef's columns. """ return self._getSourceFromColset(tableDef.params) identityRowmaker = base.makeStruct(RowmakerDef, idmaps="*") class Rowmaker(object): """A callable that arranges for the mapping of key/value pairs to other key/value pairs. Within DaCHS, Rowmakers generate database rows (and parameter dictionaries) from the output of grammars. It is constructed with the source of the mapping function, a dictionary of globals the function should see, a dictionary of defaults, giving keys to be inserted into the incoming rowdict before the mapping function is called, and a map of line numbers to names handled in that line. It is called with a dictionary of locals for the functions (i.e.,
def completeElement(self, ctx):
if not self.values:
self.values = base.makeStruct(Values, parent_=self)
self._completeElementNext(ParamBase, ctx)
def fromTest(cls, test, **kwargs): """constructs a TestRunner for a single RegTest """ return cls([base.makeStruct(RegTestSuite, tests=[test], parent_=test.parent.parent)], **kwargs)
def fromOptions(cls, labels):
"""returns Values with the elements of labels as valid options.
"""
return base.makeStruct(
cls, options=[base.makeStruct(Option, content_=l) for l in labels])
def makeOptions(*args):
"""returns a list of Option instances with values given in args.
"""
return [base.makeStruct(Option, content_=arg) for arg in args]
def completeElement(self, ctx): if self.ignoredSources is base.Undefined: self.ignoredSources = base.makeStruct(IgnoreSpec) self._completeElementNext(SourceSpec, ctx)
