def _iterResourceMeta(ctx, dataSet):
"""adds resource metadata to the Resource parent.
"""
yield V.DESCRIPTION[base.getMetaText(dataSet, "description",
macroPackage=dataSet.dd.rd, propagate=False)]
for el in itertools.chain(
_iterInfoInfos(dataSet), _iterWarningInfos(dataSet)):
yield el
sourcesSeen, citeLinksSeen = set(), set()
for table in dataSet.tables.values():
for m in table.iterMeta("source", propagate="True"):
src = m.getContent("text")
if src not in sourcesSeen:
yield V.INFO(name="source", value=src)[
"This resource contains data associated with the publication"
" %s."%src]
sourcesSeen.add(src)
for m in table.iterMeta("howtociteLink"):
link = m.getContent("text")
if link not in citeLinksSeen:
yield V.INFO(name="howtocite", value=link)[
"For advice on how to cite the resource(s)"
" that contributed to this result, see %s"%link]
citeLinksSeen.add(link)
def getVOT(self, ctx):
# the main trouble here is: What if there's multiple foreign keys
# into destTD? To prevent multiple inclusions of a single
# table, we add a reference to our serialised VOTable stan in
# destTD's _FKR_serializedVOT attribute. That will fail
# if we produce two VOTables from the same table at the same time,
# but let's worry about that later.
destTD = self.value.inTable
srcTD = self.value.parent
pkDecl = V.GROUP[V.VODML[V.ROLE["vo-dml:ObjectTypeInstance.ID"]], [
V.FIELDref(
ref=ctx.getOrMakeIdFor(destTD.tableDef.getColumnByName(colName)
)) for colName in self.foreignKey.dest
]]
pkDecl(ID=ctx.getOrMakeIdFor(pkDecl))
fkDecl = V.GROUP(ref=ctx.getOrMakeIdFor(pkDecl))[
V.VODML[V.TYPE["vo-dml:ORMReference"]], [
V.FIELDref(ref=ctx.getIdFor(srcTD.getColumnByName(colName)))
for colName in self.foreignKey.source
]]
targetVOT = getattr(destTD, "_FKR_serializedVOT", lambda: None)()
# weakrefs are None if expired
if targetVOT is None:
targetVOT = ctx.makeTable(destTD)
destTD._FKR_serializedVOT = weakref.ref(targetVOT)
ctx.getEnclosingResource()[targetVOT]
targetVOT[pkDecl]
return fkDecl
def _makeValuesForColDesc(colDesc):
"""returns a VALUES element for a column description.
This just stringifies whatever is in colDesc's respective columns,
so for anything fancy pass in byte strings to begin with.
"""
valEl = V.VALUES()
if colDesc.get("min") is None:
colDesc["min"] = getattr(colDesc.original.values, "min", None)
if colDesc.get("max") is None:
colDesc["max"] = getattr(colDesc.original.values, "max", None)
if colDesc["max"] is utils.Infimum:
colDesc["max"] = None
if colDesc["min"] is utils.Supremum:
colDesc["min"] = None
if colDesc["min"] is not None:
valEl[V.MIN(value=str(colDesc["min"]))]
if colDesc["max"] is not None:
valEl[V.MAX(value=str(colDesc["max"]))]
if colDesc["nullvalue"] is not None:
valEl(null=colDesc["nullvalue"])
for option in getattr(colDesc.original.values, "options", []):
valEl[V.OPTION(value=option.content_, name=option.title)]
return valEl
def makeVOTable(data, ctx=None, **kwargs):
"""returns a votable.V.VOTABLE object representing data.
data can be an rsc.Data or an rsc.Table. data can be a data or a table
instance, tablecoding any key in votable.tableEncoders.
You may pass a VOTableContext object; if you don't a context
with all defaults will be used.
A deprecated alternative is to directly pass VOTableContext constructor
arguments as additional keyword arguments. Don't do this, though,
we'll probably remove the option to do so at some point.
You will usually pass the result to votable.write. The object returned
contains DelayedTables, i.e., most of the content will only be realized at
render time.
"""
ctx = ctx or VOTableContext(**kwargs)
data = rsc.wrapTable(data)
if ctx.version==(1,1):
vot = V.VOTABLE11()
elif ctx.version==(1,2):
vot = V.VOTABLE12()
elif ctx.version==(1,3):
vot = V.VOTABLE()
elif ctx.version==(1,4):
vot = V.VOTABLE() # TODO: When 1.4 XSD comes out, actually implement
else:
raise votable.VOTableError("No toplevel element for VOTable version %s"%
ctx.version)
vot[_iterToplevelMeta(ctx, data)]
vot[_makeResource(ctx, data)]
if ctx.produceVODML:
if ctx.modelsUsed:
# if we declare any models, we'll need vo-dml
ctx.addVODMLPrefix("vo-dml")
for model in ctx.modelsUsed.values():
vot[model.getVOT(ctx)]
if ctx.suppressNamespace:
# use this for "simple" table with nice element names
vot._fixedTagMaterial = ""
# What follows is a hack around the insanity of stuffing
# unused namespaces and similar detritus into VOTable's roots.
rootAttrs = data.getMeta("_votableRootAttributes")
if rootAttrs:
rootHacks = [vot._fixedTagMaterial]+[
item.getContent() for item in rootAttrs]
vot._fixedTagMaterial = " ".join(s for s in rootHacks if s)
return vot
def asVOT(self, ctx, accessURL, linkIdTo=None):
"""returns VOTable stanxml for a description of this service.
This is a RESOURCE as required by Datalink.
linkIdTo is used to support data access descriptors embedded
in descovery queries. It is the id of the column containing
the identifiers. SSA can already provide this. It ends up
in a LINK child of the ID parameter.
"""
paramsByName, stcSpecs = {}, set()
for param in self.inputKeys:
paramsByName[param.name] = param
if param.stc:
stcSpecs.add(param.stc)
def getIdFor(colRef):
colRef.toParam = True
return ctx.makeIdFor(paramsByName[colRef.dest])
res = V.RESOURCE(
ID=ctx.getOrMakeIdFor(self), type="meta", utype="adhoc:service")[
[modelgroups.marshal_STC(ast, getIdFor) for ast in stcSpecs],
V.PARAM(arraysize="*",
datatype="char",
name="accessURL",
ucd="meta.ref.url",
value=accessURL)]
standardId = {
"dlasync": "ivo://ivoa.net/std/SODA#async-1.0",
"dlget": "ivo://ivoa.net/std/SODA#sync-1.0"
}.get(self.rendName)
if standardId:
res[V.PARAM(arraysize="*",
datatype="char",
name="standardID",
value=standardId)]
inputParams = V.GROUP(name="inputParams")
res = res[inputParams]
for ik in self.inputKeys:
param = ctx.addID(
ik, votablewrite.makeFieldFromColumn(ctx, V.PARAM, ik))
if linkIdTo and ik.name == "ID":
param = param(ref=linkIdTo)
inputParams[param]
return res
def _iterNotes(serManager): """yields GROUPs for table notes. The idea is that the note is in the group's description, and the FIELDrefs give the columns that the note applies to. """ # add notes as a group with FIELDrefs, but don't fail on them for key, note in serManager.notes.iteritems(): noteId = serManager.getOrMakeIdFor(note) noteGroup = V.GROUP(name="note-%s"%key, ID=noteId)[ V.DESCRIPTION[note.getContent(targetFormat="text")]] for col in serManager: if col["note"] is note: noteGroup[V.FIELDref(ref=col["id"])] yield noteGroup
def _iterFields(ctx, serManager): """iterates over V.FIELDs based on serManger's columns. """ for colDesc in serManager: el = V.FIELD() defineField(ctx, el, colDesc) yield el
def _iterWarningInfos(dataSet):
"""yields INFO items containing warnings from the tables in dataSet.
"""
for table in dataSet.tables.values():
for warning in table.getMeta("_warning", propagate=False, default=[]):
yield V.INFO(name="warning", value="In table %s: %s"%(
table.tableDef.id, warning.getContent("text", macroPackage=table)))
def _iterGroups(ctx, container, serManager): """yields GROUPs for the RD groups within container, taking params and fields from serManager's table. container can be a tableDef or a group. """ for group in container.groups: votGroup = V.GROUP(ucd=group.ucd, utype=group.utype, name=group.name) votGroup[V.DESCRIPTION[group.description]] for ref in group.columnRefs: votGroup[_makeRef(V.FIELDref, ref, serManager.table.tableDef, serManager)] for ref in group.paramRefs: votGroup[_makeRef(V.PARAMref, ref, serManager.table, serManager)] for param in group.params: votGroup[_makeVOTParam(ctx, param)] for subgroup in _iterGroups(ctx, group, serManager): votGroup[subgroup] yield votGroup
def getVOT(self, ctx):
if id(self.objectReferenced) not in ctx.alreadyInTree:
ctx.getEnclosingContainer(
)[_the( # fix this: dmvot.getSubtrees(ctx, self.objectReferenced))(
ID=ctx.getOrMakeIdFor(self.objectReferenced))]
ctx.alreadyInTree.add(id(self.objectReferenced))
return V.GROUP(ref=ctx.getIdFor(self.objectReferenced))[
V.VODML[V.TYPE["vo-dml:GROUPref"]],
V.VODML[V.ROLE[common.completeVODMLId(ctx, self.name)]]]
def getVOT(self, ctx):
attrs = votable.guessParamAttrsForValue(self.value)
attrs.update({"unit": self.unit, "ucd": self.ucd})
param = V.PARAM(name=self.name,
id=ctx.getOrMakeIdFor(self.value),
**attrs)[V.VODML[V.ROLE[completeVODMLId(
ctx, self.name)]]]
votable.serializeToParam(param, self.value)
return param
def _writeErrorTable(self, ctx, msg, code=200, queryStatus="ERROR"):
request = inevow.IRequest(ctx)
request.setHeader("content-type", base.votableType)
votable.write(
V.VOTABLE11[
V.DESCRIPTION[base.getMetaText(self.service, "description")],
V.INFO(ID="Error",
name="Error",
value=str(msg).replace('"', '\\"'))], request)
request.write("\n")
return ""
def _iterToplevelMeta(ctx, dataSet):
"""yields meta elements for the entire VOTABLE from dataSet's RD.
"""
rd = dataSet.dd.rd
if rd is None:
return
yield V.DESCRIPTION[base.getMetaText(rd, "description",
macroPackage=dataSet.dd.rd)]
for infoItem in rd.iterMeta("copyright"):
yield V.INFO(name="legal", value=infoItem.getContent("text",
macroPackage=dataSet.dd.rd))
def _makeResource(ctx, data): """returns a Resource node for the rsc.Data instance data. """ res = V.RESOURCE() with ctx.activeContainer(res): res(type=base.getMetaText(data, "_type"), utype=base.getMetaText(data, "utype"))[ _iterResourceMeta(ctx, data), _iterParams(ctx, data), [ _makeVOTParam(ctx, param) for param in data.iterParams()], _linkBuilder.build(data.dd), ] for table in data: with ctx.buildingFromTable(table): res[makeTable(ctx, table)] res[ctx.overflowElement] return res
def _makeVOTParam(ctx, param): """returns VOTable stan for param. """ # note that we're usually accessing the content, i.e., the string # serialization we got. The only exception is when we're seeing # nulls or null-equivalents. if param.content_ is base.NotGiven or param.value is None: content = None else: content = param.content_ el = V.PARAM() defineField(ctx, el, valuemappers.AnnotatedColumn(param)) if content is None: el.value = "" else: el.value = content return el
def makeTable(ctx, table):
"""returns a Table node for the table.Table instance table.
"""
sm = valuemappers.SerManager(table, mfRegistry=ctx.mfRegistry,
idManager=ctx, acquireSamples=ctx.acquireSamples)
# this must happen before FIELDs and such are serialised to ensure
# referenced things have IDs.
result = V.TABLE()
with ctx.activeContainer(result):
result(
name=table.tableDef.id,
utype=base.getMetaText(table, "utype", macroPackage=table.tableDef,
propagate=False))[
# _iterGroups must run before _iterFields and _iterParams since it
# may need to add ids to the respective items. XSD-correct ordering of
# the elements is done by xmlstan.
V.DESCRIPTION[base.getMetaText(table, "description",
macroPackage=table.tableDef, propagate=False)],
_iterGroups(ctx, table.tableDef, sm),
_iterFields(ctx, sm),
_iterTableParams(ctx, sm),
_iterNotes(sm),
_linkBuilder.build(table.tableDef),
]
if ctx.version>(1,1):
result[_iterSTC(table.tableDef, sm)]
if ctx.produceVODML:
for ann in table.tableDef.annotations:
try:
result[ann.getVOT(ctx)]
except Exception, msg:
# never fail just because stupid DM annotation doesn't work out
base.ui.notifyError("DM annotation failed: %s"%msg)
return votable.DelayedTable(result,
sm.getMappedTuples(),
tableEncoders[ctx.tablecoding],
overflowElement=ctx.overflowElement)
def _iterParams(ctx, dataSet):
"""iterates over the entries in the parameters table of dataSet.
"""
# deprecate this. The parameters table of a data object was a grave
# mistake.
# Let's see who's using it and then remove this in favor of actual
# data parameters (or table parameters)
try:
parTable = dataSet.getTableWithRole("parameters")
except base.DataError: # no parameter table
return
warnings.warn("Parameters table used. You shouldn't do that any more.")
values = {}
if parTable: # no data for parameters: keep empty values.
values = parTable.rows[0]
for item in parTable.tableDef:
colDesc = valuemappers.AnnotatedColumn(item)
el = V.PARAM()
el(value=ctx.mfRegistry.getMapper(colDesc)(values.get(item.name)))
defineField(ctx, el, colDesc)
ctx.addID(el, item)
yield el
def _makeLinkForMeta(args, localattrs=None):
localattrs.update({"href": args[0]})
return V.LINK(**localattrs)
def getVOT(self, ctx):
return V.FIELDref(ref=ctx.getOrMakeIdFor(self.value))[V.VODML[V.ROLE[
common.completeVODMLId(ctx, self.name)]]]
def _run_queryData(self, service, inputTable, queryMeta):
format = inputTable.getParam("FORMAT") or ""
if format.lower() == "metadata":
return self._makeMetadata(service)
limits = [
q for q in (inputTable.getParam("MAXREC", None),
inputTable.getParam("TOP")) if q
]
if not limits:
limits = [base.getConfig("ivoa", "dalDefaultLimit")]
limit = min(min(limits), base.getConfig("ivoa", "dalHardLimit"))
queryMeta["dbLimit"] = limit
res = svcs.DBCore.run(self, service, inputTable, queryMeta)
if len(res) == limit:
queryStatus = "OVERFLOW"
queryStatusBody = (
"Exactly %s rows were returned. This means your"
" query probably reached the match limit. Increase MAXREC." %
limit)
else:
queryStatus = "OK"
queryStatusBody = ""
self._addPreviewLinks(res)
# We wrap our result into a data instance since we need to set the
# result type
data = rsc.wrapTable(res)
data.setMeta("_type", "results")
data.addMeta("_votableRootAttributes",
'xmlns:ssa="http://www.ivoa.net/xml/DalSsap/v1.0"')
# The returnRaw property is a hack, mainly for unit testing;
# The renderer would have to add the QUERY_STATUS here.
if service.getProperty("returnData", False):
return data
# we fix tablecoding to td for now since nobody seems to like
# binary tables and we don't have huge tables here.
votCtx = votablewrite.VOTableContext(tablecoding="td")
vot = votablewrite.makeVOTable(data, votCtx)
pubDIDId = votCtx.getIdFor(res.tableDef.getColumnByName("ssa_pubDID"))
resElement = vot.getChildDict()["RESOURCE"][0]
resElement[
V.INFO(name="SERVICE_PROTOCOL", value=self.ssapVersion)["SSAP"],
V.INFO(name="QUERY_STATUS", value=queryStatus)[queryStatusBody]]
datalinkId = service.getProperty("datalink", None)
if datalinkId and res:
dlService = self.rd.getById(datalinkId)
dlCore = getDatalinkCore(dlService, res)
# new and shiny datalink (keep)
# (we're just using endpoint 0; it should a the sync service)
dlEndpoint = dlCore.datalinkEndpoints[0]
vot[dlEndpoint.asVOT(votCtx,
dlService.getURL(dlEndpoint.rendName),
linkIdTo=pubDIDId)]
# Also point to the dl metadata service
vot[V.RESOURCE(type="meta", utype="adhoc:service")[
V.PARAM(name="standardID",
datatype="char",
arraysize="*",
value="ivo://ivoa.net/std/DataLink#links-1.0"),
V.PARAM(name="accessURL",
datatype="char",
arraysize="*",
value=self.rd.getById(datalinkId).getURL("dlmeta")),
V.GROUP(name="inputParams")[V.PARAM(
name="ID",
datatype="char",
arraysize="*",
ref=pubDIDId,
ucd="meta.id;meta.main"
)[V.LINK(content_role="ddl:id-source", value="#" +
pubDIDId)]]]]
return "application/x-votable+xml", votable.asString(vot)
def _iterInfoInfos(dataSet):
"""returns a sequence of V.INFO items from the info meta of dataSet.
"""
for infoItem in dataSet.getMeta("info", default=[]):
name, value, id = infoItem.infoName, infoItem.infoValue, infoItem.infoId
yield V.INFO(name=name, value=value, ID=id)[infoItem.getContent()]
def _makeErrorTable(self, ctx, msg, queryStatus="ERROR"):
# FatalFault, DefaultFault
return V.VOTABLE[V.RESOURCE(type="results")[V.INFO(
name="QUERY_STATUS", value=queryStatus)[str(msg)]]]
