def u_000_updateObscore(cls, connection):
"""update obscore to work even when the table is empty"""
rsc.TableForDef(base.caches.getRD("//obscore").getById("emptyobscore"),
connection=connection,
create=True)
rsc.makeData(base.caches.getRD("//obscore").getById("create"),
connection=connection,
runCommit=False)
def u_000_siapDateObsToMJD(cls, connection):
"""change SIAP and SSAP dateObs columns to MJD"""
mth = base.caches.getMTH(None)
connection.execute("DROP VIEW IF EXISTS ivoa.obscore")
for tableName, fieldName in connection.query(
"SELECT tableName, fieldName FROM dc.columnmeta"
" WHERE type='timestamp' AND"
" fieldName LIKE '%%dateObs'"):
cls._upgradeTable(mth.getTableDefForTable(tableName), fieldName,
connection)
from gavo import rsc
rsc.makeData(base.caches.getRD("//obscore").getById("create"),
connection=connection, runCommit=False)
def _getMetadataData(self, queryMeta):
"""returns a SIAP-style metadata data item.
"""
# XXX TODO: build VOTable directly (rather than from data)
inputFields = []
for param in self.service.getInputKeysFor(self):
# Caution: UPLOAD mangling is a *renderer* thing -- the core
# doesn't know anything about it. Hence, parameter adaption
# is *not* done by adapting the core. Instead:
if param.type == "file":
inputFields.append(dali.getUploadKeyFor(param))
else:
inputFields.append(param.change(name="INPUT:" + param.name))
outputTable = MS(rscdef.TableDef,
columns=self.service.getCurOutputFields(queryMeta),
id="result")
nullRowmaker = MS(rscdef.RowmakerDef)
dataDesc = MS(
svcs.InputDescriptor,
makes=[MS(rscdef.Make, table=outputTable, rowmaker=nullRowmaker)],
params=inputFields,
parent_=self.service.rd)
data = rsc.makeData(dataDesc)
data.tables["result"].votCasts = self._outputTableCasts
data.setMeta("_type", "results")
data.addMeta("info", "OK", infoName="QUERY_STATUS", infoValue="OK")
data.addMeta("info",
base.getMetaText(self.service, "title") or "Unnamed",
infoName="serviceInfo",
infoValue=str(self.service.getURL(self.name)))
return data
def getPreviewData(self, accref):
table = rsc.makeData(self.sdmDD, forceSource={
"accref": accref}).getPrimaryTable()
data = [(r["spectral"], r["flux"]) for r in table.rows]
data.sort()
fig = pyplot.figure(figsize=(4,2))
ax = fig.add_axes([0,0,1,1], frameon=False)
if self.linearFluxes:
plotter = ax.plot
else:
plotter = ax.semilogy
plotter(
[r[0] for r in data],
[r[1] for r in data],
color="black")
ax.xaxis.set_major_locator(matplotlib.ticker.NullLocator())
ax.yaxis.set_major_locator(matplotlib.ticker.NullLocator())
ax.yaxis.set_minor_locator(matplotlib.ticker.NullLocator())
rendered = StringIO()
pyplot.savefig(rendered, format="png", dpi=50)
pyplot.close()
rendered = StringIO(rendered.getvalue())
im = Image.open(rendered)
im = im.convert("L")
im = im.convert("P", palette=Image.ADAPTIVE, colors=8)
compressed = StringIO()
im.save(compressed, format="png", bits=3)
return compressed.getvalue()
def u_020_updatePublished(cls, connection):
"""ingesting column_index for TAP-published tables."""
toDo = [r[0] for r in
connection.query(
"SELECT DISTINCT sourceRD FROM TAP_SCHEMA.columns")]
dd = base.resolveCrossId("//tap#importColumnsFromRD")
for rdId in toDo:
try:
rd = base.caches.getRD(rdId)
except Exception, msg:
base.ui.notifyWarning("RD %s couldn't be loaded (%s)."
" Fix and run gavo imp -m on it to get"
" column indices in TAP_SCHEMA"%(rdId, msg))
continue
rsc.makeData(dd, forceSource=rd, connection=connection)
def _makeMetadata(self, service):
metaTD = self.outputTable.change(id="results")
for param in metaTD.params:
param.name = "OUTPUT:" + param.name
dd = base.makeStruct(rscdef.DataDescriptor,
parent_=self.rd,
makes=[
base.makeStruct(rscdef.Make,
table=metaTD,
rowmaker=base.makeStruct(
rscdef.RowmakerDef))
])
dd.setMetaParent(service)
for inP in self.inputTable.params:
dd.feedObject("param", inP.change(name="INPUT:" + inP.name))
dd.setMeta("_type", "meta")
dd.addMeta("info", "", infoName="QUERY_STATUS", infoValue="OK")
dd.addMeta("info",
"SSAP",
infoName="SERVICE_PROTOCOL",
infoValue="1.04")
data = rsc.makeData(dd)
return base.votableType, votablewrite.getAsVOTable(data)
def updateServiceList(rds, metaToo=False, connection=None, onlyWarn=True,
keepTimestamp=False):
"""updates the services defined in rds in the services table in the database.
"""
recordsWritten = 0
parseOptions = rsc.getParseOptions(validateRows=True, batchSize=20)
if connection is None:
connection = base.getDBConnection("admin")
dd = common.getServicesRD().getById("tables")
dd.grammar = _rdRscRecGrammar
dd.grammar.keepTimestamp = keepTimestamp
depDD = common.getServicesRD().getById("deptable")
msg = None
for rd in rds:
if rd.sourceId.startswith("/"):
raise base.Error("Resource descriptor ID must not be absolute, but"
" '%s' seems to be."%rd.sourceId)
deletedUpdater = getDeletedIdentifiersUpdater(connection, rd)
try:
data = rsc.makeData(dd, forceSource=rd, parseOptions=parseOptions,
connection=connection)
recordsWritten += data.nAffected
rsc.makeData(depDD, forceSource=rd, connection=connection)
if metaToo:
from gavo.protocols import tap
tap.unpublishFromTAP(rd, connection)
for dependentDD in rd:
rsc.Data.create(dependentDD, connection=connection).updateMeta()
tap.publishToTAP(rd, connection)
deletedUpdater()
except base.MetaValidationError, ex:
msg = ("Aborting publication of rd '%s' since meta structure of"
" %s (id='%s') is invalid:\n * %s")%(
rd.sourceId, repr(ex.carrier), ex.carrier.id, "\n * ".join(ex.failures))
except base.NoMetaKey, ex:
msg = ("Aborting publication of '%s' at service '%s': Resource"
" record generation failed: %s"%(
rd.sourceId, ex.carrier.id, str(ex)))
def run(self, service, inputTable, queryMeta):
"""returns a data set containing product sources for the keys mentioned in
inputTable.
"""
authGroups = self._getGroups(queryMeta["user"], queryMeta["password"])
dd = MS(rscdef.DataDescriptor,
grammar=MS(ProductsGrammar, groups=authGroups),
make=[MS(rscdef.Make, table=self.outputTable)])
return rsc.makeData(dd, forceSource=self._getRAccrefs(inputTable))
def _makeInputTableFor(self, renderer, args, core=None):
"""returns an input table for this service through renderer, filled
from contextData.
"""
if isinstance(args, PreparsedInput) and not self.inputDD:
return self._hackInputTableFromPreparsed(renderer, args, core=core)
else:
return rsc.makeData(
self.getInputDDFor(renderer, core=core),
parseOptions=rsc.parseValidating,
forceSource=args,
connection=base.NullConnection()).getPrimaryTable()
def publishToTAP(rd, connection):
"""publishes info for all ADQL-enabled tables of rd to the TAP_SCHEMA.
"""
# first check if we have any adql tables at all, and don't attempt
# anything if we don't (this is cheap optimizing and keeps TAP_SCHEMA
# from being created on systems that don't do ADQL.
for table in rd.tables:
if table.adql:
break
else:
return
tapRD = base.caches.getRD(RD_ID)
for ddId in [
"importTablesFromRD", "importDMsFromRD", "importColumnsFromRD",
"importFkeysFromRD", "importGroupsFromRD"
]:
dd = tapRD.getById(ddId)
rsc.makeData(dd,
forceSource=rd,
parseOptions=rsc.parseValidating,
connection=connection)
def _importData(self, sourcePath, mode):
"""parses the input file at sourcePath and writes the result to the DB.
"""
base.ui.notifyInfo("Web upload ingesting %s in %s mode" %
(sourcePath, mode))
try:
parseOptions = rsc.getParseOptions(validateRows=True,
updateMode=True,
doTableUpdates=mode == "u")
with base.getWritableAdminConn() as conn:
res = rsc.makeData(self.destDD,
parseOptions=parseOptions,
forceSource=sourcePath,
connection=conn)
except Exception, msg:
raise base.ui.logOldExc(
base.ValidationError(
"Cannot enter %s in"
" database: %s" % (os.path.basename(sourcePath), str(msg)),
"File"))
def makeSDMDataForSSARow(ssaRow,
spectrumData,
sdmVersion=base.getConfig("ivoa", "sdmVersion")):
"""returns a rsc.Data instance containing an SDM compliant spectrum
for the spectrum described by ssaRow.
spectrumData is a data element making a primary table containing
the spectrum data from an SSA row (typically, this is going to be
the tablesource property of an SSA service).
You'll usually use this via //datalink#sdm_genData
"""
with base.getTableConn() as conn:
resData = rsc.makeData(spectrumData,
forceSource=ssaRow,
connection=conn)
resTable = resData.getPrimaryTable()
resTable.setMeta(
"description",
"Spectrum from %s" % products.makeProductLink(ssaRow["accref"]))
# fudge accref into a full URL
resTable.setParam("accref",
products.makeProductLink(resTable.getParam("accref")))
resData.DACHS_SDM_VERSION = sdmVersion
# fudge spoint params into 2-arrays
for param in resTable.iterParams():
# Bad, bad: In-place changes; we should think how such things
# can be done better in a rewrite
if param.type == "spoint":
val = param.value
param.type = "double precision(2)"
param.xtype = None
param.unit = "deg"
if val:
param.set([val.x / utils.DEG, val.y / utils.DEG])
if sdmVersion == "2":
hackSDM1ToSDM2(resData)
return resData
def getDatalinksResource(self, ctx, service):
"""returns a VOTable RESOURCE element with the data links.
This does not contain the actual service definition elements, but it
does contain references to them.
You must pass in a VOTable context object ctx (for the management
of ids). If this is the entire content of the VOTable, use
votablewrite.VOTableContext() there.
"""
internalLinks = []
internalLinks.extend(
LinkDef(s.pubDID,
service.getURL(s.rendName),
serviceType=ctx.getOrMakeIdFor(s),
semantics="#proc") for s in self.datalinkEndpoints)
for d in self.descriptors:
# for all descriptors that are products, make a full dataset
# available through the data access, possibly also adding a preview.
if not isinstance(d, ProductDescriptor):
continue
if hasattr(d, "suppressAutoLinks"):
continue
# if the accref is a datalink document, go through dlget itself.
if d.mime == "application/x-votable+xml;content=datalink":
internalLinks.append(
LinkDef(d.pubDID,
service.getURL("dlget") +
"?ID=%s" % urllib.quote(d.pubDID),
description="The full dataset.",
contentType=products.guessMediaType(d.accref),
contentLength=d.estimateSize(),
semantics="#this"))
else:
internalLinks.append(
LinkDef(d.pubDID,
products.makeProductLink(d.accref),
description="The full dataset.",
contentType=d.mime,
contentLength=d.estimateSize(),
semantics="#this"))
if getattr(d, "preview", None):
if d.preview.startswith("http"):
previewLink = d.preview
else:
previewLink = products.makeProductLink(
products.RAccref(d.accref, inputDict={"preview":
True}))
# TODO: preview mime is None for AUTO previews, and there's
# not much we can do about it. Or is there?
internalLinks.append(
LinkDef(d.pubDID,
previewLink,
description="A preview for the dataset.",
contentType=d.previewMime,
semantics="#preview"))
data = rsc.makeData(
base.caches.getRD("//datalink").getById("make_response"),
forceSource=self.datalinkLinks + internalLinks + self.errors)
data.setMeta("_type", "results")
return votablewrite.makeResource(
votablewrite.VOTableContext(tablecoding="td"), data)
def run(self, service, inputTable, queryMeta): """starts the computing process if this is a computed data set. """ res = rsc.makeData(self.resultParse, forceSource=StringIO(self._runAndCapture(inputTable))) return res.getPrimaryTable()
def u_010_createDLAsyncTable(cls, connection):
"""import job table for async datalink"""
from gavo import rsc
rsc.makeData(base.caches.getRD("//datalink").getById("import"),
connection=connection,
runCommit=False)
def u_010_updateObscore(cls, connection):
"""upgrade ivoa.obscore to obscore 1.1.
"""
dd = base.caches.getRD(
"//obscore", doQueries=False).getById("refreshAfterSchemaUpdate")
rsc.makeData(dd, connection=connection)
def u_000_remetaObscore(cls, connection):
"""update obscore metadata to fix the erroneous id"""
rsc.makeData(base.caches.getRD("//obscore").getById("create"),
connection=connection,
runCommit=False,
parseOptions=rsc.getParseOptions(metaOnly=True))
os.symlink(os.path.join(TEST_BASE, "test_data"),
os.path.join(base.getConfig("inputsDir"), "data"))
os.rmdir(os.path.join(base.getConfig("inputsDir"), "__system"))
os.symlink(os.path.join(TEST_BASE, "test_data", "__system"),
os.path.join(base.getConfig("inputsDir"), "__system"))
os.mkdir(os.path.join(base.getConfig("inputsDir"), "test"))
initdachs.initDB(dsn)
from gavo.registry import publication
from gavo import rsc
from gavo import rscdesc #noflake: caches registration
from gavo import base
publication.updateServiceList([base.caches.getRD("//services")])
# Import some resources necessary in trial tests
rsc.makeData(
base.caches.getRD("data/ssatest").getById("test_import"))
rsc.makeData(
base.caches.getRD("//obscore").getById("create"))
rsc.makeData(
base.resolveCrossId("//uws#enable_useruws"))
except:
traceback.print_exc()
sys.stderr.write("Creation of test environment failed. Remove %s\n"
" before trying again.\n"%(base.getConfig("rootDir")))
sys.exit(1)
else:
# run any pending upgrades (that's a test for them, too... of sorts)
from gavo.user import upgrade
upgrade.upgrade()
def u_000_adqlfunctions(cls, connection):
"""update ADQL GAVO-defined functions for the postgres planner's benefit"""
rsc.makeData(base.caches.getRD("//adql").getById("make_udfs"),
connection=connection,
runCommit=False)
def u_000_tapSchema(cls, connection):
"""add supportedmodels table to tap_schema"""
rsc.makeData(base.caches.getRD("//tap").getById("createSchema"),
connection=connection,
runCommit=False)
def u_000_update_funcs(cls, connection):
"""update GAVO server-side functions"""
rsc.makeData(base.caches.getRD("//adql").getById("make_udfs"),
connection=connection,
runCommit=False)
