def _startServer():
"""runs a detached server, dropping privileges and all.
"""
try:
reactor.listenTCP(int(base.getConfig("web", "serverPort")),
root.site,
interface=base.getConfig("web", "bindAddress"))
except CannotListenError:
raise base.ReportableError(
"Someone already listens on the"
" configured port %s." % base.getConfig("web", "serverPort"),
hint="This could mean that a DaCHS server is already running."
" You would have to manually kill it then since its PID file"
" got lost somehow. It's more likely that some"
" other server is already taking up this port; you may want to change"
" the [web] serverPort setting in that case.")
_dropPrivileges()
root.site.webLog = _configureTwistedLog()
PIDManager.setPID()
try:
setupServer(root)
signal.signal(signal.SIGHUP,
lambda sig, stack: reactor.callLater(0, _reloadConfig))
_preloadRDs()
reactor.run()
finally:
PIDManager.clearPID()
def render_commonhead(self, ctx, data):
# we do not want to blindly append these things to the tag since user-
# provided items in the header should have access to this environment,
# in particular to jquery; thus, this stuff must be as early as possible.
originalChildren = ctx.tag.children[:]
ctx.tag.clear()
res = ctx.tag[
T.meta(**{"charset": "utf-8"}),
T.link(rel="icon",
type="image/png",
href=base.makeSitePath("/static/img/favicon.png")),
T.link(rel="stylesheet",
href=base.makeSitePath("/static/css/style.css"),
type="text/css"),
T.link(rel="stylesheet",
href=base.makeSitePath("/formal.css"),
type="text/css"),
T.link(rel="stylesheet",
href=base.makeSitePath("/static/css/gavo_dc.css"),
type="text/css"),
T.script(src=base.makeSitePath("/static/js/jquery-gavo.js"),
type="text/javascript"),
T.script(type='text/javascript',
src=base.makeSitePath('static/js/formal' + JSEXT)),
T.script(type="text/javascript",
src=base.makeSitePath("/static/js/gavo" + JSEXT)),
T.script(type="text/javascript",
src=base.makeSitePath("/static/js/script.js")),
originalChildren, ]
if base.getConfig("web", "operatorCSS"):
res[T.link(rel="stylesheet",
type="text/css",
href=base.getConfig("web", "operatorCSS"))]
return res
def getStandardPubDID(path):
"""returns the standard DaCHS PubDID for path.
The publisher dataset identifier (PubDID) is important in protocols like
SSAP and obscore. If you use this function, the PubDID will be your
authority, the path compontent ~, and the inputs-relative path of
the input file as the parameter.
path can be relative, in which case it is interpreted relative to
the DaCHS inputsDir.
You *can* define your PubDIDs in a different way, but you'd then need
to provide a custom descriptorGenerator to datalink services (and
might need other tricks). If your data comes from plain files, use
this function.
In a rowmaker, you'll usually use the \\standardPubDID macro.
"""
# Why add inputsDir first and remove it again? Well, I want to keep
# getInputsRelativePath in the look since it does some validation
# and may, at some point, do more.
if path[0] != "/":
path = os.path.join(base.getConfig("inputsDir"), path)
return "ivo://%s/~?%s" % (base.getConfig(
"ivoa", "authority"), getInputsRelativePath(path, liberalChars=True))
def getRDInputStream(srcId):
"""returns a read-open stream for the XML source of the resource
descriptor with srcId.
srcId is already normalized; that means that absolute paths must
point to a file (sans possibly .rd), relative paths are relative
to inputsDir or pkg_resources(/resources/inputs).
This function prefers files with .rd to those without, and
inputsDir to pkg_resources (the latter allowing the user to
override built-in system RDs).
"""
for fName in _getFilenamesForId(srcId):
if os.path.isfile(fName):
# We don't want RDs from outside of inputs and config, as
# these make referencing really messy.
filePath = os.path.abspath(fName)
if not (filePath.startswith(base.getConfig("inputsDir"))
or filePath.startswith(base.getConfig("configDir"))):
raise base.ReportableError(
"%s: Only RDs below inputsDir (%s) are"
" allowed." % (fName, base.getConfig("inputsDir")))
return fName, open(fName)
if (pkg_resources.resource_exists('gavo', fName)
and not pkg_resources.resource_isdir('gavo', fName)):
return (PkgResourcePath(fName),
pkg_resources.resource_stream('gavo', fName))
raise base.RDNotFound(srcId)
def _replaceConfigStrings(srcPath, registry):
src = open(srcPath).read().decode("utf-8")
src = src.replace("__site_path__", base.getConfig("web", "nevowRoot"))
src = src.replace(
"__site_url__",
os.path.join(
base.getConfig("web", "serverURL") +
base.getConfig("web", "nevowRoot")))
return src.encode("utf-8")
def debug(args):
log.startLogging(sys.stderr)
base.DEBUG = True
root.root.child_exit = ExitPage()
reactor.listenTCP(int(base.getConfig("web", "serverPort")),
root.site,
interface=base.getConfig("web", "bindAddress"))
setupServer(root)
reactor.run()
def getURL(self, rendName, absolute=True):
# there's no sensible URL for DDs; thus, let people browse
# the RD info. At least they should find links to any tables
# included here there.
basePath = "%sbrowse/%s"%(
base.getConfig("web", "nevowRoot"),
self.rd.sourceId)
if absolute:
return base.getConfig("web", "serverURL")+basePath
return basePath
def getLogFile(baseName):
"""returns a log file group-writable by gavo.
"""
fName = os.path.join(base.getConfig("logDir"), baseName)
f = open(fName, "a")
try:
os.chmod(fName, 0664)
os.chown(fName, -1, grp.getgrnam(base.getConfig("gavoGroup"))[2])
except (KeyError, os.error): # let someone else worry about it
pass
return f
def render_getconfig(self, ctx, data):
"""looks up the text child in the DaCHS configuration and inserts
the value as a (unicode) string.
The config key is either [section]item or just item for something
in [general]. Behaviour for undefined config items is undefined.
"""
configKey = ctx.tag.children[0].strip()
mat = re.match("\[([^]]*)\](.*)", configKey)
if mat:
content = base.getConfig(mat.group(1), mat.group(2))
else:
content = base.getConfig(configKey)
ctx.tag.clear()
return ctx.tag[content]
def _makeCapability(self, publication):
service = publication.parent
return CapabilityMaker._makeCapability(self, publication)[
# XXX TODO: see what we need for "full"
SSAP.complianceLevel[service.getMeta("ssap.complianceLevel",
default="minimal")], SSAP.
dataSource[service.getMeta("ssap.dataSource", raiseOnFail=True)],
SSAP.creationType[service.getMeta("ssap.creationType",
default="archival")],
SSAP.supportedFrame["ICRS"], SSAP.maxSearchRadius["180"],
SSAP.maxRecords[str(base.getConfig("ivoa", "dalHardLimit"))], SSAP.
defaultMaxRecords[str(base.getConfig("ivoa", "dalDefaultLimit"))],
SSAP.maxAperture["180"], SSAP.testQuery[SSAP.queryDataCmd[
base.getMetaText(service, "ssap.testQuery", raiseOnFail=True) +
"&REQUEST=queryData"]], ]
def _doRender(self, ctx):
jobId = tap.WORKER_SYSTEM.getNewIdFromRequest(inevow.IRequest(ctx),
self.service)
try:
with tap.WORKER_SYSTEM.changeableJob(jobId) as job:
job.change(executionDuration=base.getConfig(
"async", "defaultExecTimeSync"))
taprunner.runTAPJob(jobId)
job = tap.WORKER_SYSTEM.getJob(jobId)
if job.phase == uws.COMPLETED:
# This is TAP, so there's exactly one result
res = job.getResults()[0]
name, type = res["resultName"], res["resultType"]
# hold on to the result fd so its inode is not lost when we delete
# the job.
f = open(os.path.join(job.getWD(), name))
return (f, type)
elif job.phase == uws.ERROR:
exc = job.error
raise base.Error(exc["msg"], hint=exc["hint"])
elif job.phase == uws.ABORTED:
raise uws.UWSError(
"Job was manually aborted. For synchronous"
" jobs, this probably means the operators killed it.",
jobId)
else:
raise uws.UWSError("Internal error. Invalid UWS phase.",
jobId)
finally:
tap.WORKER_SYSTEM.destroy(jobId)
def testFile(name, content, writeGz=False, inDir=base.getConfig("tempDir")):
"""a context manager that creates a file name with content in inDir.
The full path name is returned.
With writeGz=True, content is gzipped on the fly (don't do this if
the data already is gzipped).
You can pass in name=None to get a temporary file name if you don't care
about the name.
"""
if name is None:
handle, destName = tempfile.mkstemp(dir=inDir)
os.close(handle)
else:
if not os.path.isdir(inDir):
os.makedirs(inDir)
destName = os.path.join(inDir, name)
if writeGz:
f = gzip.GzipFile(destName, mode="wb")
else:
f = open(destName, "w")
f.write(content)
f.close()
try:
yield destName
finally:
try:
os.unlink(destName)
except os.error:
pass
def declaredel(querier, args):
import datetime
from gavo import registry
from gavo import rsc
authority, path = registry.parseIdentifier(args.svcId)
if authority != base.getConfig("ivoa", "authority"):
raise base.ReportableError("You can only declare ivo ids from your"
" own authority as deleted.")
idParts = path.split("/")
svcsRD = base.caches.getRD("//services")
# mark in resources table
resTable = rsc.TableForDef(svcsRD.getById("resources"),
connection=querier.connection)
newRow = resTable.tableDef.getDefaults()
newRow["sourceRD"] = "/".join(idParts[:-1])
newRow["resId"] = idParts[-1]
newRow["deleted"] = True
newRow["title"] = "Ex " + args.svcId
newRow["dateUpdated"] = newRow["recTimestamp"] = datetime.datetime.utcnow()
resTable.addRow(newRow)
# mark in sets table
resTable = rsc.TableForDef(svcsRD.getById("sets"),
connection=querier.connection)
newRow = resTable.tableDef.getDefaults()
newRow["sourceRD"] = "/".join(idParts[:-1])
newRow["renderer"] = "null"
newRow["resId"] = idParts[-1]
newRow["setName"] = "ivo_managed"
newRow["deleted"] = True
resTable.addRow(newRow)
def fromRAccref(cls, rAccref, grammar=None):
if not rAccref.params.get("preview"):
return None
# no static previews on cutouts
if rAccref.params.get("sra"):
return None
previewPath = rAccref.productsRow["preview"]
localName = None
if previewPath is None:
return None
elif previewPath == "AUTO":
localName = PreviewCacheManager.getCachedPreviewPath(
rAccref.accref)
else:
# remote URLs can't happen here as RemotePreview is checked
# before us.
localName = os.path.join(base.getConfig("inputsDir"), previewPath)
if localName is None:
return None
elif os.path.exists(localName):
rAccref.productsRow["accessPath"] = localName
rAccref.productsRow[
"mime"] = rAccref.productsRow["preview_mime"] or "image/jpeg"
return cls(rAccref)
def localpath(self):
try:
return self._localpathCache
except AttributeError:
self._localpathCache = os.path.join(base.getConfig("inputsDir"),
self.productsRow["accessPath"])
return self._localpathCache
def _fillDbOptions(self, args):
self["dbSortKeys"] = [s.strip()
for s in args.get("_DBOPTIONS_ORDER", []) if s.strip()]
self["direction"] = {"ASC": "ASC", "DESC": "DESC"}[
args.get("_DBOPTIONS_DIR", "ASC")]
try:
self["dbLimit"] = int(args["MAXREC"])
except (ValueError, KeyError):
self["dbLimit"] = base.getConfig("db", "defaultLimit")
try:
self["timeout"] = int(args["_TIMEOUT"])
except (ValueError, KeyError):
self["timeout"] = base.getConfig("web", "sqlTimeout")
def cacheprev(querier, args):
from gavo import api
from gavo.web.productrender import PreviewCacheManager
from twisted.internet import reactor
basePath = base.getConfig("inputsDir")
td = base.resolveId(None, args.tableId)
table = api.TableForDef(td, connection=querier.connection)
select = [td.getColumnByName("accref"), td.getColumnByName("mime")]
rows = table.iterQuery(select, "")
def runNext(token):
try:
row = rows.next()
res = PreviewCacheManager.getPreviewFor(
row["mime"],
[str(os.path.join(basePath, row["accref"])),
str(args.width)])
if getattr(res, "result", None): # don't add a callback on a
# fired deferred or you'll exhaust the stack
reactor.callLater(0.1, runNext, "continue")
else:
res.addCallback(runNext)
return res
except StopIteration:
pass
except:
import traceback
traceback.print_exc()
reactor.stop()
return ""
reactor.callLater(0, runNext, "startup")
reactor.run()
def writeDataAsCSV(table,
target,
acquireSamples=True,
dialect=base.getConfig("async", "csvDialect"),
headered=False):
"""writes table to the file target in CSV.
The CSV format chosen is controlled through the async/csvDialect
config item.
If headered is True, we also include table params (if present)
in comments.
"""
if isinstance(table, rsc.Data):
table = table.getPrimaryTable()
sm = base.SerManager(table, acquireSamples=acquireSamples)
writer = csv.writer(target, dialect)
if headered:
for param in table.iterParams():
if param.value is not None:
target.write(("# %s = %s // %s\r\n" %
(param.name, param.getStringValue(),
param.description)).encode("utf-8"))
writer.writerow([c["name"] for c in sm])
for row in sm.getMappedTuples():
try:
writer.writerow(_encodeRow(row))
except UnicodeEncodeError:
writer.writerow(row)
def onElementComplete(self):
if self.adql:
self.readProfiles = (self.readProfiles
| base.getConfig("db", "adqlProfiles"))
self.dictKeys = [c.key for c in self]
self.indexedColumns = set()
for index in self.indices:
for col in index.columns:
if "\\" in col:
try:
self.indexedColumns.add(self.expand(col))
except (base.Error,
ValueError): # cannot expand yet, ignore
pass
else:
self.indexedColumns.add(col)
if self.primary:
self.indexedColumns |= set(self.primary)
self._defineFixupFunction()
self._onElementCompleteNext(TableDef)
if self.registration:
self.registration.register()
def getReferencedElement(refString, forceType=None, **kwargs):
"""returns the element for the DaCHS reference refString.
refString would be rdId[#subRef]; rdId can be filesystem-relative,
but the RD referenced must be below inputsDir anyway.
You can pass a structure class into forceType, and a StructureError
will be raised if what's pointed to by the id isn't of that type.
You should usually use base.resolveCrossId instead of this from *within*
DaCHS. This is intended for code handling RD ids from users.
This supports further keyword arguments to getRD.
"""
# get the inputs postfix now so we don't pollute the current exception later
try:
cwdInInputs = utils.getRelativePath(os.getcwd(),
base.getConfig("inputsDir"),
liberalChars=True)
except ValueError:
# not in inputs
cwdInInputs = None
try:
return base.resolveCrossId(refString, forceType=forceType, **kwargs)
except base.RDNotFound:
if cwdInInputs:
return base.resolveCrossId("%s/%s" % (cwdInInputs, refString),
forceType=forceType)
raise
def morphADQL(query,
metaProfile=None,
tdsForUploads=[],
externalLimit=None,
hardLimit=None):
"""returns an postgres query and an (empty) result table for the
ADQL in query.
"""
ctx, t = adql.parseAnnotating(
query, getFieldInfoGetter(metaProfile, tdsForUploads))
if t.setLimit is None:
if externalLimit is None:
t.setLimit = str(base.getConfig("adql", "webDefaultLimit"))
else:
t.setLimit = str(int(externalLimit))
table = rsc.TableForDef(_getTableDescForOutput(t))
if hardLimit and int(t.setLimit) > hardLimit:
table.addMeta(
"_warning", "This service as a hard row limit"
" of %s. Your row limit was decreased to this value." % hardLimit)
t.setLimit = str(hardLimit)
morphStatus, morphedTree = adql.morphPG(t)
for warning in morphStatus.warnings:
table.addMeta("_warning", warning)
# escape % to hide them form dbapi replacing
query = adql.flatten(morphedTree).replace("%", "%%")
table.tableDef.setLimit = t.setLimit and int(t.setLimit)
_addTableMeta(query, t, table)
return query, table
def _meta_identifier(self):
# Special case the authority
if base.getMetaText(self, "resType") == "authority":
localPart = ""
else:
localPart = "/%s/%s" % (self.rd.sourceId, self.id)
return "ivo://%s%s" % (base.getConfig("ivoa", "authority"), localPart)
def makeProfiles(dsn, userPrefix=""):
"""writes profiles with made-up passwords to DaCHS' config dir.
This will mess everything up when the users already exist. We
should probably provide an option to drop standard users.
userPrefix is mainly for the test infrastructure.
"""
profilePath = base.getConfig("configDir")
dsnContent = ["database = %s" % (dsn.parsed["dbname"])]
if "host" in dsn.parsed:
dsnContent.append("host = %s" % dsn.parsed["host"])
else:
dsnContent.append("host = localhost")
if "port" in dsn.parsed:
dsnContent.append("port = %s" % dsn.parsed["port"])
else:
dsnContent.append("port = 5432")
for fName, content in [
("dsn", "\n".join(dsnContent) + "\n"),
("feed", "include dsn\nuser = %sgavoadmin\npassword = %s\n" %
(userPrefix, _genPW())),
("trustedquery", "include dsn\nuser = %sgavo\npassword = %s\n" %
(userPrefix, _genPW())),
("untrustedquery", "include dsn\nuser = %suntrusted\npassword = %s\n" %
(userPrefix, _genPW())),
]:
destPath = os.path.join(profilePath, fName)
if not os.path.exists(destPath):
with open(destPath, "w") as f:
f.write(content)
def _makeCapability(self, publication):
service = publication.parent
return CapabilityMaker._makeCapability(
self, publication
)[SIA.imageServiceType[service.getMeta("sia.type", raiseOnFail=True)],
SIA.maxQueryRegionSize[
SIA.long[service.
getMeta("sia.maxQueryRegionSize.long", default=None)],
SIA.lat[service.
getMeta("sia.maxQueryRegionSize.lat", default=None)], ],
SIA.maxImageExtent[
SIA.long[service.getMeta("sia.maxImageExtent.long", default=None
)], SIA.
lat[service.getMeta("sia.maxImageExtent.lat", default=None)], ],
SIA.maxImageSize[
service.getMeta("sia.maxImageSize", default=None), ],
SIA.maxFileSize[service.getMeta("sia.maxFileSize", default=None), ],
SIA.maxRecords[
service.
getMeta("sia.maxRecords",
default=str(base.getConfig("ivoa", "dalHardLimit"))), ],
SIA.testQuery[
SIA.pos[SIA.long[service.getMeta("testQuery.pos.ra")],
SIA.lat[service.getMeta("testQuery.pos.dec")], ],
SIA.size[SIA.long[service.getMeta("testQuery.size.ra")],
SIA.lat[service.getMeta("testQuery.size.dec")], ], ], ]
def runTAPJobNoState(parameters, jobId, queryProfile, timeout):
"""executes a TAP job defined by parameters and writes the
result to the job's working directory.
This does not do state management. Use runTAPJob if you need it.
"""
_hangIfMagic(jobId, parameters, timeout)
# The following makes us bail out if a bad format was passed -- no
# sense spending the CPU on executing the query then, so we get the
# format here.
defaultFormat = "votable"
if base.getConfig("ivoa", "votDefaultEncoding")=="td":
defaultFormat = "votable/td"
format = normalizeTAPFormat(parameters.get("format", defaultFormat))
res = _makeDataFor(getQTableFromJob(
parameters, jobId, queryProfile, timeout))
try:
job = tap.WORKER_SYSTEM.getJob(jobId)
destF = job.openResult(
formats.getMIMEFor(format, job.parameters.get("format")), "result")
writeResultTo(format, res, destF)
destF.close()
except Exception:
# DB errors can occur here since we're streaming directly from
# the database.
svcs.mapDBErrors(*sys.exc_info())
def parseCommandLine(args=None):
"""parses the command line for main()
"""
parser = argparse.ArgumentParser(description="Run tests embedded in RDs")
parser.add_argument("id", type=str,
help="RD id or cross-RD identifier for a testable thing.")
parser.add_argument("-v", "--verbose", help="Talk while working",
action="store_true", dest="verbose")
parser.add_argument("-d", "--dump-negative", help="Dump the content of"
" failing tests to stdout",
action="store_true", dest="dumpNegative")
parser.add_argument("-t", "--tag", help="Also run tests tagged with TAG.",
action="store", dest="tag", default=None, metavar="TAG")
parser.add_argument("-R", "--n-repeat", help="Run each test N times",
action="store", dest="nRepeat", type=int, default=None, metavar="N")
parser.add_argument("-T", "--timeout", help="Abort and fail requests"
" after inactivity of SECONDS",
action="store", dest="timeout", type=int, default=15, metavar="SECONDS")
parser.add_argument("-D", "--dump-to", help="Dump the content of"
" last failing test to FILE", metavar="FILE",
action="store", type=str, dest="failFile",
default=None)
parser.add_argument("-w", "--wait", help="Wait SECONDS before executing"
" a request", metavar="SECONDS", action="store",
dest="execDelay", type=int, default=0)
parser.add_argument("-u", "--serverURL", help="URL of the DaCHS root"
" at the server to test",
action="store", type=str, dest="serverURL",
default=base.getConfig("web", "serverURL"))
parser.add_argument("-n", "--number-par", help="Number of requests"
" to be run in parallel",
action="store", type=int, dest="nThreads",
default=8)
return parser.parse_args(args)
def u_010_updateTAPRecord(cls, connection):
"""prettify the TAP record's IVORN"""
from gavo.registry import publication
publication.updateServiceList([base.caches.getRD("//services")],
connection=connection)
publication.makeDeletedRecord(
"ivo://" + base.getConfig("ivoa", "authority") +
"/__system__/tap/run", connection)
def __init__(self, dest, hint=None):
self.rawDest = dest
dest = str(dest)
if not dest.startswith("http"):
dest = base.getConfig("web", "serverURL")+base.makeSitePath(dest)
self.dest = dest
Error.__init__(self, "This is supposed to redirect to %s"%dest,
hint=hint)
def __init__(self, *args, **kwargs):
ProductDescriptor.__init__(self, *args, **kwargs)
with open(os.path.join(base.getConfig("inputsDir"),
self.accessPath)) as f:
self.hdr = utils.readPrimaryHeaderQuick(f, maxHeaderBlocks=100)
self.slices = []
self.dataIsPristine = True
self._axesTouched = set()
def render_prependsite(self, ctx, data):
"""prepends a site id to the body.
This is intended for titles and similar; it puts the string in
[web]sitename in front of anything that already is in ctx.tag.
"""
ctx.tag.children = [base.getConfig("web", "sitename")
] + ctx.tag.children
return ctx.tag
