def _startServer(): """runs a detached server, dropping privileges and all. """ try: reactor.listenTCP(int(base.getConfig("web", "serverPort")), root.site, interface=base.getConfig("web", "bindAddress")) except CannotListenError: raise base.ReportableError( "Someone already listens on the" " configured port %s." % base.getConfig("web", "serverPort"), hint="This could mean that a DaCHS server is already running." " You would have to manually kill it then since its PID file" " got lost somehow. It's more likely that some" " other server is already taking up this port; you may want to change" " the [web] serverPort setting in that case.") _dropPrivileges() root.site.webLog = _configureTwistedLog() PIDManager.setPID() try: setupServer(root) signal.signal(signal.SIGHUP, lambda sig, stack: reactor.callLater(0, _reloadConfig)) _preloadRDs() reactor.run() finally: PIDManager.clearPID()
def render_commonhead(self, ctx, data): # we do not want to blindly append these things to the tag since user- # provided items in the header should have access to this environment, # in particular to jquery; thus, this stuff must be as early as possible. originalChildren = ctx.tag.children[:] ctx.tag.clear() res = ctx.tag[ T.meta(**{"charset": "utf-8"}), T.link(rel="icon", type="image/png", href=base.makeSitePath("/static/img/favicon.png")), T.link(rel="stylesheet", href=base.makeSitePath("/static/css/style.css"), type="text/css"), T.link(rel="stylesheet", href=base.makeSitePath("/formal.css"), type="text/css"), T.link(rel="stylesheet", href=base.makeSitePath("/static/css/gavo_dc.css"), type="text/css"), T.script(src=base.makeSitePath("/static/js/jquery-gavo.js"), type="text/javascript"), T.script(type='text/javascript', src=base.makeSitePath('static/js/formal' + JSEXT)), T.script(type="text/javascript", src=base.makeSitePath("/static/js/gavo" + JSEXT)), T.script(type="text/javascript", src=base.makeSitePath("/static/js/script.js")), originalChildren, ] if base.getConfig("web", "operatorCSS"): res[T.link(rel="stylesheet", type="text/css", href=base.getConfig("web", "operatorCSS"))] return res
def getStandardPubDID(path): """returns the standard DaCHS PubDID for path. The publisher dataset identifier (PubDID) is important in protocols like SSAP and obscore. If you use this function, the PubDID will be your authority, the path compontent ~, and the inputs-relative path of the input file as the parameter. path can be relative, in which case it is interpreted relative to the DaCHS inputsDir. You *can* define your PubDIDs in a different way, but you'd then need to provide a custom descriptorGenerator to datalink services (and might need other tricks). If your data comes from plain files, use this function. In a rowmaker, you'll usually use the \\standardPubDID macro. """ # Why add inputsDir first and remove it again? Well, I want to keep # getInputsRelativePath in the look since it does some validation # and may, at some point, do more. if path[0] != "/": path = os.path.join(base.getConfig("inputsDir"), path) return "ivo://%s/~?%s" % (base.getConfig( "ivoa", "authority"), getInputsRelativePath(path, liberalChars=True))
def getRDInputStream(srcId): """returns a read-open stream for the XML source of the resource descriptor with srcId. srcId is already normalized; that means that absolute paths must point to a file (sans possibly .rd), relative paths are relative to inputsDir or pkg_resources(/resources/inputs). This function prefers files with .rd to those without, and inputsDir to pkg_resources (the latter allowing the user to override built-in system RDs). """ for fName in _getFilenamesForId(srcId): if os.path.isfile(fName): # We don't want RDs from outside of inputs and config, as # these make referencing really messy. filePath = os.path.abspath(fName) if not (filePath.startswith(base.getConfig("inputsDir")) or filePath.startswith(base.getConfig("configDir"))): raise base.ReportableError( "%s: Only RDs below inputsDir (%s) are" " allowed." % (fName, base.getConfig("inputsDir"))) return fName, open(fName) if (pkg_resources.resource_exists('gavo', fName) and not pkg_resources.resource_isdir('gavo', fName)): return (PkgResourcePath(fName), pkg_resources.resource_stream('gavo', fName)) raise base.RDNotFound(srcId)
def _replaceConfigStrings(srcPath, registry): src = open(srcPath).read().decode("utf-8") src = src.replace("__site_path__", base.getConfig("web", "nevowRoot")) src = src.replace( "__site_url__", os.path.join( base.getConfig("web", "serverURL") + base.getConfig("web", "nevowRoot"))) return src.encode("utf-8")
def debug(args): log.startLogging(sys.stderr) base.DEBUG = True root.root.child_exit = ExitPage() reactor.listenTCP(int(base.getConfig("web", "serverPort")), root.site, interface=base.getConfig("web", "bindAddress")) setupServer(root) reactor.run()
def getURL(self, rendName, absolute=True): # there's no sensible URL for DDs; thus, let people browse # the RD info. At least they should find links to any tables # included here there. basePath = "%sbrowse/%s"%( base.getConfig("web", "nevowRoot"), self.rd.sourceId) if absolute: return base.getConfig("web", "serverURL")+basePath return basePath
def getLogFile(baseName): """returns a log file group-writable by gavo. """ fName = os.path.join(base.getConfig("logDir"), baseName) f = open(fName, "a") try: os.chmod(fName, 0664) os.chown(fName, -1, grp.getgrnam(base.getConfig("gavoGroup"))[2]) except (KeyError, os.error): # let someone else worry about it pass return f
def render_getconfig(self, ctx, data): """looks up the text child in the DaCHS configuration and inserts the value as a (unicode) string. The config key is either [section]item or just item for something in [general]. Behaviour for undefined config items is undefined. """ configKey = ctx.tag.children[0].strip() mat = re.match("\[([^]]*)\](.*)", configKey) if mat: content = base.getConfig(mat.group(1), mat.group(2)) else: content = base.getConfig(configKey) ctx.tag.clear() return ctx.tag[content]
def _makeCapability(self, publication): service = publication.parent return CapabilityMaker._makeCapability(self, publication)[ # XXX TODO: see what we need for "full" SSAP.complianceLevel[service.getMeta("ssap.complianceLevel", default="minimal")], SSAP. dataSource[service.getMeta("ssap.dataSource", raiseOnFail=True)], SSAP.creationType[service.getMeta("ssap.creationType", default="archival")], SSAP.supportedFrame["ICRS"], SSAP.maxSearchRadius["180"], SSAP.maxRecords[str(base.getConfig("ivoa", "dalHardLimit"))], SSAP. defaultMaxRecords[str(base.getConfig("ivoa", "dalDefaultLimit"))], SSAP.maxAperture["180"], SSAP.testQuery[SSAP.queryDataCmd[ base.getMetaText(service, "ssap.testQuery", raiseOnFail=True) + "&REQUEST=queryData"]], ]
def _doRender(self, ctx): jobId = tap.WORKER_SYSTEM.getNewIdFromRequest(inevow.IRequest(ctx), self.service) try: with tap.WORKER_SYSTEM.changeableJob(jobId) as job: job.change(executionDuration=base.getConfig( "async", "defaultExecTimeSync")) taprunner.runTAPJob(jobId) job = tap.WORKER_SYSTEM.getJob(jobId) if job.phase == uws.COMPLETED: # This is TAP, so there's exactly one result res = job.getResults()[0] name, type = res["resultName"], res["resultType"] # hold on to the result fd so its inode is not lost when we delete # the job. f = open(os.path.join(job.getWD(), name)) return (f, type) elif job.phase == uws.ERROR: exc = job.error raise base.Error(exc["msg"], hint=exc["hint"]) elif job.phase == uws.ABORTED: raise uws.UWSError( "Job was manually aborted. For synchronous" " jobs, this probably means the operators killed it.", jobId) else: raise uws.UWSError("Internal error. Invalid UWS phase.", jobId) finally: tap.WORKER_SYSTEM.destroy(jobId)
def testFile(name, content, writeGz=False, inDir=base.getConfig("tempDir")): """a context manager that creates a file name with content in inDir. The full path name is returned. With writeGz=True, content is gzipped on the fly (don't do this if the data already is gzipped). You can pass in name=None to get a temporary file name if you don't care about the name. """ if name is None: handle, destName = tempfile.mkstemp(dir=inDir) os.close(handle) else: if not os.path.isdir(inDir): os.makedirs(inDir) destName = os.path.join(inDir, name) if writeGz: f = gzip.GzipFile(destName, mode="wb") else: f = open(destName, "w") f.write(content) f.close() try: yield destName finally: try: os.unlink(destName) except os.error: pass
def declaredel(querier, args): import datetime from gavo import registry from gavo import rsc authority, path = registry.parseIdentifier(args.svcId) if authority != base.getConfig("ivoa", "authority"): raise base.ReportableError("You can only declare ivo ids from your" " own authority as deleted.") idParts = path.split("/") svcsRD = base.caches.getRD("//services") # mark in resources table resTable = rsc.TableForDef(svcsRD.getById("resources"), connection=querier.connection) newRow = resTable.tableDef.getDefaults() newRow["sourceRD"] = "/".join(idParts[:-1]) newRow["resId"] = idParts[-1] newRow["deleted"] = True newRow["title"] = "Ex " + args.svcId newRow["dateUpdated"] = newRow["recTimestamp"] = datetime.datetime.utcnow() resTable.addRow(newRow) # mark in sets table resTable = rsc.TableForDef(svcsRD.getById("sets"), connection=querier.connection) newRow = resTable.tableDef.getDefaults() newRow["sourceRD"] = "/".join(idParts[:-1]) newRow["renderer"] = "null" newRow["resId"] = idParts[-1] newRow["setName"] = "ivo_managed" newRow["deleted"] = True resTable.addRow(newRow)
def fromRAccref(cls, rAccref, grammar=None): if not rAccref.params.get("preview"): return None # no static previews on cutouts if rAccref.params.get("sra"): return None previewPath = rAccref.productsRow["preview"] localName = None if previewPath is None: return None elif previewPath == "AUTO": localName = PreviewCacheManager.getCachedPreviewPath( rAccref.accref) else: # remote URLs can't happen here as RemotePreview is checked # before us. localName = os.path.join(base.getConfig("inputsDir"), previewPath) if localName is None: return None elif os.path.exists(localName): rAccref.productsRow["accessPath"] = localName rAccref.productsRow[ "mime"] = rAccref.productsRow["preview_mime"] or "image/jpeg" return cls(rAccref)
def localpath(self): try: return self._localpathCache except AttributeError: self._localpathCache = os.path.join(base.getConfig("inputsDir"), self.productsRow["accessPath"]) return self._localpathCache
def _fillDbOptions(self, args): self["dbSortKeys"] = [s.strip() for s in args.get("_DBOPTIONS_ORDER", []) if s.strip()] self["direction"] = {"ASC": "ASC", "DESC": "DESC"}[ args.get("_DBOPTIONS_DIR", "ASC")] try: self["dbLimit"] = int(args["MAXREC"]) except (ValueError, KeyError): self["dbLimit"] = base.getConfig("db", "defaultLimit") try: self["timeout"] = int(args["_TIMEOUT"]) except (ValueError, KeyError): self["timeout"] = base.getConfig("web", "sqlTimeout")
def cacheprev(querier, args): from gavo import api from gavo.web.productrender import PreviewCacheManager from twisted.internet import reactor basePath = base.getConfig("inputsDir") td = base.resolveId(None, args.tableId) table = api.TableForDef(td, connection=querier.connection) select = [td.getColumnByName("accref"), td.getColumnByName("mime")] rows = table.iterQuery(select, "") def runNext(token): try: row = rows.next() res = PreviewCacheManager.getPreviewFor( row["mime"], [str(os.path.join(basePath, row["accref"])), str(args.width)]) if getattr(res, "result", None): # don't add a callback on a # fired deferred or you'll exhaust the stack reactor.callLater(0.1, runNext, "continue") else: res.addCallback(runNext) return res except StopIteration: pass except: import traceback traceback.print_exc() reactor.stop() return "" reactor.callLater(0, runNext, "startup") reactor.run()
def writeDataAsCSV(table, target, acquireSamples=True, dialect=base.getConfig("async", "csvDialect"), headered=False): """writes table to the file target in CSV. The CSV format chosen is controlled through the async/csvDialect config item. If headered is True, we also include table params (if present) in comments. """ if isinstance(table, rsc.Data): table = table.getPrimaryTable() sm = base.SerManager(table, acquireSamples=acquireSamples) writer = csv.writer(target, dialect) if headered: for param in table.iterParams(): if param.value is not None: target.write(("# %s = %s // %s\r\n" % (param.name, param.getStringValue(), param.description)).encode("utf-8")) writer.writerow([c["name"] for c in sm]) for row in sm.getMappedTuples(): try: writer.writerow(_encodeRow(row)) except UnicodeEncodeError: writer.writerow(row)
def onElementComplete(self): if self.adql: self.readProfiles = (self.readProfiles | base.getConfig("db", "adqlProfiles")) self.dictKeys = [c.key for c in self] self.indexedColumns = set() for index in self.indices: for col in index.columns: if "\\" in col: try: self.indexedColumns.add(self.expand(col)) except (base.Error, ValueError): # cannot expand yet, ignore pass else: self.indexedColumns.add(col) if self.primary: self.indexedColumns |= set(self.primary) self._defineFixupFunction() self._onElementCompleteNext(TableDef) if self.registration: self.registration.register()
def getReferencedElement(refString, forceType=None, **kwargs): """returns the element for the DaCHS reference refString. refString would be rdId[#subRef]; rdId can be filesystem-relative, but the RD referenced must be below inputsDir anyway. You can pass a structure class into forceType, and a StructureError will be raised if what's pointed to by the id isn't of that type. You should usually use base.resolveCrossId instead of this from *within* DaCHS. This is intended for code handling RD ids from users. This supports further keyword arguments to getRD. """ # get the inputs postfix now so we don't pollute the current exception later try: cwdInInputs = utils.getRelativePath(os.getcwd(), base.getConfig("inputsDir"), liberalChars=True) except ValueError: # not in inputs cwdInInputs = None try: return base.resolveCrossId(refString, forceType=forceType, **kwargs) except base.RDNotFound: if cwdInInputs: return base.resolveCrossId("%s/%s" % (cwdInInputs, refString), forceType=forceType) raise
def morphADQL(query, metaProfile=None, tdsForUploads=[], externalLimit=None, hardLimit=None): """returns an postgres query and an (empty) result table for the ADQL in query. """ ctx, t = adql.parseAnnotating( query, getFieldInfoGetter(metaProfile, tdsForUploads)) if t.setLimit is None: if externalLimit is None: t.setLimit = str(base.getConfig("adql", "webDefaultLimit")) else: t.setLimit = str(int(externalLimit)) table = rsc.TableForDef(_getTableDescForOutput(t)) if hardLimit and int(t.setLimit) > hardLimit: table.addMeta( "_warning", "This service as a hard row limit" " of %s. Your row limit was decreased to this value." % hardLimit) t.setLimit = str(hardLimit) morphStatus, morphedTree = adql.morphPG(t) for warning in morphStatus.warnings: table.addMeta("_warning", warning) # escape % to hide them form dbapi replacing query = adql.flatten(morphedTree).replace("%", "%%") table.tableDef.setLimit = t.setLimit and int(t.setLimit) _addTableMeta(query, t, table) return query, table
def _meta_identifier(self): # Special case the authority if base.getMetaText(self, "resType") == "authority": localPart = "" else: localPart = "/%s/%s" % (self.rd.sourceId, self.id) return "ivo://%s%s" % (base.getConfig("ivoa", "authority"), localPart)
def makeProfiles(dsn, userPrefix=""): """writes profiles with made-up passwords to DaCHS' config dir. This will mess everything up when the users already exist. We should probably provide an option to drop standard users. userPrefix is mainly for the test infrastructure. """ profilePath = base.getConfig("configDir") dsnContent = ["database = %s" % (dsn.parsed["dbname"])] if "host" in dsn.parsed: dsnContent.append("host = %s" % dsn.parsed["host"]) else: dsnContent.append("host = localhost") if "port" in dsn.parsed: dsnContent.append("port = %s" % dsn.parsed["port"]) else: dsnContent.append("port = 5432") for fName, content in [ ("dsn", "\n".join(dsnContent) + "\n"), ("feed", "include dsn\nuser = %sgavoadmin\npassword = %s\n" % (userPrefix, _genPW())), ("trustedquery", "include dsn\nuser = %sgavo\npassword = %s\n" % (userPrefix, _genPW())), ("untrustedquery", "include dsn\nuser = %suntrusted\npassword = %s\n" % (userPrefix, _genPW())), ]: destPath = os.path.join(profilePath, fName) if not os.path.exists(destPath): with open(destPath, "w") as f: f.write(content)
def _makeCapability(self, publication): service = publication.parent return CapabilityMaker._makeCapability( self, publication )[SIA.imageServiceType[service.getMeta("sia.type", raiseOnFail=True)], SIA.maxQueryRegionSize[ SIA.long[service. getMeta("sia.maxQueryRegionSize.long", default=None)], SIA.lat[service. getMeta("sia.maxQueryRegionSize.lat", default=None)], ], SIA.maxImageExtent[ SIA.long[service.getMeta("sia.maxImageExtent.long", default=None )], SIA. lat[service.getMeta("sia.maxImageExtent.lat", default=None)], ], SIA.maxImageSize[ service.getMeta("sia.maxImageSize", default=None), ], SIA.maxFileSize[service.getMeta("sia.maxFileSize", default=None), ], SIA.maxRecords[ service. getMeta("sia.maxRecords", default=str(base.getConfig("ivoa", "dalHardLimit"))), ], SIA.testQuery[ SIA.pos[SIA.long[service.getMeta("testQuery.pos.ra")], SIA.lat[service.getMeta("testQuery.pos.dec")], ], SIA.size[SIA.long[service.getMeta("testQuery.size.ra")], SIA.lat[service.getMeta("testQuery.size.dec")], ], ], ]
def runTAPJobNoState(parameters, jobId, queryProfile, timeout): """executes a TAP job defined by parameters and writes the result to the job's working directory. This does not do state management. Use runTAPJob if you need it. """ _hangIfMagic(jobId, parameters, timeout) # The following makes us bail out if a bad format was passed -- no # sense spending the CPU on executing the query then, so we get the # format here. defaultFormat = "votable" if base.getConfig("ivoa", "votDefaultEncoding")=="td": defaultFormat = "votable/td" format = normalizeTAPFormat(parameters.get("format", defaultFormat)) res = _makeDataFor(getQTableFromJob( parameters, jobId, queryProfile, timeout)) try: job = tap.WORKER_SYSTEM.getJob(jobId) destF = job.openResult( formats.getMIMEFor(format, job.parameters.get("format")), "result") writeResultTo(format, res, destF) destF.close() except Exception: # DB errors can occur here since we're streaming directly from # the database. svcs.mapDBErrors(*sys.exc_info())
def parseCommandLine(args=None): """parses the command line for main() """ parser = argparse.ArgumentParser(description="Run tests embedded in RDs") parser.add_argument("id", type=str, help="RD id or cross-RD identifier for a testable thing.") parser.add_argument("-v", "--verbose", help="Talk while working", action="store_true", dest="verbose") parser.add_argument("-d", "--dump-negative", help="Dump the content of" " failing tests to stdout", action="store_true", dest="dumpNegative") parser.add_argument("-t", "--tag", help="Also run tests tagged with TAG.", action="store", dest="tag", default=None, metavar="TAG") parser.add_argument("-R", "--n-repeat", help="Run each test N times", action="store", dest="nRepeat", type=int, default=None, metavar="N") parser.add_argument("-T", "--timeout", help="Abort and fail requests" " after inactivity of SECONDS", action="store", dest="timeout", type=int, default=15, metavar="SECONDS") parser.add_argument("-D", "--dump-to", help="Dump the content of" " last failing test to FILE", metavar="FILE", action="store", type=str, dest="failFile", default=None) parser.add_argument("-w", "--wait", help="Wait SECONDS before executing" " a request", metavar="SECONDS", action="store", dest="execDelay", type=int, default=0) parser.add_argument("-u", "--serverURL", help="URL of the DaCHS root" " at the server to test", action="store", type=str, dest="serverURL", default=base.getConfig("web", "serverURL")) parser.add_argument("-n", "--number-par", help="Number of requests" " to be run in parallel", action="store", type=int, dest="nThreads", default=8) return parser.parse_args(args)
def u_010_updateTAPRecord(cls, connection): """prettify the TAP record's IVORN""" from gavo.registry import publication publication.updateServiceList([base.caches.getRD("//services")], connection=connection) publication.makeDeletedRecord( "ivo://" + base.getConfig("ivoa", "authority") + "/__system__/tap/run", connection)
def __init__(self, dest, hint=None): self.rawDest = dest dest = str(dest) if not dest.startswith("http"): dest = base.getConfig("web", "serverURL")+base.makeSitePath(dest) self.dest = dest Error.__init__(self, "This is supposed to redirect to %s"%dest, hint=hint)
def __init__(self, *args, **kwargs): ProductDescriptor.__init__(self, *args, **kwargs) with open(os.path.join(base.getConfig("inputsDir"), self.accessPath)) as f: self.hdr = utils.readPrimaryHeaderQuick(f, maxHeaderBlocks=100) self.slices = [] self.dataIsPristine = True self._axesTouched = set()
def render_prependsite(self, ctx, data): """prepends a site id to the body. This is intended for titles and similar; it puts the string in [web]sitename in front of anything that already is in ctx.tag. """ ctx.tag.children = [base.getConfig("web", "sitename") ] + ctx.tag.children return ctx.tag