def getMetaForDescriptor(self, descriptor):
"""returns a pair of linkDefs, inputKeys for a datalink desriptor
and this core.
"""
linkDefs, inputKeys, errors = [], self.inputKeys[:], []
for metaMaker in self.metaMakers:
try:
for item in metaMaker.compile(self)(self, descriptor):
if isinstance(item, LinkDef):
linkDefs.append(item)
elif isinstance(item, DatalinkFault):
errors.append(item)
else:
inputKeys.append(item)
except Exception, ex:
if base.DEBUG:
base.ui.notifyError(
"Error in datalink meta generator %s: %s" %
(metaMaker, repr(ex)))
base.ui.notifyError("Failing source: \n%s" %
metaMaker.getFuncCode())
errors.append(
DatalinkFault.Fault(
descriptor.pubDID, "Unexpected failure while creating"
" datalink: %s" % utils.safe_str(ex)))
def processSource(data, source, feeder, opts, connection=None):
"""ingests source into the Data instance data.
If you pass in a connection, you can set opts.keepGoing to true
and make the system continue importing even if a particular source
has caused an error. In that case, everything contributed by
the bad source is rolled back.
"""
if not opts.keepGoing:
# simple shortcut if we don't want to recover from bad sources
_processSourceReal(data, source, feeder, opts)
else: # recover from bad sources, be more careful
if connection is None:
raise base.ReportableError(
"Can only ignore source errors"
" with an explicit connection",
hint="This is a programming error.")
try:
with connection.savepoint():
_processSourceReal(data, source, feeder, opts)
feeder.flush()
except Exception, ex:
feeder.reset()
if not isinstance(ex, base.ExecutiveAction):
base.ui.notifyError(
"Error while importing source; changes from"
" this source will be rolled back, processing will continue."
" (%s)" % utils.safe_str(ex))
def doMetaOverride(container, metaKey, metaValue, extraArgs={}):
"""creates the representation of metaKey/metaValue in container.
If metaKey does not need any special action, this returns None.
This gets called from one central point in MetaMixin.addMeta, and
essentially all magic involved should be concentrated here.
"""
if metaKey in META_CLASSES_FOR_KEYS and not isinstance(
metaValue, MetaValue):
try:
container.addMeta(
metaKey, META_CLASSES_FOR_KEYS[metaKey](metaValue,
**extraArgs))
return True
except TypeError:
raise utils.logOldExc(
MetaError(
"Invalid arguments for %s meta items: %s" %
(metaKey, utils.safe_str(extraArgs)), None))
# let's see if there's some way to rationalise this kind of thing
# later.
if metaKey == "creator":
return _doCreatorMetaOverride(container, metaValue)
def ensureOnDiskMatches(self):
"""raises a DataError if the on-disk structure of a table doesn't
match DaCHS' idea of it.
If the table doesn't exist on disk, this will raise a NotFoundError.
"""
dbCols = self.getColumnsFromDB(self.tableDef.getQName())
mismatches = []
if len(self.tableDef.columns) < len(dbCols):
mismatches.append(
"extra columns in DB (%s)" %
", ".join(name
for name, _ in dbCols[len(self.tableDef.columns):]))
for index, col in enumerate(self.tableDef):
try:
name, type = dbCols[index]
except IndexError:
mismatches.append("from column %s on: No matches in DB" %
col.name)
break
if col.name.lower() != name:
mismatches.append("mismatching name of %s (DB: %s)" %
(col.name, name))
continue
try:
base.sqltypeToPgValidator(col.type)(type)
except TypeError, ex:
mismatches.append("type mismatch in column %s (%s)" %
(col.name, utils.safe_str(ex)))
def _reportError(self, failure, request):
# Do not trap svcs.WebRedirect here!
failure.trap(base.ValidationError)
request.setHeader("content-type", "text/plain")
request.setResponseCode(422)
return "%s: %s\n" % (failure.value.__class__.__name__,
utils.safe_str(failure.value))
def run(self, service, inputData, queryMeta):
rows = []
for par in inputData.iterParams():
if par.type == "file":
value = par.value[1].read()
else:
value = par.value
rows.append({"key": par.name, "value": utils.safe_str(value)})
return rsc.TableForDef(self.outputTable, rows=rows)
def _makeExtension(serMan):
"""returns a pyfits hdu for the valuemappers.SerManager instance table.
"""
values = list(serMan.getMappedTuples())
columns = []
utypes = []
descriptions = []
for colInd, colDesc in enumerate(serMan):
descriptions.append(colDesc["description"])
if colDesc["datatype"]=="char" or colDesc["datatype"]=="unicodeChar":
makeArray = _makeStringArray
else:
makeArray = _makeValueArray
typecode, arr = makeArray(values, colInd, colDesc)
if typecode in 'ED':
nullValue = None # (NaN implied)
else:
nullValue = _getNullValue(colDesc)
columns.append(pyfits.Column(name=str(colDesc["name"]),
unit=str(colDesc["unit"]), format=typecode,
null=nullValue, array=arr))
if colDesc["utype"]:
utypes.append((colInd, str(colDesc["utype"].lower())))
hdu = pyfits.new_table(pyfits.ColDefs(columns))
for colInd, utype in utypes:
hdu.header.update("TUTYP%d"%(colInd+1), utype)
cards = hdu.header.ascard
for colInd, desc in enumerate(descriptions):
cards["TTYPE%d"%(colInd+1)].comment = desc.encode("ascii", "ignore")
if not hasattr(serMan.table, "IgnoreTableParams"):
for param in serMan.table.iterParams():
if param.value is None:
continue
key, value, comment = str(param.name), param.value, param.description
if isinstance(value, unicode):
value = value.encode('ascii', "xmlcharrefreplace")
if isinstance(comment, unicode):
comment = comment.encode('ascii', "xmlcharrefreplace")
if len(key)>8:
key = "hierarch "+key
try:
hdu.header.update(key=key, value=value, comment=comment)
except ValueError, ex:
# do not fail just because some header couldn't be serialised
base.ui.notifyWarning(
"Failed to serialise param %s to a FITS header (%s)"%(
param.name,
utils.safe_str(ex)))
def _getRunnerForAll(runnerArgs):
from gavo.registry import publication
from gavo import api
suites = []
for rdId in publication.findAllRDs():
try:
rd = api.getRD(rdId, doQueries=False)
except Exception, msg:
base.ui.notifyError("Error loading RD %s (%s). Ignoring."%(
rdId, utils.safe_str(msg)))
suites.extend(rd.tests)
def run(self):
"""runs callable under somewhat reliable circumstances.
"""
try:
self.callable()
except Exception:
utils.sendUIEvent(
"Error",
"Failure in timed job %s. Trying to send maintainer a mail." %
utils.safe_str(self))
self.reportCronFailure("".join(
traceback.format_exception(*sys.exc_info())))
def iterLimitsForRD(rd):
"""returns a list of values to fill in for an entire RD.
See iterLimitsForTable.
"""
for td in rd.tables:
if td.onDisk:
try:
for limits in iterLimitsForTable(td):
yield limits
except base.ReportableError, msg:
base.ui.notifyError("Skipping %s: %s" %
(td.id, utils.safe_str(msg)))
def printTableInfo(td):
"""tries to obtain various information on the properties of the
database table described by td.
"""
annotateDBTable(td)
propTable = [("col", ) + _PROP_SEQ]
for col in td:
row = [col.name]
for prop in _PROP_SEQ:
if prop in col.annotations:
row.append(
utils.makeEllipsis(utils.safe_str(col.annotations[prop]),
30))
else:
row.append("-")
propTable.append(tuple(row))
print utils.formatSimpleTable(propTable)
def sendMailToAdmin(subject, message):
"""tries to send a mail to the configured administrator.
This relies on a functional mail infrastructure on the local host.
"""
if not config.get("maintainerAddress"):
utils.sendUIEvent(
"Error", "Wanted to send mail with subject '%s', but no"
" maintainerAddress is given" % subject)
return
osinter.sendMail("\n".join([
"To: " + config.get("maintainerAddress"), "Subject: " + subject,
"From: DaCHS server <%s>" % config.get("maintainerAddress"),
"Content-Type: text/plain", "",
utils.safe_str(message)
]))
def validateTables(rd, args):
"""does some sanity checks on the (top-level) tables within rd.
"""
valid = True
identifierSymbol = adql.getSymbols()["identifier"]
for td in rd.tables:
for col in td:
try:
if col.unit:
parsedUnit = api.parseUnit(col.unit)
if parsedUnit.isUnknown and not args.acceptFreeUnits:
outputWarning(
rd.sourceId,
"Column %s.%s: Unit %s is not interoperable" %
(td.getQName(), col.name, col.unit))
except api.BadUnit:
valid = False
outputError(
rd.sourceId, "Bad unit in table %s, column %s: %s" %
(td.getQName(), col.name, repr(col.unit)))
try:
identifierSymbol.parseString(str(col.name), parseAll=True)
except base.ParseException, msg:
outputWarning(
rd.sourceId, "Column %s.%s: Name is not a regular"
" ADQL identifier." % (td.id, col.name))
if td.onDisk and args.compareDB:
with base.getTableConn() as conn:
q = base.UnmanagedQuerier(conn)
if q.tableExists(td.getQName()):
t = api.TableForDef(td, connection=conn)
try:
t.ensureOnDiskMatches()
except api.DataError, msg:
outputError(
rd.sourceId,
utils.makeEllipsis(utils.safe_str(msg), 160))
def _do_dropRD(opts, rdId, selectedIds=()):
"""drops the data and services defined in the RD selected by rdId.
"""
try:
rd = api.getReferencedElement(rdId, forceType=api.RD)
except api.NotFoundError:
if opts.force:
rd = None
else:
raise
with base.AdhocQuerier(base.getWritableAdminConn) as querier:
if rd is not None:
if opts.dropAll:
dds = rd.dds
else:
dds = common.getPertainingDDs(rd, selectedIds)
parseOptions = api.getParseOptions(systemImport=opts.systemImport)
for dd in dds:
api.Data.drop(dd,
connection=querier.connection,
parseOptions=parseOptions)
if not selectedIds or opts.dropAll:
from gavo.registry import servicelist
servicelist.cleanServiceTablesFor(rd, querier.connection)
tap.unpublishFromTAP(rd, querier.connection)
try:
with querier.connection.savepoint():
querier.query("drop schema %s" % rd.schema)
except Exception, msg:
api.ui.notifyWarning(
"Cannot drop RD %s's schema %s because:"
" %s." % (rd.sourceId, rd.schema, utils.safe_str(msg)))
else:
for dd in dds:
gatherDependents(dd)
if parseOptions.buildDependencies:
parseOptions = parseOptions.change(buildDependencies=False)
try:
buildSequence = utils.topoSort(edges)
except ValueError, ex:
raise utils.logOldExc(
base.ReportableError(
"Could not sort"
" dependent DDs topologically (use --hints to learn more).",
hint="This is most likely because there's a cyclic dependency."
" Please check your dependency structure. The original message"
" is: %s" % utils.safe_str(ex)))
# note that the original DD is the first item in the build sequence,
# and we don't want to re-make it here
if parseOptions.metaOnly:
if len(buildSequence) > 1:
base.ui.notifyWarning(
"Only importing metadata, not rebulding"
" dependencies. Depending on your changes, it may be"
" necessary to manually re-make one of these: %s" %
", ".join(dd.id for dd in buildSequence[1:]))
else:
for dd in buildSequence[1:]:
base.ui.notifyInfo("Making dependent %s" % dd.id)
makeData(dd, parseOptions=parseOptions, connection=connection)
# to just call output(someString) to write to the user directly.
#
# To write messages, append strings to the messages list. An empty string
# would produce a paragraph. Append unicode or ASCII.
retval = 1
messages = []
try:
raise
except SystemExit, msg:
retval = msg.code
except KeyboardInterrupt:
retval = 2
except grammars.ParseError, msg:
if msg.location:
messages.append("Parse error at %s: %s" %
(msg.location, utils.safe_str(msg)))
else:
messages.append("Parse error: %s" % utils.safe_str(msg))
if msg.record:
messages.append("")
messages.append("Offending input was:\n")
messages.append(repr(msg.record) + "\n")
except base.SourceParseError, msg:
messages.append("While parsing source %s, near %s:\n" %
(msg.source, msg.location))
messages.append((msg.msg + "\n").decode("iso-8859-1", "ignore"))
if msg.offending:
messages.append("Offending literal: %s\n" % repr(msg.offending))
except base.BadCode, msg:
def reportCronFailure(self, message):
sendMailToAdmin(
"DaCHS %s job failed" % self.name, "\n".join([
"DaCHS job %s failed" % utils.safe_str(self), "\nDetails:\n",
message
]))
def writeErrorElement(self, outputFile, exception):
outputFile.write(
"""<INFO name="QUERY_STATUS" value="ERROR">%s</INFO>""" %
escapePCDATA("Error while serializing VOTable,"
" content is probably incomplete: %s" %
utils.safe_str(exception)))
def adaptForRenderer(self, renderer):
"""returns a core for a specific product.
The ugly thing about datalink in DaCHS' architecture is that its
interface (in terms of, e.g., inputKeys' values children) depends
on the arguments themselves, specifically the pubDID.
The workaround is to abuse the renderer-specific getCoreFor,
ignore the renderer and instead steal an "args" variable from
somewhere upstack. Nasty, but for now an acceptable solution.
It is particularly important to never let service cache the
cores returned for the dl* renderers; hence to "nocache" magic.
This tries to generate all datalink-relevant metadata in one go
and avoid calling the descriptorGenerator(s) more than once per
pubDID. It therefore adds datalinkLinks, datalinkEndpoints,
and datalinkDescriptors attributes. These are used later
in either metadata generation or data processing.
The latter will in general use only the last pubDID passed in.
Therefore, this last pubDID determines the service interface
for now. Perhaps we should be joining the inputKeys in some way,
though, e.g., if we want to allow retrieving multiple datasets
in a tar file? Or to re-use the same service for all pubdids?
"""
# if we're not speaking real datalink, return right away (this will
# be cached, so this must never happen for actual data)
if not renderer.name in self.datalinkAdaptingRenderers:
return self
try:
args = utils.stealVar("args")
if not isinstance(args, dict):
# again, we're not being called in a context with a pubdid
raise ValueError("No pubdid")
except ValueError:
# no arguments found: decide later on whether to fault out.
args = {"ID": []}
pubDIDs = self._getPubDIDs(args)
descGen = self.descriptorGenerator.compile(self)
descriptors = []
for pubDID in pubDIDs:
try:
descriptors.append(descGen(pubDID, args))
except Exception, ex:
# non-dlmeta exception should go right through to let people redirect
# (and also because messages might be better).
if renderer.name != "dlmeta":
raise
else:
if isinstance(ex, base.NotFoundError):
descriptors.append(
DatalinkFault.NotFoundFault(
pubDID, utils.safe_str(ex)))
else:
if base.DEBUG:
base.ui.notifyError(
"Error in datalink descriptor generator: %s" %
utils.safe_str(ex))
descriptors.append(
DatalinkFault.Fault(pubDID, utils.safe_str(ex)))
