def inputReceived(self, inPars, queryMeta):
"""returns True if all inputKeys can be filled from inPars.
As a side effect, inPars will receive defaults form the input keys
if there are any.
"""
if not self._isActive(inPars):
return False
keysFound, keysMissing = [], []
for f in self.inputKeys:
if inPars.get(f.name) is None:
keysMissing.append(f)
else:
if f.value!=inPars.get(f.name): # non-defaulted
keysFound.append(f)
if not keysMissing:
return True
# keys are missing. That's ok if none were found and we're not required
if not self.required and not keysFound:
return False
if self.required:
raise base.ValidationError("is mandatory but was not provided.",
colName=keysMissing[0].name)
# we're optional, but a value was given and others are missing
if not self.combining:
raise base.ValidationError("When you give a value for %s,"
" you must give value(s) for %s, too"%(keysFound[0].getLabel(),
", ".join(k.name for k in keysMissing)),
colName=keysMissing[0].name)
return True
def validateValue(self, value):
"""raises a ValidationError if value does not match the constraints
given here.
"""
if value is None:
if self.required:
raise base.ValidationError(
"Field %s is empty but non-optional" % self.name,
self.name)
return
# Only validate these if we're not a database column
if not isinstance(self, Column):
vals = self.values
if vals:
if vals.options:
if value and not vals.validateOptions(value):
raise base.ValidationError(
"Value %s not consistent with"
" legal values %s" % (value, vals.options),
self.name)
else:
if vals.min and value < vals.min:
raise base.ValidationError(
"%s too small (must be at least %s)" %
(value, vals.min), self.name)
if vals.max and value > vals.max:
raise base.ValidationError(
"%s too large (must be less than %s)" %
(value, vals.max), self.name)
def _writeFile(self, srcFile, fName):
"""writes the contents of srcFile to fName in destDD's staging dir.
"""
try:
targetDir = os.path.join(self.rd.resdir,
self.destDD.getProperty("stagingDir"))
except KeyError:
raise base.ui.logOldExc(
base.ValidationError(
"Uploading is only"
" supported for data having a staging directory.", "File"))
if not os.path.exists(targetDir):
raise base.ValidationError("Staging directory does not exist.",
"File")
targetFName = fName.split("/")[-1].encode("iso-8859-1")
if not targetFName:
raise base.ValidationError("Bad file name", "File")
targetPath = os.path.join(targetDir, targetFName)
with open(targetPath, "w") as f:
f.write(srcFile.read())
try:
self._fixPermissions(targetPath)
except os.error:
# Nothing we can do, and it may not even hurt
pass
return targetPath
def getQuery(queriedTable, parameters, sqlPars, prefix="sia"):
"""returns an SQL fragment for a SIAP query for bboxes.
The SQL is returned as a WHERE-fragment in a string. The parameters
are added in the sqlPars dictionary.
parameters is a dictionary that maps the SIAP keywords to the
values in the query. Parameters not defined by SIAP are ignored.
"""
posStr = urllib.unquote(parameters["POS"])
try:
ra, dec = dissectPositions(posStr)
except (ValueError, TypeError):
raise base.ui.logOldExc(
base.ValidationError("%s is not a RA,DEC pair." % posStr, "POS",
posStr))
try:
sizes = map(float, parameters["SIZE"].split(","))
except ValueError:
raise base.ui.logOldExc(
base.ValidationError(
"Size specification"
" has to be <degs> or <degs>,<degs>", "SIZE",
parameters["SIZE"]))
if len(sizes) == 1:
sizes = sizes * 2
intersect = parameters.get("INTERSECT", "OVERLAPS")
query = _getQueryMaker(queriedTable)(intersect, ra, dec, sizes, prefix,
sqlPars)
# the following are for the benefit of cutout queries.
sqlPars["_ra"], sqlPars["_dec"] = ra, dec
sqlPars["_sra"], sqlPars["_sdec"] = sizes
return query
def getConeSQL(self, colName, sqlPars, coneSize):
if self.qualifier and self.qualifier != 'ICRS':
# XXX TODO: implement at least a couple of common frames
raise base.ValidationError(
"Cannot match against coordinates"
" given in %s frame" % self.qualifier, self.destName)
sizeName = base.getSQLKey("size", coneSize * DEG, sqlPars)
parts = []
if len(self.ranges) % 2:
raise base.ValidationError(
"PQL position values must be lists of"
" length divisible by 2.", self.destName)
lastCoo = None
for r in self.ranges:
if r.value is None:
raise base.ValidationError(
"Ranges are not allowed as cone centers", self.destName)
if lastCoo is None:
lastCoo = r.value
else:
parts.append(
"%s <-> %%(%s)s < %%(%s)s" %
(colName,
base.getSQLKey(
"pos", pgsphere.SPoint.fromDegrees(lastCoo, r.value),
sqlPars), sizeName))
lastCoo = None
return "(%s)" % " OR ".join(parts)
def makeName(self, field):
self.index += 1
res = getattr(field, "name", None)
if res is None:
raise base.ValidationError("Field without name in upload.",
"UPLOAD")
if res in self.seenNames:
raise base.ValidationError("Duplicate column name illegal in"
" uploaded tables (%s)"%res, "UPLOAD")
self.seenNames.add(res)
return utils.QuotedName(res)
def _realSubmitAction(self, ctx, form, data):
"""helps submitAction by doing the real work.
It is here so we can add an error handler in submitAction.
"""
# TODO: There's significant overlap here with
# grend.runServiceWithFormalData; refactor?
queryMeta = svcs.QueryMeta.fromContext(ctx)
queryMeta["formal_data"] = data
if (self.service.core.outputTable.columns
and not self.service.getCurOutputFields(queryMeta)):
raise base.ValidationError(
"These output settings yield no"
" output fields", "_OUTPUT")
if queryMeta["format"] in ("HTML", ""):
resultWriter = self
else:
resultWriter = serviceresults.getFormat(queryMeta["format"])
if resultWriter.compute:
d = self.runService(svcs.PreparsedInput(data), queryMeta)
else:
d = defer.succeed(None)
return d.addCallback(resultWriter._formatOutput, ctx)
def iterUploads(request):
"""iterates over DALI uploads in request.
This yields pairs of (file name, file object), where file name
is the file name requested (sanitized to have no slashes and non-ASCII).
The UPLOAD and inline-file keys are removed from request's args
member. file object is a cgi-style thing with file, filename,
etc. attributes.
"""
# UWS auto-downcases things (it probably shouldn't)
uploads = request.args.pop("UPLOAD", [])+request.args.pop("upload", [])
if not uploads:
return
for uploadString in uploads:
destName, uploadSource = parseUploadString(uploadString)
# mangle the future file name such that we hope it's representable
# in the file system
destName = str(destName).replace("/", "_")
try:
if uploadSource.startswith("param:"):
fileKey = uploadSource[6:]
upload = request.fields[fileKey]
# remove upload in string form from args to remove clutter
request.args.pop(fileKey, None)
else:
upload = URLUpload(uploadSource, destName)
yield destName, upload
except (KeyError, AttributeError):
raise base.ui.logOldExc(base.ValidationError(
"%s references a non-existing"
" file upload."%uploadSource, "UPLOAD",
hint="If you pass UPLOAD=foo,param:x,"
" you must pass a file upload under the key x."))
def _completeRow(self, rawRow):
caseNormalized = dict((k.lower(), v) for k, v in rawRow.iteritems())
procRow = {}
if self.grammar.rejectExtras:
extraNames = set(caseNormalized) - set(
p.name.lower() for p in self.grammar.inputTable.params)
if extraNames:
raise base.ValidationError(
"The following parameter(s) are"
" not accepted by this service: %s" %
",".join(sorted(extraNames)), "(various)")
for ik in self.grammar.inputTable.params:
if ik.name in rawRow:
val = rawRow[ik.name]
else:
val = self.grammar.defaults.get(ik.name, None)
# TODO: we probably should avoid having tuples here in the first place
if isinstance(val, tuple):
val = list(val)
if val is not None and not isinstance(val, list):
val = [val]
procRow[ik.name] = val
return procRow
def makeName(self, field):
name = getattr(field, "name", None)
if name is None:
raise base.ValidationError("Field without name in upload.",
"UPLOAD")
if valuemappers.needsQuoting(name):
if name in self.seenNames:
raise base.ValidationError("Duplicate column name illegal in"
" uploaded tables (%s)"%name, "UPLOAD")
self.seenNames.add(name)
return utils.QuotedName(name)
else:
if name.lower() in self.seenNames:
raise base.ValidationError("Duplicate column name illegal in"
" uploaded tables (%s)"%name, "UPLOAD")
self.seenNames.add(name.lower())
return name
def requireValue(val, fieldName):
"""returns val unless it is None, in which case a ValidationError
for fieldName will be raised.
"""
if val is None:
raise base.ValidationError("Value is required but was not provided",
fieldName)
return val
def validateParams(self): """raises a ValidationError if any required parameters of this tables are None. """ for par in self.iterParams(): if par.required and par.value is None: raise base.ValidationError( "Value is required but was not provided", par.name)
def normalizeTAPFormat(rawFmt):
format = rawFmt.lower()
try:
return tap.FORMAT_CODES[format][0]
except KeyError:
raise base.ValidationError(
"Unsupported format '%s'."%format, colName="FORMAT",
hint="Legal format codes include %s"%(", ".join(tap.FORMAT_CODES)))
def _getArgs(self, inputTable):
args = [base.getBinaryName(self.computer)]
for par in inputTable.iterParams():
if par.content_ is base.NotGiven:
raise base.ValidationError("Command line argument %s must not"
" be undefined"%par.name, par.name, base.NotGiven)
args.append(par.content_)
return args
def parseUploadString(uploadString):
"""returns resourceName, uploadSource from a DALI upload string.
"""
try:
destName, uploadSource = uploadString.split(",", 1)
except (TypeError, ValueError):
raise base.ValidationError("Invalid UPLOAD string",
"UPLOAD", hint="UPLOADs look like my_upload,http://foo.bar/up"
" or inline_upload,param:foo.")
return destName, uploadSource
def mapADQLErrors(excType, excValue, excTb):
if (isinstance(excValue, adql.ParseException)
or isinstance(excValue, adql.ParseSyntaxException)):
raise base.ui.logOldExc(
base.ValidationError(
"Could not parse your query: %s" % unicode(excValue), "query"))
elif isinstance(excValue, adql.ColumnNotFound):
raise base.ui.logOldExc(
base.ValidationError("No such field known: %s" % unicode(excValue),
"query"))
elif isinstance(excValue, adql.AmbiguousColumn):
raise base.ui.logOldExc(
base.ValidationError(
"%s needs to be qualified." % unicode(excValue), "query"))
elif isinstance(excValue, adql.Error):
raise base.ui.logOldExc(
base.ValidationError(unicode(excValue), "query"))
else:
svcs.mapDBErrors(excType, excValue, excTb)
def run(self, service, inputTable, queryMeta):
defaultRequest = service.getProperty("defaultRequest", "")
requestType = (inputTable.getParam("REQUEST")
or defaultRequest).upper()
if requestType == "QUERYDATA":
return self._run_queryData(service, inputTable, queryMeta)
elif requestType == "GETTARGETNAMES":
return self._run_getTargetNames(service, inputTable, queryMeta)
else:
raise base.ValidationError("Missing or invalid value for REQUEST.",
"REQUEST")
def _parseInputDict(self, inputDict):
res = {}
for key, val in inputDict.iteritems():
if val is not None and key in self._buildKeys:
try:
res[key] = self._buildKeys[key](val)
except (ValueError, TypeError):
raise base.ValidationError(
"Invalid value for constructor argument to %s:"
" %s=%r" % (self.__class__.__name__, key, val),
"accref")
return res
def _saveUpload(cls, job, uploadName):
try:
uploadData = codetricks.stealVar("request").files[uploadName]
except KeyError:
raise base.ui.logOldExc(
base.ValidationError("No upload '%s' found" % uploadName,
"UPLOAD"))
destFName = cls._cleanName(uploadData.filename)
with job.openFile(destFName, "w") as f:
f.write(uploadData.file.read())
return LocalFile(job.jobId, job.getWD(), destFName)
def parseSIAP2Geometry(aString, fieldName="POS"):
"""parses a SIAPv2 geometry spec to a pgsphere object.
Parse errors raise validation errors for fieldName.
"""
mat = re.match("(CIRCLE|RANGE|POLYGON) (.*)", aString)
if not mat:
raise base.ValidationError(
"Invalid SIAPv2 geometry: '%s'"
" (expected a SIAPv2 shape name)" %
utils.makeEllipsis(aString, 20), fieldName)
geoName = mat.group(1)
try:
args = [float(s) for s in mat.group(2).split()]
except ValueError:
raise base.ValidationError(
"Invalid SIAPv2 coordinates: '%s'"
" (bad floating point literal '%s')" %
(utils.makeEllipsis(mat.group(2), 20), s), fieldName)
if geoName == "CIRCLE":
if len(args) != 3:
raise base.ValidationError(
"Invalid SIAPv2 CIRCLE: '%s'"
" (need exactly three numbers)" %
(utils.makeEllipsis(aString, 20)), fieldName)
return pgsphere.SCircle(pgsphere.SPoint.fromDegrees(args[0], args[1]),
args[2] * utils.DEG)
elif geoName == "RANGE":
# SBox isn't really RANGE, but RANGE shouldn't have been
# part of the standard and people that use it deserve
# to get bad results.
if len(args) != 4:
raise base.ValidationError(
"Invalid SIAPv2 RANGE: '%s'"
" (need exactly four numbers)" %
(utils.makeEllipsis(aString, 20)), fieldName)
if args[0] > args[1] or args[2] > args[3]:
raise base.ValidationError(
"Invalid SIAPv2 RANGE: '%s'"
" (lower limits must be smaller than upper limits)" %
(utils.makeEllipsis(aString, 20)), fieldName)
return pgsphere.SBox(pgsphere.SPoint.fromDegrees(args[0], args[2]),
pgsphere.SPoint.fromDegrees(args[1], args[3]))
elif geoName == "POLYGON":
if len(args) < 6 or len(args) % 2:
raise base.ValidationError(
"Invalid SIAPv2 POLYGON: '%s'"
" (need more than three coordinate *pairs*)" %
(utils.makeEllipsis(mat.group(2), 20)), fieldName)
return pgsphere.SPoly([
pgsphere.SPoint.fromDegrees(*pair)
for pair in utils.iterConsecutivePairs(args)
])
else:
assert False
def parseUploadString(uploadString):
"""iterates over pairs of tableName, uploadSource from a TAP upload string.
"""
try:
res = utils.pyparseString(getUploadGrammar(), uploadString).asList()
return res
except ParseException, ex:
raise base.ValidationError(
"Syntax error in UPLOAD parameter (near %s)" % (ex.loc),
"UPLOAD",
hint=
"Note that we only allow regular SQL identifiers as table names,"
" i.e., basically only alphanumerics are allowed.")
def addRow(self, row):
"""adds a row to the table.
Use this only to add one or two rows, otherwise go for getFeeder.
"""
try:
self.query(self.addCommand, row)
except sqlsupport.IntegrityError:
raise base.ui.logOldExc(
base.ValidationError(
"Row %s cannot be added since it clashes"
" with an existing record on the primary key" % row,
row=row,
colName="unknown"))
def mapDBErrors(excType, excValue, excTb):
"""translates exception into something we can display properly.
"""
# This is a helper to all DB-based cores -- it probably should go
# into TableBasedCore.
if getattr(excValue, "cursor", None) is not None:
base.ui.notifyWarning("Failed DB query: %s"%excValue.cursor.query)
if isinstance(excValue, sqlsupport.QueryCanceledError):
message = "Query timed out (took too long).\n"
if base.getConfig("maintainerAddress"):
message = message+("Unless you know why the query took that"
" long, please contact %s.\n"%base.getConfig("maintainerAddress"))
message += ("Meanwhile, if this failure happened with a cross match,"
" please try exchanging the large and the small catalog in POINT"
" and CIRCLE.\n")
raise base.ui.logOldExc(base.ValidationError(message, "query"))
elif isinstance(excValue, base.NotFoundError):
raise base.ui.logOldExc(base.ValidationError("Could not locate %s '%s'"%(
excValue.what, excValue.lookedFor), "query"))
elif isinstance(excValue, base.DBError):
raise base.ui.logOldExc(base.ValidationError(unicode(excValue), "query"))
else:
raise
def run(self, service, inputTable, queryMeta):
"""does the DB query and returns an InMemoryTable containing
the result.
"""
resultTableDef = self._makeResultTableDef(
service, inputTable, queryMeta)
resultTableDef.copyMetaFrom(self.queriedTable)
if not resultTableDef.columns:
raise base.ValidationError("No output columns with these settings."
"_OUTPUT")
sortKeys = None
if self.sortKey:
sortKeys = self.sortKey.split(",")
queryMeta.overrideDbOptions(limit=self.limit, sortKeys=sortKeys,
sortFallback=self.getProperty("defaultSortKey", None))
try:
fragment, pars = self._getSQLWhere(inputTable, queryMeta)
except base.LiteralParseError, ex:
raise base.ui.logOldExc(base.ValidationError(str(ex),
colName=ex.attName))
def _ingestUploads(uploads, connection):
tds = []
for destName, src in uploads:
if isinstance(src, tap.LocalFile):
srcF = open(src.fullPath)
else:
try:
srcF = utils.urlopenRemote(src)
except IOError, ex:
raise base.ui.logOldExc(
base.ValidationError("Upload '%s' cannot be retrieved"%(
src), "UPLOAD", hint="The I/O operation failed with the message: "+
str(ex)))
if valuemappers.needsQuoting(destName):
raise base.ValidationError("'%s' is not a valid table name on"
" this site"%destName, "UPLOAD", hint="It either contains"
" non-alphanumeric characters or conflicts with an ADQL"
" reserved word. Quoted table names are not supported"
" at this site.")
uploadedTable = votableread.uploadVOTable(destName, srcF, connection,
nameMaker=votableread.AutoQuotedNameMaker())
if uploadedTable is not None:
tds.append(uploadedTable.tableDef)
srcF.close()
def changingAxis(self, axisIndex, parName):
"""must be called before cutting out along axisIndex.
axIndex is a FITS (1-based) axis index axIndex, parName the name of the
parameter that causes the cutout.
This will simply return if nobody has called changingAxis with that index
before and raise a ValidationError otherwise. Data functions doing a cutout
must call this before doing so; if they don't the cutout will probably be
wrong when two conflicting constraints are given.
"""
if axisIndex in self._axesTouched:
raise base.ValidationError(
"Attempt to cut out along axis %d that"
" has been modified before." % axisIndex, parName)
self._axesTouched.add(axisIndex)
def query(self, ident): try: return self.cache.getItem(ident) except KeyError: try: f = urllib.urlopen(self.SVC_URL+urllib.quote(ident)) response = f.read() f.close() newOb = self._parseXML(response) self.cache.addItem(ident, newOb, save=self.saveNew) return newOb except socket.error: # Simbad is offline raise base.ui.logOldExc(base.ValidationError( "Simbad is offline, cannot query.", "hscs_pos", # really, this should be added by the widget hint="If this problem persists, complain to us rather than simbad."))
def parseHumanSpoint(cooSpec, colName=None):
"""tries to interpret cooSpec as some sort of cone center.
Attempted interpretations include various forms of coordinate pairs
and simbad objects; hence, this will in general cause network traffic.
If no sense can be made, a ValidationError on colName is raised.
"""
try:
cooPair = base.parseCooPair(cooSpec)
except ValueError:
simbadData = base.caches.getSesame("web").query(cooSpec)
if not simbadData:
raise base.ValidationError(
"%s is neither a RA,DEC"
" pair nor a simbad resolvable object." % cooSpec, colName)
cooPair = simbadData["RA"], simbadData["dec"]
return cooPair
def _ensureRowIdentity(self, row, key):
"""raises an exception if row is not equivalent to the row stored
for key.
This is one strategy for resolving primary key conflicts.
"""
storedRow = self.rowIndex[key]
if row.keys()!=storedRow.keys():
raise Error("Differing rows for primary key %s: %s vs. %s"%(
key, self.rowIndex[key], row))
for colName in row:
if row[colName] is None or storedRow[colName] is None:
continue
if row[colName]!=storedRow[colName]:
raise base.ValidationError(
"Differing rows for primary key %s;"
" %s vs. %s"%(key, row[colName],
storedRow[colName]), colName=colName, row=row)
def _runAndCapture(self, inputTable):
# if we wanted to get really fancy, it shouldn't be hard to pipe that stuff
# directly into the grammar.
pipe = subprocess.Popen(self._getArgs(inputTable), 2**16,
stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True,
cwd=os.path.dirname(self.computer))
writeThread = self._feedInto(self._getInput(inputTable), pipe.stdin)
data = pipe.stdout.read()
pipe.stdout.close()
writeThread.join(0.1)
retcode = pipe.wait()
if retcode!=0:
raise base.ValidationError("The subprocess %s returned %s. This"
" indicates an external executable could not be run or failed"
" with your parameters. You should probably report this to the"
" operators."%(os.path.basename(self.computer), retcode),
"query")
return data
