def inputReceived(self, inPars, queryMeta): """returns True if all inputKeys can be filled from inPars. As a side effect, inPars will receive defaults form the input keys if there are any. """ if not self._isActive(inPars): return False keysFound, keysMissing = [], [] for f in self.inputKeys: if inPars.get(f.name) is None: keysMissing.append(f) else: if f.value!=inPars.get(f.name): # non-defaulted keysFound.append(f) if not keysMissing: return True # keys are missing. That's ok if none were found and we're not required if not self.required and not keysFound: return False if self.required: raise base.ValidationError("is mandatory but was not provided.", colName=keysMissing[0].name) # we're optional, but a value was given and others are missing if not self.combining: raise base.ValidationError("When you give a value for %s," " you must give value(s) for %s, too"%(keysFound[0].getLabel(), ", ".join(k.name for k in keysMissing)), colName=keysMissing[0].name) return True
def validateValue(self, value): """raises a ValidationError if value does not match the constraints given here. """ if value is None: if self.required: raise base.ValidationError( "Field %s is empty but non-optional" % self.name, self.name) return # Only validate these if we're not a database column if not isinstance(self, Column): vals = self.values if vals: if vals.options: if value and not vals.validateOptions(value): raise base.ValidationError( "Value %s not consistent with" " legal values %s" % (value, vals.options), self.name) else: if vals.min and value < vals.min: raise base.ValidationError( "%s too small (must be at least %s)" % (value, vals.min), self.name) if vals.max and value > vals.max: raise base.ValidationError( "%s too large (must be less than %s)" % (value, vals.max), self.name)
def _writeFile(self, srcFile, fName): """writes the contents of srcFile to fName in destDD's staging dir. """ try: targetDir = os.path.join(self.rd.resdir, self.destDD.getProperty("stagingDir")) except KeyError: raise base.ui.logOldExc( base.ValidationError( "Uploading is only" " supported for data having a staging directory.", "File")) if not os.path.exists(targetDir): raise base.ValidationError("Staging directory does not exist.", "File") targetFName = fName.split("/")[-1].encode("iso-8859-1") if not targetFName: raise base.ValidationError("Bad file name", "File") targetPath = os.path.join(targetDir, targetFName) with open(targetPath, "w") as f: f.write(srcFile.read()) try: self._fixPermissions(targetPath) except os.error: # Nothing we can do, and it may not even hurt pass return targetPath
def getQuery(queriedTable, parameters, sqlPars, prefix="sia"): """returns an SQL fragment for a SIAP query for bboxes. The SQL is returned as a WHERE-fragment in a string. The parameters are added in the sqlPars dictionary. parameters is a dictionary that maps the SIAP keywords to the values in the query. Parameters not defined by SIAP are ignored. """ posStr = urllib.unquote(parameters["POS"]) try: ra, dec = dissectPositions(posStr) except (ValueError, TypeError): raise base.ui.logOldExc( base.ValidationError("%s is not a RA,DEC pair." % posStr, "POS", posStr)) try: sizes = map(float, parameters["SIZE"].split(",")) except ValueError: raise base.ui.logOldExc( base.ValidationError( "Size specification" " has to be <degs> or <degs>,<degs>", "SIZE", parameters["SIZE"])) if len(sizes) == 1: sizes = sizes * 2 intersect = parameters.get("INTERSECT", "OVERLAPS") query = _getQueryMaker(queriedTable)(intersect, ra, dec, sizes, prefix, sqlPars) # the following are for the benefit of cutout queries. sqlPars["_ra"], sqlPars["_dec"] = ra, dec sqlPars["_sra"], sqlPars["_sdec"] = sizes return query
def getConeSQL(self, colName, sqlPars, coneSize): if self.qualifier and self.qualifier != 'ICRS': # XXX TODO: implement at least a couple of common frames raise base.ValidationError( "Cannot match against coordinates" " given in %s frame" % self.qualifier, self.destName) sizeName = base.getSQLKey("size", coneSize * DEG, sqlPars) parts = [] if len(self.ranges) % 2: raise base.ValidationError( "PQL position values must be lists of" " length divisible by 2.", self.destName) lastCoo = None for r in self.ranges: if r.value is None: raise base.ValidationError( "Ranges are not allowed as cone centers", self.destName) if lastCoo is None: lastCoo = r.value else: parts.append( "%s <-> %%(%s)s < %%(%s)s" % (colName, base.getSQLKey( "pos", pgsphere.SPoint.fromDegrees(lastCoo, r.value), sqlPars), sizeName)) lastCoo = None return "(%s)" % " OR ".join(parts)
def makeName(self, field): self.index += 1 res = getattr(field, "name", None) if res is None: raise base.ValidationError("Field without name in upload.", "UPLOAD") if res in self.seenNames: raise base.ValidationError("Duplicate column name illegal in" " uploaded tables (%s)"%res, "UPLOAD") self.seenNames.add(res) return utils.QuotedName(res)
def _realSubmitAction(self, ctx, form, data): """helps submitAction by doing the real work. It is here so we can add an error handler in submitAction. """ # TODO: There's significant overlap here with # grend.runServiceWithFormalData; refactor? queryMeta = svcs.QueryMeta.fromContext(ctx) queryMeta["formal_data"] = data if (self.service.core.outputTable.columns and not self.service.getCurOutputFields(queryMeta)): raise base.ValidationError( "These output settings yield no" " output fields", "_OUTPUT") if queryMeta["format"] in ("HTML", ""): resultWriter = self else: resultWriter = serviceresults.getFormat(queryMeta["format"]) if resultWriter.compute: d = self.runService(svcs.PreparsedInput(data), queryMeta) else: d = defer.succeed(None) return d.addCallback(resultWriter._formatOutput, ctx)
def iterUploads(request): """iterates over DALI uploads in request. This yields pairs of (file name, file object), where file name is the file name requested (sanitized to have no slashes and non-ASCII). The UPLOAD and inline-file keys are removed from request's args member. file object is a cgi-style thing with file, filename, etc. attributes. """ # UWS auto-downcases things (it probably shouldn't) uploads = request.args.pop("UPLOAD", [])+request.args.pop("upload", []) if not uploads: return for uploadString in uploads: destName, uploadSource = parseUploadString(uploadString) # mangle the future file name such that we hope it's representable # in the file system destName = str(destName).replace("/", "_") try: if uploadSource.startswith("param:"): fileKey = uploadSource[6:] upload = request.fields[fileKey] # remove upload in string form from args to remove clutter request.args.pop(fileKey, None) else: upload = URLUpload(uploadSource, destName) yield destName, upload except (KeyError, AttributeError): raise base.ui.logOldExc(base.ValidationError( "%s references a non-existing" " file upload."%uploadSource, "UPLOAD", hint="If you pass UPLOAD=foo,param:x," " you must pass a file upload under the key x."))
def _completeRow(self, rawRow): caseNormalized = dict((k.lower(), v) for k, v in rawRow.iteritems()) procRow = {} if self.grammar.rejectExtras: extraNames = set(caseNormalized) - set( p.name.lower() for p in self.grammar.inputTable.params) if extraNames: raise base.ValidationError( "The following parameter(s) are" " not accepted by this service: %s" % ",".join(sorted(extraNames)), "(various)") for ik in self.grammar.inputTable.params: if ik.name in rawRow: val = rawRow[ik.name] else: val = self.grammar.defaults.get(ik.name, None) # TODO: we probably should avoid having tuples here in the first place if isinstance(val, tuple): val = list(val) if val is not None and not isinstance(val, list): val = [val] procRow[ik.name] = val return procRow
def makeName(self, field): name = getattr(field, "name", None) if name is None: raise base.ValidationError("Field without name in upload.", "UPLOAD") if valuemappers.needsQuoting(name): if name in self.seenNames: raise base.ValidationError("Duplicate column name illegal in" " uploaded tables (%s)"%name, "UPLOAD") self.seenNames.add(name) return utils.QuotedName(name) else: if name.lower() in self.seenNames: raise base.ValidationError("Duplicate column name illegal in" " uploaded tables (%s)"%name, "UPLOAD") self.seenNames.add(name.lower()) return name
def requireValue(val, fieldName): """returns val unless it is None, in which case a ValidationError for fieldName will be raised. """ if val is None: raise base.ValidationError("Value is required but was not provided", fieldName) return val
def validateParams(self): """raises a ValidationError if any required parameters of this tables are None. """ for par in self.iterParams(): if par.required and par.value is None: raise base.ValidationError( "Value is required but was not provided", par.name)
def normalizeTAPFormat(rawFmt): format = rawFmt.lower() try: return tap.FORMAT_CODES[format][0] except KeyError: raise base.ValidationError( "Unsupported format '%s'."%format, colName="FORMAT", hint="Legal format codes include %s"%(", ".join(tap.FORMAT_CODES)))
def _getArgs(self, inputTable): args = [base.getBinaryName(self.computer)] for par in inputTable.iterParams(): if par.content_ is base.NotGiven: raise base.ValidationError("Command line argument %s must not" " be undefined"%par.name, par.name, base.NotGiven) args.append(par.content_) return args
def parseUploadString(uploadString): """returns resourceName, uploadSource from a DALI upload string. """ try: destName, uploadSource = uploadString.split(",", 1) except (TypeError, ValueError): raise base.ValidationError("Invalid UPLOAD string", "UPLOAD", hint="UPLOADs look like my_upload,http://foo.bar/up" " or inline_upload,param:foo.") return destName, uploadSource
def mapADQLErrors(excType, excValue, excTb): if (isinstance(excValue, adql.ParseException) or isinstance(excValue, adql.ParseSyntaxException)): raise base.ui.logOldExc( base.ValidationError( "Could not parse your query: %s" % unicode(excValue), "query")) elif isinstance(excValue, adql.ColumnNotFound): raise base.ui.logOldExc( base.ValidationError("No such field known: %s" % unicode(excValue), "query")) elif isinstance(excValue, adql.AmbiguousColumn): raise base.ui.logOldExc( base.ValidationError( "%s needs to be qualified." % unicode(excValue), "query")) elif isinstance(excValue, adql.Error): raise base.ui.logOldExc( base.ValidationError(unicode(excValue), "query")) else: svcs.mapDBErrors(excType, excValue, excTb)
def run(self, service, inputTable, queryMeta): defaultRequest = service.getProperty("defaultRequest", "") requestType = (inputTable.getParam("REQUEST") or defaultRequest).upper() if requestType == "QUERYDATA": return self._run_queryData(service, inputTable, queryMeta) elif requestType == "GETTARGETNAMES": return self._run_getTargetNames(service, inputTable, queryMeta) else: raise base.ValidationError("Missing or invalid value for REQUEST.", "REQUEST")
def _parseInputDict(self, inputDict): res = {} for key, val in inputDict.iteritems(): if val is not None and key in self._buildKeys: try: res[key] = self._buildKeys[key](val) except (ValueError, TypeError): raise base.ValidationError( "Invalid value for constructor argument to %s:" " %s=%r" % (self.__class__.__name__, key, val), "accref") return res
def _saveUpload(cls, job, uploadName): try: uploadData = codetricks.stealVar("request").files[uploadName] except KeyError: raise base.ui.logOldExc( base.ValidationError("No upload '%s' found" % uploadName, "UPLOAD")) destFName = cls._cleanName(uploadData.filename) with job.openFile(destFName, "w") as f: f.write(uploadData.file.read()) return LocalFile(job.jobId, job.getWD(), destFName)
def parseSIAP2Geometry(aString, fieldName="POS"): """parses a SIAPv2 geometry spec to a pgsphere object. Parse errors raise validation errors for fieldName. """ mat = re.match("(CIRCLE|RANGE|POLYGON) (.*)", aString) if not mat: raise base.ValidationError( "Invalid SIAPv2 geometry: '%s'" " (expected a SIAPv2 shape name)" % utils.makeEllipsis(aString, 20), fieldName) geoName = mat.group(1) try: args = [float(s) for s in mat.group(2).split()] except ValueError: raise base.ValidationError( "Invalid SIAPv2 coordinates: '%s'" " (bad floating point literal '%s')" % (utils.makeEllipsis(mat.group(2), 20), s), fieldName) if geoName == "CIRCLE": if len(args) != 3: raise base.ValidationError( "Invalid SIAPv2 CIRCLE: '%s'" " (need exactly three numbers)" % (utils.makeEllipsis(aString, 20)), fieldName) return pgsphere.SCircle(pgsphere.SPoint.fromDegrees(args[0], args[1]), args[2] * utils.DEG) elif geoName == "RANGE": # SBox isn't really RANGE, but RANGE shouldn't have been # part of the standard and people that use it deserve # to get bad results. if len(args) != 4: raise base.ValidationError( "Invalid SIAPv2 RANGE: '%s'" " (need exactly four numbers)" % (utils.makeEllipsis(aString, 20)), fieldName) if args[0] > args[1] or args[2] > args[3]: raise base.ValidationError( "Invalid SIAPv2 RANGE: '%s'" " (lower limits must be smaller than upper limits)" % (utils.makeEllipsis(aString, 20)), fieldName) return pgsphere.SBox(pgsphere.SPoint.fromDegrees(args[0], args[2]), pgsphere.SPoint.fromDegrees(args[1], args[3])) elif geoName == "POLYGON": if len(args) < 6 or len(args) % 2: raise base.ValidationError( "Invalid SIAPv2 POLYGON: '%s'" " (need more than three coordinate *pairs*)" % (utils.makeEllipsis(mat.group(2), 20)), fieldName) return pgsphere.SPoly([ pgsphere.SPoint.fromDegrees(*pair) for pair in utils.iterConsecutivePairs(args) ]) else: assert False
def parseUploadString(uploadString): """iterates over pairs of tableName, uploadSource from a TAP upload string. """ try: res = utils.pyparseString(getUploadGrammar(), uploadString).asList() return res except ParseException, ex: raise base.ValidationError( "Syntax error in UPLOAD parameter (near %s)" % (ex.loc), "UPLOAD", hint= "Note that we only allow regular SQL identifiers as table names," " i.e., basically only alphanumerics are allowed.")
def addRow(self, row): """adds a row to the table. Use this only to add one or two rows, otherwise go for getFeeder. """ try: self.query(self.addCommand, row) except sqlsupport.IntegrityError: raise base.ui.logOldExc( base.ValidationError( "Row %s cannot be added since it clashes" " with an existing record on the primary key" % row, row=row, colName="unknown"))
def mapDBErrors(excType, excValue, excTb): """translates exception into something we can display properly. """ # This is a helper to all DB-based cores -- it probably should go # into TableBasedCore. if getattr(excValue, "cursor", None) is not None: base.ui.notifyWarning("Failed DB query: %s"%excValue.cursor.query) if isinstance(excValue, sqlsupport.QueryCanceledError): message = "Query timed out (took too long).\n" if base.getConfig("maintainerAddress"): message = message+("Unless you know why the query took that" " long, please contact %s.\n"%base.getConfig("maintainerAddress")) message += ("Meanwhile, if this failure happened with a cross match," " please try exchanging the large and the small catalog in POINT" " and CIRCLE.\n") raise base.ui.logOldExc(base.ValidationError(message, "query")) elif isinstance(excValue, base.NotFoundError): raise base.ui.logOldExc(base.ValidationError("Could not locate %s '%s'"%( excValue.what, excValue.lookedFor), "query")) elif isinstance(excValue, base.DBError): raise base.ui.logOldExc(base.ValidationError(unicode(excValue), "query")) else: raise
def run(self, service, inputTable, queryMeta): """does the DB query and returns an InMemoryTable containing the result. """ resultTableDef = self._makeResultTableDef( service, inputTable, queryMeta) resultTableDef.copyMetaFrom(self.queriedTable) if not resultTableDef.columns: raise base.ValidationError("No output columns with these settings." "_OUTPUT") sortKeys = None if self.sortKey: sortKeys = self.sortKey.split(",") queryMeta.overrideDbOptions(limit=self.limit, sortKeys=sortKeys, sortFallback=self.getProperty("defaultSortKey", None)) try: fragment, pars = self._getSQLWhere(inputTable, queryMeta) except base.LiteralParseError, ex: raise base.ui.logOldExc(base.ValidationError(str(ex), colName=ex.attName))
def _ingestUploads(uploads, connection): tds = [] for destName, src in uploads: if isinstance(src, tap.LocalFile): srcF = open(src.fullPath) else: try: srcF = utils.urlopenRemote(src) except IOError, ex: raise base.ui.logOldExc( base.ValidationError("Upload '%s' cannot be retrieved"%( src), "UPLOAD", hint="The I/O operation failed with the message: "+ str(ex))) if valuemappers.needsQuoting(destName): raise base.ValidationError("'%s' is not a valid table name on" " this site"%destName, "UPLOAD", hint="It either contains" " non-alphanumeric characters or conflicts with an ADQL" " reserved word. Quoted table names are not supported" " at this site.") uploadedTable = votableread.uploadVOTable(destName, srcF, connection, nameMaker=votableread.AutoQuotedNameMaker()) if uploadedTable is not None: tds.append(uploadedTable.tableDef) srcF.close()
def changingAxis(self, axisIndex, parName): """must be called before cutting out along axisIndex. axIndex is a FITS (1-based) axis index axIndex, parName the name of the parameter that causes the cutout. This will simply return if nobody has called changingAxis with that index before and raise a ValidationError otherwise. Data functions doing a cutout must call this before doing so; if they don't the cutout will probably be wrong when two conflicting constraints are given. """ if axisIndex in self._axesTouched: raise base.ValidationError( "Attempt to cut out along axis %d that" " has been modified before." % axisIndex, parName) self._axesTouched.add(axisIndex)
def query(self, ident): try: return self.cache.getItem(ident) except KeyError: try: f = urllib.urlopen(self.SVC_URL+urllib.quote(ident)) response = f.read() f.close() newOb = self._parseXML(response) self.cache.addItem(ident, newOb, save=self.saveNew) return newOb except socket.error: # Simbad is offline raise base.ui.logOldExc(base.ValidationError( "Simbad is offline, cannot query.", "hscs_pos", # really, this should be added by the widget hint="If this problem persists, complain to us rather than simbad."))
def parseHumanSpoint(cooSpec, colName=None): """tries to interpret cooSpec as some sort of cone center. Attempted interpretations include various forms of coordinate pairs and simbad objects; hence, this will in general cause network traffic. If no sense can be made, a ValidationError on colName is raised. """ try: cooPair = base.parseCooPair(cooSpec) except ValueError: simbadData = base.caches.getSesame("web").query(cooSpec) if not simbadData: raise base.ValidationError( "%s is neither a RA,DEC" " pair nor a simbad resolvable object." % cooSpec, colName) cooPair = simbadData["RA"], simbadData["dec"] return cooPair
def _ensureRowIdentity(self, row, key): """raises an exception if row is not equivalent to the row stored for key. This is one strategy for resolving primary key conflicts. """ storedRow = self.rowIndex[key] if row.keys()!=storedRow.keys(): raise Error("Differing rows for primary key %s: %s vs. %s"%( key, self.rowIndex[key], row)) for colName in row: if row[colName] is None or storedRow[colName] is None: continue if row[colName]!=storedRow[colName]: raise base.ValidationError( "Differing rows for primary key %s;" " %s vs. %s"%(key, row[colName], storedRow[colName]), colName=colName, row=row)
def _runAndCapture(self, inputTable): # if we wanted to get really fancy, it shouldn't be hard to pipe that stuff # directly into the grammar. pipe = subprocess.Popen(self._getArgs(inputTable), 2**16, stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True, cwd=os.path.dirname(self.computer)) writeThread = self._feedInto(self._getInput(inputTable), pipe.stdin) data = pipe.stdout.read() pipe.stdout.close() writeThread.join(0.1) retcode = pipe.wait() if retcode!=0: raise base.ValidationError("The subprocess %s returned %s. This" " indicates an external executable could not be run or failed" " with your parameters. You should probably report this to the" " operators."%(os.path.basename(self.computer), retcode), "query") return data