class Option(base.Structure):
"""A value for enumerated columns.
For presentation purposes, an option can have a title, defaulting to
the option's value.
"""
name_ = "option"
_title = base.UnicodeAttribute(
"title",
default=base.NotGiven,
description="A Label for presentation purposes; defaults to val.",
copyable=True)
_val = base.DataContent(
copyable=True,
description="The value of"
" the option; this is what is used in, e.g., queries and the like.")
def __repr__(self):
# may occur in user messages from formal, so we use title.
return self.title
def completeElement(self, ctx):
if self.title is base.NotGiven:
self.title = self.content_
self._completeElementNext(Option, ctx)
class MixinPar(procdef.RDParameter):
"""A parameter definition for mixins.
The (optional) body provides a default for the parameter.
"""
name_ = "mixinPar"
_expr = base.DataContent(description="The default for the parameter."
" A __NULL__ here does not directly mean None/NULL, but since the"
" content will frequently end up in attributes, it will ususally work"
" as presetting None."
" An empty content means a non-preset parameter, which must be filled"
" in applications. The magic value __EMPTY__ allows presetting an"
" empty string.",
# mixinPars must not evaluate __NULL__; this stuff ends up in
# macro expansions, where an actual None is not desirable.
null=None,
copyable=True, strip=True, default=base.NotGiven)
def validate(self):
self._validateNext(MixinPar)
if len(self.key)<2:
raise base.LiteralParseError("name", self.key, hint="Names of"
" mixin parameters must have at least two characters (since"
" they are exposed as macros")
class RDParameter(base.Structure):
"""A base class for parameters.
"""
_name = base.UnicodeAttribute("key", default=base.Undefined,
description="The name of the parameter", copyable=True, strip=True,
aliases=["name"])
_descr = base.NWUnicodeAttribute("description", default=None,
description="Some human-readable description of what the"
" parameter is about", copyable=True, strip=True)
_expr = base.DataContent(description="The default for the parameter."
" The special value __NULL__ indicates a NULL (python None) as usual."
" An empty content means a non-preset parameter, which must be filled"
" in applications. The magic value __EMPTY__ allows presetting an"
" empty string.",
copyable=True, strip=True, default=base.NotGiven)
_late = base.BooleanAttribute("late", default=False,
description="Bind the name not at setup time but at applying"
" time. In rowmaker procedures, for example, this allows you to"
" refer to variables like vars or rowIter in the bindings.")
def isDefaulted(self):
return self.content_ is not base.NotGiven
def validate(self):
self._validateNext(RDParameter)
if not utils.identifierPattern.match(self.key):
raise base.LiteralParseError("name", self.key, hint=
"The name you supplied was not defined by any procedure definition.")
def completeElement(self, ctx):
if self.content_=="__EMPTY__":
self.content_ = ""
self._completeElementNext(RDParameter, ctx)
class STCDef(base.Structure):
"""A definition of a space-time coordinate system using STC-S.
"""
# Programmatically, you have
# * compiled -- an AST of the entire specification
# * iterColTypes -- iterates over column name/utype pairs, for the
# embedding table only; all others must not touch it
name_ = "stc"
_source = base.DataContent(
copyable=True,
description="An STC-S string"
" with column references (using quote syntax) instead of values")
def completeElement(self, ctx):
self._completeElementNext(STCDef, ctx)
try:
self.compiled = stc.parseQSTCS(self.content_)
except stc.STCSParseError, msg:
raise base.ui.logOldExc(
base.StructureError("Bad stc definition: %s" % str(msg)))
self.compiled.stripUnits()
self._origFields = dict(
(value.dest, utype)
for utype, value in stc.getUtypes(self.compiled)
if isinstance(value, stc.ColRef))
class BinaryRecordDef(base.Structure):
"""A definition of a binary record.
A binary records consists of a number of binary fields, each of which
is defined by a name and a format code. The format codes supported
here are a subset of what python's struct module supports. The
widths given below are for big, little, and packed binfmts.
For native (which is the default), it depends on your platform.
* <number>s -- <number> characters making up a string
* b,B -- signed and unsigned byte (8 bit)
* h,H -- signed and unsigned short (16 bit)
* i,I -- signed and unsigned int (32 bit)
* q,Q -- signed and unsigned long (64 bit)
* f,d -- float and double.
The content of this element gives the record structure in the format
<name>(<code>){<whitespace><name>(<code>)} where <name> is a c-style
identifier.
"""
name_ = "binaryRecordDef"
_fieldsGrammar = _getFieldsGrammar()
_binfmt = base.EnumeratedUnicodeAttribute("binfmt",
default="native",
validValues=["big", "little", "native", "packed"],
description="Binary format of the input data; big and little stand"
" for msb first and lsb first, and"
" packed is like native except no alignment takes place.")
_fields = base.DataContent(description="The enumeration of"
" the record fields.")
_binfmtToStructCode = {
"native": "",
"packed": "=",
"big": ">",
"little": "<"}
def completeElement(self, ctx):
try:
parsedFields = utils.pyparseString(self._fieldsGrammar, self.content_)
except pyparsing.ParseBaseException, ex:
raise base.ui.logOldExc(base.LiteralParseError("binaryRecordDef",
re.sub("\s+", " ", self.content_),
pos=str(ex.loc), hint="The parser said: '%s'"%str(ex)))
# XXX TODO: Position should probably be position during XML parse.
# Fix when we have source positions on parsed elements.
self.structFormat = (self._binfmtToStructCode[self.binfmt]+
str("".join(f["formatCode"] for f in parsedFields)))
self.recordLength = struct.calcsize(self.structFormat)
self.fieldNames = tuple(f["identifier"] for f in parsedFields)
self._completeElementNext(BinaryRecordDef, ctx)
class Script(base.Structure, base.RestrictionMixin):
"""A script, i.e., some executable item within a resource descriptor.
The content of scripts is given by their type -- usually, they are
either python scripts or SQL with special rules for breaking the
script into individual statements (which are basically like python's).
The special language AC_SQL is like SQL, but execution errors are
ignored. This is not what you want for most data RDs (it's intended
for housekeeping scripts).
See `Scripting`_.
"""
name_ = "script"
typeDesc_ = "Embedded executable code with a type definition"
_lang = base.EnumeratedUnicodeAttribute(
"lang",
default=base.Undefined,
description="Language of the script.",
validValues=["SQL", "python", "AC_SQL"],
copyable=True)
_type = base.EnumeratedUnicodeAttribute(
"type",
default=base.Undefined,
description="Point of time at which script is to run.",
validValues=[
"preImport", "newSource", "preIndex", "postCreation", "beforeDrop",
"sourceDone"
],
copyable=True)
_name = base.UnicodeAttribute(
"name",
default="anonymous",
description="A human-consumable designation of the script.",
copyable=True)
_notify = base.BooleanAttribute(
"notify",
default=True,
description="Send out a notification when running this"
" script.",
copyable=True)
_content = base.DataContent(copyable=True, description="The script body.")
_original = base.OriginalAttribute()
def getSource(self):
"""returns the content with all macros expanded.
"""
return self.parent.getExpander().expand(self.content_)
class DataModelRoles(base.Structure):
"""an annotation of a table in terms of data models.
The content of this element is a Simple Instance Language term.
"""
# We defer the parsing of the contained element to (hopefully) the
# end of the parsing of the RD to enable forward references with
# too many headaches (stubs don't cut it: we need to know types).
#
# There's an additional complication in that we may want to
# access parsed annotations while parsing other annotations
# (e.g., when processing foreign keys).
# To allow the thing to "parse itself" in such situations, we do
# all the crazy magic with the _buildAnnotation function.
name_ = "dm"
_sil = base.DataContent(description="SIL (simple instance language)"
" annotation.",
copyable=True)
def completeElement(self, ctx):
def _buildAnnotation():
self._parsedAnnotation = sil.getAnnotation(
self.content_, getAnnotationMaker(self.parent))
self.parent.annotations.append(self._parsedAnnotation)
self._buildAnnotation = lambda: None
self._buildAnnotation = _buildAnnotation
ctx.addExitFunc(lambda rd, ctx: self._buildAnnotation())
self._completeElementNext(DataModelRoles, ctx)
def parse(self):
"""returns a parsed version of the embedded annotation.
Do not call this while the RD is still being built, as dm
elements may contain forward references, and these might
not yet be available during the parse.
"""
self._buildAnnotation()
return self._parsedAnnotation
def getCopy(self, instance, newParent):
# we'll have to re-parse since we want to reference the new columns
self.parent.annotations.append(
sil.getAnnotation(
self.content_,
functools.partial(makeAttributeAnnotation, newParent)))
class Upload(base.Structure):
"""An upload going with a URL.
"""
name_ = "httpUpload"
_src = common.ResdirRelativeAttribute("source",
default=base.NotGiven,
description="Path to a file containing the data to be uploaded.",
copyable=True)
_name = base.UnicodeAttribute("name",
default=base.Undefined,
description="Name of the upload parameter",
copyable=True)
_filename = base.UnicodeAttribute("fileName",
default=None,
description="Remote file name for the uploaded file.",
copyable=True)
_content = base.DataContent(description="Inline data to be uploaded"
" (conflicts with source)")
@property
def rd(self):
return self.parent.rd
def addToForm(self, form):
"""sets up a _Form instance to upload the data.
"""
if self.content_:
data = self.content_
else:
with open(self.source) as f:
data = f.read()
form.addFile(self.name, self.fileName, data)
def validate(self):
if (self.content_ and self.source
or not (self.content_ or self.source)):
raise base.StructureError("Exactly one of element content and source"
" attribute must be given for an upload.")
class CustomPageFunction(base.Structure, base.RestrictionMixin):
"""An abstract base for nevow.rend.Page-related functions on services.
"""
_name = base.UnicodeAttribute(
"name",
default=base.Undefined,
description="Name of the render function (use this in the"
" n:render or n:data attribute in custom templates).",
copyable=True,
strip=True)
_code = base.DataContent(description="Function body of the renderer; the"
" arguments are named ctx and data.",
copyable=True)
def onElementComplete(self):
self._onElementCompleteNext(CustomPageFunction)
vars = globals().copy()
vars["service"] = self.parent
exec("def %s(ctx, data):\n%s" %
(self.name,
utils.fixIndentation(self.content_,
newIndent=" ",
governingLine=1).rstrip())) in vars
self.func = vars[self.name]
class MappedExpression(base.Structure):
"""a base class for map and var.
You must give a destDict class attribute to make these work.
"""
destDict = None
restrictedMode = False
_dest = base.UnicodeAttribute("key",
default=base.Undefined,
description="Name of the column the value is to end up in.",
copyable=True,
strip=True,
aliases=["dest", "name"])
_src = base.UnicodeAttribute("source",
default=None,
description="Source key name to convert to column value (either a grammar"
" key or a var).",
copyable=True,
strip=True,
aliases=["src"])
_nullExcs = base.UnicodeAttribute("nullExcs",
default=base.NotGiven,
description="Exceptions that should be caught and"
" cause the value to be NULL, separated by commas.")
_expr = base.DataContent(
description="A python expression giving the value for key.",
copyable=True,
strip=True)
_nullExpr = base.UnicodeAttribute("nullExpr",
default=base.NotGiven,
description="A python expression for a value that is mapped to"
" NULL (None). Equality is checked after building the value, so"
" this expression has to be of the column type. Use map with"
" the parseWithNull function to catch null values before type"
" conversion.")
def completeElement(self, ctx):
self.restrictedMode = getattr(ctx, "restricted", False)
if self.restrictedMode and (
self.content_
or self.nullExpr
or self.nullValue):
raise base.RestrictedElement("map", hint="In restricted mode, only"
" maps with a source attribute are allowed; nullExpr or nullValue"
" are out, too, since they can be used to inject raw code.")
if not self.content_ and not self.source:
self.source = self.key
if self.content_ and "\\" in self.content_:
self.content_ = self.parent.expand(self.content_)
def validate(self):
"""checks that code content is a parseable python expression and that
the destination exists in the tableDef
"""
self._validateNext(MappedExpression)
if (self.content_ and self.source) or not (self.content_ or self.source):
raise base.StructureError("Map must have exactly one of source attribute"
" or element content")
if not utils.identifierPattern.match(self.key):
raise base.LiteralParseError("name", self.key,
hint="Var keys must be valid python"
" identifiers, and '%s' is not"%self.key)
if self.source:
if not utils.identifierPattern.match(self.source):
raise base.LiteralParseError("source", self.source,
hint="Map sources must be (python)"
" identifiers, and '%s' is not"%self.source)
if self.nullExpr is not base.NotGiven:
utils.ensureExpression(self.nullExpr)
if self.content_:
utils.ensureExpression(common.replaceProcDefAt(self.content_), self.name_)
if self.nullExcs is not base.NotGiven:
utils.ensureExpression(self.nullExcs, "%s.nullExcs"%(self.name_))
def getCode(self, columns):
"""returns python source code for this map.
"""
code = []
if self.content_:
code.append('%s["%s"] = %s'%(self.destDict, self.key, self.content_))
else:
colDef = columns.getColumnByName(self.key)
try:
code.append('%s["%s"] = %s'%(self.destDict,
self.key,
base.sqltypeToPythonCode(colDef.type)%'vars["%s"]'%self.source))
except base.ConversionError:
raise base.ui.logOldExc(base.LiteralParseError("map", colDef.type,
hint="Auto-mapping to %s is impossible since"
" no default map for %s is known"%(self.key, colDef.type)))
if self.nullExpr is not base.NotGiven:
code.append('\nif %s["%s"]==%s: %s["%s"] = None'%(
self.destDict,
self.key,
self.nullExpr,
self.destDict,
self.key))
code = "".join(code)
if self.nullExcs is not base.NotGiven:
code = 'try:\n%s\nexcept (%s): %s["%s"] = None'%(
re.sub("(?m)^", " ", code),
self.nullExcs,
self.destDict,
self.key)
return code
class ParamBase(ColumnBase):
"""A basic parameter.
This is the base for both Param and InputKey.
"""
_value = base.DataContent(
description="The value of parameter."
" It is parsed according to the param's type using the default"
" parser for the type VOTable tabledata.",
default=base.NotGiven,
copyable=True,
expand=True)
_valueCache = base.Undefined
__contentStore = base.NotGiven
nullLiteral = ""
unprocessedTypes = set(["raw", "file"])
def __repr__(self):
return "<%s %s=%s>" % (self.__class__.__name__, repr(
self.name), repr(self.content_))
def __set_content(self, val):
self.__contentStore = val
def __get_content(self):
if (self.__contentStore is base.NotGiven
and self._valueCache is not base.Undefined):
self.__contentStore = self._unparse(self._valueCache)
return self.__contentStore
content_ = property(__get_content, __set_content)
def expand(self, value):
"""hands up macro expansion requests to a parent, if there is one
and it can handle expansions.
"""
if hasattr(self.parent, "expand"):
return self.parent.expand(value)
return value
def completeElement(self, ctx):
if not self.values:
self.values = base.makeStruct(Values, parent_=self)
self._completeElementNext(ParamBase, ctx)
def onElementComplete(self):
self._onElementCompleteNext(ParamBase)
if self.content_ is base.NotGiven:
if self.values.default:
self.set(self.values.default)
else:
self.set(self.content_)
@property
def value(self):
"""returns a typed value for the parameter.
Unset items give None here.
"""
if self._valueCache is base.Undefined:
if self.content_ is base.NotGiven:
self._valueCache = None
else:
self._valueCache = self._parse(self.content_)
return self._valueCache
def getStringValue(self):
"""returns a string serialisation of the value.
This is what would reproduce the value if embedded in an XML
serialisation of the param.
"""
if self.type in self.unprocessedTypes:
return "(Unrepresentable %s)" % self.type
return self.content_
def set(self, val):
"""sets this parameter's value.
val can be a python value, or string literal. In the second
case, this string literal will be preserved in string serializations
of this param.
If val is an invalid value for this item, a ValidationError is
raised and the item's value will be Undefined.
"""
if isinstance(val, basestring):
self.content_ = val
else:
self.content_ = base.NotGiven
self._valueCache = self._parse(val)
def _parse(self, literal, atom=False):
"""parses literal using the default value parser for this param's
type.
If literal is not a string or a list of strings, it will be returned
unchanged.
In lists of strings, each element will be treated individually,
and the result will be our value. This is mainly a service
for InputKeys fed from nevow args.
The method also makes sure literal matches any constraints
set by a values child and raises a ValidationError if not.
"""
if self.type in self.unprocessedTypes:
return literal
elif not isinstance(literal, basestring):
# the awful atom thing is for when array-like things are
# specified in multiple HTTP parameters rather than
# using VOTable serialisation. It's probably wrong
# to accept that in the first place, except of course
# we have that with strings. Perhaps only turn arrays off
# for strings?
if _isStringList(literal):
value = [
self._parse(l, atom=self.xtype != "interval")
for l in literal
]
else:
value = literal
elif literal == "__NULL__" or literal == "":
value = None
elif ((self.type == "text" or self.type == "unicode")
and literal == "__EMPTY__"):
value = ""
else:
if literal == self.values.nullLiteral:
value = None
else:
try:
type, arraysize, xtype = self._getVOTableType()
if atom:
arraysize = None
value = paramval.getVOTParser(type, arraysize, self.xtype
or xtype)(literal)
# make NaNs NULL here for consistent VOTable practice
if value != value:
value = None
except ValueError:
raise base.ValidationError(
"%s is not a valid literal for %s" %
(repr(literal), self.name), self.name)
if not self.values.validateOptions(value):
raise base.ValidationError(
"%s is not a valid value for %s" % (repr(literal), self.name),
self.name)
# unify NULL value representation to the empty string
if value is None:
self.content_ = ""
return value
def _unparse(self, value):
"""returns a string representation of value appropriate for this
type.
Actually, for certain types only handled internally (like file or raw),
this is not a string representation at all but just the python stuff.
Plus, right now, for sequences we're not doing anything. We probably
should; but we'll need to be much more careful in ContextGramar then.
"""
if self.type in self.unprocessedTypes:
return value
if value is None:
return ""
else:
type, arraysize, xtype = self._getVOTableType()
val = paramval.getVOTSerializer(type, arraysize, self.xtype
or xtype)(value)
return val
class SourceSpec(base.Structure):
"""A Specification of a data descriptor's inputs.
This will typcially be files taken from a file system. If so, DaCHS will,
in each directory, process the files in alphabetical order. No guarantees
are made as to the sequence directories are processed in.
Multiple patterns are processed in the order given in the RD.
"""
name_ = "sources"
_patterns = base.ListOfAtomsAttribute("patterns", description=
"Paths to the source files. You can use shell patterns here.",
itemAttD=base.UnicodeAttribute("pattern", description="Shell pattern"
" for source file(s), relative to resource directory."),
copyable=True)
_items = base.ListOfAtomsAttribute("items", description=
"String literals to pass to grammars. In contrast to patterns,"
" they are not interpreted as file names but passed to the"
" grammar verbatim. Normal grammars do not like this. It is"
" mainly intended for use with custom or null grammars.",
itemAttD=base.UnicodeAttribute("item",
description="Grammar-specific string"), copyable=True)
_recurse = base.BooleanAttribute("recurse", default=False,
description="Search for pattern(s) recursively in their directory"
" part(s)?", copyable=True)
_ignore = base.StructAttribute("ignoredSources", childFactory=
IgnoreSpec, description="Specification of sources that should not"
" be processed although they match patterns. Typically used"
" in update-type data descriptors.", copyable=True)
_file = base.DataContent(description="A single"
" file name (this is for convenience)", copyable="True")
_original = base.OriginalAttribute()
def __iter__(self):
return self.iterSources()
def completeElement(self, ctx):
if self.ignoredSources is base.Undefined:
self.ignoredSources = base.makeStruct(IgnoreSpec)
self._completeElementNext(SourceSpec, ctx)
def _expandDirParts(self, dirParts, ignoreDotDirs=True):
"""expands a list of directories into a list of them and all their
descendants.
It follows symbolic links but doesn't do any bookkeeping, so bad
things will happen if the directory graph contains cycles.
"""
res = []
for root in dirParts:
for root, dirs, files in os.walk(root):
if ignoreDotDirs:
if os.path.basename(root).startswith("."):
continue
dirs = [dir for dir in dirs if not dir.startswith(".")]
dirs = (os.path.join(root, dir) for dir in dirs)
res.extend(dir for dir in dirs if os.path.isdir(dir))
for child in files:
destName = os.path.join(root, child)
if os.path.islink(destName) and not os.path.isfile(destName):
res.extend(self._expandDirParts(destName))
return res
def iterSources(self, connection=None):
self.ignoredSources.prepare(connection)
for item in self.items:
if not self.ignoredSources.isIgnored(item):
yield item
baseDir = ""
if self.parent.rd:
baseDir = self.parent.rd.resdir
for pattern in self.patterns:
dirPart, baseName = os.path.split(pattern)
if self.parent.rd:
dirParts = [os.path.join(baseDir, dirPart)]
else:
dirParts = [dirPart]
if self.recurse:
dirParts = dirParts+self._expandDirParts(dirParts)
for dir in sorted(dirParts):
for name in sorted(glob.glob(os.path.join(dir, baseName))):
fullName = os.path.abspath(name)
if not self.ignoredSources.isIgnored(fullName):
yield fullName
if self.content_:
yield os.path.abspath(os.path.join(baseDir, self.content_))
def __nonzero__(self):
return (not not self.patterns) or (not not self.items
) or (not not self.content_)
class DataURL(base.Structure):
"""A source document for a regression test.
As string URLs, they specify where to get data from, but the additionally
let you specify uploads, authentication, headers and http methods,
while at the same time saving you manual escaping of parameters.
The bodies is the path to run the test against. This is
interpreted as relative to the RD if there's no leading slash,
relative to the server if there's a leading slash, and absolute
if there's a scheme.
The attributes are translated to parameters, except for a few
pre-defined names. If you actually need those as URL parameters,
should at us and we'll provide some way of escaping these.
We don't actually parse the URLs coming in here. GET parameters
are appended with a & if there's a ? in the existing URL, with a ?
if not. Again, shout if this is too dumb for you (but urlparse
really isn't all that robust either...)
"""
name_ = "url"
# httpURL will be set to the URL actually used in retrieveResource
# Only use this to report the source of the data for, e.g., failing
# tests.
httpURL = "(not retrieved)"
_base = base.DataContent(description="Base for URL generation; embedded"
" whitespace will be removed, so you're free to break those whereever"
" you like.",
copyable=True)
_httpMethod = base.UnicodeAttribute("httpMethod",
description="Request method; usually one of GET or POST",
default="GET")
_httpPost = common.ResdirRelativeAttribute("postPayload",
default=base.NotGiven,
description="Path to a file containing material that should go"
" with a POST request (conflicts with additional parameters).",
copyable=True)
_parset = base.EnumeratedUnicodeAttribute("parSet",
description="Preselect a default parameter set; form gives what"
" our framework adds to form queries.", default=base.NotGiven,
validValues=["form"],
copyable=True)
_httpHeaders = base.DictAttribute("httpHeader",
description="Additional HTTP headers to pass.",
copyable=True)
_httpAuthKey = base.UnicodeAttribute("httpAuthKey",
description="A key into ~/.gavo/test.creds to find a user/password"
" pair for this request.",
default=base.NotGiven,
copyable=True)
_httpUploads = base.StructListAttribute("uploads",
childFactory=Upload,
description='HTTP uploads to add to request (must have httpMethod="POST")',
copyable=True)
_httpHonorRedirects = base.BooleanAttribute("httpHonorRedirects",
default=False,
description="Follow 30x redirects instead of just using"
" status, headers, and payload of the initial request.",
copyable="True")
_rd = common.RDAttribute()
_open = DynamicOpenVocAttribute("open")
def getValue(self, serverURL):
"""returns a pair of full request URL and postable payload for this
test.
"""
urlBase = re.sub(r"\s+", "", self.content_)
if "://" in urlBase:
# we believe there's a scheme in there
pass
elif urlBase.startswith("/"):
urlBase = serverURL+urlBase
else:
urlBase = serverURL+"/"+self.parent.rd.sourceId+"/"+urlBase
if self.httpMethod=="POST":
return urlBase
else:
return self._addParams(urlBase, urllib.urlencode(self.getParams()))
def getParams(self):
"""returns the URL parameters as a sequence of kw, value pairs.
"""
params = getattr(self, "freeAttrs", [])
if self.parSet=="form":
params.extend([("__nevow_form__", "genForm"), ("submit", "Go"),
("_charset_", "UTF-8")])
return params
def retrieveResource(self, serverURL, timeout):
"""returns a triple of status, headers, and content for retrieving
this URL.
"""
self.httpURL, payload = self.getValue(serverURL), None
headers = {
"user-agent": "DaCHS regression tester"}
headers.update(self.httpHeader)
if self.httpMethod=="POST":
if self.postPayload:
with open(self.postPayload) as f:
payload = f.read()
elif self.uploads:
form = _FormData()
for key, value in self.getParams():
form.addParam(key, value)
for upload in self.uploads:
upload.addToForm(form)
boundary = "========== roughtest deadbeef"
form.set_param("boundary", boundary)
headers["Content-Type"] = form.get_content_type(
)+'; boundary="%s"'%boundary
payload = form.as_string()
else:
payload = urllib.urlencode(self.getParams())
headers["Content-Type"] = "application/x-www-form-urlencoded"
scheme, host, path, _, query, _ = urlparse.urlparse(str(self.httpURL))
assert scheme=="http"
if self.httpAuthKey is not base.NotGiven:
headers.update(getAuthFor(self.httpAuthKey))
status, respHeaders, content = doHTTPRequest(str(self.httpMethod),
host, path, query, payload, headers, timeout)
while self.httpHonorRedirects and status in [301, 302, 303]:
scheme, host, path, _, query, _ = urlparse.urlparse(
getHeaderValue(respHeaders, "location"))
status, respHeaders, content = doHTTPRequest("GET",
host, path, query, None, {}, timeout)
return status, respHeaders, content
def _addParams(self, urlBase, params):
"""a brief hack to add query parameters to GET-style URLs.
This is a workaround for not trusting urlparse and is fairly easy to
fool.
Params must already be fully encoded.
"""
if not params:
return urlBase
if "?" in urlBase:
return urlBase+"&"+params
else:
return urlBase+"?"+params
def validate(self):
if self.postPayload is not base.NotGiven:
if self.getParams():
raise base.StructureError("No parameters (or parSets) are"
" possible with postPayload")
if self.httpMethod!="POST":
raise base.StructureError("Only POST is allowed as httpMethod"
" together with postPayload")
if self.uploads:
if self.httpMethod!="POST":
raise base.StructureError("Only POST is allowed as httpMethod"
" together with upload")
self._validateNext(DataURL)
class DBIndex(base.Structure):
"""A description of an index in the database.
In real databases, indices may be fairly complex things; still, the
most common usage here will be to just index a single column::
<index columns="my_col"/>
To index over functions, use the character content; parentheses are added
by DaCHS, so don't have them in the content. An explicit specification
of the index expression is also necessary to allow RE pattern matches using
indices in character columns (outside of the C locale). That would be::
<index columns="uri">uri text_pattern_ops</index>
(you still want to give columns so the metadata engine is aware of the
index). See section "Operator Classes and Operator Families" in
the Postgres documentation for details.
"""
name_ = "index"
_name = base.UnicodeAttribute(
"name",
default=base.Undefined,
description="Name of the index (defaults to something computed from"
" columns; the name of the parent table will be prepended in the DB)",
copyable=True)
_columns = base.StringListAttribute(
"columns",
description=
"Table columns taking part in the index (must be given even if there"
" is an expression building the index and mention all columns taking"
" part in the index generated by it",
copyable=True)
_cluster = base.BooleanAttribute(
"cluster",
default=False,
description="Cluster the table according to this index?",
copyable=True)
_code = base.DataContent(
copyable=True,
description="Raw SQL specifying an expression the table should be"
" indexed for. If not given, the expression will be generated from"
" columns (which is what you usually want).")
_method = base.UnicodeAttribute(
"method",
default=None,
description="The indexing method, like an index type. In the 8.x,"
" series of postgres, you need to set method=GIST for indices"
" over pgsphere columns; otherwise, you should not need to"
" worry about this.",
copyable=True)
def completeElement(self, ctx):
if self.content_ and getattr(ctx, "restricted", False):
raise base.RestrictedElement("index",
hint="Free-form SQL on indices"
" is not allowed in restricted mode")
self._completeElementNext(DBIndex, ctx)
if not self.columns:
raise base.StructureError("Index without columns is verboten.")
if self.name is base.Undefined:
self.name = "%s" % (re.sub("[^\w]+", "_", "_".join(self.columns)))
if not self.content_:
self.content_ = "%s" % ",".join(self.columns)
def iterCode(self):
destTableName = self.parent.getQName()
usingClause = ""
if self.method is not None:
usingClause = " USING %s" % self.method
yield self.parent.expand(
"CREATE INDEX %s ON %s%s (%s)" %
(self.dbname, destTableName, usingClause, self.content_))
if self.cluster:
yield self.parent.expand("CLUSTER %s ON %s" %
(self.dbname, destTableName))
def create(self, querier):
"""creates the index on the parent table if necessary.
querier is an object mixing in the DBMethodsMixin, usually the
DBTable object the index should be created on.
"""
if not querier.hasIndex(self.parent.getQName(), self.dbname):
if not self.parent.system:
base.ui.notifyIndexCreation(self.parent.expand(self.dbname))
for statement in self.iterCode():
querier.query(statement)
def drop(self, querier):
"""drops the index if it exists.
querier is an object mixing in the DBMethodsMixin, usually the
DBTable object the index possibly exists on.
"""
iName = self.parent.expand(self.dbname)
if querier.hasIndex(self.parent.getQName(), iName):
querier.query("DROP INDEX %s.%s" % (self.parent.rd.schema, iName))
@property
def dbname(self):
return "%s_%s" % (self.parent.id, self.name)