class EditCore(standardcores.TableBasedCore):
"""A core that allows POSTing records into database tables.
"""
name_ = "editCore"
_queriedTable = base.ReferenceAttribute(
"queriedTable",
default=base.Undefined,
description="Reference to the table to"
" be edited",
forceType=rscdef.TableDef)
def completeElement(self, ctx):
if self.outputTable is base.Undefined:
self.outputTable = base.parseFromString(outputdef.OutputTableDef,
uploadOutputDef)
self._completeElementNext(EditCore, ctx)
def run(self, service, inputTable, queryMeta):
conn = base.getDBConnection(base.getDBProfile("admin"))
table = rsc.TableForDef(self.queriedTable, connection=conn)
table.addRow(inputTable.getParamDict())
conn.commit()
conn.close()
return rsc.TableForDef(self.outputTable, rows=[{"nAffected": 1}])
class FieldRef(base.Structure):
"""A reference to a table column for building simple views.
"""
name_ = "columnRef"
docName_ = "columnRef (view)"
aliases = ["fieldRef"]
_srcTable = base.ReferenceAttribute(
"table",
default=base.Undefined,
description="Reference to the table the field comes from.",
forceType=TableDef)
_srcCol = base.UnicodeAttribute(
"key",
default=base.Undefined,
description="Column name within the referenced table.",
aliases=["column"])
def onElementComplete(self):
self._onElementCompleteNext(FieldRef)
if not self.key in self.table:
raise base.StructureError("No field '%s' in table %s" %
(self.key, self.table.getQName()))
def getColumn(self):
return self.table.getColumnByName(self.key)
def getQName(self):
name = "%s.%s" % (self.table.getQName(), self.key)
if "\\" in name:
name = self.expand(name)
return name
class RowsetGrammar(common.Grammar):
"""A grammar handling sequences of tuples.
To add semantics to the field, it must know the "schema" of the
data. This is defined via the table it is supposed to get the input
from.
This grammar probably is only useful for internal purposes.
"""
name_ = "rowsetGrammar"
rowIterator = RowsetIterator
_fieldsFrom = base.ReferenceAttribute(
"fieldsFrom",
description="the table defining the columns in the tuples.",
copyable=True)
def onElementComplete(self):
self._onElementCompleteNext(RowsetGrammar)
self.names = [c.name for c in self.fieldsFrom]
class SDMCore(svcs.Core):
"""A core for making (VO)Tables according to the Spectral Data Model.
Do *not* use this any more, use datalink to do this.
Here, the input table consists of the accref of the data to be generated.
The data child of an SDMVOTCore prescribes how to come up with the
table. The output table is the (primary) table of the data instance.
If you find yourself using this, please let the authors know. We
tend to believe SDMCores should no longer be necessary in the presence
of getData, and hence we might want to remove this at some point.
"""
name_ = "sdmCore"
inputTableXML = """<inputTable id="inFields">
<inputKey name="accref" type="text" required="True"
description="Accref of the data within the SSAP table."/>
<inputKey name="dm" type="text" description="Data model to
generate the table for (sdm or sed)">sdm</inputKey>
</inputTable>"""
_queriedTable = base.ReferenceAttribute(
"queriedTable",
default=base.Undefined,
description="A reference to the SSAP table"
" to search the accrefs in",
copyable=True)
_sdmDD = base.StructAttribute(
"sdmDD",
default=base.Undefined,
childFactory=rscdef.DataDescriptor,
description="A data instance that builds the SDM table. You'll need"
" a custom or embedded grammar for those that accepts an SDM row"
" as input.",
copyable=True)
def onElementComplete(self):
warnings.warn(
"SDMCore is deprecated, and chances are you don't need"
" need it. See http://docs.g-vo.org/DaCHS/commonproblems.html for"
" details.", DeprecationWarning)
self._onElementCompleteNext(SDMCore)
if self.sdmDD.getMeta("utype", default=None) is None:
self.sdmDD.setMeta("utype", "spec:Spectrum")
def run(self, service, inputTable, queryMeta):
with base.getTableConn() as conn:
ssaTable = rsc.TableForDef(self.queriedTable, connection=conn)
try:
# XXX TODO: Figure out why the unquote here is required.
accref = urllib.unquote(inputTable.getParam("accref"))
res = list(
ssaTable.iterQuery(ssaTable.tableDef, "accref=%(accref)s",
{"accref": accref}))
if not res:
raise svcs.UnknownURI(
"No spectrum with accref %s known here" %
inputTable.getParam("accref"))
ssaRow = res[0]
finally:
ssaTable.close()
resData = makeSDMDataForSSARow(ssaRow, self.sdmDD)
votContextArgs = {}
if queryMeta["tdEnc"]:
votContextArgs["tablecoding"] = "td"
# This is for VOSpec, in particular the tablecoding; I guess once
# we actually support the sed DM, this should go, and the
# specview links should use sed dcc sourcePaths.
if inputTable.getParam("dm") == "sed":
hackSDMToSED(resData)
votContextArgs["tablecoding"] = "td"
return (base.votableType,
votable.asString(makeSDMVOT(resData, **votContextArgs)))
class ContextGrammar(grammars.Grammar):
"""A grammar for web inputs.
These are almost exclusively in InputDDs. They hold InputKeys
defining what they take from the context.
For DBCores, the InputDDs are generally defined implicitely
via CondDescs. Thus, only for other cores will you ever need
to bother with ContextGrammars (unless you're going for special
effects).
The source tokens for context grammars are dictionaries; these
are either typed dictionaries from nevow, where the values
usually are atomic, or, preferably, the dictionaries of lists
from request.args.
ContextGrammars only yield rows if there's a rowKey defined.
In that case, an outer join of all other parameters is returned;
with rowKey defined, the input keys are obtained from the table's
columns.
In normal usage, they just yield a single parameter row,
corresponding to the source dictionary possibly completed with
defaults, where non-requried input keys get None defaults where not
given. Missing required parameters yield errors.
Since most VO protocols require case-insensitive matching of parameter
names, matching of input key names and the keys of the input dictionary
is attempted first literally, then disregarding case.
"""
name_ = "contextGrammar"
_inputTable = base.ReferenceAttribute(
"inputTable",
default=base.NotGiven,
description="The table that is to be built using this grammar",
copyable=True)
_inputKeys = rscdef.ColumnListAttribute(
"inputKeys",
childFactory=InputKey,
description="Input keys this context grammar should parse."
" These must not be given if there is an input table defined.")
_rowKey = base.UnicodeAttribute(
"rowKey",
default=base.NotGiven,
description="The name of a key that is used to generate"
" rows from the input",
copyable=True)
_rejectExtras = base.BooleanAttribute(
"rejectExtras",
default=False,
description="If true, the grammar will reject extra input parameters."
" Note that for form-based services, there *are* extra parameters"
" not declared in the services' input tables. Right now,"
" contextGrammar does not ignore those.",
copyable=True)
_original = base.OriginalAttribute("original")
rowIterator = ContextRowIterator
def onElementComplete(self):
if self.inputTable is not base.NotGiven:
if self.inputKeys != []:
raise base.StructureError(
"InputKeys and inputTable must not"
" both be given in a context grammar")
else:
if self.rowKey:
self.inputKeys = [
InputKey.fromColumn(c) for c in self.inputTable.columns
]
else:
self.inputKeys = self.inputTable.params
else:
columns = []
if self.rowKey:
columns = self.inputKeys
self.inputTable = MS(InputTable,
params=self.inputKeys,
columns=columns)
self.defaults = {}
for ik in self.iterInputKeys():
if not ik.required:
self.defaults[ik.name] = None
if ik.value is not None:
self.defaults[ik.name] = ik.value
self._onElementCompleteNext(ContextGrammar)
def iterInputKeys(self):
for ik in self.inputKeys:
yield ik
class Make(base.Structure, scripting.ScriptingMixin):
"""A build recipe for tables belonging to a data descriptor.
All makes belonging to a DD will be processed in the order in which they
appear in the file.
"""
name_ = "make"
_table = base.ReferenceAttribute("table",
description="Reference to the table to be embedded",
default=base.Undefined,
copyable=True,
forceType=tabledef.TableDef)
_rowmaker = base.ReferenceAttribute("rowmaker",
default=base.NotGiven,
forceType=rmkdef.RowmakerDef,
description="The rowmaker (i.e., mapping rules from grammar keys to"
" table columns) for the table being made.",
copyable=True)
_parmaker = base.ReferenceAttribute("parmaker",
default=base.NotGiven,
forceType=rmkdef.ParmakerDef,
description="The parmaker (i.e., mapping rules from grammar parameters"
" to table parameters) for the table being made. You will usually"
" not give a parmaker.",
copyable=True)
_role = base.UnicodeAttribute("role",
default=None,
description="The role of the embedded table within the data set",
copyable=True)
_rowSource = base.EnumeratedUnicodeAttribute("rowSource",
default="rows",
validValues=["rows", "parameters"],
description="Source for the raw rows processed by this rowmaker.",
copyable=True,
strip=True)
def __repr__(self):
return "Make(table=%r, rowmaker=%r)"%(
self.table and self.table.id, self.rowmaker and self.rowmaker.id)
def onParentComplete(self):
if self.rowmaker is base.NotGiven:
self.rowmaker = rmkdef.RowmakerDef.makeIdentityFromTable(self.table)
def getExpander(self):
"""used by the scripts of expanding their source.
We always return the expander of the table being made.
"""
return self.table.getExpander()
def create(self, connection, parseOptions, tableFactory, **kwargs):
"""returns a new empty instance of the table this is making.
"""
newTable = tableFactory(self.table,
parseOptions=parseOptions, connection=connection, role=self.role,
create=True, **kwargs)
if (self.table.onDisk
and not parseOptions.updateMode
and not getattr(self.parent, "updating", False)):
newTable._runScripts = self.getRunner()
return newTable
def runParmakerFor(self, grammarParameters, destTable):
"""feeds grammarParameter to destTable.
"""
if self.parmaker is base.NotGiven:
return
parmakerFunc = self.parmaker.compileForTableDef(destTable.tableDef)
destTable.setParams(parmakerFunc(grammarParameters, destTable),
raiseOnBadKeys=False)
class Publication(base.Structure, base.ComputedMetaMixin):
"""A specification of how a service should be published.
This contains most of the metadata for what is an interface in
registry speak.
"""
name_ = "publish"
_rd = rscdef.RDAttribute()
_render = base.UnicodeAttribute(
"render",
default=base.Undefined,
description="The renderer the publication will point at.",
copyable=True)
_sets = base.StringSetAttribute(
"sets",
description="Comma-separated list of sets this service will be"
" published in. Predefined are: local=publish on front page,"
" ivo_managed=register with the VO registry. If you leave it"
" empty, 'local' publication is assumed.",
copyable="True")
_service = base.ReferenceAttribute(
"service",
default=base.NotGiven,
description="Reference for a service actually implementing the"
" capability corresponding to this publication. This is"
" mainly when there is a vs:WebBrowser service accompanying a VO"
" protocol service, and this other service should be published"
" in the same resource record. See also the operator's guide.",
copyable="True")
_auxiliary = base.BooleanAttribute(
"auxiliary",
default=False,
description="Auxiliary publications are for capabilities"
" not intended to be picked up for all-VO queries, typically"
" because they are already registered with other services."
" This is mostly used internally; you probably have no reason"
" to touch it.")
def completeElement(self, ctx):
if self.render is base.Undefined:
self.render = "form"
if not self.sets:
self.sets.add("local")
if self.service is base.NotGiven:
self.service = self.parent
self.setMetaParent(self.service)
self._completeElementNext(Publication, ctx)
def validate(self):
self._validateNext(Publication)
try:
renderers.getRenderer(self.render)
except KeyError:
raise base.StructureError("Unknown renderer: %s" % self.render)
def _meta_accessURL(self):
return self.service.getURL(self.render)
def _meta_urlUse(self):
return renderers.getRenderer(self.render).urlUse
def _meta_requestMethod(self):
return renderers.getRenderer(self.render).preferredMethod
def _meta_resultType(self):
return renderers.getRenderer(self.render).resultType
class ProcApp(ProcDef):
"""An abstract base for procedure applications.
Deriving classes need to provide:
- a requiredType attribute specifying what ProcDefs can be applied.
- a formalArgs attribute containing a (python) formal argument list
- of course, a name_ for XML purposes.
They can, in addition, give a class attribute additionalNamesForProcs,
which is a dictionary that is joined into the global namespace during
procedure compilation.
"""
_procDef = base.ReferenceAttribute("procDef", forceType=ProcDef,
default=base.NotGiven, description="Reference to the procedure"
" definition to apply", copyable=True)
_bindings = base.StructListAttribute("bindings", description=
"Values for parameters of the procedure definition",
childFactory=Binding, copyable=True)
_name = base.UnicodeAttribute("name", default=base.NotGiven,
description="A name of the proc. ProcApps compute their (python)"
" names to be somwhat random strings. Set a name manually to"
" receive more easily decipherable error messages. If you do that,"
" you have to care about name clashes yourself, though.", strip=True)
requiredType = None
additionalNamesForProcs = {}
def validate(self):
if self.procDef and self.procDef.type and self.requiredType:
if self.procDef.type!=self.requiredType:
raise base.StructureError("The procDef %s has type %s, but"
" here %s procDefs are required."%(self.procDef.id,
self.procDef.type, self.requiredType))
if self.procDef:
if self.procDef.deprecated:
if self.getSourcePosition()!="<internally built>":
# for now, don't warn about these; they typically
# originate when copying/adapting cores and will just
# confuse operators
procId = "unnamed procApp"
if self.name:
procId = "procApp %s"%self.name
base.ui.notifyWarning("%s, %s: %s"%(
self.getSourcePosition(),
procId,
utils.fixIndentation(self.procDef.deprecated, "")))
self._validateNext(ProcApp)
self._ensureParsBound()
def completeElement(self, ctx):
self._completeElementNext(ProcApp, ctx)
if self.name is base.NotGiven: # make up a name from self's id
self.name = ("proc%x"%id(self)).replace("-", "")
@utils.memoized
def getSetupPars(self):
"""returns the setup parameters for the proc app, where procDef
parameters may be overridden by self's parameters.
"""
allSetups = []
if self.procDef is not base.NotGiven:
allSetups.extend(self.procDef.setups)
allSetups.extend(self.setups)
return unionByKey(*[s.pars for s in allSetups])
def _ensureParsBound(self):
"""raises an error if non-defaulted pars of procDef are not filled
by the bindings.
"""
bindNames = set(b.key for b in self.bindings)
for p in self.getSetupPars():
if not p.isDefaulted():
if not p.key in bindNames:
raise base.StructureError("Parameter %s is not defaulted in"
" %s and thus must be bound."%(p.key, self.name))
if p.key in bindNames:
bindNames.remove(p.key)
if bindNames:
raise base.StructureError("May not bind non-existing parameter(s)"
" %s."%(", ".join(bindNames)))
def onElementComplete(self):
self._onElementCompleteNext(ProcApp)
self._boundNames = dict((b.key, b.content_) for b in self.bindings)
def _combineWithProcDef(self, methodName, boundNames):
# A slightly tricky helper method for the implementation of get*SetupCode:
# this combines the results of calling methodName on a procDef
# (where applicable) with calling it on ProcDef for self.
parts = []
if self.procDef is not base.NotGiven:
parts.append(getattr(self.procDef, methodName)(boundNames))
parts.append(getattr(ProcDef, methodName)(self, boundNames))
return "\n".join(parts)
def getLateSetupCode(self, boundNames):
return self._combineWithProcDef("getLateSetupCode", boundNames)
def getParSetupCode(self, boundNames):
return self._combineWithProcDef("getParSetupCode", boundNames)
def getBodySetupCode(self, boundNames):
return self._combineWithProcDef("getBodySetupCode", boundNames)
def getSetupCode(self):
code = "\n".join((
self.getParSetupCode(self._boundNames),
self.getBodySetupCode(self._boundNames)))
if "\\" in code:
code = self.parent.expand(code)
return code
def _getFunctionDefinition(self, mainSource):
"""returns mainSource in a function definition with proper
signature including setup of late code.
"""
parts = [self.getLateSetupCode(self._boundNames)]
parts.append(mainSource)
body = "\n".join(parts)
if not body.strip():
body = " pass"
return "def %s(%s):\n%s"%(self.name, self.formalArgs,
body)
def getFuncCode(self):
"""returns a function definition for this proc application.
This includes bindings of late parameters.
Locally defined code overrides code defined in a procDef.
"""
mainCode = ""
if self.code is base.NotGiven:
if self.procDef is not base.NotGiven:
mainCode = self.procDef.getCode()
else:
mainCode = self.getCode()
code = self._getFunctionDefinition(mainCode)
if "\\" in code:
code = self.parent.expand(code)
return code
def _compileForParent(self, parent):
"""helps compile.
"""
# go get the RD for parent; it's always handy in this kind
# of code
curEl = parent
while not hasattr(curEl, "rd"):
if curEl.parent:
curEl = curEl.parent
else:
break
try:
rd = curEl.rd
except AttributeError:
# maybe an unrooted element
rd = None
return rmkfuncs.makeProc(
self.name, self.getFuncCode(),
self.getSetupCode(), parent,
rd=rd,
**self.additionalNamesForProcs)
def breakCircles(self):
# overridden to undo additional memoization
ProcDef.breakCircles(self)
utils.forgetMemoized(self)
def compile(self, parent=None):
"""returns a callable for this procedure application.
You can pass a different parent; it will then be used to
expand macros. If you do not give it, the embedding structure will
be used.
"""
if parent is None:
parent = self.parent
return utils.memoizeOn(parent, self, self._compileForParent, parent)
class ForeignKey(base.Structure):
"""A description of a foreign key relation between this table and another
one.
"""
name_ = "foreignKey"
_inTable = base.ReferenceAttribute(
"inTable",
default=base.Undefined,
description="Reference to the table the foreign key points to.",
copyable=True)
_source = base.UnicodeAttribute(
"source",
default=base.Undefined,
description="Comma-separated list of local columns corresponding"
" to the foreign key. No sanity checks are performed here.",
copyable=True)
_dest = base.UnicodeAttribute(
"dest",
default=base.NotGiven,
description="Comma-separated list of columns in the target table"
" belonging to its key. No checks for their existence, uniqueness,"
" etc. are done here. If not given, defaults to source.")
_metaOnly = base.BooleanAttribute(
"metaOnly",
default=False,
description="Do not tell the database to actually create the foreign"
" key, just declare it in the metadata. This is for when you want"
" to document a relationship but don't want the DB to actually"
" enforce this. This is typically a wise thing to do when you have, say"
" a gigarecord of flux/density pairs and only several thousand metadata"
" records -- you may want to update the latter without having"
" to tear down the former.")
def getDescription(self):
return "%s:%s -> %s:%s" % (self.parent.getQName(), ",".join(
self.source), self.destTableName, ".".join(self.dest))
def _parseList(self, raw):
if isinstance(raw, list):
# we're being copied
return raw
return [s.strip() for s in raw.split(",") if s.strip()]
def onElementComplete(self):
self.destTableName = self.inTable.getQName()
self.isADQLKey = self.inTable.adql and self.inTable.adql != 'hidden'
self.source = self._parseList(self.source)
if self.dest is base.NotGiven:
self.dest = self.source
else:
self.dest = self._parseList(self.dest)
self._onElementCompleteNext(ForeignKey)
def create(self, querier):
if self.metaOnly:
return
if not querier.foreignKeyExists(self.parent.getQName(),
self.destTableName, self.source,
self.dest):
return querier.query(
"ALTER TABLE %s ADD FOREIGN KEY (%s)"
" REFERENCES %s (%s)"
" ON DELETE CASCADE"
" DEFERRABLE INITIALLY DEFERRED" %
(self.parent.getQName(), ",".join(
self.source), self.destTableName, ",".join(self.dest)))
def delete(self, querier):
if self.metaOnly:
return
try:
constraintName = querier.getForeignKeyName(self.parent.getQName(),
self.destTableName,
self.source, self.dest)
except base.DBError: # key does not exist.
return
querier.query("ALTER TABLE %s DROP CONSTRAINT %s" %
(self.parent.getQName(), constraintName))
def getAnnotation(self, roleName, container):
"""returns a dm annotation for this foreign key.
"""
return dm.ForeignKeyAnnotation(roleName, self)
class CondDesc(base.Structure):
"""A query specification for cores talking to the database.
CondDescs define inputs as a sequence of InputKeys (see `Element InputKey`_).
Internally, the values in the InputKeys can be translated to SQL.
"""
name_ = "condDesc"
_inputKeys = rscdef.ColumnListAttribute("inputKeys",
childFactory=inputdef.InputKey,
description="One or more InputKeys defining the condition's input.",
copyable=True)
_silent = base.BooleanAttribute("silent",
default=False,
description="Do not produce SQL from this CondDesc. This"
" can be used to convey meta information to the core. However,"
" in general, a service is a more appropriate place to deal with"
" such information, and thus you should prefer service InputKeys"
" to silent CondDescs.",
copyable=True)
_required = base.BooleanAttribute("required",
default=False,
description="Reject queries not filling the InputKeys of this CondDesc",
copyable=True)
_fixedSQL = base.UnicodeAttribute("fixedSQL",
default=None,
description="Always insert this SQL statement into the query. Deprecated.",
copyable=True)
_buildFrom = base.ReferenceAttribute("buildFrom",
description="A reference to a column or an InputKey to define"
" this CondDesc",
default=None)
_phraseMaker = base.StructAttribute("phraseMaker",
default=None,
description="Code to generate custom SQL from the input keys",
childFactory=PhraseMaker,
copyable=True)
_combining = base.BooleanAttribute("combining",
default=False,
description="Allow some input keys to be missing when others are given?"
" (you want this for pseudo-condDescs just collecting random input"
" keys)", # (and I wish I had a better idea)
copyable="True")
_group = base.StructAttribute("group",
default=None,
childFactory=rscdef.Group,
description="Group child input keys in the input table (primarily"
" interesting for web forms, where this grouping is shown graphically;"
" Set the style property to compact to have a one-line group there)")
_joiner = base.UnicodeAttribute("joiner",
default="OR",
description="When yielding multiple fragments, join them"
" using this operator (probably the only thing besides OR is"
" AND).",
copyable=True)
_original = base.OriginalAttribute()
def __init__(self, parent, **kwargs):
base.Structure.__init__(self, parent, **kwargs)
# copy parent's resolveName if present for buildFrom resolution
if hasattr(self.parent, "resolveName"):
self.resolveName = self.parent.resolveName
def __repr__(self):
return "<CondDesc %s>"%",".join(ik.name for ik in self.inputKeys)
@classmethod
def fromInputKey(cls, ik, **kwargs):
return base.makeStruct(CondDesc, inputKeys=[ik], **kwargs)
@classmethod
def fromColumn(cls, col, **kwargs):
return base.makeStruct(cls, buildFrom=col, **kwargs)
@property
def name(self):
"""returns some key for uniqueness of condDescs.
"""
# This is necessary for ColumnLists that are used
# for CondDescs as well. Ideally, we'd do this on an
# InputKeys basis and yield their names (because that's what
# formal counts on), but it's probably not worth the effort.
return "+".join([f.name for f in self.inputKeys])
def completeElement(self, ctx):
if self.buildFrom and not self.inputKeys:
# use the column as input key; special renderers may want
# to do type mapping, but the default is to have plain input
self.inputKeys = [inputdef.InputKey.fromColumn(self.buildFrom)]
self._completeElementNext(CondDesc, ctx)
def expand(self, *args, **kwargs):
"""hands macro expansion requests (from phraseMakers) upwards.
This is to the queried table if the parent has one (i.e., we're
part of a core), or to the RD if not (i.e., we're defined within
an rd).
"""
if hasattr(self.parent, "queriedTable"):
return self.parent.queriedTable.expand(*args, **kwargs)
else:
return self.parent.rd.expand(*args, **kwargs)
def _makePhraseDefault(self, ignored, inputKeys, inPars, outPars, core):
# the default phrase maker uses whatever the individual input keys
# come up with.
for ik in self.inputKeys:
yield base.getSQLForField(ik, inPars, outPars)
# We only want to compile the phraseMaker if actually necessary.
# condDescs may be defined within resource descriptors (e.g., in
# scs.rd), and they can't be compiled there (since macros may
# be missing); thus, we dispatch on the first call.
def _getPhraseMaker(self):
try:
return self.__compiledPhraseMaker
except AttributeError:
if self.phraseMaker is not None:
val = self.phraseMaker.compile()
else:
val = self._makePhraseDefault
self.__compiledPhraseMaker = val
return self.__compiledPhraseMaker
makePhrase = property(_getPhraseMaker)
def _isActive(self, inPars):
"""returns True if the dict inPars contains input to all our input keys.
"""
for f in self.inputKeys:
if f.name not in inPars:
return False
return True
def inputReceived(self, inPars, queryMeta):
"""returns True if all inputKeys can be filled from inPars.
As a side effect, inPars will receive defaults form the input keys
if there are any.
"""
if not self._isActive(inPars):
return False
keysFound, keysMissing = [], []
for f in self.inputKeys:
if inPars.get(f.name) is None:
keysMissing.append(f)
else:
if f.value!=inPars.get(f.name): # non-defaulted
keysFound.append(f)
if not keysMissing:
return True
# keys are missing. That's ok if none were found and we're not required
if not self.required and not keysFound:
return False
if self.required:
raise base.ValidationError("is mandatory but was not provided.",
colName=keysMissing[0].name)
# we're optional, but a value was given and others are missing
if not self.combining:
raise base.ValidationError("When you give a value for %s,"
" you must give value(s) for %s, too"%(keysFound[0].getLabel(),
", ".join(k.name for k in keysMissing)),
colName=keysMissing[0].name)
return True
def asSQL(self, inPars, sqlPars, queryMeta):
if self.silent or not self.inputReceived(inPars, queryMeta):
return ""
res = list(self.makePhrase(
self, self.inputKeys, inPars, sqlPars, self.parent))
sql = base.joinOperatorExpr(self.joiner, res)
if self.fixedSQL:
sql = base.joinOperatorExpr(self.joiner, [sql, self.fixedSQL])
return sql
def adaptForRenderer(self, renderer):
"""returns a changed version of self if renderer suggests such a
change.
This only happens if buildFrom is non-None. The method must
return a "defused" version that has buildFrom None (or self,
which will do because core.adaptForRenderer stops adapting if
the condDescs are stable).
The adaptors may also raise a Replace exception and return a
full CondDesc; this is done, e.g., for spoints for the form
renderer, since they need two input keys and a completely modified
phrase.
"""
if not self.buildFrom:
return self
adaptor = inputdef.getRendererAdaptor(renderer)
if adaptor is None:
return self
try:
newInputKeys = []
for ik in self.inputKeys:
newInputKeys.append(adaptor(ik))
if self.inputKeys==newInputKeys:
return self
else:
return self.change(inputKeys=newInputKeys, buildFrom=None)
except base.Replace, ex:
return ex.newOb
class TableBasedCore(core.Core):
"""A core knowing a DB table it operates on and allowing the definition
of condDescs.
"""
_queriedTable = base.ReferenceAttribute("queriedTable",
default=base.Undefined, description="A reference to the table"
" this core queries.", copyable=True, callbacks=["_fixNamePath"])
_condDescs = base.StructListAttribute("condDescs", childFactory=CondDesc,
description="Descriptions of the SQL and input generating entities"
" for this core; if not given, they will be generated from the"
" table columns.", copyable=True)
_namePath = rscdef.NamePathAttribute(description="Id of an element"
" that will be used to located names in id references. Defaults"
" to the queriedTable's id.")
def _buildInputTableFromCondDescs(self):
groups, iks = [], []
for cd in self.condDescs:
for ik in cd.inputKeys:
iks.append(ik)
if cd.group:
groups.append(cd.group.change(
paramRefs=[MS(rscdef.ParameterReference, dest=ik.name)
for ik in cd.inputKeys],
parent_=None))
self.inputTable = MS(inputdef.InputTable,
params=iks,
groups=groups)
def completeElement(self, ctx):
# if no condDescs have been given, make them up from the table columns.
if not self.condDescs and self.queriedTable:
self.condDescs = [self.adopt(CondDesc.fromColumn(c))
for c in self.queriedTable]
# if an inputTable is given, trust it fits the condDescs, else
# build the input table
if self.inputTable is base.NotGiven:
self._buildInputTableFromCondDescs()
# if no outputTable has been given, make it up from the columns
# of the queried table unless a prototype is defined (which is
# handled by core itself).
if (self.outputTableXML is None
and self.outputTable is base.NotGiven
and self.queriedTable):
self.outputTable = outputdef.OutputTableDef.fromTableDef(
self.queriedTable, ctx)
self._completeElementNext(TableBasedCore, ctx)
def _fixNamePath(self, qTable):
# callback from queriedTable to make it the default for namePath as well
if self.namePath is None:
self.namePath = qTable.getFullId()
def _getSQLWhere(self, inputTable, queryMeta):
"""returns a where fragment and the appropriate parameters
for the query defined by inputTable and queryMeta.
"""
sqlPars = {}
inputPars = dict((p.name, p.value) for p in inputTable.iterParams())
return base.joinOperatorExpr("AND",
[cd.asSQL(inputPars, sqlPars, queryMeta)
for cd in self.condDescs]), sqlPars
def _makeTable(self, rowIter, resultTableDef, queryMeta):
"""returns a table from the row iterator rowIter, updating queryMeta
as necessary.
"""
rows = list(rowIter)
isOverflowed = len(rows)>queryMeta.get("dbLimit", 1e10)
if isOverflowed:
del rows[-1]
queryMeta["Matched"] = len(rows)
res = rsc.TableForDef(resultTableDef, rows=rows)
if isOverflowed:
queryMeta["Overflow"] = True
res.addMeta("_warning", "The query limit was reached. Increase it"
" to retrieve more matches. Note that unsorted truncated queries"
" are not reproducible (i.e., might return a different result set"
" at a later time).")
res.addMeta("_queryStatus", "Overflowed")
else:
res.addMeta("_queryStatus", "Ok")
return res
def adaptForRenderer(self, renderer):
"""returns a core tailored to renderer renderers.
This mainly means asking the condDescs to build themselves for
a certain renderer. If no polymorphuous condDescs are there,
self is returned.
"""
newCondDescs = []
for cd in self.condDescs:
newCondDescs.append(cd.adaptForRenderer(renderer))
if newCondDescs!=self.condDescs:
return self.change(condDescs=newCondDescs, inputTable=base.NotGiven
).adaptForRenderer(renderer)
else:
return core.Core.adaptForRenderer(self, renderer)
class UploadCore(core.Core):
"""A core handling uploads of files to the database.
It allows users to upload individual files into a special staging
area (taken from the stagingDir property of the destination data descriptor)
and causes these files to be parsed using destDD.
You can tell UploadCores to either insert or update the incoming data using
the "mode" input key.
"""
name_ = "uploadCore"
_destDD = base.ReferenceAttribute(
"destDD",
default=base.Undefined,
description="Reference to the data we are uploading into.")
inputTableXML = """
<inputTable id="inFields">
<inputKey name="File" type="file" required="True"
tablehead="Source to upload"/>
<inputKey name="Mode" type="text" tablehead="Upload mode"
required="True" multiplicity="forced-single">
<values default="i">
<option title="Insert">i</option>
<option title="Update">u</option>
</values>
</inputKey>
</inputTable>
"""
outputTableXML = uploadOutputDef
def _fixPermissions(self, fName):
"""tries to chmod the newly created file to 0664 and change the group
to config.gavoGroup.
"""
try:
os.chmod(fName, 0664)
os.chown(fName, -1, grp.getgrnam(base.getConfig("gavoGroup"))[2])
except (KeyError, os.error): # let someone else worry about it
pass
def _writeFile(self, srcFile, fName):
"""writes the contents of srcFile to fName in destDD's staging dir.
"""
try:
targetDir = os.path.join(self.rd.resdir,
self.destDD.getProperty("stagingDir"))
except KeyError:
raise base.ui.logOldExc(
base.ValidationError(
"Uploading is only"
" supported for data having a staging directory.", "File"))
if not os.path.exists(targetDir):
raise base.ValidationError("Staging directory does not exist.",
"File")
targetFName = fName.split("/")[-1].encode("iso-8859-1")
if not targetFName:
raise base.ValidationError("Bad file name", "File")
targetPath = os.path.join(targetDir, targetFName)
with open(targetPath, "w") as f:
f.write(srcFile.read())
try:
self._fixPermissions(targetPath)
except os.error:
# Nothing we can do, and it may not even hurt
pass
return targetPath
def _importData(self, sourcePath, mode):
"""parses the input file at sourcePath and writes the result to the DB.
"""
base.ui.notifyInfo("Web upload ingesting %s in %s mode" %
(sourcePath, mode))
try:
parseOptions = rsc.getParseOptions(validateRows=True,
updateMode=True,
doTableUpdates=mode == "u")
with base.getWritableAdminConn() as conn:
res = rsc.makeData(self.destDD,
parseOptions=parseOptions,
forceSource=sourcePath,
connection=conn)
except Exception, msg:
raise base.ui.logOldExc(
base.ValidationError(
"Cannot enter %s in"
" database: %s" % (os.path.basename(sourcePath), str(msg)),
"File"))
return res.nAffected
