class BinaryGrammar(Grammar):
"""A grammar that builds rowdicts from binary data.
The grammar expects the input to be in fixed-length records.
the actual specification of the fields is done via a binaryRecordDef
element.
"""
name_ = "binaryGrammar"
rowIterator = BinaryRowIterator
_til = base.IntAttribute("skipBytes",
default=0,
description="Number of bytes to skip before parsing records.")
_fdefs = base.StructAttribute("fieldDefs",
description="Definition of the record.",
childFactory=BinaryRecordDef)
_armoring = base.EnumeratedUnicodeAttribute("armor",
default=None,
validValues=["fortran"],
description="Record armoring; by default it's None meaning the"
" data was dumped to the file sequentially. Set it to fortran"
" for fortran unformatted files (4 byte length before and after"
" the payload).")
class REGrammar(Grammar, FileRowAttributes):
"""A grammar that builds rowdicts from records and fields specified
via REs separating them.
There is also a simple facility for "cleaning up" records. This can be
used to remove standard shell-like comments; use
``recordCleaner="(?:#.*)?(.*)"``.
"""
name_ = "reGrammar"
rowIterator = REIterator
_til = base.IntAttribute(
"topIgnoredLines",
default=0,
description="Skip this many lines at the top of each source file.",
copyable=True)
_stopPat = REAttribute(
"stopPat",
default=None,
description="Stop parsing when a record *matches* this RE (this"
" is for skipping non-data footers",
copyable=True)
_recordSep = REAttribute(
"recordSep",
default=re.compile("\n"),
description="RE for separating two records in the source.",
copyable=True)
_fieldSep = REAttribute(
"fieldSep",
default=re.compile(r"\s+"),
description="RE for separating two fields in a record.",
copyable=True)
_commentPat = REAttribute(
"commentPat",
default=None,
description="RE inter-record material to be ignored (note: make this"
" match the entire comment, or you'll get random mess from partly-matched"
" comments. Use '(?m)^#.*$' for beginning-of-line hash-comments.",
copyable=True)
_recordCleaner = REAttribute(
"recordCleaner",
default=None,
description="A regular expression matched against each record."
" The matched groups in this RE are joined by blanks and used"
" as the new pattern. This can be used for simple cleaning jobs;"
" However, records not matching recordCleaner are rejected.",
copyable=True)
_names = base.StringListAttribute(
"names",
description="Names for the parsed fields, in matching sequence. You can"
r" use macros here, e.g., \\colNames{someTable}.",
expand=True,
copyable=True)
_lax = base.BooleanAttribute(
"lax",
description="allow more or less"
" fields in source records than there are names",
default=False,
copyable=True)
class FITSTableGrammar(common.Grammar):
"""A grammar parsing from FITS tables.
fitsTableGrammar result in typed records, i.e., values normally come
in the types they are supposed to have. Of course, that won't work
for datetimes, STC-S regions, and the like.
The keys of the result dictionaries are simpily the names given in
the FITS.
"""
name_ = "fitsTableGrammar"
_hduIndex = base.IntAttribute(
"hdu",
default=1,
description="Take the data from this extension (primary=0)."
" Tabular data typically resides in the first extension.")
rowIterator = FITSTableIterator
class CSVGrammar(Grammar, FileRowAttributes):
"""A grammar that uses python's csv module to parse files.
Note that these grammars by default interpret the first line of
the input file as the column names. When your files don't follow
that convention, you *must* give names (as in ``names='raj2000,
dej2000, magV'``), or you'll lose the first line and have silly
column names.
CSVGrammars currently do not support non-ASCII inputs.
Contact the authors if you need that.
"""
name_ = "csvGrammar"
_delimiter = base.UnicodeAttribute("delimiter",
description="CSV delimiter",
default=",",
copyable=True)
_names = base.StringListAttribute(
"names",
default=None,
description="Names for the parsed fields, in sequence of the"
" comma separated values. The default is to read the field names"
" from the first line of the csv file. You can use macros here,"
r" e.g., \\colNames{someTable}.",
expand=True,
copyable=True)
_strip = base.BooleanAttribute(
"strip",
default=False,
description="If True, whitespace immediately following a delimiter"
" is ignored.",
copyable=True)
_til = base.IntAttribute(
"topIgnoredLines",
default=0,
description="Skip this many lines at the top of each source file.")
rowIterator = CSVIterator
class InputKey(column.ParamBase):
"""A description of a piece of input.
Think of inputKeys as abstractions for input fields in forms, though
they are used for services not actually exposing HTML forms as well.
Some of the DDL-type attributes (e.g., references) only make sense here
if columns are being defined from the InputKey.
You can give a "defaultForForm" property on inputKeys to supply
a string literal default that will be pre-filled in the form
renderer and is friends but not for other renderers (like S*AP).
Properties evaluated:
* defaultForForm -- a value entered into form fields by default
(be stingy with those; while it's nice to not have to set things
presumably right for almost everyone, having to delete stuff
you don't want over and over is really annoying).
* adaptToRenderer -- a true boolean literal here causes the param
to be adapted for the renderer (e.g., float could become vizierexpr-float).
You'll usually not want this, because the expressions are
generally evaluated by the database, and the condDescs do the
adaptation themselves. This is mainly for rare situations like
file uploads in custom cores.
"""
name_ = "inputKey"
# XXX TODO: make widgetFactory and showItems properties.
_widgetFactory = base.UnicodeAttribute(
"widgetFactory",
default=None,
description="A python expression for a custom widget"
" factory for this input,"
" e.g., 'Hidden' or 'widgetFactory(TextArea, rows=15, cols=30)'",
copyable=True)
_showItems = base.IntAttribute(
"showItems",
default=3,
description="Number of items to show at one time on selection widgets.",
copyable=True)
_inputUnit = base.UnicodeAttribute(
"inputUnit",
default=None,
description="Override unit of the table column with this.",
copyable=True)
_std = base.BooleanAttribute(
"std",
default=False,
description="Is this input key part of a standard interface for"
" registry purposes?",
copyable=True)
_multiplicity = base.UnicodeAttribute(
"multiplicity",
default=None,
copyable=True,
description="Set"
" this to single to have an atomic value (chosen at random"
" if multiple input values are given),"
" forced-single to have an atomic value"
" and raise an exception if multiple values come in, or"
" multiple to receive lists. On the form renderer, this is"
" ignored, and the values are what nevow formal passes in."
" If not given, it is single unless there is a values element with"
" options, in which case it's multiple.")
# Don't validate meta for these -- while they are children
# of validated structures (services), they don't need any
# meta at all. This should go as soon as we have a sane
# inheritance hierarchy for tables.
metaModel = None
def completeElement(self, ctx):
self._completeElementNext(InputKey, ctx)
if self.restrictedMode and self.widgetFactory:
raise base.RestrictedElement("widgetFactory")
def onElementComplete(self):
self._onElementCompleteNext(InputKey)
# compute scaling if an input unit is given
self.scaling = None
if self.inputUnit:
self.scaling = base.computeConversionFactor(
self.inputUnit, self.unit)
if self.multiplicity is None:
self.multiplicity = "single"
if self.isEnumerated():
# these almost always want lists returned.
self.multiplicity = "multiple"
def onParentComplete(self):
if self.parent and hasattr(self.parent, "required"):
# children of condDescs inherit their requiredness
# (unless defaulted)
self.required = self.parent.required
# but if there's a default, never require an input
if self.value:
self.required = False
def validateValue(self, literal):
"""raises a ValidationError if literal cannot be deserialised into
an acceptable value for self.
"""
self._parse(literal)
def _getVOTableType(self):
"""returns the VOTable type for the param.
The reason this is overridden is that historically, we've been
cavalier about letting in multiple values for a single param
(as in enumerated values and such).
It's probably too late to fix this now, so for InputKeys with
multiplicity multiple we're allowing arrays, too.
"""
type, arraysize, xtype = column.ParamBase._getVOTableType(self)
if self.multiplicity == "multiple" and arraysize == '1':
arraysize = "*"
return type, arraysize, xtype
@classmethod
def fromColumn(cls, column, **kwargs):
"""returns an InputKey for query input to column.
"""
if isinstance(column, InputKey):
if kwargs:
return column.change(**kwargs)
else:
return column
instance = cls(None)
instance.feedObject("original", column)
if column.isEnumerated():
instance.feedObject("multiplicity", "multiple")
for k, v in kwargs.iteritems():
instance.feed(k, v)
if not "required" in kwargs:
instance.feedObject("required", False)
return instance.finishElement(None)
class DirectGrammar(base.Structure, base.RestrictionMixin):
"""A user-defined external grammar.
See the separate document on user-defined code on more on direct grammars.
Also note the program gavomkboost that can help you generate core for
the C boosters used by direct grammars.
"""
name_ = "directGrammar"
_cbooster = rscdef.ResdirRelativeAttribute(
"cBooster",
default=base.Undefined,
description="resdir-relative path to the booster C source.",
copyable=True)
_gzippedInput = base.BooleanAttribute(
"gzippedInput",
default=False,
description="Pipe gzip before booster? (will not work for FITS)",
copyable=True)
_autoNull = base.UnicodeAttribute(
"autoNull",
default=None,
description="Use this string as general NULL value (when reading"
" from plain text).",
copyable=True)
_ignoreBadRecords = base.BooleanAttribute(
"ignoreBadRecords",
default=False,
description="Let booster ignore invalid records?",
copyable=True)
_recordSize = base.IntAttribute(
"recordSize",
default=4000,
description="For bin boosters, read this many bytes to make"
" up a record; for line-based boosters, this is the maximum"
" length of an input line.",
copyable=True)
_preFilter = base.UnicodeAttribute(
"preFilter",
default=None,
description="Pipe input through this program before handing it to"
" the booster; this string is shell-expanded (will not work for FITS).",
copyable=True)
_customFlags = base.UnicodeAttribute(
"customFlags",
default="",
description="Pass these flags to the C compiler when building the"
" booster.",
copyable=True)
_type = base.EnumeratedUnicodeAttribute(
"type",
default="col",
validValues=["col", "bin", "fits", "split"],
description="Make code for a booster parsing by column indices (col),"
" by splitting along separators (split), by reading fixed-length"
" binary records (bin), for from FITS binary tables (fits).",
copyable=True)
_splitChar = base.UnicodeAttribute(
"splitChar",
default="|",
description="For split boosters, use this as the separator.",
copyable=True)
_ext = base.IntAttribute(
"extension",
default=1,
description=
"For FITS table boosters, get the table from this extension.",
copyable=True)
_mapKeys = base.StructAttribute(
"mapKeys",
childFactory=common.MapKeys,
default=None,
copyable=True,
description="For a FITS booster, map DB table column names"
" to FITS column names (e.g., if the FITS table name flx is to"
" end up in the DB column flux, say flux:flx).")
_rd = rscdef.RDAttribute()
isDispatching = False
def validate(self):
self._validateNext(DirectGrammar)
if self.type == 'bin':
if not self.recordSize:
raise base.StructureError(
"DirectGrammars reading from binary need"
" a recordSize attribute")
if self.mapKeys is not None:
if self.type != "fits":
raise base.StructureError("mapKeys is only allowed for FITS"
" boosters.")
def onElementComplete(self):
if self.type == "fits":
if self.mapKeys:
self.keyMap = self.mapKeys.maps
else:
self.keyMap = {}
def getBooster(self):
return CBooster(self.cBooster,
self.parent,
gzippedInput=self.gzippedInput,
preFilter=self.preFilter,
autoNull=self.autoNull,
ignoreBadRecords=self.ignoreBadRecords,
customFlags=self.customFlags)
def parse(self, sourceToken, targetData=None):
booster = self.getBooster()
makes = self.parent.makes
if len(makes) != 1:
raise base.StructureError(
"Directgrammar only works for data having"
" exactly one table, but data '%s' has %d" %
(self.parent.id, len(makes)))
def copyIn(data):
data.tables.values()[0].copyIn(booster.getOutput(sourceToken))
if booster.getStatus():
raise base.SourceParseError("Booster returned error signature",
source=sourceToken)
return copyIn
class ColumnGrammar(Grammar, FileRowAttributes):
"""A grammar that builds rowdicts out of character index ranges.
This works by using the colRanges attribute like <col key="mag">12-16</col>,
which will take the characters 12 through 16 inclusive from each input
line to build the input column mag.
As a shortcut, you can also use the colDefs attribute; it contains
a string of the form {<key>:<range>}, i.e.,
a whitespace-separated list of colon-separated items of key and range
as accepted by cols, e.g.::
<colDefs>
a: 3-4
_u: 7
</colDefs>
"""
name_ = "columnGrammar"
_til = base.IntAttribute(
"topIgnoredLines",
default=0,
description="Skip this many lines at the top of each source file.",
copyable=True)
_cols = base.DictAttribute("colRanges",
description="Mapping of"
" source keys to column ranges.",
itemAttD=ColRangeAttribute("col"),
copyable=True)
_colDefs = base.ActionAttribute("colDefs",
description="Shortcut"
" way of defining cols",
methodName="_parseColDefs")
_commentIntroducer = base.UnicodeAttribute(
"commentIntroducer",
default=base.NotGiven,
description="A character sequence"
" that, when found at the beginning of a line makes this line"
" ignored",
copyable=True)
def _getColDefGrammar(self):
with utils.pyparsingWhitechars("\n\t\r "):
intLiteral = pyparsing.Word(pyparsing.nums)
# need to manually swallow whitespace after literals
blindWhite = pyparsing.Suppress(
pyparsing.Optional(pyparsing.White()))
dash = blindWhite + pyparsing.Literal("-") + blindWhite
range = pyparsing.Combine(
dash + blindWhite + intLiteral
| intLiteral +
pyparsing.Optional(dash + pyparsing.Optional(intLiteral)))
range.setName("Column range")
identifier = pyparsing.Regex(utils.identifierPattern.pattern[:-1])
identifier.setName("Column key")
clause = (identifier + pyparsing.Literal(":") + blindWhite +
range).addParseAction(lambda s, p, t: (t[0], t[2]))
colDefs = pyparsing.ZeroOrMore(clause) + pyparsing.StringEnd()
# range.setDebug(True);identifier.setDebug(True);clause.setDebug(True)
return colDefs
def _parseColDefs(self, ctx):
# the handler for colDefs -- parse shortcut colDefs
try:
for key, range in utils.pyparseString(self._getColDefGrammar(),
self.colDefs):
self.colRanges[key] = self._cols.itemAttD.parse(range)
except pyparsing.ParseException, ex:
raise base.LiteralParseError(
"colDefs",
self.colDefs,
hint="colDefs is a whitespace-separated list of key:range pairs."
" Your literal doesn't look like this, and here's what the"
" parser had to complain: %s" % ex)
class ColumnBase(base.Structure, base.MetaMixin):
"""A base class for columns, parameters, output fields, etc.
Actually, right now there's far too much cruft in here that
should go into Column proper or still somewhere else. Hence:
XXX TODO: Refactor.
See also Column for a docstring that still applies to all we've in
here.
"""
_name = ParamNameAttribute("name",
default=base.Undefined,
description="Name of the param",
copyable=True,
before="type")
_type = TypeNameAttribute(
"type",
default="real",
description="datatype for the column (SQL-like type system)",
copyable=True,
before="unit")
_unit = base.UnicodeAttribute("unit",
default="",
description="Unit of the values",
copyable=True,
before="ucd",
strip=True)
_ucd = base.UnicodeAttribute("ucd",
default="",
description="UCD of the column",
copyable=True,
before="description")
_description = base.NWUnicodeAttribute(
"description",
default="",
copyable=True,
description=
"A short (one-line) description of the values in this column.")
_tablehead = base.UnicodeAttribute(
"tablehead",
default=None,
description="Terse phrase to put into table headers for this"
" column",
copyable=True)
_utype = base.UnicodeAttribute("utype",
default=None,
description="utype for this column",
copyable=True)
_required = base.BooleanAttribute(
"required",
default=False,
description="Record becomes invalid when this column is NULL",
copyable=True)
_displayHint = DisplayHintAttribute(
"displayHint",
description="Suggested presentation; the format is "
" <kw>=<value>{,<kw>=<value>}, where what is interpreted depends"
" on the output format. See, e.g., documentation on HTML renderers"
" and the formatter child of outputFields.",
copyable=True)
_verbLevel = base.IntAttribute(
"verbLevel",
default=20,
description="Minimal verbosity level at which to include this column",
copyable=True)
_values = base.StructAttribute("values",
default=None,
childFactory=Values,
description="Specification of legal values",
copyable=True)
_fixup = base.UnicodeAttribute(
"fixup",
description=
"A python expression the value of which will replace this column's"
" value on database reads. Write a ___ to access the original"
' value. You can use macros for the embedding table.'
' This is for, e.g., simple URL generation'
' (fixup="\'\\internallink{/this/svc}\'+___").'
' It will *only* kick in when tuples are deserialized from the'
" database, i.e., *not* for values taken from tables in memory.",
default=None,
copyable=True)
_note = base.UnicodeAttribute(
"note",
description="Reference to a note meta"
" on this table explaining more about this column",
default=None,
copyable=True)
_xtype = base.UnicodeAttribute("xtype",
description="VOTable xtype giving"
" the serialization form",
default=None,
copyable=True)
_stc = TableManagedAttribute(
"stc",
description="Internally used"
" STC information for this column (do not assign to unless instructed"
" to do so)",
default=None,
copyable=True)
_stcUtype = TableManagedAttribute(
"stcUtype",
description="Internally used"
" STC information for this column (do not assign to)",
default=None,
copyable=True)
_properties = base.PropertyAttribute(copyable=True)
_original = base.OriginalAttribute()
restrictedMode = False
def __repr__(self):
return "<Column %s>" % repr(self.name)
def setMetaParent(self, parent):
# columns should *not* take part in meta inheritance. The reason is
# that there are usually many columns to a table, and there's no
# way I can see that any piece of metadata should be repeated in
# all of them. On the other hand, for votlinks (no name an example),
# meta inheritance would have disastrous consequences.
# So, we bend the rules a bit.
raise base.StructureError(
"Columns may not have meta parents.",
hint="The rationale for this is explained in rscdef/column.py,"
" look for setMetaParent.")
def onParentComplete(self):
# we need to resolve note on construction since columns are routinely
# copied to other tables and meta info does not necessarily follow.
if isinstance(self.note, basestring):
try:
self.note = self.parent.getNote(self.note)
except base.NotFoundError: # non-existing notes silently ignored
self.note = None
def completeElement(self, ctx):
self.restrictedMode = getattr(ctx, "restricted", False)
if isinstance(self.name, utils.QuotedName):
self.key = self.name.name
if ')' in self.key:
# No '()' allowed in key for that breaks the %()s syntax (sigh!).
# Work around with the following quick hack that would break
# if people carefully chose proper names. Anyone using delim.
# ids in SQL deserves a good spanking anyway.
self.key = self.key.replace(')', "__").replace('(', "__")
else:
self.key = self.name
self._completeElementNext(ColumnBase, ctx)
def isEnumerated(self):
return self.values and self.values.options
def validate(self):
self._validateNext(ColumnBase)
if self.restrictedMode and self.fixup:
raise base.RestrictedElement("fixup")
def validateValue(self, value):
"""raises a ValidationError if value does not match the constraints
given here.
"""
if value is None:
if self.required:
raise base.ValidationError(
"Field %s is empty but non-optional" % self.name,
self.name)
return
# Only validate these if we're not a database column
if not isinstance(self, Column):
vals = self.values
if vals:
if vals.options:
if value and not vals.validateOptions(value):
raise base.ValidationError(
"Value %s not consistent with"
" legal values %s" % (value, vals.options),
self.name)
else:
if vals.min and value < vals.min:
raise base.ValidationError(
"%s too small (must be at least %s)" %
(value, vals.min), self.name)
if vals.max and value > vals.max:
raise base.ValidationError(
"%s too large (must be less than %s)" %
(value, vals.max), self.name)
def isIndexed(self):
"""returns a guess as to whether this column is part of an index.
This may return True, False, or None (unknown).
"""
if self.parent and hasattr(self.parent, "indexedColumns"):
# parent is something like a TableDef
if self.name in self.parent.indexedColumns:
return True
else:
return False
def isPrimary(self):
"""returns a guess as to whether this column is a primary key of the
embedding table.
This may return True, False, or None (unknown).
"""
if self.parent and hasattr(self.parent, "primary"):
# parent is something like a TableDef
if self.name in self.parent.primary:
return True
else:
return False
_indexedCleartext = {
True: "indexed",
False: "notIndexed",
None: "unknown",
}
def asInfoDict(self):
"""returns a dictionary of certain, "user-interesting" properties
of the data field, in a dict of strings.
"""
return {
"name": unicode(self.name),
"description": self.description or "N/A",
"tablehead": self.getLabel(),
"unit": self.unit or "N/A",
"ucd": self.ucd or "N/A",
"verbLevel": self.verbLevel,
"indexState": self._indexedCleartext[self.isIndexed()],
"note": self.note,
}
def getDDL(self):
"""returns an SQL fragment describing this column ready for
inclusion in a DDL statement.
"""
type = self.type
# we have one "artificial" type, and it shouldn't become more than
# one; so, a simple hack should do it.
if type.upper() == "UNICODE":
type = "TEXT"
# The "str" does magic for delimited identifiers, so it's important.
items = [str(self.name), type]
if self.required:
items.append("NOT NULL")
return " ".join(items)
def getDisplayHintAsString(self):
return self._displayHint.unparse(self.displayHint)
def getLabel(self):
"""returns a short label for this column.
The label is either the tablehead or, missing it, the capitalized
column name.
"""
if self.tablehead is not None:
return self.tablehead
return str(self.name).capitalize()
def _getVOTableType(self):
"""returns the VOTable type, arraysize and xtype for this
column-like thing.
"""
type, arraysize, xtype = base.sqltypeToVOTable(self.type)
if self.type == "date":
xtype = "dachs:DATE"
return type, arraysize, xtype
class FITSProdGrammar(Grammar):
r"""A grammar that returns FITS-headers as dictionaries.
This is the grammar you want when one FITS file corresponds to one
row in the destination table.
The keywords of the grammar record are the cards in the primary
header (or some other hdu using the same-named attribute). "-" in
keywords is replaced with an underscore for easier @-referencing.
You can use a mapKeys element to effect further name cosmetics.
This grammar should handle compressed FITS images transparently if
set qnd="False". This means that you will essentially get the readers
from the second extension for those even if you left hdu="0".
The original header is preserved as the value of the header\_ key. This
is mainly intended for use WCS use, as in ``pywcs.WCS(@header_)``.
If you have more complex structures in your FITS files, you can get access
to the pyfits HDU using the hdusField attribute. With
``hdusField="_H"``, you could say things like ``@_H[1].data[10][0]``
to get the first data item in the tenth row in the second HDU.
"""
name_ = "fitsProdGrammar"
_qnd = base.BooleanAttribute(
"qnd",
default=True,
description="Use a hack to read the FITS header more quickly. This only"
" works for the primary HDU",
copyable=True)
_hduIndex = base.IntAttribute(
"hdu",
default=0,
description="Take the header from this HDU. You must say qnd='False'"
" for this to take effect.",
copyable=True)
_mapKeys = base.StructAttribute(
"mapKeys",
childFactory=MapKeys,
default=None,
copyable=True,
description="Prescription for how to"
" map header keys to grammar dictionary keys")
_hdusAttr = base.UnicodeAttribute(
"hdusField",
default=None,
description="If set, the complete pyfits HDU list for the FITS"
" file is returned in this grammar field.",
copyable=True)
_maxHeaderBlocks = base.IntAttribute(
"maxHeaderBlocks",
default=40,
copyable=True,
description="Stop looking for"
" FITS END cards and raise an error after this many blocks."
" You may need to raise this for people dumping obscene amounts"
" of data or history into headers.")
rowIterator = FITSProdIterator
def onElementComplete(self):
if self.mapKeys is None:
self.mapKeys = base.makeStruct(MapKeys)
self._onElementCompleteNext(FITSProdGrammar)
class Execute(base.Structure, base.ExpansionDelegator):
"""a container for calling code.
This is a cron-like functionality. The jobs are run in separate
threads, so they need to be thread-safe with respect to the
rest of DaCHS. DaCHS serializes calls, though, so that your
code should never run twice at the same time.
At least on CPython, you must make sure your code does not
block with the GIL held; this is still in the server process.
If you do daring things, fork off (note that you must not use
any database connections you may have after forking, which means
you can't safely use the RD passed in). See the docs on `Element job`_.
Then testing/debugging such code, use ``gavo admin execute rd#id``
to immediately run the jobs.
"""
name_ = "execute"
_title = base.UnicodeAttribute(
"title",
default=base.Undefined,
description="Some descriptive title for the job; this is used"
" in diagnostics.",
copyable=False,
)
_at = base.StringListAttribute(
"at",
description="One or more hour:minute pairs at which to run"
" the code each day. This conflicts with every. Optionally,"
" you can prefix each time by one of m<dom> or w<dow> for"
" jobs only to be exectued at some day of the month or week, both"
" counted from 1. So, 'm22 7:30, w3 15:02' would execute on"
" the 22nd of each month at 7:30 UTC and on every wednesday at 15:02.",
default=base.NotGiven,
copyable=True,
)
_every = base.IntAttribute(
"every",
default=base.NotGiven,
description="Run the job roughly every this many seconds."
" This conflicts with at. Note that the first execution of"
" such a job is after every/10 seconds, and that the timers"
" start anew at every server restart. So, if you restart"
" often, these jobs may run much more frequently or not at all"
" if the interval is large. If every is smaller than zero, the"
" job will be executed immediately when the RD is being loaded and is"
" then run every abs(every) seconds",
copyable=True,
)
_job = base.StructAttribute(
"job",
childFactory=CronJob,
default=base.Undefined,
description="The code to run.",
copyable=True,
)
_debug = base.BooleanAttribute(
"debug",
description="If true, on execution of external processes (span or"
" spawnPython), the output will be accumulated and mailed to"
" the administrator. Note that output of the actual cron job"
" itself is not caught (it might turn up in serverStderr)."
" You could use execDef.outputAccum.append(<stuff>) to have"
" information from within the code included.",
default=False)
_properties = base.PropertyAttribute()
_rd = common.RDAttribute()
def spawn(self, cliList):
"""spawns an external command, capturing the output and mailing it
to the admin if it failed.
Output is buffered and mailed, so it shouldn't be too large.
This does not raise an exception if it failed (in normal usage,
this would cause two mails to be sent). Instead, it returns the
returncode of the spawned process; if that's 0, you're ok. But
in general, you wouldn't want to check it.
"""
p = subprocess.Popen(cliList,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=True)
childOutput, _ = p.communicate()
if p.returncode:
cron.sendMailToAdmin(
"A process spawned by %s failed with %s" %
(self.title, p.returncode),
"Output of %s:\n\n%s" % (cliList, childOutput))
elif self.debug:
if childOutput:
self.outputAccum.append("\n\n%s -> %s\n" %
(cliList, p.returncode))
self.outputAccum.append(childOutput)
return p.returncode
def spawnPython(self, pythonFile):
"""spawns a new python interpreter executing pythonFile.
pythonFile may be resdir-relative.
"""
self.spawn(["python", os.path.join(self.rd.resdir, pythonFile)])
def _parseAt(self, atSpec, ctx):
"""returns a tuple ready for cron.repeatAt from atSpec.
see the at StringListAttribute for how it would look like; this
parses one element of that string list.
"""
mat = re.match(
r"(?P<dow>w\d\s+)?"
r"(?P<dom>m\d\d?\s+)?"
r"(?P<hr>\d+):(?P<min>\d+)", atSpec)
if not mat:
raise base.LiteralParseError("at", atSpec, pos=ctx.pos, hint=
"This is hour:minute optionally prefixed by either w<weekday> or"\
" m<day of month>, each counted from 1.")
hour, minute = int(mat.group("hr")), int(mat.group("min"))
if not (0 <= hour <= 23 and 0 <= minute <= 59):
raise base.LiteralParseError(
"at",
atSpec,
pos=ctx.pos,
hint=
"This must be hour:minute with 0<=hour<=23 or 0<=minute<=59")
dom = None
if mat.group("dom"):
dom = int(mat.group("dom")[1:])
if not 1 <= dom <= 28:
raise base.LiteralParseError(
"at",
atSpec,
pos=ctx.pos,
hint="day-of-month in at must be between 1 and 28.")
dow = None
if mat.group("dow"):
dow = int(mat.group("dow")[1:])
if not 1 <= dow <= 7:
raise base.LiteralParseError(
"at",
atSpec,
pos=ctx.pos,
hint="day-of-week in at must be between 1 and 7.")
return (dom, dow, hour, minute)
def completeElement(self, ctx):
self._completeElementNext(Execute, ctx)
if len([s for s in [self.at, self.every] if s is base.NotGiven]) != 1:
raise base.StructureError(
"Exactly one of at and every required"
" for Execute",
pos=ctx.pos)
if self.at is not base.NotGiven:
self.parsedAt = []
for literal in self.at:
self.parsedAt.append(self._parseAt(literal, ctx))
def onElementComplete(self):
self._onElementCompleteNext(Execute)
self.jobName = "%s#%s" % (self.rd.sourceId, self.title)
self.callable = _guardedFunctionFactory.makeGuardedThreaded(
self.job.compile(), self)
if self.at is not base.NotGiven:
cron.repeatAt(self.parsedAt, self.jobName, self.callable)
else:
cron.runEvery(self.every, self.jobName, self.callable)
class DBCore(TableBasedCore):
"""A core performing database queries on one table or view.
DBCores ask the service for the desired output schema and adapt their
output. The DBCore's output table, on the other hand, lists all fields
available from the queried table.
"""
name_ = "dbCore"
_sortKey = base.UnicodeAttribute("sortKey",
description="A pre-defined sort order (suppresses DB options widget)."
" The sort key accepts multiple columns, separated by commas.",
copyable=True)
_limit = base.IntAttribute("limit", description="A pre-defined"
" match limit (suppresses DB options widget).", copyable=True)
_distinct = base.BooleanAttribute("distinct", description="Add a"
" 'distinct' modifier to the query?", default=False, copyable=True)
_groupBy = base.UnicodeAttribute("groupBy", description=
"A group by clause. You shouldn't generally need this, and if"
" you use it, you must give an outputTable to your core.",
default=None)
def wantsTableWidget(self):
return self.sortKey is None and self.limit is None
def getQueryCols(self, service, queryMeta):
"""returns the fields we need in the output table.
The normal DbBased core just returns whatever the service wants.
Derived cores, e.g., for special protocols, could override this
to make sure they have some fields in the result they depend on.
"""
return service.getCurOutputFields(queryMeta)
def _runQuery(self, resultTableDef, fragment, pars, queryMeta,
**kwargs):
with base.getTableConn() as conn:
queriedTable = rsc.TableForDef(self.queriedTable, nometa=True,
create=False, connection=conn)
queriedTable.setTimeout(queryMeta["timeout"])
if fragment and pars:
resultTableDef.addMeta("info", repr(pars),
infoName="queryPars", infoValue=fragment)
iqArgs = {"limits": queryMeta.asSQL(), "distinct": self.distinct,
"groupBy": self.groupBy}
iqArgs.update(kwargs)
try:
try:
return self._makeTable(
queriedTable.iterQuery(resultTableDef, fragment, pars,
**iqArgs), resultTableDef, queryMeta)
except:
mapDBErrors(*sys.exc_info())
finally:
queriedTable.close()
def _makeResultTableDef(self, service, inputTable, queryMeta):
"""returns an OutputTableDef object for querying our table with queryMeta.
"""
return base.makeStruct(outputdef.OutputTableDef,
parent_=self.queriedTable.parent, id="result",
onDisk=False, columns=self.getQueryCols(service, queryMeta),
params=self.queriedTable.params)
def run(self, service, inputTable, queryMeta):
"""does the DB query and returns an InMemoryTable containing
the result.
"""
resultTableDef = self._makeResultTableDef(
service, inputTable, queryMeta)
resultTableDef.copyMetaFrom(self.queriedTable)
if not resultTableDef.columns:
raise base.ValidationError("No output columns with these settings."
"_OUTPUT")
sortKeys = None
if self.sortKey:
sortKeys = self.sortKey.split(",")
queryMeta.overrideDbOptions(limit=self.limit, sortKeys=sortKeys,
sortFallback=self.getProperty("defaultSortKey", None))
try:
fragment, pars = self._getSQLWhere(inputTable, queryMeta)
except base.LiteralParseError, ex:
raise base.ui.logOldExc(base.ValidationError(str(ex),
colName=ex.attName))
queryMeta["sqlQueryPars"] = pars
return self._runQuery(resultTableDef, fragment, pars, queryMeta)
class OutputTableDef(rscdef.TableDef):
"""A table that has outputFields for columns.
"""
name_ = "outputTable"
# Don't validate meta for these -- while they are children
# of validated structures (services), they don't need any
# meta at all. This should go as soon as we have a sane
# inheritance hierarchy for tables.
metaModel = None
_cols = rscdef.ColumnListAttribute(
"columns",
childFactory=OutputField,
description="Output fields for this table.",
aliases=["column"],
copyable=True)
_verbLevel = base.IntAttribute(
"verbLevel",
default=None,
description="Copy over columns from fromTable not"
" more verbose than this.")
_autocols = base.StringListAttribute(
"autoCols",
description="Column names obtained from fromTable; you can use"
" shell patterns into the output table's parent table (in a table"
" core, that's the queried table; in a service, it's the core's"
" output table) here.")
def __init__(self, parent, **kwargs):
rscdef.TableDef.__init__(self, parent, **kwargs)
self.parentTable = None
try:
# am I in a table-based core?
self.parentTable = self.parent.queriedTable
except (AttributeError, base.StructureError):
# no.
pass
if not self.parentTable:
try:
# am I in a service with a core with output table?
self.parentTable = self.parent.core.outputTable
except (AttributeError, base.StructureError):
# no.
pass
if not self.parentTable:
# no suitable column source, use an empty table:
self.parentTable = _EMPTY_TABLE
self.namePath = None
def _adoptColumn(self, sourceColumn):
# Do not overwrite existing fields here to let the user
# override individually
try:
self.getColumnByName(sourceColumn.name)
except base.NotFoundError:
self.feedObject("outputField",
OutputField.fromColumn(sourceColumn))
def _addNames(self, ctx, names):
# since autoCols is not copyable, we can require
# that _addNames only be called when there's a real parse context.
if ctx is None:
raise base.StructureError("outputTable autocols is"
" only available with a parse context")
for name in names:
self._addName(ctx, name)
def _addName(self, ctx, name):
"""adopts a param or column name into the outputTable.
name may be a reference or a param or column name in the parent
table (as determined in the constructor, i.e., the queried table
of a core or the output table of a service's core.
You can also use shell patterns into parent columns.
"""
if utils.identifierPattern.match(name):
refOb = ctx.resolveId(name, self)
if refOb.name_ == "param":
self.feedObject("param", refOb.copy(self))
else:
self._adoptColumn(refOb)
else:
# it's a shell pattern into parent table
for col in self.parentTable:
if fnmatch.fnmatch(col.name, name):
self._adoptColumn(col)
def completeElement(self, ctx):
if self.autoCols:
self._addNames(ctx, self.autoCols)
if self.verbLevel:
table = self.parentTable
for col in table.columns:
if col.verbLevel <= self.verbLevel:
self._adoptColumn(col)
for par in table.params:
if par.verbLevel <= self.verbLevel:
self.feedObject("param", par.copy(self))
self._completeElementNext(OutputTableDef, ctx)
@classmethod
def fromColumns(cls, columns, **kwargs):
return rscdef.TableDef.fromColumns(
[OutputField.fromColumn(c) for c in columns])
@classmethod
def fromTableDef(cls, tableDef, ctx):
return cls(None,
columns=[OutputField.fromColumn(c) for c in tableDef],
forceUnique=tableDef.forceUnique,
dupePolicy=tableDef.dupePolicy,
primary=tableDef.primary,
params=tableDef.params).finishElement(ctx)