def runForData(self, service, inputTable, queryMeta):
"""returns a data set processed according to inputTable's parameters.
"""
try:
args = inputTable.getParamDict()
if not self.dataFunctions:
raise base.DataError(
"This datalink service cannot process data")
descriptor = self.descriptors[-1]
self.dataFunctions[0].compile(self)(descriptor, args)
if descriptor.data is None:
raise base.ReportableError(
"Internal Error: a first data function did"
" not create data.")
for func in self.dataFunctions[1:]:
try:
func.compile(self)(descriptor, args)
except FormatNow:
break
except DeliverNow:
return descriptor.data
res = self.dataFormatter.compile(self)(descriptor, args)
return res
finally:
self.finalize()
def _iterRows(self):
dumpContents = self.inputFile.read()
tableName, fieldNames, curPos = guessFieldNames(dumpContents)
insertionPat = re.compile("INSERT INTO `%s` VALUES "%tableName)
# TODO: handle embedded quotes ('')
valueRE = "('[^']*'|[^',][^,]*)"
rowPat = re.compile(r"\s*\(%s\),?"%(",".join(valueRE for i in fieldNames)))
while True:
mat = insertionPat.search(dumpContents, curPos)
if not mat:
break
curPos = mat.end()
while True:
mat = rowPat.match(dumpContents, curPos)
if not mat:
# sanity check that we really reached the end of the VALUE
# statement
if not dumpContents[curPos:curPos+30].strip().startswith(";"):
raise base.DataError("Expected VALUES-ending ; char %s;"
" found %s instead."%(
curPos, repr(dumpContents[curPos: curPos+30])))
break
yield makeRecord(fieldNames, mat.groups())
curPos = mat.end()
def createUniquenessRules(self):
if not self.tableDef.forceUnique:
return
def getMatchCondition():
return " AND ".join("%s=new.%s" % (n, n)
for n in self.tableDef.primary)
if self.tableDef.dupePolicy == "drop":
self.query("CREATE OR REPLACE RULE updatePolicy AS"
" ON INSERT TO %s WHERE"
" EXISTS(SELECT * FROM %s WHERE %s)"
" DO INSTEAD NOTHING" %
(self.tableName, self.tableName, getMatchCondition()))
elif self.tableDef.dupePolicy == "check":
# This one is tricky: if the inserted column is *different*,
# the rule does not fire and we get a pkey violation.
# Furthermore, special NULL handling is required -- we
# do not check columns that have NULLs in new or old.
self.query("CREATE OR REPLACE RULE updatePolicy AS"
" ON INSERT TO %s WHERE"
" EXISTS(SELECT 1 FROM %s WHERE %s)"
" DO INSTEAD NOTHING" %
(self.tableName, self.tableName, " AND ".join(
"(new.%s IS NULL OR %s IS NULL OR %s=new.%s)" %
(c.name, c.name, c.name, c.name)
for c in self.tableDef)))
elif self.tableDef.dupePolicy == "dropOld":
args = {"table": self.tableName, "matchCond": getMatchCondition()}
self.query(
'CREATE OR REPLACE FUNCTION "dropOld_%(table)s"()'
' RETURNS trigger AS $body$\n'
" BEGIN\n"
" IF (EXISTS(SELECT 1 FROM %(table)s WHERE %(matchCond)s)) THEN\n"
" DELETE FROM %(table)s WHERE %(matchCond)s;\n"
" END IF;\n"
" RETURN NEW;\nEND\n$body$ LANGUAGE plpgsql" % args)
self.query(
'CREATE TRIGGER "dropOld_%(table)s" BEFORE INSERT OR UPDATE'
' ON %(table)s FOR EACH ROW EXECUTE PROCEDURE "dropOld_%(table)s"()'
% args)
elif self.tableDef.dupePolicy == "overwrite":
self.query(
"CREATE OR REPLACE RULE updatePolicy AS"
" ON INSERT TO %s WHERE"
" EXISTS(SELECT %s FROM %s WHERE %s)"
" DO INSTEAD UPDATE %s SET %s WHERE %s" %
(self.tableName, ",".join(self.tableDef.primary),
self.tableName, getMatchCondition(), self.tableName,
", ".join("%s=new.%s" % (c.name, c.name)
for c in self.tableDef), getMatchCondition()))
else:
raise base.DataError("Invalid dupePolicy: %s" %
self.tableDef.dupePolicy)
def getPrimaryTable(self):
"""returns the table contained if there is only one, or the one
with the role primary.
If no matching table can be found, raise a DataError.
"""
try:
return self.tables[self.dd.getPrimary().id]
except (KeyError, base.StructureError):
raise base.DataError("No primary table in this data")
def getRow(self, *key):
"""returns the row with the primary key key from the table.
This will raise a DataError on tables without primaries.
"""
if not self.tableDef.primary:
raise base.DataError("Table %s has no primary key and thus does"
" not support getRow" % self.tableName)
res = list(
self.iterQuery(self.tableDef,
" AND ".join("%s=%%(%s)s" % (n, n)
for n in self.tableDef.primary),
pars=dict(zip(self.tableDef.primary, key))))
if not res:
raise KeyError(key)
return res[0]
def computePreviewFor(product):
"""returns image/jpeg bytes containing a preview of product.
This only works for a select subset of products. You're usually
better off using static previews.
"""
if hasattr(product, "makePreview"):
return product.makePreview()
sourceMime = product.pr["mime"]
if sourceMime == 'image/fits':
return makePreviewFromFITS(product)
elif sourceMime in _PIL_COMPATIBLE_MIMES:
return makePreviewWithPIL(product)
else:
raise base.DataError("Cannot make automatic preview for %s" %
sourceMime)
def guessFieldNames(dump):
"""returns the table name and the column names for the first
CREATE TABLE statement in a MySQL dump.
"""
mat = re.search("CREATE TABLE `([^`]*)` \(", dump)
if not mat:
raise base.DataError("No proper CREATE TABLE statement found")
tableName = mat.group(1)
curPos = mat.end()
names = []
rowPat = re.compile(
r"\s*`(?P<name>[^`]*)` (?P<type>[^ ]*) (?P<flags>[^,)]*),?")
while True:
mat = rowPat.match(dump, curPos)
if not mat:
# sanity check would be great here.
break
names.append(mat.group("name"))
curPos = mat.end()
return tableName, names, curPos
def getTableWithRole(self, role):
try:
return self.tables[self.dd.getTableDefWithRole(role).id]
except (KeyError, base.StructureError):
raise base.DataError("No table with role %s known here" %
repr(role))
def _assertActive(self):
if not self.active:
raise base.DataError("Trying to feed a dormant feeder.")
def addRow(self, row):
raise base.DataError("You cannot add data to views")
def add(self, data):
raise base.DataError("Attempt to feed to a read-only table")
col.name)
break
if col.name.lower() != name:
mismatches.append("mismatching name of %s (DB: %s)" %
(col.name, name))
continue
try:
base.sqltypeToPgValidator(col.type)(type)
except TypeError, ex:
mismatches.append("type mismatch in column %s (%s)" %
(col.name, utils.safe_str(ex)))
if mismatches:
raise base.DataError(
"Table %s: %s" %
(self.tableDef.getQName(), "; ".join(mismatches)))
def close(self):
"""cleans up connection if it is owned.
"""
if self.ownedConnection and not self.connection.closed:
self.connection.close()
class DBTable(DBMethodsMixin, table.BaseTable, MetaTableMixin):
"""is a table in the database.
It is created, if necessary, on construction, but indices and primary
keys will only be created if a feeder finishes, or on a manual makeIndices
call.