def upgrade(forceDBVersion=None, dryRun=False):
"""runs all updates necessary to bring a database to the
CURRENT_SCHEMAVERSION.
Everything is run in one transaction. Errors lead to the rollback of
the whole thing.
"""
if forceDBVersion is None:
startVersion = getDBSchemaVersion()
else:
startVersion = forceDBVersion
if startVersion == CURRENT_SCHEMAVERSION:
return
with base.getWritableAdminConn() as conn:
for statement in iterStatements(startVersion, CURRENT_SCHEMAVERSION):
if callable(statement):
if statement.__doc__:
showProgress("> %s..." % statement.__doc__)
# if no docstring is present, we assume the function will output
# custom user feedback
statement(conn)
else:
showProgress(
"> " +
getattr(statement, "annotation", "executing %s" %
utils.makeEllipsis(statement, 60)) + "... ")
conn.execute(statement)
showProgress(" ok\n")
if dryRun:
conn.rollback()
conn.commit()
def debugReferenceChain(ob):
"""a sort-of-interactive way to investigate where ob is referenced.
"""
import gc
while True:
print repr(ob)
refs = gc.get_referrers(ob)
while refs:
nob = refs.pop()
print len(refs), utils.makeEllipsis(repr(nob))
res = raw_input()
if res == "h":
print "d,x,<empty>,g"
elif res == "d":
import ipdb
ipdb.set_trace()
elif res == "x":
return
elif res:
ob = nob
break
def sendMail(mailText):
"""sends mailText (which has to have all the headers) via sendmail.
(which is configured in [general]sendmail).
This will return True when sendmail has accepted the mail, False
otherwise.
"""
mailText = formatMail(mailText)
pipe = subprocess.Popen(config.get("sendmail"),
shell=True,
stdin=subprocess.PIPE)
pipe.stdin.write(mailText)
pipe.stdin.close()
if pipe.wait():
utils.sendUIEvent(
"Error", "Wanted to send mail starting with"
" '%s', but sendmail returned an error message"
" (check the [general]sendmail setting)." %
utils.makeEllipsis(mailText, 300))
return False
return True
def parseSIAP2Geometry(aString, fieldName="POS"):
"""parses a SIAPv2 geometry spec to a pgsphere object.
Parse errors raise validation errors for fieldName.
"""
mat = re.match("(CIRCLE|RANGE|POLYGON) (.*)", aString)
if not mat:
raise base.ValidationError(
"Invalid SIAPv2 geometry: '%s'"
" (expected a SIAPv2 shape name)" %
utils.makeEllipsis(aString, 20), fieldName)
geoName = mat.group(1)
try:
args = [float(s) for s in mat.group(2).split()]
except ValueError:
raise base.ValidationError(
"Invalid SIAPv2 coordinates: '%s'"
" (bad floating point literal '%s')" %
(utils.makeEllipsis(mat.group(2), 20), s), fieldName)
if geoName == "CIRCLE":
if len(args) != 3:
raise base.ValidationError(
"Invalid SIAPv2 CIRCLE: '%s'"
" (need exactly three numbers)" %
(utils.makeEllipsis(aString, 20)), fieldName)
return pgsphere.SCircle(pgsphere.SPoint.fromDegrees(args[0], args[1]),
args[2] * utils.DEG)
elif geoName == "RANGE":
# SBox isn't really RANGE, but RANGE shouldn't have been
# part of the standard and people that use it deserve
# to get bad results.
if len(args) != 4:
raise base.ValidationError(
"Invalid SIAPv2 RANGE: '%s'"
" (need exactly four numbers)" %
(utils.makeEllipsis(aString, 20)), fieldName)
if args[0] > args[1] or args[2] > args[3]:
raise base.ValidationError(
"Invalid SIAPv2 RANGE: '%s'"
" (lower limits must be smaller than upper limits)" %
(utils.makeEllipsis(aString, 20)), fieldName)
return pgsphere.SBox(pgsphere.SPoint.fromDegrees(args[0], args[2]),
pgsphere.SPoint.fromDegrees(args[1], args[3]))
elif geoName == "POLYGON":
if len(args) < 6 or len(args) % 2:
raise base.ValidationError(
"Invalid SIAPv2 POLYGON: '%s'"
" (need more than three coordinate *pairs*)" %
(utils.makeEllipsis(mat.group(2), 20)), fieldName)
return pgsphere.SPoly([
pgsphere.SPoint.fromDegrees(*pair)
for pair in utils.iterConsecutivePairs(args)
])
else:
assert False
def notifyNewSource(self, sourceToken): """is called when a new source is being operated on. The callbacks are passed some, hopefully useful, token string. For file source, this is the file name, otherwise we try to make up something. As side effects, the curSource attribute is set to this value. """ if isinstance(sourceToken, basestring): sourceName = utils.makeLeftEllipsis(sourceToken) else: sourceName = utils.makeEllipsis(repr(sourceToken), maxLen=160) self.curSource = sourceName self.sourceStack.append(sourceName) return sourceName
def printTableInfo(td):
"""tries to obtain various information on the properties of the
database table described by td.
"""
annotateDBTable(td)
propTable = [("col", ) + _PROP_SEQ]
for col in td:
row = [col.name]
for prop in _PROP_SEQ:
if prop in col.annotations:
row.append(
utils.makeEllipsis(utils.safe_str(col.annotations[prop]),
30))
else:
row.append("-")
propTable.append(tuple(row))
print utils.formatSimpleTable(propTable)
def validateRSTOne(el):
validSoFar = True
for key, val in getattr(el, "getAllMetaPairs", lambda: [])():
if val.format == 'rst':
content = val.getExpandedContent(macroPackage=el)
_, msg = utils.rstxToHTMLWithWarning(content)
if msg:
outputWarning(
rd.sourceId,
"%s metadata on %s (%s) has an RST problem: %s" %
(key, el, utils.makeEllipsis(content, 80), msg))
for child in el.iterChildren():
if child:
validSoFar = validSoFar and validateRSTOne(child)
return validSoFar
def validateTables(rd, args):
"""does some sanity checks on the (top-level) tables within rd.
"""
valid = True
identifierSymbol = adql.getSymbols()["identifier"]
for td in rd.tables:
for col in td:
try:
if col.unit:
parsedUnit = api.parseUnit(col.unit)
if parsedUnit.isUnknown and not args.acceptFreeUnits:
outputWarning(
rd.sourceId,
"Column %s.%s: Unit %s is not interoperable" %
(td.getQName(), col.name, col.unit))
except api.BadUnit:
valid = False
outputError(
rd.sourceId, "Bad unit in table %s, column %s: %s" %
(td.getQName(), col.name, repr(col.unit)))
try:
identifierSymbol.parseString(str(col.name), parseAll=True)
except base.ParseException, msg:
outputWarning(
rd.sourceId, "Column %s.%s: Name is not a regular"
" ADQL identifier." % (td.id, col.name))
if td.onDisk and args.compareDB:
with base.getTableConn() as conn:
q = base.UnmanagedQuerier(conn)
if q.tableExists(td.getQName()):
t = api.TableForDef(td, connection=conn)
try:
t.ensureOnDiskMatches()
except api.DataError, msg:
outputError(
rd.sourceId,
utils.makeEllipsis(utils.safe_str(msg), 160))