def putAnnotation ( anno, annodb, options ):
"""Return an annotation object by identifier"""
# for updates, make sure the annotation exists and is of the right type
if 'update' in options:
oldanno = getAnnotation ( anno.annid, annodb )
# can't update annotations that don't exist
if oldanno == None:
raise ANNError ( "During update no annotation found at id %d" % anno.annid )
# can update if they are the same type
elif oldanno.__class__ == anno.__class__:
anno.update(annodb)
# need to delete and then insert if we're changing the annotation type
# only from the base type
elif oldanno.__class__ == Annotation:
oldanno.delete(annodb)
anno.store(annodb)
# otherwise an illegal update
else:
raise ANNError ( "Cannot change the type of annotation from %s to %s" % (oldanno.__class__,anno.__class__))
# Write the user chosen annotation id
else:
anno.store(annodb)
class Annotation:
"""Metdata common to all annotations."""
def __init__ ( self ):
"""Initialize the fields to zero or null"""
# metadata fields
self.annid = 0
self.status = 0
self.confidence = 0.0
self.author = ""
self.kvpairs = defaultdict(list)
def setID ( self, db ):
"""if annid == 0, create a new identifier"""
if self.annid == 0:
self.annid = db.nextID()
else:
db.setID(self.annid)
def store ( self, annodb, annotype=ANNO_ANNOTATION ):
"""Store the annotation to the annotations database"""
cursor = annodb.conn.cursor()
sql = "INSERT INTO %s VALUES ( %s, %s, %s, %s )"\
% ( anno_dbtables['annotation'], self.annid, annotype, self.confidence, self.status )
try:
cursor.execute(sql)
except MySQLdb.Error, e:
logger.warning ( "Error inserting annotation %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
raise ANNError ( "Error inserting annotation: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
# author: make a KV pair
if self.author != "":
self.kvpairs['ann_author'] = self.author
if len(self.kvpairs) != 0:
try:
kvclause = ','.join(['(' + str(self.annid) +',\'' + k + '\',\'' + v +'\')' for (k,v) in self.kvpairs.iteritems()])
except:
raise ANNError ( "Improperly formatted key/value csv string:" + kvclause )
sql = "INSERT INTO %s VALUES %s" % ( anno_dbtables['kvpairs'], kvclause )
try:
cursor.execute(sql)
except MySQLdb.Error, e:
logger.warning ( "Error inserting kvpairs %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
raise ANNError ( "Error inserting kvpairs: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def getAnnoObjects(self, predicates):
"""Return a list of annotation object ids that match equality predicates.
Legal predicates are currently:
type
status
Predicates are given in a dictionary.
"""
# legal fields
fields = ('type', 'status')
# start of the SQL clause
sql = "SELECT annoid FROM " + annotation.anno_dbtables['annotation']
clause = ''
# probably need to avoid SQL injection attacks.
# throw an error or build the sql clause
for field in predicates.keys():
if field not in fields:
raise ANNError("Illegal field in URL: %s" % (field))
elif clause == '':
clause += " WHERE "
else:
clause += ' AND '
clause += '%s = %s' % (field, predicates[field])
sql += clause + ';'
try:
self.cursor.execute(sql)
except MySQLdb.Error, e:
logger.error("Error retrieving ids: %d: %s. sql=%s" %
(e.args[0], e.args[1], sql))
raise
def newEMCAProj(self, token, openid, dbhost, project, dbtype, dataset,
dataurl, resolution, readonly, exceptions):
"""Create a new emca project"""
# TODO need to undo the project creation if not totally sucessful
dbcfg = dbconfig.switchDataset(dataset)
# Insert the project entry into the database
sql = "INSERT INTO {0} VALUES (\'{1}\',\'{2}\',\'{3}\',\'{4}\',{5},\'{6}\',\'{7}\',{8},{9},{10})".format (\
emcaprivate.table, token, openid, dbhost, project, dbtype, dataset, dataurl, resolution, readonly, exceptions )
logger.info("Creating new project. Host %s. Project %s. SQL=%s" %
(dbhost, project, sql))
try:
cursor = self.conn.cursor()
cursor.execute(sql)
except MySQLdb.Error, e:
logger.error(
"Could not query emca projects database %d: %s. sql=%s" %
(e.args[0], e.args[1], sql))
raise ANNError(
"Could not query emca projects database %d: %s. sql=%s" %
(e.args[0], e.args[1], sql))
class AnnSynapse (Annotation):
"""Metadata specific to synapses"""
def __init__(self ):
"""Initialize the fields to zero or null"""
self.weight = 0.0
self.synapse_type = 0
self.seeds = []
self.segments = []
# Call the base class constructor
Annotation.__init__(self)
def store ( self, annodb ):
"""Store the synapse to the annotations databae"""
cursor = annodb.conn.cursor()
sql = "INSERT INTO %s VALUES ( %s, %s, %s )"\
% ( anno_dbtables['synapse'], self.annid, self.synapse_type, self.weight )
try:
cursor.execute ( sql )
except MySQLdb.Error, e:
logger.warning ( "Error inserting synapse %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
raise ANNError ( "Error inserting synapse: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
# synapse_seeds: pack into a kv pair
if len(self.seeds)!=0:
try:
self.kvpairs['synapse_seeds'] = ','.join([str(i) for i in self.seeds])
except:
raise ANNError ("Improperly formatted seeds: %s " % (self.seeds) )
# synapse_segments: pack into a kv pair
if len(self.segments)!=0:
try:
self.kvpairs['synapse_segments'] = ','.join([str(i) + ':' + str(j) for i,j in self.segments])
except:
raise ANNError ("Improperly formatted segments. Should be nx2 matrix: %s" % (self.segments) )
cursor.close()
# and call store on the base classs
Annotation.store ( self, annodb, ANNO_SYNAPSE)
def listAnnoObjects(webargs, postdata=None):
""" Return a list of anno ids restricted by equality predicates.
Equalities are alternating in field/value in the url.
"""
[token, dontuse, restargs] = webargs.split('/', 2)
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
db = emcadb.EMCADB(dbcfg, proj)
# Split the URL and get the args
args = restargs.split('/')
predicates = dict(zip(args[::2], args[1::2]))
annoids = db.getAnnoObjects(predicates)
# We have a cutout as well
if postdata:
# RB this is a brute force implementation. This probably needs to be
# optimized to use several different execution strategies based on the
# cutout size and the number of objects.
# Make a named temporary file for the HDF5
tmpfile = tempfile.NamedTemporaryFile()
tmpfile.write(postdata)
tmpfile.seek(0)
h5f = h5py.File(tmpfile.name, driver='core', backing_store=False)
corner = h5f['XYZOFFSET'][:]
dim = h5f['CUTOUTSIZE'][:]
resolution = h5f['RESOLUTION'][0]
if not dbcfg.checkCube(resolution, corner[0], corner[0] + dim[0],
corner[1], corner[1] + dim[1], corner[2],
corner[2] + dim[2]):
logger.warning("Illegal cutout corner=%s, dim=%s" % (corner, dim))
raise ANNError("Illegal cutout corner=%s, dim=%s" % (corner, dim))
# RBFIX this a hack
#
# the zstart in dbconfig is sometimes offset to make it aligned.
# Probably remove the offset is the best idea. and align data
# to zero regardless of where it starts. For now.
corner[2] -= dbcfg.slicerange[0]
cutout = db.cutout(corner, dim, resolution)
annoids = np.intersect1d(annoids, np.unique(cutout.data))
if postdata:
h5f.close()
tmpfile.close()
return h5ann.PackageIDs(annoids)
def retrieve ( self, annid, annodb ):
"""Retrieve the synapse by annid"""
cursor = annodb.conn.cursor()
# Call the base class retrieve
annotype = Annotation.retrieve ( self, annid, annodb )
# verify the annotation object type
if annotype != ANNO_SEGMENT:
raise ANNError ( "Incompatible annotation type. Expected SEGMENT got %s" % annotype )
sql = "SELECT segmentclass, parentseed, neuron FROM %s WHERE annoid = %s" % ( anno_dbtables['segment'], annid )
try:
cursor.execute ( sql )
except MySQLdb.Error, e:
logger.warning ( "Error retrieving synapse %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
raise ANNError ( "Error retrieving segment: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def retrieve ( self, annid, annodb ):
"""Retrieve the organelle by annid"""
cursor = annodb.conn.cursor()
# Call the base class retrieve
annotype = Annotation.retrieve ( self, annid, annodb )
# verify the annotation object type
if annotype != ANNO_ORGANELLE:
raise ANNError ( "Incompatible annotation type. Expected ORGANELLE got %s" % annotype )
sql = "SELECT organelleclass, parentseed, centroidx, centroidy, centroidz FROM %s WHERE annoid = %s" % ( anno_dbtables['organelle'], annid )
try:
cursor.execute ( sql )
except MySQLdb.Error, e:
logger.warning ( "Error retrieving organelle %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
raise ANNError ( "Error retrieving organelle: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def listIds(imageargs, dbcfg, proj):
"""Return the list of annotation identifiers in a region"""
# Perform argument processing
try:
args = restargs.BrainRestArgs()
args.cutoutArgs(imageargs, dbcfg)
except restargs.RESTArgsError, e:
logger.warning("REST Arguments failed: %s" % (e))
raise ANNError(e)
def deleteAnnotation ( annoid, annodb, options ):
"""Polymorphically delete an annotaiton by identifier"""
oldanno = getAnnotation ( annoid, annodb )
# can't delete annotations that don't exist
if oldanno == None:
raise ANNError ( "During delete no annotation found at id %d" % annoid )
# methinks we can call polymorphically
oldanno.delete(annodb)
def retrieve ( self, annid, annodb ):
"""Retrieve the annotation by annid"""
cursor = annodb.conn.cursor()
sql = "SELECT * FROM %s WHERE annoid = %s" % ( anno_dbtables['annotation'], annid )
try:
cursor.execute ( sql )
except MySQLdb.Error, e:
logger.warning ( "Error retrieving annotation %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
raise ANNError ( "Error retrieving annotation: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def cutout(imageargs, dbcfg, proj, channel=None):
"""Build the returned cube of data. This method is called by all
of the more basic services to build the data.
They then format and refine the output."""
# Perform argument processing
try:
args = restargs.BrainRestArgs()
args.cutoutArgs(imageargs, dbcfg)
except restargs.RESTArgsError, e:
logger.warning("REST Arguments failed: %s" % (e))
raise ANNError(e)
def selectService(webargs, dbcfg, proj):
"""Parse the first arg and call service, HDF5, npz, etc."""
[service, sym, rangeargs] = webargs.partition('/')
if service == 'xy':
return xyImage(rangeargs, dbcfg, proj)
elif service == 'xz':
return xzImage(rangeargs, dbcfg, proj)
elif service == 'yz':
return yzImage(rangeargs, dbcfg, proj)
elif service == 'hdf5':
return HDF5(rangeargs, dbcfg, proj)
elif service == 'npz':
return numpyZip(rangeargs, dbcfg, proj)
elif service == 'zip':
return binZip(rangeargs, dbcfg, proj)
elif service == 'id':
return annId(rangeargs, dbcfg, proj)
elif service == 'ids':
return listIds(rangeargs, dbcfg, proj)
elif service == 'xyanno':
return xyAnno(rangeargs, dbcfg, proj)
elif service == 'xzanno':
return xzAnno(rangeargs, dbcfg, proj)
elif service == 'yzanno':
return yzAnno(rangeargs, dbcfg, proj)
# elif service == 'xytiff':
# return xyTiff ( rangeargs, dbcfg, proj )
#
# elif service == 'xztiff':
# return xzTiff ( rangeargs, dbcfg, proj)
#
# elif service == 'yztiff':
# return yzTiff ( rangeargs, dbcfg, proj )
else:
logger.warning(
"An illegal Web GET service was requested %s. Args %s" %
(service, webargs))
raise ANNError("No such Web service: %s" % service)
def yzAnno(imageargs, dbcfg, proj):
"""Return an yz plane fileobj.read()"""
[annoidstr, sym, imageargs] = imageargs.partition('/')
annoid = int(annoidstr)
# Perform argument processing
try:
args = restargs.BrainRestArgs()
args.yzArgs(imageargs, dbcfg)
except restargs.RESTArgsError, e:
logger.warning("REST Arguments failed: %s" % (e))
raise ANNError(e)
def update ( self, annodb ):
"""Update the synapse in the annotations databae"""
cursor = annodb.conn.cursor()
sql = "UPDATE %s SET synapse_type=%s, weight=%s WHERE annoid=%s "\
% (anno_dbtables['synapse'], self.synapse_type, self.weight, self.annid)
try:
cursor.execute ( sql )
except MySQLdb.Error, e:
logger.warning ( "Error updating synapse %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
raise ANNError ( "Error updating synapse: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def store ( self, annodb ):
"""Store the synapse to the annotations databae"""
cursor = annodb.conn.cursor()
sql = "INSERT INTO %s VALUES ( %s, %s, %s )"\
% ( anno_dbtables['synapse'], self.annid, self.synapse_type, self.weight )
try:
cursor.execute ( sql )
except MySQLdb.Error, e:
logger.warning ( "Error inserting synapse %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
raise ANNError ( "Error inserting synapse: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def updateBase ( self, annotype, annodb ):
"""Update the annotation in the annotations database"""
cursor = annodb.conn.cursor()
sql = "UPDATE %s SET type=%s, confidence=%s, status=%s WHERE annoid = %s"\
% ( anno_dbtables['annotation'], annotype, self.confidence, self.status, self.annid)
try:
cursor.execute ( sql )
except MySQLdb.Error, e:
logger.warning ( "Error updating annotation %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
raise ANNError ( "Error updating annotation: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def getAnnotation ( annid, annodb ):
"""Return an annotation object by identifier"""
cursor = annodb.conn.cursor()
# First, what type is it. Look at the annotation table.
sql = "SELECT type FROM %s WHERE annoid = %s" % ( anno_dbtables['annotation'], annid )
cursor = annodb.conn.cursor ()
try:
cursor.execute ( sql )
except MySQLdb.Error, e:
logger.warning ( "Error reading id %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
raise ANNError ( "Error reading id: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def store ( self, annodb, annotype=ANNO_ANNOTATION ):
"""Store the annotation to the annotations database"""
cursor = annodb.conn.cursor()
sql = "INSERT INTO %s VALUES ( %s, %s, %s, %s )"\
% ( anno_dbtables['annotation'], self.annid, annotype, self.confidence, self.status )
try:
cursor.execute(sql)
except MySQLdb.Error, e:
logger.warning ( "Error inserting annotation %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
raise ANNError ( "Error inserting annotation: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def retrieve ( self, annid, annodb ):
"""Retrieve the synapse by annid"""
# Call the base class retrieve
annotype = Annotation.retrieve ( self, annid, annodb )
# verify the annotation object type
if annotype != ANNO_NEURON:
raise ANNError ( "Incompatible annotation type. Expected NEURON got %s" % annotype )
if self.kvpairs.get('segments'):
self.segments = [int(i) for i in self.kvpairs['segments'].split(',')]
del ( self.kvpairs['segments'] )
def update ( self, annodb ):
"""Update the synapse in the annotations database"""
cursor = annodb.conn.cursor()
sql = "UPDATE %s SET segmentclass=%s, parentseed=%s, neuron=%s WHERE annoid=%s "\
% (anno_dbtables['segment'], self.segmentclass, self.parentseed, self.neuron, self.annid)
try:
cursor.execute ( sql )
except MySQLdb.Error, e:
logger.warning ( "Error updating segment %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
raise ANNError ( "Error updating segment: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def store ( self, annodb ):
"""Store the synapse to the annotations databae"""
cursor = annodb.conn.cursor()
sql = "INSERT INTO %s VALUES ( %s, %s, %s, %s )"\
% ( anno_dbtables['segment'], self.annid, self.segmentclass, self.parentseed, self.neuron )
try:
cursor.execute ( sql )
except MySQLdb.Error, e:
logger.warning ( "Error inserting segment %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
raise ANNError ( "Error inserting segment: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def deleteAnnotation(webargs):
"""Delete a RAMON object"""
## TODO add retry loop for transaction
[token, sym, otherargs] = webargs.partition('/')
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
db = emcadb.EMCADB(dbcfg, proj)
# Split the URL and get the args
args = otherargs.split('/', 2)
# if the first argument is numeric. it is an annoid
if re.match('^[\d,]+$', args[0]):
annoids = map(int, args[0].split(','))
# if not..this is not a well-formed delete request
else:
logger.warning(
"Delete did not specify a legal object identifier = %s" % args[0])
raise ANNError(
"Delete did not specify a legal object identifier = %s" % args[0])
for annoid in annoids:
db.startTxn()
tries = 0
done = False
while not done and tries < 5:
try:
db.deleteAnnotation(annoid)
done = True
# rollback if you catch an error
except MySQLdb.OperationalError, e:
logger.warning("Transaction did not complete. %s" % (e))
tries += 1
db.rollback()
continue
except MySQLdb.Error, e:
logger.warning("Put transaction rollback. %s" % (e))
db.rollback()
raise
except Exception, e:
logger.exception(
"Put transaction rollback. Unknown error. %s" % (e))
db.rollback()
raise
def getObjects(request, webargs):
"""Batch fetch of RAMON objects"""
try:
if request.method == 'GET':
raise ANNError(
"GET requested. objects Web service requires a POST of a list of identifiers."
)
elif request.method == 'POST':
return django.http.HttpResponse(emcarest.getAnnotations(
webargs, request.body),
mimetype="product/hdf5")
except ANNError, e:
return django.http.HttpResponseNotFound(e.value)
def yzSlice(imageargs, dbcfg, proj):
"""Return an yz plane as a cube"""
if proj.getDBType() == emcaproj.CHANNELS:
[channel, sym, imageargs] = imageargs.partition('/')
else:
channel = None
# Perform argument processing
try:
args = restargs.BrainRestArgs()
args.yzArgs(imageargs, dbcfg)
except restargs.RESTArgsError, e:
logger.warning("REST Arguments failed: %s" % (e))
raise ANNError(e)
def delete ( self, annodb ):
"""Delete the organelle from the database"""
cursor = annodb.conn.cursor()
sql = "DELETE FROM %s WHERE annoid = %s;"\
% ( anno_dbtables['organelle'], self.annid )
sql += "DELETE FROM %s WHERE annoid = %s" % ( anno_dbtables['kvpairs'], self.annid )
try:
cursor.execute ( sql )
except MySQLdb.Error, e:
logger.warning ( "Error deleting organelle %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
raise ANNError ( "Error deleting organelle: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def retrieve ( self, annid, annodb ):
"""Retrieve the seed by annid"""
cursor = annodb.conn.cursor()
# Call the base class retrieve
Annotation.retrieve ( self, annid, annodb )
sql = "SELECT parentid, sourceid, cube_location, positionx, positiony, positionz FROM %s WHERE annoid = %s" % ( anno_dbtables['seed'], annid )
try:
cursor.execute ( sql )
except MySQLdb.Error, e:
logger.warning ( "Error retrieving seed %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
raise ANNError ( "Error retrieving seed: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def H5GetVolume ( h5fh ):
"""Return the volume associated with the annotation"""
# assume a single annotation for now
keys = h5fh.keys()
idgrp = h5fh.get(keys[0])
if idgrp.get('XYZOFFSET'):
if idgrp.get('CUTOUT'):
return (idgrp['XYZOFFSET'], idgrp['CUTOUT'])
else:
logger.warning("Improperly formatted HDF5 file. XYZOFFSET define but no CUTOUT.")
raise ANNError("Improperly formatted HDF5 file. XYZOFFSET define but no CUTOUT.")
else:
return None
class EMCAProjectsDB:
"""Database for the annotation and cutout projects"""
def __init__(self):
"""Create the database connection"""
# Connection info in dbconfig
self.conn = MySQLdb.connect(host=emcaprivate.dbhost,
user=emcaprivate.dbuser,
passwd=emcaprivate.dbpasswd,
db=emcaprivate.db)
#
# Load the emca databse information based on the token
#
def getProj(self, token):
"""Load the annotation database information based on the token"""
# Lookup the information for the database project based on the token
sql = "SELECT * from %s where token = \'%s\'" % (emcaprivate.table,
token)
try:
cursor = self.conn.cursor()
cursor.execute(sql)
except MySQLdb.Error, e:
logger.error(
"Could not query emca projects database %d: %s. sql=%s" %
(e.args[0], e.args[1], sql))
raise ANNError(
"Could not query emca projects database %d: %s. sql=%s" %
(e.args[0], e.args[1], sql))
# get the project information
row = cursor.fetchone()
# if the project is not found. error
if (row == None):
logger.warning("Project token %s not found." % (token))
raise ANNError("Project token %s not found." % (token))
[
token, openid, host, project, dbtype, dataset, dataurl, resolution,
readonly, exceptions
] = row
return EMCAProject(project, host, dbtype, dataset, dataurl, resolution,
readonly, exceptions)
def deleteEMCAProj(self, token):
"""Create a new emca project"""
proj = self.getProj(token)
sql = "DELETE FROM %s WHERE token=\'%s\'" % (emcaprivate.table, token)
try:
cursor = self.conn.cursor()
cursor.execute(sql)
except MySQLdb.Error, e:
conn.rollback()
logging.error(
"Failed to remove project from projects tables %d: %s. sql=%s"
% (e.args[0], e.args[1], sql))
raise ANNError(
"Failed to remove project from projects tables %d: %s. sql=%s"
% (e.args[0], e.args[1], sql))