def newEMCAProj(self, token, openid, dbhost, project, dbtype, dataset,
dataurl, resolution, readonly, exceptions):
"""Create a new emca project"""
# TODO need to undo the project creation if not totally sucessful
dbcfg = dbconfig.switchDataset(dataset)
# Insert the project entry into the database
sql = "INSERT INTO {0} VALUES (\'{1}\',\'{2}\',\'{3}\',\'{4}\',{5},\'{6}\',\'{7}\',{8},{9},{10})".format (\
emcaprivate.table, token, openid, dbhost, project, dbtype, dataset, dataurl, resolution, readonly, exceptions )
logger.info("Creating new project. Host %s. Project %s. SQL=%s" %
(dbhost, project, sql))
try:
cursor = self.conn.cursor()
cursor.execute(sql)
except MySQLdb.Error, e:
logger.error(
"Could not query emca projects database %d: %s. sql=%s" %
(e.args[0], e.args[1], sql))
raise ANNError(
"Could not query emca projects database %d: %s. sql=%s" %
(e.args[0], e.args[1], sql))
def getCSV ( webargs ):
"""Fetch a RAMON object as CSV. Always includes bounding box. No data option."""
[ token, csvliteral, annoid, reststr ] = webargs.split ('/',3)
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj ( token )
dbcfg = dbconfig.switchDataset ( proj.getDataset() )
db = emcadb.EMCADB ( dbcfg, proj )
# Make the HDF5 file
# Create an in-memory HDF5 file
tmpfile = tempfile.NamedTemporaryFile()
h5f = h5py.File ( tmpfile.name )
dataoption = AR_BOUNDINGBOX
[resstr, sym, rest] = reststr.partition('/')
resolution = int(resstr) if resstr != '' else proj.getResolution()
getAnnoById ( annoid, h5f, db, dbcfg, dataoption, resolution )
# convert the HDF5 file to csv
csvstr = h5ann.h5toCSV ( h5f )
return csvstr
def getCSV(webargs):
"""Fetch a RAMON object as CSV. Always includes bounding box. No data option."""
[token, csvliteral, annoid, reststr] = webargs.split('/', 3)
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
db = emcadb.EMCADB(dbcfg, proj)
# Make the HDF5 file
# Create an in-memory HDF5 file
tmpfile = tempfile.NamedTemporaryFile()
h5f = h5py.File(tmpfile.name)
dataoption = AR_BOUNDINGBOX
[resstr, sym, rest] = reststr.partition('/')
resolution = int(resstr) if resstr != '' else proj.getResolution()
getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution)
# convert the HDF5 file to csv
csvstr = h5ann.h5toCSV(h5f)
return csvstr
def __init__(self, token):
"""Load the database and project"""
projdb = emcaproj.EMCAProjectsDB()
self.proj = projdb.getProj(token)
self.dbcfg = dbconfig.switchDataset(self.proj.getDataset())
# Bind the annotation database
self.imgDB = emcadb.EMCADB(self.dbcfg, self.proj)
def annopost ( webargs, postdata ):
"""Interface to the annotation write service
Load the annotation project and invoke the appropriate
dataset."""
[ token, sym, rangeargs ] = webargs.partition ('/')
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj ( token )
dbcfg = dbconfig.switchDataset ( proj.getDataset() )
return selectPost ( rangeargs, dbcfg, proj, postdata )
def annopost(webargs, postdata):
"""Interface to the annotation write service
Load the annotation project and invoke the appropriate
dataset."""
[token, sym, rangeargs] = webargs.partition('/')
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
return selectPost(rangeargs, dbcfg, proj, postdata)
def listAnnoObjects(webargs, postdata=None):
""" Return a list of anno ids restricted by equality predicates.
Equalities are alternating in field/value in the url.
"""
[token, dontuse, restargs] = webargs.split('/', 2)
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
db = emcadb.EMCADB(dbcfg, proj)
# Split the URL and get the args
args = restargs.split('/')
predicates = dict(zip(args[::2], args[1::2]))
annoids = db.getAnnoObjects(predicates)
# We have a cutout as well
if postdata:
# RB this is a brute force implementation. This probably needs to be
# optimized to use several different execution strategies based on the
# cutout size and the number of objects.
# Make a named temporary file for the HDF5
tmpfile = tempfile.NamedTemporaryFile()
tmpfile.write(postdata)
tmpfile.seek(0)
h5f = h5py.File(tmpfile.name, driver='core', backing_store=False)
corner = h5f['XYZOFFSET'][:]
dim = h5f['CUTOUTSIZE'][:]
resolution = h5f['RESOLUTION'][0]
if not dbcfg.checkCube(resolution, corner[0], corner[0] + dim[0],
corner[1], corner[1] + dim[1], corner[2],
corner[2] + dim[2]):
logger.warning("Illegal cutout corner=%s, dim=%s" % (corner, dim))
raise ANNError("Illegal cutout corner=%s, dim=%s" % (corner, dim))
# RBFIX this a hack
#
# the zstart in dbconfig is sometimes offset to make it aligned.
# Probably remove the offset is the best idea. and align data
# to zero regardless of where it starts. For now.
corner[2] -= dbcfg.slicerange[0]
cutout = db.cutout(corner, dim, resolution)
annoids = np.intersect1d(annoids, np.unique(cutout.data))
if postdata:
h5f.close()
tmpfile.close()
return h5ann.PackageIDs(annoids)
def __init__(self, token):
"""Load the annotation database and project"""
projdb = emcaproj.EMCAProjectsDB()
self.proj = projdb.getProj ( token )
self.dbcfg = dbconfig.switchDataset ( self.proj.getDataset() )
# Bind the annotation database
self.annoDB = emcadb.EMCADB ( self.dbcfg, self.proj )
def emcaget ( webargs ):
"""Interface to the cutout service for annotations.
Load the annotation project and invoke the appropriate
dataset."""
[ token, sym, rangeargs ] = webargs.partition ('/')
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj ( token )
dbcfg = dbconfig.switchDataset ( proj.getDataset() )
return selectService ( rangeargs, dbcfg, proj )
def emcaget(webargs):
"""Interface to the cutout service for annotations.
Load the annotation project and invoke the appropriate
dataset."""
[token, sym, rangeargs] = webargs.partition('/')
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
return selectService(rangeargs, dbcfg, proj)
def __init__(self,token,resolution):
"""DB configuration and some state"""
self._resolution = resolution
# Get DB configuration stuff
annprojdb = annproj.AnnotateProjectsDB()
annoproj = annprojdb.getAnnoProj ( token )
self.dbcfg = dbconfig.switchDataset ( annoproj.getDataset() )
self.annodb = anndb.AnnotateDB ( self.dbcfg, annoproj )
def __init__(self, token, resolution):
"""DB configuration and some state"""
self._resolution = resolution
# Get DB configuration stuff
annprojdb = annproj.AnnotateProjectsDB()
annoproj = annprojdb.getAnnoProj(token)
self.dbcfg = dbconfig.switchDataset(annoproj.getDataset())
self.annodb = anndb.AnnotateDB(self.dbcfg, annoproj)
def listAnnoObjects ( webargs, postdata=None ):
""" Return a list of anno ids restricted by equality predicates.
Equalities are alternating in field/value in the url.
"""
[ token, dontuse, restargs ] = webargs.split ('/',2)
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj ( token )
dbcfg = dbconfig.switchDataset ( proj.getDataset() )
db = emcadb.EMCADB ( dbcfg, proj )
# Split the URL and get the args
args = restargs.split('/')
predicates = dict(zip(args[::2], args[1::2]))
annoids = db.getAnnoObjects ( predicates )
# We have a cutout as well
if postdata:
# RB this is a brute force implementation. This probably needs to be
# optimized to use several different execution strategies based on the
# cutout size and the number of objects.
# Make a named temporary file for the HDF5
tmpfile = tempfile.NamedTemporaryFile ( )
tmpfile.write ( postdata )
tmpfile.seek(0)
h5f = h5py.File ( tmpfile.name, driver='core', backing_store=False )
corner = h5f['XYZOFFSET'][:]
dim = h5f['CUTOUTSIZE'][:]
resolution = h5f['RESOLUTION'][0]
if not dbcfg.checkCube( resolution, corner[0], corner[0]+dim[0], corner[1], corner[1]+dim[1], corner[2], corner[2]+dim[2] ):
logger.warning ( "Illegal cutout corner=%s, dim=%s" % ( corner, dim))
raise ANNError ( "Illegal cutout corner=%s, dim=%s" % ( corner, dim))
# RBFIX this a hack
#
# the zstart in dbconfig is sometimes offset to make it aligned.
# Probably remove the offset is the best idea. and align data
# to zero regardless of where it starts. For now.
corner[2] -= dbcfg.slicerange[0]
cutout = db.cutout ( corner, dim, resolution )
annoids = np.intersect1d ( annoids, np.unique( cutout.data ))
if postdata:
h5f.close()
tmpfile.close()
return h5ann.PackageIDs ( annoids )
def deleteAnnotation(webargs):
"""Delete a RAMON object"""
## TODO add retry loop for transaction
[token, sym, otherargs] = webargs.partition('/')
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
db = emcadb.EMCADB(dbcfg, proj)
# Split the URL and get the args
args = otherargs.split('/', 2)
# if the first argument is numeric. it is an annoid
if re.match('^[\d,]+$', args[0]):
annoids = map(int, args[0].split(','))
# if not..this is not a well-formed delete request
else:
logger.warning(
"Delete did not specify a legal object identifier = %s" % args[0])
raise ANNError(
"Delete did not specify a legal object identifier = %s" % args[0])
for annoid in annoids:
db.startTxn()
tries = 0
done = False
while not done and tries < 5:
try:
db.deleteAnnotation(annoid)
done = True
# rollback if you catch an error
except MySQLdb.OperationalError, e:
logger.warning("Transaction did not complete. %s" % (e))
tries += 1
db.rollback()
continue
except MySQLdb.Error, e:
logger.warning("Put transaction rollback. %s" % (e))
db.rollback()
raise
except Exception, e:
logger.exception(
"Put transaction rollback. Unknown error. %s" % (e))
db.rollback()
raise
def __init__(self, token, annoid, resolution):
"""DB configuration and some state"""
self._annid = annoid
self._resolution = resolution
# Get DB configuration stuff
annprojdb = annproj.AnnotateProjectsDB()
annoproj = annprojdb.getAnnoProj(token)
self.dbcfg = dbconfig.switchDataset(annoproj.getDataset())
self.annodb = anndb.AnnotateDB(self.dbcfg, annoproj)
# Now get metadata for the old object
self.seg = self.annodb.getAnnotation(self._annid)
pprint(vars(self.seg))
def __init__(self,token,annoid,resolution):
"""DB configuration and some state"""
self._annid = annoid
self._resolution = resolution
# Get DB configuration stuff
annprojdb = annproj.AnnotateProjectsDB()
annoproj = annprojdb.getAnnoProj ( token )
self.dbcfg = dbconfig.switchDataset ( annoproj.getDataset() )
self.annodb = anndb.AnnotateDB ( self.dbcfg, annoproj )
# Now get metadata for the old object
self.seg = self.annodb.getAnnotation( self._annid )
pprint (vars(self.seg))
def deleteAnnotation ( webargs ):
"""Delete a RAMON object"""
## TODO add retry loop for transaction
[ token, sym, otherargs ] = webargs.partition ('/')
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj ( token )
dbcfg = dbconfig.switchDataset ( proj.getDataset() )
db = emcadb.EMCADB ( dbcfg, proj )
# Split the URL and get the args
args = otherargs.split('/', 2)
# if the first argument is numeric. it is an annoid
if re.match ( '^[\d,]+$', args[0] ):
annoids = map(int, args[0].split(','))
# if not..this is not a well-formed delete request
else:
logger.warning ("Delete did not specify a legal object identifier = %s" % args[0] )
raise ANNError ("Delete did not specify a legal object identifier = %s" % args[0] )
for annoid in annoids:
db.startTxn()
tries = 0
done = False
while not done and tries < 5:
try:
db.deleteAnnotation ( annoid )
done = True
# rollback if you catch an error
except MySQLdb.OperationalError, e:
logger.warning ("Transaction did not complete. %s" % (e))
tries += 1
db.rollback()
continue
except MySQLdb.Error, e:
logger.warning ("Put transaction rollback. %s" % (e))
db.rollback()
raise
except Exception, e:
logger.exception ("Put transaction rollback. Unknown error. %s" % (e))
db.rollback()
raise
def newEMCAProj ( self, token, openid, dbhost, project, dbtype, dataset, dataurl, resolution, readonly, exceptions ):
"""Create a new emca project"""
# TODO need to undo the project creation if not totally sucessful
dbcfg = dbconfig.switchDataset ( dataset )
# Insert the project entry into the database
sql = "INSERT INTO {0} VALUES (\'{1}\',\'{2}\',\'{3}\',\'{4}\',{5},\'{6}\',\'{7}\',{8},{9},{10})".format (\
emcaprivate.table, token, openid, dbhost, project, dbtype, dataset, dataurl, resolution, readonly, exceptions )
logger.info ( "Creating new project. Host %s. Project %s. SQL=%s" % ( dbhost, project, sql ))
try:
cursor = self.conn.cursor()
cursor.execute ( sql )
except MySQLdb.Error, e:
logger.error ("Could not query emca projects database %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
raise ANNError ("Could not query emca projects database %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def projInfo(webargs):
"""Return information about the project and database"""
[token, projinfoliteral, otherargs] = webargs.split('/', 2)
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
# Create an in-memory HDF5 file
tmpfile = tempfile.NamedTemporaryFile()
h5f = h5py.File(tmpfile.name)
# Populate the file with project information
proj.h5Info(h5f)
dbcfg.h5Info(h5f)
h5f.close()
tmpfile.seek(0)
return tmpfile.read()
def projInfo ( webargs ):
"""Return information about the project and database"""
[ token, projinfoliteral, otherargs ] = webargs.split ('/',2)
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj ( token )
dbcfg = dbconfig.switchDataset ( proj.getDataset() )
# Create an in-memory HDF5 file
tmpfile = tempfile.NamedTemporaryFile ()
h5f = h5py.File ( tmpfile.name )
# Populate the file with project information
proj.h5Info ( h5f )
dbcfg.h5Info ( h5f )
h5f.close()
tmpfile.seek(0)
return tmpfile.read()
def putAnnotation ( webargs, postdata ):
"""Put a RAMON object as HDF5 by object identifier"""
[ token, sym, optionsargs ] = webargs.partition ('/')
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj ( token )
dbcfg = dbconfig.switchDataset ( proj.getDataset() )
db = emcadb.EMCADB ( dbcfg, proj )
# Don't write to readonly projects
if proj.getReadOnly()==1:
logger.warning("Attempt to write to read only project. %s: %s" % (proj.getDBName(),webargs))
raise ANNError("Attempt to write to read only project. %s: %s" % (proj.getDBName(),webargs))
options = optionsargs.split('/')
# return string of id values
retvals = []
# Make a named temporary file for the HDF5
tmpfile = tempfile.NamedTemporaryFile ( )
tmpfile.write ( postdata )
tmpfile.seek(0)
h5f = h5py.File ( tmpfile.name, driver='core', backing_store=False )
try:
for k in h5f.keys():
idgrp = h5f.get(k)
# Convert HDF5 to annotation
anno = h5ann.H5toAnnotation ( k, idgrp )
# set the identifier (separate transaction)
if not ('update' in options or 'dataonly' in options or 'reduce' in options):
anno.setID ( db )
tries = 0
done = False
while not done and tries < 5:
# start a transaction: get mysql out of line at a time mode
db.startTxn ()
try:
if anno.__class__ in [ annotation.AnnNeuron, annotation.AnnSeed ] and ( idgrp.get('VOXELS') or idgrp.get('CUTOUT')):
logger.warning ("Cannot write to annotation type %s" % (anno.__class__))
raise ANNError ("Cannot write to annotation type %s" % (anno.__class__))
if 'update' in options and 'dataonly' in options:
logger.warning ("Illegal combination of options. Cannot use udpate and dataonly together")
raise ANNError ("Illegal combination of options. Cannot use udpate and dataonly together")
elif not 'dataonly' in options and not 'reduce' in options:
# Put into the database
db.putAnnotation ( anno, options )
retvals.append(anno.annid)
# Is a resolution specified? or use default
h5resolution = idgrp.get('RESOLUTION')
if h5resolution == None:
resolution = proj.getResolution()
else:
resolution = h5resolution[0]
# Load the data associated with this annotation
# Is it voxel data?
voxels = idgrp.get('VOXELS')
if voxels and 'reduce' not in options:
if 'preserve' in options:
conflictopt = 'P'
elif 'exception' in options:
conflictopt = 'E'
else:
conflictopt = 'O'
# Check that the voxels have a conforming size:
if voxels.shape[1] != 3:
logger.warning ("Voxels data not the right shape. Must be (:,3). Shape is %s" % str(voxels.shape))
raise ANNError ("Voxels data not the right shape. Must be (:,3). Shape is %s" % str(voxels.shape))
exceptions = db.annotate ( anno.annid, resolution, voxels, conflictopt )
# Otherwise this is a shave operation
elif voxels and 'reduce' in options:
# Check that the voxels have a conforming size:
if voxels.shape[1] != 3:
logger.warning ("Voxels data not the right shape. Must be (:,3). Shape is %s" % str(voxels.shape))
raise ANNError ("Voxels data not the right shape. Must be (:,3). Shape is %s" % str(voxels.shape))
db.shave ( anno.annid, resolution, voxels )
# Is it dense data?
cutout = idgrp.get('CUTOUT')
h5xyzoffset = idgrp.get('XYZOFFSET')
if cutout != None and h5xyzoffset != None and 'reduce' not in options:
if 'preserve' in options:
conflictopt = 'P'
elif 'exception' in options:
conflictopt = 'E'
else:
conflictopt = 'O'
# the zstart in dbconfig is sometimes offset to make it aligned.
# Probably remove the offset is the best idea. and align data
# to zero regardless of where it starts. For now.
corner = h5xyzoffset[:]
corner[2] -= dbcfg.slicerange[0]
db.annotateEntityDense ( anno.annid, corner, resolution, np.array(cutout), conflictopt )
elif cutout != None and h5xyzoffset != None and 'reduce' in options:
corner = h5xyzoffset[:]
corner[2] -= dbcfg.slicerange[0]
db.shaveEntityDense ( anno.annid, corner, resolution, np.array(cutout))
elif cutout != None or h5xyzoffset != None:
#TODO this is a loggable error
pass
# Commit if there is no error
db.commit()
# Here with no error is successful
done = True
# rollback if you catch an error
except MySQLdb.OperationalError, e:
logger.warning ("Transaction did not complete. %s" % (e))
tries += 1
db.rollback()
continue
except MySQLdb.Error, e:
logger.warning ("Put transaction rollback. %s" % (e))
db.rollback()
raise
def mcFalseColor(webargs):
"""False color image of multiple channels"""
[token, mcfcstr, chanstr, service, imageargs] = webargs.split('/', 4)
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
if proj.getDBType() != emcaproj.CHANNELS:
logger.warning("Not a multiple channel project.")
raise ANNError("Not a multiple channel project.")
channels = chanstr.split(",")
combined_img = None
for i in range(len(channels)):
if service == 'xy':
cb = xySlice(str(channels[i]) + "/" + imageargs, dbcfg, proj)
elif service == 'xz':
cb = xzSlice(str(channels[i]) + "/" + imageargs, dbcfg, proj)
elif service == 'yz':
cb = yzSlice(str(channels[i]) + "/" + imageargs, dbcfg, proj)
else:
logger.warning("No such service %s. Args: %s" % (service, webargs))
raise ANNError("No such service %s" % (service))
# First channel is cyan
if i == 0:
data32 = np.array(cb.data * (1. / 256), dtype=np.uint32)
combined_img = 0xFF000000 + np.left_shift(
data32, 8) + np.left_shift(data32, 16)
# Second is yellow
elif i == 1:
data32 = np.array(cb.data * (1. / 256), dtype=np.uint32)
combined_img += np.left_shift(data32, 8) + data32
# Third is Magenta
elif i == 2:
data32 = np.array(cb.data * (1. / 256), dtype=np.uint32)
combined_img += np.left_shift(data32, 16) + data32
# Fourth is Red
elif i == 3:
data32 = np.array(cb.data * (1. / 256), dtype=np.uint32)
combined_img += data32
# Fifth is Green
elif i == 4:
data32 = np.array(cb.data * (1. / 256), dtype=np.uint32)
combined_img += np.left_shift(data32, 8)
# Sixth is Blue
elif i == 5:
data32 = np.array(cb.data * (1. / 256), dtype=np.uint32)
combined_img += np.left_shift(data32, 16)
else:
logger.warning(
"Only support six channels at a time. You requested %s " %
(chanstr))
raise ANNError(
"Only support six channels at a time. You requested %s " %
(chanstr))
if service == 'xy':
ydim, xdim = combined_img.shape[1:3]
outimage = Image.frombuffer('RGBA', (xdim, ydim),
combined_img[0, :, :].flatten(), 'raw',
'RGBA', 0, 1)
elif service == 'xz':
ydim = combined_img.shape[0]
xdim = combined_img.shape[2]
outimage = Image.frombuffer('RGBA', (xdim, ydim),
combined_img[:, 0, :].flatten(), 'raw',
'RGBA', 0, 1)
elif service == 'yz':
ydim = combined_img.shape[0]
xdim = combined_img.shape[1]
outimage = Image.frombuffer('RGBA', (xdim, ydim),
combined_img[:, :, 0].flatten(), 'raw',
'RGBA', 0, 1)
# Enhance the image
import ImageEnhance
enhancer = ImageEnhance.Brightness(outimage)
outimage = enhancer.enhance(4.0)
fileobj = cStringIO.StringIO()
outimage.save(fileobj, "PNG")
fileobj.seek(0)
return fileobj.read()
# include the paths EM_BASE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), ".." )) EM_UTIL_PATH = os.path.join(EM_BASE_PATH, "util" ) EM_EMCA_PATH = os.path.join(EM_BASE_PATH, "emca" ) EM_DBCONFIG_PATH = os.path.join(EM_BASE_PATH, "dbconfig" ) sys.path += [ EM_UTIL_PATH, EM_DBCONFIG_PATH, EM_EMCA_PATH ] import emcaproj import emcadb import dbconfig # load the project projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj ( 'kasthuri11' ) dbcfg = dbconfig.switchDataset ( proj.getDataset() ) #Load the database db = emcadb.EMCADB ( dbcfg, proj ) _ximagesz = 4096 _yimagesz = 4096 fid = open ( "/data/formisha/rasdamancutout", "w" ) for zstart in range(1,1024,16): zend = min(zstart+16,1025) corner = [1024,1024,zstart] resolution = 1
def emcacatmaid(webargs):
"""Interface to the cutout service for catmaid request. It does address translation."""
CM_TILESIZE = 256
token, plane, resstr, xtilestr, ytilestr, zslicestr, rest = webargs.split(
'/', 7)
xtile = int(xtilestr)
ytile = int(ytilestr)
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
# datatype from the project
if proj.getDBType() == emcaproj.IMAGES:
datatype = np.uint8
else:
datatype = np.uint32
resolution = int(resstr)
# build the cutout request
if plane == 'xy':
# figure out the cutout (limit to max image size)
xstart = xtile * CM_TILESIZE
ystart = ytile * CM_TILESIZE
xend = min((xtile + 1) * CM_TILESIZE, dbcfg.imagesz[resolution][0])
yend = min((ytile + 1) * CM_TILESIZE, dbcfg.imagesz[resolution][1])
# Return empty data if request is outside bounds. don't like it.
if xstart >= xend or ystart >= yend:
cutoutdata = np.zeros([CM_TILESIZE, CM_TILESIZE], dtype=datatype)
else:
imageargs = '%s/%s,%s/%s,%s/%s/' % (resstr, xstart, xend, ystart,
yend, zslicestr)
cb = xySlice(imageargs, dbcfg, proj)
# reshape (if it's not a full cutout)
if cb.data.shape != [1, CM_TILESIZE, CM_TILESIZE]:
cutoutdata = np.zeros([CM_TILESIZE, CM_TILESIZE],
dtype=cb.data.dtype)
cutoutdata[0:cb.data.shape[1],
0:cb.data.shape[2]] = cb.data.reshape(
[cb.data.shape[1], cb.data.shape[2]])
else:
cutoutdata = cb.data.reshape([CM_TILESIZE, CM_TILESIZE])
elif plane == 'xz' or plane == 'yz':
# The x or y plane is normal. The z plane needs some translation.
# the ytilestr actually represents data in the z-plane
pixelsperslice = dbcfg.zscale[int(resstr)]
if plane == 'xz':
# figure out the cutout (limit to max image size)
xstart = xtile * CM_TILESIZE
xend = min((xtile + 1) * CM_TILESIZE, dbcfg.imagesz[resolution][0])
# Now we need the ytile'th set of CM_TILESZ
ystart = ytile * int(
float(CM_TILESIZE) / pixelsperslice) + dbcfg.slicerange[0]
# get more data so that we always have 512 pixels
yend = min(
(ytile + 1) * int(float(CM_TILESIZE) / pixelsperslice + 1),
dbcfg.slicerange[1] - dbcfg.slicerange[0] +
1) + dbcfg.slicerange[0]
# Return empty data if request is outside bounds. don't like it.
if xstart == xend or ystart == yend:
cutoutdata = np.zeros([CM_TILESIZE, CM_TILESIZE],
dtype=datatype)
else:
imageargs = '%s/%s,%s/%s/%s,%s/' % (resstr, xstart, xend,
zslicestr, ystart, yend)
cb = xzSlice(imageargs, dbcfg, proj)
if cb.data.shape != [CM_TILESIZE, 1, CM_TILESIZE]:
cutoutdata = np.zeros([CM_TILESIZE, CM_TILESIZE],
dtype=cb.data.dtype)
cutoutdata[0:cb.data.shape[0],
0:cb.data.shape[2]] = cb.data.reshape(
[cb.data.shape[0], cb.data.shape[2]])
else:
# reshape
cutoutdata = cb.data.reshape([CM_TILESIZE, CM_TILESIZE])
elif plane == 'yz':
# figure out the cutout (limit to max image size)
xtart = xtile * CM_TILESIZE
xend = min((xtile + 1) * CM_TILESIZE, dbcfg.imagesz[resolution][1])
ystart = ytile * int(
float(CM_TILESIZE) / pixelsperslice) + dbcfg.slicerange[0]
yend = min(
(ytile + 1) * int(float(CM_TILESIZE) / pixelsperslice + 1),
dbcfg.slicerange[1] - dbcfg.slicerange[0] +
1) + dbcfg.slicerange[0]
# Return empty data if request is outside bounds. don't like it.
if xstart == xend or ystart == yend:
cutoutdata = np.zeros([CM_TILESIZE, CM_TILESIZE],
dtype=datatype)
else:
imageargs = '%s/%s/%s,%s/%s,%s/' % (
resstr, zslicestr, xtile * CM_TILESIZE,
(xtile + 1) * CM_TILESIZE, ystart, yend)
cb = yzSlice(imageargs, dbcfg, proj)
if cb.data.shape != [CM_TILESIZE, CM_TILESIZE, 1]:
cutoutdata = np.zeros([CM_TILESIZE, CM_TILESIZE],
dtype=cb.data.dtype)
cutoutdata[0:cb.data.shape[0],
0:cb.data.shape[1]] = cb.data.reshape(
[cb.data.shape[0], cb.data.shape[1]])
else:
cutoutdata = cb.data.reshape([CM_TILESIZE, CM_TILESIZE])
else:
logger.warning(
"No such cutout plane: %s. Must be (xy|xz|yz).. Args %s" %
(service, webargs))
raise ANNError("No such cutout plane: %s. Must be (xy|xz|yz)." %
plane)
# Write the image to a readable stream
if cutoutdata.dtype == np.uint8:
outimage = Image.frombuffer('L', [CM_TILESIZE, CM_TILESIZE],
cutoutdata, 'raw', 'L', 0, 1)
elif cutoutdata.dtype == np.uint32:
recolor_cy(cutoutdata, cutoutdata)
outimage = Image.frombuffer('RGBA', [CM_TILESIZE, CM_TILESIZE],
cutoutdata, 'raw', 'RGBA', 0, 1)
return outimage
def main():
parser = argparse.ArgumentParser(description='Ingest a tiff stack.')
parser.add_argument('token', action="store" )
parser.add_argument('channel', type=int, action="store" )
parser.add_argument('path', action="store" )
parser.add_argument('numslices', type=int, action="store" )
result = parser.parse_args()
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj ( result.token )
dbcfg = dbconfig.switchDataset ( proj.getDataset() )
_ximgsz = None
_yimgsz = None
for sl in range(result.numslices):
filenm = result.path + '/' + '{:0>4}'.format(sl) + '.png'
print filenm
img = Image.open ( filenm, "r" )
if _ximgsz==None and _yimgsz==None:
_ximgsz,_yimgsz = img.size
imarray = np.zeros ( [result.numslices, _yimgsz, _ximgsz], dtype=np.uint16 )
else:
assert _ximgsz == img.size[0] and _yimgsz == img.size[1]
imarray[sl,:,:] = np.asarray ( img )
# get the size of the cube
xcubedim,ycubedim,zcubedim = dbcfg.cubedim[0]
# and the limits of iteration
xlimit = (_ximgsz-1) / xcubedim + 1
ylimit = (_yimgsz-1) / ycubedim + 1
zlimit = (result.numslices-1) / zcubedim + 1
# open the database
db = emcadb.EMCADB ( dbcfg, proj )
# get a db cursor
cursor = db.conn.cursor()
for z in range(zlimit):
db.commit()
for y in range(ylimit):
for x in range(xlimit):
zmin = z*zcubedim
zmax = min((z+1)*zcubedim,result.numslices)
zmaxrel = ((zmax-1)%zcubedim)+1
ymin = y*ycubedim
ymax = min((y+1)*ycubedim,_yimgsz)
ymaxrel = ((ymax-1)%ycubedim)+1
xmin = x*xcubedim
xmax = min((x+1)*xcubedim,_ximgsz)
xmaxrel = ((xmax-1)%xcubedim)+1
# morton key
key = zindex.XYZMorton ( [x,y,z] )
# Create a channel cube
cube = chancube.ChanCube ( [xcubedim,ycubedim,zcubedim] )
# data for this key
cube.data[0:zmaxrel,0:ymaxrel,0:xmaxrel] = imarray[zmin:zmax,ymin:ymax,xmin:xmax]
# compress the cube
npz = cube.toNPZ ()
# add the cube to the database
sql = "INSERT INTO " + proj.getTable(RESOLUTION) + "(zindex, channel, cube) VALUES (%s, %s, %s)"
print sql
try:
cursor.execute ( sql, (key, result.channel, npz))
except MySQLdb.Error, e:
raise ANNError ( "Error updating data cube: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def getAnnotations ( webargs, postdata ):
"""Get multiple annotations. Takes an HDF5 that lists ids in the post."""
[ token, objectsliteral, otherargs ] = webargs.split ('/',2)
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj ( token )
dbcfg = dbconfig.switchDataset ( proj.getDataset() )
db = emcadb.EMCADB ( dbcfg, proj )
# Read the post data HDF5 and get a list of identifiers
tmpinfile = tempfile.NamedTemporaryFile ( )
tmpinfile.write ( postdata )
tmpinfile.seek(0)
h5in = h5py.File ( tmpinfile.name )
# IDENTIFIERS
if not h5in.get('ANNOIDS'):
logger.warning ("Requesting multiple annotations. But no HDF5 \'ANNOIDS\' field specified.")
raise ANNError ("Requesting multiple annotations. But no HDF5 \'ANNOIDS\' field specified.")
# GET the data out of the HDF5 file. Never operate on the data in place.
annoids = h5in['ANNOIDS'][:]
# set variables to None: need them in call to getAnnoByID, but not all paths set all
corner = None
dim = None
resolution = None
dataarg = ''
# process options
# Split the URL and get the args
if otherargs != '':
( dataarg, cutout ) = otherargs.split('/', 1)
if dataarg =='' or dataarg == 'nodata':
dataoption = AR_NODATA
elif dataarg == 'voxels':
dataoption = AR_VOXELS
# only arg to voxels is resolution
[resstr, sym, rest] = cutout.partition('/')
resolution = int(resstr) if resstr != '' else proj.getResolution()
elif dataarg == 'cutout':
# if blank of just resolution then a tightcutout
if cutout == '' or re.match('^\d+[\/]*$', cutout):
dataoption = AR_TIGHTCUTOUT
[resstr, sym, rest] = cutout.partition('/')
resolution = int(resstr) if resstr != '' else proj.getResolution()
else:
dataoption = AR_CUTOUT
# Perform argument processing
brargs = restargs.BrainRestArgs ();
brargs.cutoutArgs ( cutout, dbcfg )
# Extract the relevant values
corner = brargs.getCorner()
dim = brargs.getDim()
resolution = brargs.getResolution()
# RBTODO test this interface
elif dataarg == 'boundingbox':
# if blank of just resolution then a tightcutout
if cutout == '' or re.match('^\d+[\/]*$', cutout):
dataoption = AR_BOUNDINGBOX
[resstr, sym, rest] = cutout.partition('/')
resolution = int(resstr) if resstr != '' else proj.getResolution()
else:
logger.warning ("In getAnnotations: Error: no such data option %s " % ( dataarg ))
raise ANNError ("In getAnnotations: Error: no such data option %s " % ( dataarg ))
# Make the HDF5 output file
# Create an in-memory HDF5 file
tmpoutfile = tempfile.NamedTemporaryFile()
h5fout = h5py.File ( tmpoutfile.name )
# get annotations for each identifier
for annoid in annoids:
# the int here is to prevent using a numpy value in an inner loop. This is a 10x performance gain.
getAnnoById ( int(annoid), h5fout, db, dbcfg, dataoption, resolution, corner, dim )
# close temporary file
h5in.close()
tmpinfile.close()
# Transmit back the populated HDF5 file
h5fout.flush()
tmpoutfile.seek(0)
return tmpoutfile.read()
def getAnnotation(webargs):
"""Fetch a RAMON object as HDF5 by object identifier"""
[token, sym, otherargs] = webargs.partition('/')
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
db = emcadb.EMCADB(dbcfg, proj)
# Split the URL and get the args
args = otherargs.split('/', 2)
# Make the HDF5 file
# Create an in-memory HDF5 file
tmpfile = tempfile.NamedTemporaryFile()
h5f = h5py.File(tmpfile.name)
# if the first argument is numeric. it is an annoid
if re.match('^[\d,]+$', args[0]):
annoids = map(int, args[0].split(','))
for annoid in annoids:
# default is no data
if args[1] == '' or args[1] == 'nodata':
dataoption = AR_NODATA
getAnnoById(annoid, h5f, db, dbcfg, dataoption)
# if you want voxels you either requested the resolution id/voxels/resolution
# or you get data from the default resolution
elif args[1] == 'voxels':
dataoption = AR_VOXELS
[resstr, sym, rest] = args[2].partition('/')
resolution = int(
resstr) if resstr != '' else proj.getResolution()
getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution)
elif args[1] == 'cutout':
# if there are no args or only resolution, it's a tight cutout request
if args[2] == '' or re.match('^\d+[\/]*$', args[2]):
dataoption = AR_TIGHTCUTOUT
[resstr, sym, rest] = args[2].partition('/')
resolution = int(
resstr) if resstr != '' else proj.getResolution()
getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution)
else:
dataoption = AR_CUTOUT
# Perform argument processing
brargs = restargs.BrainRestArgs()
brargs.cutoutArgs(args[2], dbcfg)
# Extract the relevant values
corner = brargs.getCorner()
dim = brargs.getDim()
resolution = brargs.getResolution()
getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution,
corner, dim)
elif args[1] == 'boundingbox':
dataoption = AR_BOUNDINGBOX
[resstr, sym, rest] = args[2].partition('/')
resolution = int(
resstr) if resstr != '' else proj.getResolution()
getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution)
else:
logger.warning(
"Fetch identifier %s. Error: no such data option %s " %
(annoid, args[1]))
raise ANNError(
"Fetch identifier %s. Error: no such data option %s " %
(annoid, args[1]))
# the first argument is not numeric. it is a service other than getAnnotation
else:
logger.warning(
"Get interface %s requested. Illegal or not implemented. Args: %s"
% (args[0], webargs))
raise ANNError(
"Get interface %s requested. Illegal or not implemented" %
(args[0]))
h5f.flush()
tmpfile.seek(0)
return tmpfile.read()
def getAnnotation ( webargs ):
"""Fetch a RAMON object as HDF5 by object identifier"""
[ token, sym, otherargs ] = webargs.partition ('/')
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj ( token )
dbcfg = dbconfig.switchDataset ( proj.getDataset() )
db = emcadb.EMCADB ( dbcfg, proj )
# Split the URL and get the args
args = otherargs.split('/', 2)
# Make the HDF5 file
# Create an in-memory HDF5 file
tmpfile = tempfile.NamedTemporaryFile()
h5f = h5py.File ( tmpfile.name )
# if the first argument is numeric. it is an annoid
if re.match ( '^[\d,]+$', args[0] ):
annoids = map(int, args[0].split(','))
for annoid in annoids:
# default is no data
if args[1] == '' or args[1] == 'nodata':
dataoption = AR_NODATA
getAnnoById ( annoid, h5f, db, dbcfg, dataoption )
# if you want voxels you either requested the resolution id/voxels/resolution
# or you get data from the default resolution
elif args[1] == 'voxels':
dataoption = AR_VOXELS
[resstr, sym, rest] = args[2].partition('/')
resolution = int(resstr) if resstr != '' else proj.getResolution()
getAnnoById ( annoid, h5f, db, dbcfg, dataoption, resolution )
elif args[1] =='cutout':
# if there are no args or only resolution, it's a tight cutout request
if args[2] == '' or re.match('^\d+[\/]*$', args[2]):
dataoption = AR_TIGHTCUTOUT
[resstr, sym, rest] = args[2].partition('/')
resolution = int(resstr) if resstr != '' else proj.getResolution()
getAnnoById ( annoid, h5f, db, dbcfg, dataoption, resolution )
else:
dataoption = AR_CUTOUT
# Perform argument processing
brargs = restargs.BrainRestArgs ();
brargs.cutoutArgs ( args[2], dbcfg )
# Extract the relevant values
corner = brargs.getCorner()
dim = brargs.getDim()
resolution = brargs.getResolution()
getAnnoById ( annoid, h5f, db, dbcfg, dataoption, resolution, corner, dim )
elif args[1] == 'boundingbox':
dataoption = AR_BOUNDINGBOX
[resstr, sym, rest] = args[2].partition('/')
resolution = int(resstr) if resstr != '' else proj.getResolution()
getAnnoById ( annoid, h5f, db, dbcfg, dataoption, resolution )
else:
logger.warning ("Fetch identifier %s. Error: no such data option %s " % ( annoid, args[1] ))
raise ANNError ("Fetch identifier %s. Error: no such data option %s " % ( annoid, args[1] ))
# the first argument is not numeric. it is a service other than getAnnotation
else:
logger.warning("Get interface %s requested. Illegal or not implemented. Args: %s" % ( args[0], webargs ))
raise ANNError ("Get interface %s requested. Illegal or not implemented" % ( args[0] ))
h5f.flush()
tmpfile.seek(0)
return tmpfile.read()
def emcacatmaid ( webargs ):
"""Interface to the cutout service for catmaid request. It does address translation."""
CM_TILESIZE=256
token, plane, resstr, xtilestr, ytilestr, zslicestr, rest = webargs.split('/',7)
xtile = int(xtilestr)
ytile = int(ytilestr)
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj ( token )
dbcfg = dbconfig.switchDataset ( proj.getDataset() )
# datatype from the project
if proj.getDBType() == emcaproj.IMAGES:
datatype = np.uint8
else:
datatype = np.uint32
resolution = int(resstr)
# build the cutout request
if plane=='xy':
# figure out the cutout (limit to max image size)
xstart = xtile*CM_TILESIZE
ystart = ytile*CM_TILESIZE
xend = min ((xtile+1)*CM_TILESIZE,dbcfg.imagesz[resolution][0])
yend = min ((ytile+1)*CM_TILESIZE,dbcfg.imagesz[resolution][1])
# Return empty data if request is outside bounds. don't like it.
if xstart>=xend or ystart>=yend:
cutoutdata = np.zeros ( [CM_TILESIZE,CM_TILESIZE], dtype=datatype )
else:
imageargs = '%s/%s,%s/%s,%s/%s/' % ( resstr, xstart, xend, ystart, yend, zslicestr )
cb = xySlice ( imageargs, dbcfg, proj )
# reshape (if it's not a full cutout)
if cb.data.shape != [1,CM_TILESIZE,CM_TILESIZE]:
cutoutdata = np.zeros ( [CM_TILESIZE,CM_TILESIZE], dtype=cb.data.dtype )
cutoutdata[0:cb.data.shape[1],0:cb.data.shape[2]] = cb.data.reshape([cb.data.shape[1],cb.data.shape[2]])
else:
cutoutdata = cb.data.reshape([CM_TILESIZE,CM_TILESIZE])
elif plane=='xz' or plane=='yz':
# The x or y plane is normal. The z plane needs some translation.
# the ytilestr actually represents data in the z-plane
pixelsperslice = dbcfg.zscale[int(resstr)]
if plane=='xz':
# figure out the cutout (limit to max image size)
xstart = xtile*CM_TILESIZE
xend = min ((xtile+1)*CM_TILESIZE,dbcfg.imagesz[resolution][0])
# Now we need the ytile'th set of CM_TILESZ
ystart = ytile*int(float(CM_TILESIZE)/pixelsperslice) + dbcfg.slicerange[0]
# get more data so that we always have 512 pixels
yend = min((ytile+1)*int(float(CM_TILESIZE)/pixelsperslice+1),dbcfg.slicerange[1]-dbcfg.slicerange[0]+1) + dbcfg.slicerange[0]
# Return empty data if request is outside bounds. don't like it.
if xstart==xend or ystart==yend:
cutoutdata = np.zeros ( [CM_TILESIZE,CM_TILESIZE], dtype=datatype )
else:
imageargs = '%s/%s,%s/%s/%s,%s/' % ( resstr, xstart, xend, zslicestr, ystart, yend )
cb = xzSlice ( imageargs, dbcfg, proj )
if cb.data.shape != [CM_TILESIZE,1,CM_TILESIZE]:
cutoutdata = np.zeros ( [CM_TILESIZE,CM_TILESIZE], dtype=cb.data.dtype )
cutoutdata[0:cb.data.shape[0],0:cb.data.shape[2]] = cb.data.reshape([cb.data.shape[0],cb.data.shape[2]])
else:
# reshape
cutoutdata = cb.data.reshape([CM_TILESIZE,CM_TILESIZE])
elif plane=='yz':
# figure out the cutout (limit to max image size)
xtart = xtile*CM_TILESIZE
xend = min ((xtile+1)*CM_TILESIZE,dbcfg.imagesz[resolution][1])
ystart = ytile*int(float(CM_TILESIZE)/pixelsperslice)+ dbcfg.slicerange[0]
yend = min((ytile+1)*int(float(CM_TILESIZE)/pixelsperslice+1),dbcfg.slicerange[1]-dbcfg.slicerange[0]+1)+ dbcfg.slicerange[0]
# Return empty data if request is outside bounds. don't like it.
if xstart==xend or ystart==yend:
cutoutdata = np.zeros ( [CM_TILESIZE,CM_TILESIZE], dtype=datatype )
else:
imageargs = '%s/%s/%s,%s/%s,%s/' % ( resstr, zslicestr, xtile*CM_TILESIZE, (xtile+1)*CM_TILESIZE, ystart, yend )
cb = yzSlice ( imageargs, dbcfg, proj )
if cb.data.shape != [CM_TILESIZE,CM_TILESIZE,1]:
cutoutdata = np.zeros ( [CM_TILESIZE,CM_TILESIZE], dtype=cb.data.dtype)
cutoutdata[0:cb.data.shape[0],0:cb.data.shape[1]] = cb.data.reshape([cb.data.shape[0],cb.data.shape[1]])
else:
cutoutdata = cb.data.reshape([CM_TILESIZE,CM_TILESIZE])
else:
logger.warning("No such cutout plane: %s. Must be (xy|xz|yz).. Args %s" % ( service, webargs ))
raise ANNError ( "No such cutout plane: %s. Must be (xy|xz|yz)." % plane )
# Write the image to a readable stream
if cutoutdata.dtype==np.uint8:
outimage = Image.frombuffer ( 'L', [CM_TILESIZE,CM_TILESIZE], cutoutdata, 'raw', 'L', 0, 1 )
elif cutoutdata.dtype==np.uint32:
recolor_cy (cutoutdata, cutoutdata)
outimage = Image.frombuffer ( 'RGBA', [CM_TILESIZE,CM_TILESIZE], cutoutdata, 'raw', 'RGBA', 0, 1 )
return outimage
def getAnnotations(webargs, postdata):
"""Get multiple annotations. Takes an HDF5 that lists ids in the post."""
[token, objectsliteral, otherargs] = webargs.split('/', 2)
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
db = emcadb.EMCADB(dbcfg, proj)
# Read the post data HDF5 and get a list of identifiers
tmpinfile = tempfile.NamedTemporaryFile()
tmpinfile.write(postdata)
tmpinfile.seek(0)
h5in = h5py.File(tmpinfile.name)
# IDENTIFIERS
if not h5in.get('ANNOIDS'):
logger.warning(
"Requesting multiple annotations. But no HDF5 \'ANNOIDS\' field specified."
)
raise ANNError(
"Requesting multiple annotations. But no HDF5 \'ANNOIDS\' field specified."
)
# GET the data out of the HDF5 file. Never operate on the data in place.
annoids = h5in['ANNOIDS'][:]
# set variables to None: need them in call to getAnnoByID, but not all paths set all
corner = None
dim = None
resolution = None
dataarg = ''
# process options
# Split the URL and get the args
if otherargs != '':
(dataarg, cutout) = otherargs.split('/', 1)
if dataarg == '' or dataarg == 'nodata':
dataoption = AR_NODATA
elif dataarg == 'voxels':
dataoption = AR_VOXELS
# only arg to voxels is resolution
[resstr, sym, rest] = cutout.partition('/')
resolution = int(resstr) if resstr != '' else proj.getResolution()
elif dataarg == 'cutout':
# if blank of just resolution then a tightcutout
if cutout == '' or re.match('^\d+[\/]*$', cutout):
dataoption = AR_TIGHTCUTOUT
[resstr, sym, rest] = cutout.partition('/')
resolution = int(resstr) if resstr != '' else proj.getResolution()
else:
dataoption = AR_CUTOUT
# Perform argument processing
brargs = restargs.BrainRestArgs()
brargs.cutoutArgs(cutout, dbcfg)
# Extract the relevant values
corner = brargs.getCorner()
dim = brargs.getDim()
resolution = brargs.getResolution()
# RBTODO test this interface
elif dataarg == 'boundingbox':
# if blank of just resolution then a tightcutout
if cutout == '' or re.match('^\d+[\/]*$', cutout):
dataoption = AR_BOUNDINGBOX
[resstr, sym, rest] = cutout.partition('/')
resolution = int(resstr) if resstr != '' else proj.getResolution()
else:
logger.warning("In getAnnotations: Error: no such data option %s " %
(dataarg))
raise ANNError("In getAnnotations: Error: no such data option %s " %
(dataarg))
# Make the HDF5 output file
# Create an in-memory HDF5 file
tmpoutfile = tempfile.NamedTemporaryFile()
h5fout = h5py.File(tmpoutfile.name)
# get annotations for each identifier
for annoid in annoids:
# the int here is to prevent using a numpy value in an inner loop. This is a 10x performance gain.
getAnnoById(int(annoid), h5fout, db, dbcfg, dataoption, resolution,
corner, dim)
# close temporary file
h5in.close()
tmpinfile.close()
# Transmit back the populated HDF5 file
h5fout.flush()
tmpoutfile.seek(0)
return tmpoutfile.read()
def main():
parser = argparse.ArgumentParser(description='Ingest a tiff stack.')
parser.add_argument('token', action="store" )
parser.add_argument('channel', type=int, action="store" )
parser.add_argument('path', action="store" )
parser.add_argument('numslices', type=int, action="store" )
result = parser.parse_args()
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj ( result.token )
dbcfg = dbconfig.switchDataset ( proj.getDataset() )
_ximgsz = None
_yimgsz = None
for sl in range(result.numslices):
filenm = result.path + '/' + '{:0>4}'.format(sl) + '.png'
print filenm
img = Image.open ( filenm, "r" )
if _ximgsz==None and _yimgsz==None:
_ximgsz,_yimgsz = img.size
imarray = np.zeros ( [result.numslices, _yimgsz, _ximgsz], dtype=np.uint16 )
else:
assert _ximgsz == img.size[0] and _yimgsz == img.size[1]
imarray[sl,:,:] = np.asarray ( img )
# get the size of the cube
xcubedim,ycubedim,zcubedim = dbcfg.cubedim[0]
# and the limits of iteration
xlimit = (_ximgsz-1) / xcubedim + 1
ylimit = (_yimgsz-1) / ycubedim + 1
zlimit = (result.numslices-1) / zcubedim + 1
# open the database
db = emcadb.EMCADB ( dbcfg, proj )
# get a db cursor
cursor = db.conn.cursor()
for z in range(zlimit):
db.commit()
for y in range(ylimit):
for x in range(xlimit):
zmin = z*zcubedim
zmax = min((z+1)*zcubedim,result.numslices)
zmaxrel = ((zmax-1)%zcubedim)+1
ymin = y*ycubedim
ymax = min((y+1)*ycubedim,_yimgsz)
ymaxrel = ((ymax-1)%ycubedim)+1
xmin = x*xcubedim
xmax = min((x+1)*xcubedim,_ximgsz)
xmaxrel = ((xmax-1)%xcubedim)+1
# morton key
key = zindex.XYZMorton ( [x,y,z] )
# Create a channel cube
cube = imagecube.ImageCube16 ( [xcubedim,ycubedim,zcubedim] )
# data for this key
cube.data[0:zmaxrel,0:ymaxrel,0:xmaxrel] = imarray[zmin:zmax,ymin:ymax,xmin:xmax]
# compress the cube
npz = cube.toNPZ ()
# add the cube to the database
sql = "INSERT INTO " + proj.getTable(RESOLUTION) + "(channel, zindex, cube) VALUES (%s, %s, %s)"
print sql
try:
cursor.execute ( sql, (result.channel, key, npz))
except MySQLdb.Error, e:
raise ANNError ( "Error updating data cube: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def mcFalseColor ( webargs ):
"""False color image of multiple channels"""
[ token, mcfcstr, chanstr, service, imageargs ] = webargs.split ('/', 4)
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj ( token )
dbcfg = dbconfig.switchDataset ( proj.getDataset() )
if proj.getDBType() != emcaproj.CHANNELS:
logger.warning ( "Not a multiple channel project." )
raise ANNError ( "Not a multiple channel project." )
channels = chanstr.split(",")
combined_img = None
for i in range(len(channels)):
if service == 'xy':
cb = xySlice ( str(channels[i]) + "/" + imageargs, dbcfg, proj )
elif service == 'xz':
cb = xzSlice ( str(channels[i]) + "/" + imageargs, dbcfg, proj )
elif service == 'yz':
cb = yzSlice ( str(channels[i]) + "/" + imageargs, dbcfg, proj )
else:
logger.warning ( "No such service %s. Args: %s" % (service,webargs))
raise ANNError ( "No such service %s" % (service) )
# First channel is cyan
if i == 0:
data32 = np.array ( cb.data * (1./256), dtype=np.uint32 )
combined_img = 0xFF000000 + np.left_shift(data32,8) + np.left_shift(data32,16)
# Second is yellow
elif i == 1:
data32 = np.array ( cb.data * (1./256), dtype=np.uint32 )
combined_img += np.left_shift(data32,8) + data32
# Third is Magenta
elif i == 2:
data32 = np.array ( cb.data * (1./256), dtype=np.uint32 )
combined_img += np.left_shift(data32,16) + data32
# Fourth is Red
elif i == 3:
data32 = np.array ( cb.data * (1./256), dtype=np.uint32 )
combined_img += data32
# Fifth is Green
elif i == 4:
data32 = np.array ( cb.data * (1./256), dtype=np.uint32 )
combined_img += np.left_shift(data32,8)
# Sixth is Blue
elif i == 5:
data32 = np.array ( cb.data * (1./256), dtype=np.uint32 )
combined_img += np.left_shift(data32,16)
else:
logger.warning ( "Only support six channels at a time. You requested %s " % (chanstr))
raise ANNError ( "Only support six channels at a time. You requested %s " % (chanstr))
if service == 'xy':
ydim, xdim = combined_img.shape[1:3]
outimage = Image.frombuffer ( 'RGBA', (xdim,ydim), combined_img[0,:,:].flatten(), 'raw', 'RGBA', 0, 1 )
elif service == 'xz':
ydim = combined_img.shape[0]
xdim = combined_img.shape[2]
outimage = Image.frombuffer ( 'RGBA', (xdim,ydim), combined_img[:,0,:].flatten(), 'raw', 'RGBA', 0, 1 )
elif service == 'yz':
ydim = combined_img.shape[0]
xdim = combined_img.shape[1]
outimage = Image.frombuffer ( 'RGBA', (xdim,ydim), combined_img[:,:,0].flatten(), 'raw', 'RGBA', 0, 1 )
# Enhance the image
import ImageEnhance
enhancer = ImageEnhance.Brightness(outimage)
outimage = enhancer.enhance(4.0)
fileobj = cStringIO.StringIO ( )
outimage.save ( fileobj, "PNG" )
fileobj.seek(0)
return fileobj.read()
def putAnnotation(webargs, postdata):
"""Put a RAMON object as HDF5 by object identifier"""
[token, sym, optionsargs] = webargs.partition('/')
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
db = emcadb.EMCADB(dbcfg, proj)
# Don't write to readonly projects
if proj.getReadOnly() == 1:
logger.warning("Attempt to write to read only project. %s: %s" %
(proj.getDBName(), webargs))
raise ANNError("Attempt to write to read only project. %s: %s" %
(proj.getDBName(), webargs))
options = optionsargs.split('/')
# return string of id values
retvals = []
# Make a named temporary file for the HDF5
tmpfile = tempfile.NamedTemporaryFile()
tmpfile.write(postdata)
tmpfile.seek(0)
h5f = h5py.File(tmpfile.name, driver='core', backing_store=False)
try:
for k in h5f.keys():
idgrp = h5f.get(k)
# Convert HDF5 to annotation
anno = h5ann.H5toAnnotation(k, idgrp)
# set the identifier (separate transaction)
if not ('update' in options or 'dataonly' in options
or 'reduce' in options):
anno.setID(db)
tries = 0
done = False
while not done and tries < 5:
# start a transaction: get mysql out of line at a time mode
db.startTxn()
try:
if anno.__class__ in [
annotation.AnnNeuron, annotation.AnnSeed
] and (idgrp.get('VOXELS') or idgrp.get('CUTOUT')):
logger.warning("Cannot write to annotation type %s" %
(anno.__class__))
raise ANNError("Cannot write to annotation type %s" %
(anno.__class__))
if 'update' in options and 'dataonly' in options:
logger.warning(
"Illegal combination of options. Cannot use udpate and dataonly together"
)
raise ANNError(
"Illegal combination of options. Cannot use udpate and dataonly together"
)
elif not 'dataonly' in options and not 'reduce' in options:
# Put into the database
db.putAnnotation(anno, options)
retvals.append(anno.annid)
# Is a resolution specified? or use default
h5resolution = idgrp.get('RESOLUTION')
if h5resolution == None:
resolution = proj.getResolution()
else:
resolution = h5resolution[0]
# Load the data associated with this annotation
# Is it voxel data?
voxels = idgrp.get('VOXELS')
if voxels and 'reduce' not in options:
if 'preserve' in options:
conflictopt = 'P'
elif 'exception' in options:
conflictopt = 'E'
else:
conflictopt = 'O'
# Check that the voxels have a conforming size:
if voxels.shape[1] != 3:
logger.warning(
"Voxels data not the right shape. Must be (:,3). Shape is %s"
% str(voxels.shape))
raise ANNError(
"Voxels data not the right shape. Must be (:,3). Shape is %s"
% str(voxels.shape))
exceptions = db.annotate(anno.annid, resolution,
voxels, conflictopt)
# Otherwise this is a shave operation
elif voxels and 'reduce' in options:
# Check that the voxels have a conforming size:
if voxels.shape[1] != 3:
logger.warning(
"Voxels data not the right shape. Must be (:,3). Shape is %s"
% str(voxels.shape))
raise ANNError(
"Voxels data not the right shape. Must be (:,3). Shape is %s"
% str(voxels.shape))
db.shave(anno.annid, resolution, voxels)
# Is it dense data?
cutout = idgrp.get('CUTOUT')
h5xyzoffset = idgrp.get('XYZOFFSET')
if cutout != None and h5xyzoffset != None and 'reduce' not in options:
if 'preserve' in options:
conflictopt = 'P'
elif 'exception' in options:
conflictopt = 'E'
else:
conflictopt = 'O'
# the zstart in dbconfig is sometimes offset to make it aligned.
# Probably remove the offset is the best idea. and align data
# to zero regardless of where it starts. For now.
corner = h5xyzoffset[:]
corner[2] -= dbcfg.slicerange[0]
db.annotateEntityDense(anno.annid, corner, resolution,
np.array(cutout), conflictopt)
elif cutout != None and h5xyzoffset != None and 'reduce' in options:
corner = h5xyzoffset[:]
corner[2] -= dbcfg.slicerange[0]
db.shaveEntityDense(anno.annid, corner, resolution,
np.array(cutout))
elif cutout != None or h5xyzoffset != None:
#TODO this is a loggable error
pass
# Commit if there is no error
db.commit()
# Here with no error is successful
done = True
# rollback if you catch an error
except MySQLdb.OperationalError, e:
logger.warning("Transaction did not complete. %s" % (e))
tries += 1
db.rollback()
continue
except MySQLdb.Error, e:
logger.warning("Put transaction rollback. %s" % (e))
db.rollback()
raise
