def getCSV(webargs):
"""Fetch a RAMON object as CSV. Always includes bounding box. No data option."""
[token, csvliteral, annoid, reststr] = webargs.split('/', 3)
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
db = emcadb.EMCADB(dbcfg, proj)
# Make the HDF5 file
# Create an in-memory HDF5 file
tmpfile = tempfile.NamedTemporaryFile()
h5f = h5py.File(tmpfile.name)
dataoption = AR_BOUNDINGBOX
[resstr, sym, rest] = reststr.partition('/')
resolution = int(resstr) if resstr != '' else proj.getResolution()
getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution)
# convert the HDF5 file to csv
csvstr = h5ann.h5toCSV(h5f)
return csvstr
def __init__(self, token):
"""Load the database and project"""
projdb = emcaproj.EMCAProjectsDB()
self.proj = projdb.getProj(token)
self.dbcfg = dbconfig.switchDataset(self.proj.getDataset())
# Bind the annotation database
self.imgDB = emcadb.EMCADB(self.dbcfg, self.proj)
def annId(imageargs, dbcfg, proj):
"""Return the annotation identifier of a voxel"""
# Perform argument processing
(resolution, voxel) = restargs.voxel(imageargs, dbcfg)
# Get the identifier
db = emcadb.EMCADB(dbcfg, proj)
return db.getVoxel(resolution, voxel)
def listAnnoObjects(webargs, postdata=None):
""" Return a list of anno ids restricted by equality predicates.
Equalities are alternating in field/value in the url.
"""
[token, dontuse, restargs] = webargs.split('/', 2)
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
db = emcadb.EMCADB(dbcfg, proj)
# Split the URL and get the args
args = restargs.split('/')
predicates = dict(zip(args[::2], args[1::2]))
annoids = db.getAnnoObjects(predicates)
# We have a cutout as well
if postdata:
# RB this is a brute force implementation. This probably needs to be
# optimized to use several different execution strategies based on the
# cutout size and the number of objects.
# Make a named temporary file for the HDF5
tmpfile = tempfile.NamedTemporaryFile()
tmpfile.write(postdata)
tmpfile.seek(0)
h5f = h5py.File(tmpfile.name, driver='core', backing_store=False)
corner = h5f['XYZOFFSET'][:]
dim = h5f['CUTOUTSIZE'][:]
resolution = h5f['RESOLUTION'][0]
if not dbcfg.checkCube(resolution, corner[0], corner[0] + dim[0],
corner[1], corner[1] + dim[1], corner[2],
corner[2] + dim[2]):
logger.warning("Illegal cutout corner=%s, dim=%s" % (corner, dim))
raise ANNError("Illegal cutout corner=%s, dim=%s" % (corner, dim))
# RBFIX this a hack
#
# the zstart in dbconfig is sometimes offset to make it aligned.
# Probably remove the offset is the best idea. and align data
# to zero regardless of where it starts. For now.
corner[2] -= dbcfg.slicerange[0]
cutout = db.cutout(corner, dim, resolution)
annoids = np.intersect1d(annoids, np.unique(cutout.data))
if postdata:
h5f.close()
tmpfile.close()
return h5ann.PackageIDs(annoids)
def deleteAnnotation(webargs):
"""Delete a RAMON object"""
## TODO add retry loop for transaction
[token, sym, otherargs] = webargs.partition('/')
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
db = emcadb.EMCADB(dbcfg, proj)
# Split the URL and get the args
args = otherargs.split('/', 2)
# if the first argument is numeric. it is an annoid
if re.match('^[\d,]+$', args[0]):
annoids = map(int, args[0].split(','))
# if not..this is not a well-formed delete request
else:
logger.warning(
"Delete did not specify a legal object identifier = %s" % args[0])
raise ANNError(
"Delete did not specify a legal object identifier = %s" % args[0])
for annoid in annoids:
db.startTxn()
tries = 0
done = False
while not done and tries < 5:
try:
db.deleteAnnotation(annoid)
done = True
# rollback if you catch an error
except MySQLdb.OperationalError, e:
logger.warning("Transaction did not complete. %s" % (e))
tries += 1
db.rollback()
continue
except MySQLdb.Error, e:
logger.warning("Put transaction rollback. %s" % (e))
db.rollback()
raise
except Exception, e:
logger.exception(
"Put transaction rollback. Unknown error. %s" % (e))
db.rollback()
raise
def __init__(self, token, resolution, path):
self.path = path
self.resolution = resolution
self.projdb = emcaproj.EMCAProjectsDB()
self.proj = self.projdb.loadProject(token)
self._ximgsz = self.proj.datasetcfg.imagesz[resolution][0]
self._yimgsz = self.proj.datasetcfg.imagesz[resolution][1]
self.startslice = self.proj.datasetcfg.slicerange[0]
self.endslice = self.proj.datasetcfg.slicerange[1]
self.batchsz = self.proj.datasetcfg.cubedim[resolution][2]
self.alldirs = os.listdir(path)
# open the database
self.db = emcadb.EMCADB(self.proj)
# get a db cursor
self.cursor = self.db.conn.cursor()
EM_EMCA_PATH = os.path.join(EM_BASE_PATH, "emca" ) EM_DBCONFIG_PATH = os.path.join(EM_BASE_PATH, "dbconfig" ) sys.path += [ EM_UTIL_PATH, EM_DBCONFIG_PATH, EM_EMCA_PATH ] import emcaproj import emcadb import dbconfig # load the project projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj ( 'kasthuri11' ) dbcfg = dbconfig.switchDataset ( proj.getDataset() ) #Load the database db = emcadb.EMCADB ( dbcfg, proj ) _ximagesz = 4096 _yimagesz = 4096 fid = open ( "/data/formisha/rasdamancutout", "w" ) for zstart in range(1,1024,16): zend = min(zstart+16,1025) corner = [1024,1024,zstart] resolution = 1 dim = [ _ximagesz, _yimagesz, zend-zstart ] print "Corner %s dim %s." % ( corner, dim ) cube = db.cutout ( corner, dim, resolution )
def main():
parser = argparse.ArgumentParser(description='Ingest a tiff stack.')
parser.add_argument('token', action="store" )
parser.add_argument('channel', type=int, action="store" )
parser.add_argument('path', action="store" )
parser.add_argument('numslices', type=int, action="store" )
result = parser.parse_args()
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj ( result.token )
dbcfg = dbconfig.switchDataset ( proj.getDataset() )
_ximgsz = None
_yimgsz = None
for sl in range(result.numslices):
filenm = result.path + '/' + '{:0>4}'.format(sl) + '.png'
print filenm
img = Image.open ( filenm, "r" )
if _ximgsz==None and _yimgsz==None:
_ximgsz,_yimgsz = img.size
imarray = np.zeros ( [result.numslices, _yimgsz, _ximgsz], dtype=np.uint16 )
else:
assert _ximgsz == img.size[0] and _yimgsz == img.size[1]
imarray[sl,:,:] = np.asarray ( img )
# get the size of the cube
xcubedim,ycubedim,zcubedim = dbcfg.cubedim[0]
# and the limits of iteration
xlimit = (_ximgsz-1) / xcubedim + 1
ylimit = (_yimgsz-1) / ycubedim + 1
zlimit = (result.numslices-1) / zcubedim + 1
# open the database
db = emcadb.EMCADB ( dbcfg, proj )
# get a db cursor
cursor = db.conn.cursor()
for z in range(zlimit):
db.commit()
for y in range(ylimit):
for x in range(xlimit):
zmin = z*zcubedim
zmax = min((z+1)*zcubedim,result.numslices)
zmaxrel = ((zmax-1)%zcubedim)+1
ymin = y*ycubedim
ymax = min((y+1)*ycubedim,_yimgsz)
ymaxrel = ((ymax-1)%ycubedim)+1
xmin = x*xcubedim
xmax = min((x+1)*xcubedim,_ximgsz)
xmaxrel = ((xmax-1)%xcubedim)+1
# morton key
key = zindex.XYZMorton ( [x,y,z] )
# Create a channel cube
cube = chancube.ChanCube ( [xcubedim,ycubedim,zcubedim] )
# data for this key
cube.data[0:zmaxrel,0:ymaxrel,0:xmaxrel] = imarray[zmin:zmax,ymin:ymax,xmin:xmax]
# compress the cube
npz = cube.toNPZ ()
# add the cube to the database
sql = "INSERT INTO " + proj.getTable(RESOLUTION) + "(zindex, channel, cube) VALUES (%s, %s, %s)"
print sql
try:
cursor.execute ( sql, (key, result.channel, npz))
except MySQLdb.Error, e:
raise ANNError ( "Error updating data cube: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def getAnnotations(webargs, postdata):
"""Get multiple annotations. Takes an HDF5 that lists ids in the post."""
[token, objectsliteral, otherargs] = webargs.split('/', 2)
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
db = emcadb.EMCADB(dbcfg, proj)
# Read the post data HDF5 and get a list of identifiers
tmpinfile = tempfile.NamedTemporaryFile()
tmpinfile.write(postdata)
tmpinfile.seek(0)
h5in = h5py.File(tmpinfile.name)
# IDENTIFIERS
if not h5in.get('ANNOIDS'):
logger.warning(
"Requesting multiple annotations. But no HDF5 \'ANNOIDS\' field specified."
)
raise ANNError(
"Requesting multiple annotations. But no HDF5 \'ANNOIDS\' field specified."
)
# GET the data out of the HDF5 file. Never operate on the data in place.
annoids = h5in['ANNOIDS'][:]
# set variables to None: need them in call to getAnnoByID, but not all paths set all
corner = None
dim = None
resolution = None
dataarg = ''
# process options
# Split the URL and get the args
if otherargs != '':
(dataarg, cutout) = otherargs.split('/', 1)
if dataarg == '' or dataarg == 'nodata':
dataoption = AR_NODATA
elif dataarg == 'voxels':
dataoption = AR_VOXELS
# only arg to voxels is resolution
[resstr, sym, rest] = cutout.partition('/')
resolution = int(resstr) if resstr != '' else proj.getResolution()
elif dataarg == 'cutout':
# if blank of just resolution then a tightcutout
if cutout == '' or re.match('^\d+[\/]*$', cutout):
dataoption = AR_TIGHTCUTOUT
[resstr, sym, rest] = cutout.partition('/')
resolution = int(resstr) if resstr != '' else proj.getResolution()
else:
dataoption = AR_CUTOUT
# Perform argument processing
brargs = restargs.BrainRestArgs()
brargs.cutoutArgs(cutout, dbcfg)
# Extract the relevant values
corner = brargs.getCorner()
dim = brargs.getDim()
resolution = brargs.getResolution()
# RBTODO test this interface
elif dataarg == 'boundingbox':
# if blank of just resolution then a tightcutout
if cutout == '' or re.match('^\d+[\/]*$', cutout):
dataoption = AR_BOUNDINGBOX
[resstr, sym, rest] = cutout.partition('/')
resolution = int(resstr) if resstr != '' else proj.getResolution()
else:
logger.warning("In getAnnotations: Error: no such data option %s " %
(dataarg))
raise ANNError("In getAnnotations: Error: no such data option %s " %
(dataarg))
# Make the HDF5 output file
# Create an in-memory HDF5 file
tmpoutfile = tempfile.NamedTemporaryFile()
h5fout = h5py.File(tmpoutfile.name)
# get annotations for each identifier
for annoid in annoids:
# the int here is to prevent using a numpy value in an inner loop. This is a 10x performance gain.
getAnnoById(int(annoid), h5fout, db, dbcfg, dataoption, resolution,
corner, dim)
# close temporary file
h5in.close()
tmpinfile.close()
# Transmit back the populated HDF5 file
h5fout.flush()
tmpoutfile.seek(0)
return tmpoutfile.read()
def getAnnotation(webargs):
"""Fetch a RAMON object as HDF5 by object identifier"""
[token, sym, otherargs] = webargs.partition('/')
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
db = emcadb.EMCADB(dbcfg, proj)
# Split the URL and get the args
args = otherargs.split('/', 2)
# Make the HDF5 file
# Create an in-memory HDF5 file
tmpfile = tempfile.NamedTemporaryFile()
h5f = h5py.File(tmpfile.name)
# if the first argument is numeric. it is an annoid
if re.match('^[\d,]+$', args[0]):
annoids = map(int, args[0].split(','))
for annoid in annoids:
# default is no data
if args[1] == '' or args[1] == 'nodata':
dataoption = AR_NODATA
getAnnoById(annoid, h5f, db, dbcfg, dataoption)
# if you want voxels you either requested the resolution id/voxels/resolution
# or you get data from the default resolution
elif args[1] == 'voxels':
dataoption = AR_VOXELS
[resstr, sym, rest] = args[2].partition('/')
resolution = int(
resstr) if resstr != '' else proj.getResolution()
getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution)
elif args[1] == 'cutout':
# if there are no args or only resolution, it's a tight cutout request
if args[2] == '' or re.match('^\d+[\/]*$', args[2]):
dataoption = AR_TIGHTCUTOUT
[resstr, sym, rest] = args[2].partition('/')
resolution = int(
resstr) if resstr != '' else proj.getResolution()
getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution)
else:
dataoption = AR_CUTOUT
# Perform argument processing
brargs = restargs.BrainRestArgs()
brargs.cutoutArgs(args[2], dbcfg)
# Extract the relevant values
corner = brargs.getCorner()
dim = brargs.getDim()
resolution = brargs.getResolution()
getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution,
corner, dim)
elif args[1] == 'boundingbox':
dataoption = AR_BOUNDINGBOX
[resstr, sym, rest] = args[2].partition('/')
resolution = int(
resstr) if resstr != '' else proj.getResolution()
getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution)
else:
logger.warning(
"Fetch identifier %s. Error: no such data option %s " %
(annoid, args[1]))
raise ANNError(
"Fetch identifier %s. Error: no such data option %s " %
(annoid, args[1]))
# the first argument is not numeric. it is a service other than getAnnotation
else:
logger.warning(
"Get interface %s requested. Illegal or not implemented. Args: %s"
% (args[0], webargs))
raise ANNError(
"Get interface %s requested. Illegal or not implemented" %
(args[0]))
h5f.flush()
tmpfile.seek(0)
return tmpfile.read()
def selectPost(webargs, dbcfg, proj, postdata):
"""Parse the first arg and call the right post service"""
[service, sym, postargs] = webargs.partition('/')
# Don't write to readonly projects
if proj.getReadOnly() == 1:
logger.warning("Attempt to write to read only project. %s: %s" %
(proj.getDBName(), webargs))
raise ANNError("Attempt to write to read only project. %s: %s" %
(proj.getDBName(), webargs))
# choose to overwrite (default), preserve, or make exception lists
# when voxels conflict
# Perform argument processing
# Bind the annotation database
db = emcadb.EMCADB(dbcfg, proj)
db.startTxn()
tries = 0
done = False
while not done and tries < 5:
try:
if service == 'npvoxels':
# get the resolution
[entity, resolution, conflictargs] = postargs.split('/', 2)
# Grab the voxel list
fileobj = cStringIO.StringIO(postdata)
voxlist = np.load(fileobj)
conflictopt = restargs.conflictOption(conflictargs)
entityid = db.annotate(int(entity), int(resolution), voxlist,
conflictopt)
elif service == 'npdense':
# Process the arguments
try:
args = restargs.BrainRestArgs()
args.cutoutArgs(postargs, dbcfg)
except restargs.RESTArgsError, e:
logger.warning("REST Arguments failed: %s" % (e))
raise ANNError(e)
corner = args.getCorner()
resolution = args.getResolution()
# This is used for ingest only now. So, overwrite conflict option.
conflictopt = restargs.conflictOption("")
# get the data out of the compressed blob
rawdata = zlib.decompress(postdata)
fileobj = cStringIO.StringIO(rawdata)
voxarray = np.load(fileobj)
# Get the annotation database
db = emcadb.EMCADB(dbcfg, proj)
if proj.getDBType() == emcaproj.IMAGES:
db.writeImageCuboid(corner, resolution, voxarray)
# this is just a status
entityid = 0
# Choose the verb, get the entity (as needed), and annotate
# Translates the values directly
else:
entityid = db.annotateDense(corner, resolution, voxarray,
conflictopt)
db.conn.commit()
else:
logger.warning(
"An illegal Web POST service was requested: %s. Args %s" %
(service, webargs))
raise ANNError("No such Web service: %s" % service)
db.commit()
done = True
def putAnnotation(webargs, postdata):
"""Put a RAMON object as HDF5 by object identifier"""
[token, sym, optionsargs] = webargs.partition('/')
# Get the annotation database
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
db = emcadb.EMCADB(dbcfg, proj)
# Don't write to readonly projects
if proj.getReadOnly() == 1:
logger.warning("Attempt to write to read only project. %s: %s" %
(proj.getDBName(), webargs))
raise ANNError("Attempt to write to read only project. %s: %s" %
(proj.getDBName(), webargs))
options = optionsargs.split('/')
# return string of id values
retvals = []
# Make a named temporary file for the HDF5
tmpfile = tempfile.NamedTemporaryFile()
tmpfile.write(postdata)
tmpfile.seek(0)
h5f = h5py.File(tmpfile.name, driver='core', backing_store=False)
try:
for k in h5f.keys():
idgrp = h5f.get(k)
# Convert HDF5 to annotation
anno = h5ann.H5toAnnotation(k, idgrp)
# set the identifier (separate transaction)
if not ('update' in options or 'dataonly' in options
or 'reduce' in options):
anno.setID(db)
tries = 0
done = False
while not done and tries < 5:
# start a transaction: get mysql out of line at a time mode
db.startTxn()
try:
if anno.__class__ in [
annotation.AnnNeuron, annotation.AnnSeed
] and (idgrp.get('VOXELS') or idgrp.get('CUTOUT')):
logger.warning("Cannot write to annotation type %s" %
(anno.__class__))
raise ANNError("Cannot write to annotation type %s" %
(anno.__class__))
if 'update' in options and 'dataonly' in options:
logger.warning(
"Illegal combination of options. Cannot use udpate and dataonly together"
)
raise ANNError(
"Illegal combination of options. Cannot use udpate and dataonly together"
)
elif not 'dataonly' in options and not 'reduce' in options:
# Put into the database
db.putAnnotation(anno, options)
retvals.append(anno.annid)
# Is a resolution specified? or use default
h5resolution = idgrp.get('RESOLUTION')
if h5resolution == None:
resolution = proj.getResolution()
else:
resolution = h5resolution[0]
# Load the data associated with this annotation
# Is it voxel data?
voxels = idgrp.get('VOXELS')
if voxels and 'reduce' not in options:
if 'preserve' in options:
conflictopt = 'P'
elif 'exception' in options:
conflictopt = 'E'
else:
conflictopt = 'O'
# Check that the voxels have a conforming size:
if voxels.shape[1] != 3:
logger.warning(
"Voxels data not the right shape. Must be (:,3). Shape is %s"
% str(voxels.shape))
raise ANNError(
"Voxels data not the right shape. Must be (:,3). Shape is %s"
% str(voxels.shape))
exceptions = db.annotate(anno.annid, resolution,
voxels, conflictopt)
# Otherwise this is a shave operation
elif voxels and 'reduce' in options:
# Check that the voxels have a conforming size:
if voxels.shape[1] != 3:
logger.warning(
"Voxels data not the right shape. Must be (:,3). Shape is %s"
% str(voxels.shape))
raise ANNError(
"Voxels data not the right shape. Must be (:,3). Shape is %s"
% str(voxels.shape))
db.shave(anno.annid, resolution, voxels)
# Is it dense data?
cutout = idgrp.get('CUTOUT')
h5xyzoffset = idgrp.get('XYZOFFSET')
if cutout != None and h5xyzoffset != None and 'reduce' not in options:
if 'preserve' in options:
conflictopt = 'P'
elif 'exception' in options:
conflictopt = 'E'
else:
conflictopt = 'O'
# the zstart in dbconfig is sometimes offset to make it aligned.
# Probably remove the offset is the best idea. and align data
# to zero regardless of where it starts. For now.
corner = h5xyzoffset[:]
corner[2] -= dbcfg.slicerange[0]
db.annotateEntityDense(anno.annid, corner, resolution,
np.array(cutout), conflictopt)
elif cutout != None and h5xyzoffset != None and 'reduce' in options:
corner = h5xyzoffset[:]
corner[2] -= dbcfg.slicerange[0]
db.shaveEntityDense(anno.annid, corner, resolution,
np.array(cutout))
elif cutout != None or h5xyzoffset != None:
#TODO this is a loggable error
pass
# Commit if there is no error
db.commit()
# Here with no error is successful
done = True
# rollback if you catch an error
except MySQLdb.OperationalError, e:
logger.warning("Transaction did not complete. %s" % (e))
tries += 1
db.rollback()
continue
except MySQLdb.Error, e:
logger.warning("Put transaction rollback. %s" % (e))
db.rollback()
raise
