def getCSV(webargs): """Fetch a RAMON object as CSV. Always includes bounding box. No data option.""" [token, csvliteral, annoid, reststr] = webargs.split('/', 3) # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj(token) dbcfg = dbconfig.switchDataset(proj.getDataset()) db = emcadb.EMCADB(dbcfg, proj) # Make the HDF5 file # Create an in-memory HDF5 file tmpfile = tempfile.NamedTemporaryFile() h5f = h5py.File(tmpfile.name) dataoption = AR_BOUNDINGBOX [resstr, sym, rest] = reststr.partition('/') resolution = int(resstr) if resstr != '' else proj.getResolution() getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution) # convert the HDF5 file to csv csvstr = h5ann.h5toCSV(h5f) return csvstr
def __init__(self, token): """Load the database and project""" projdb = emcaproj.EMCAProjectsDB() self.proj = projdb.getProj(token) self.dbcfg = dbconfig.switchDataset(self.proj.getDataset()) # Bind the annotation database self.imgDB = emcadb.EMCADB(self.dbcfg, self.proj)
def annId(imageargs, dbcfg, proj): """Return the annotation identifier of a voxel""" # Perform argument processing (resolution, voxel) = restargs.voxel(imageargs, dbcfg) # Get the identifier db = emcadb.EMCADB(dbcfg, proj) return db.getVoxel(resolution, voxel)
def listAnnoObjects(webargs, postdata=None): """ Return a list of anno ids restricted by equality predicates. Equalities are alternating in field/value in the url. """ [token, dontuse, restargs] = webargs.split('/', 2) # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj(token) dbcfg = dbconfig.switchDataset(proj.getDataset()) db = emcadb.EMCADB(dbcfg, proj) # Split the URL and get the args args = restargs.split('/') predicates = dict(zip(args[::2], args[1::2])) annoids = db.getAnnoObjects(predicates) # We have a cutout as well if postdata: # RB this is a brute force implementation. This probably needs to be # optimized to use several different execution strategies based on the # cutout size and the number of objects. # Make a named temporary file for the HDF5 tmpfile = tempfile.NamedTemporaryFile() tmpfile.write(postdata) tmpfile.seek(0) h5f = h5py.File(tmpfile.name, driver='core', backing_store=False) corner = h5f['XYZOFFSET'][:] dim = h5f['CUTOUTSIZE'][:] resolution = h5f['RESOLUTION'][0] if not dbcfg.checkCube(resolution, corner[0], corner[0] + dim[0], corner[1], corner[1] + dim[1], corner[2], corner[2] + dim[2]): logger.warning("Illegal cutout corner=%s, dim=%s" % (corner, dim)) raise ANNError("Illegal cutout corner=%s, dim=%s" % (corner, dim)) # RBFIX this a hack # # the zstart in dbconfig is sometimes offset to make it aligned. # Probably remove the offset is the best idea. and align data # to zero regardless of where it starts. For now. corner[2] -= dbcfg.slicerange[0] cutout = db.cutout(corner, dim, resolution) annoids = np.intersect1d(annoids, np.unique(cutout.data)) if postdata: h5f.close() tmpfile.close() return h5ann.PackageIDs(annoids)
def deleteAnnotation(webargs): """Delete a RAMON object""" ## TODO add retry loop for transaction [token, sym, otherargs] = webargs.partition('/') # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj(token) dbcfg = dbconfig.switchDataset(proj.getDataset()) db = emcadb.EMCADB(dbcfg, proj) # Split the URL and get the args args = otherargs.split('/', 2) # if the first argument is numeric. it is an annoid if re.match('^[\d,]+$', args[0]): annoids = map(int, args[0].split(',')) # if not..this is not a well-formed delete request else: logger.warning( "Delete did not specify a legal object identifier = %s" % args[0]) raise ANNError( "Delete did not specify a legal object identifier = %s" % args[0]) for annoid in annoids: db.startTxn() tries = 0 done = False while not done and tries < 5: try: db.deleteAnnotation(annoid) done = True # rollback if you catch an error except MySQLdb.OperationalError, e: logger.warning("Transaction did not complete. %s" % (e)) tries += 1 db.rollback() continue except MySQLdb.Error, e: logger.warning("Put transaction rollback. %s" % (e)) db.rollback() raise except Exception, e: logger.exception( "Put transaction rollback. Unknown error. %s" % (e)) db.rollback() raise
def __init__(self, token, resolution, path): self.path = path self.resolution = resolution self.projdb = emcaproj.EMCAProjectsDB() self.proj = self.projdb.loadProject(token) self._ximgsz = self.proj.datasetcfg.imagesz[resolution][0] self._yimgsz = self.proj.datasetcfg.imagesz[resolution][1] self.startslice = self.proj.datasetcfg.slicerange[0] self.endslice = self.proj.datasetcfg.slicerange[1] self.batchsz = self.proj.datasetcfg.cubedim[resolution][2] self.alldirs = os.listdir(path) # open the database self.db = emcadb.EMCADB(self.proj) # get a db cursor self.cursor = self.db.conn.cursor()
EM_EMCA_PATH = os.path.join(EM_BASE_PATH, "emca" ) EM_DBCONFIG_PATH = os.path.join(EM_BASE_PATH, "dbconfig" ) sys.path += [ EM_UTIL_PATH, EM_DBCONFIG_PATH, EM_EMCA_PATH ] import emcaproj import emcadb import dbconfig # load the project projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj ( 'kasthuri11' ) dbcfg = dbconfig.switchDataset ( proj.getDataset() ) #Load the database db = emcadb.EMCADB ( dbcfg, proj ) _ximagesz = 4096 _yimagesz = 4096 fid = open ( "/data/formisha/rasdamancutout", "w" ) for zstart in range(1,1024,16): zend = min(zstart+16,1025) corner = [1024,1024,zstart] resolution = 1 dim = [ _ximagesz, _yimagesz, zend-zstart ] print "Corner %s dim %s." % ( corner, dim ) cube = db.cutout ( corner, dim, resolution )
def main(): parser = argparse.ArgumentParser(description='Ingest a tiff stack.') parser.add_argument('token', action="store" ) parser.add_argument('channel', type=int, action="store" ) parser.add_argument('path', action="store" ) parser.add_argument('numslices', type=int, action="store" ) result = parser.parse_args() projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj ( result.token ) dbcfg = dbconfig.switchDataset ( proj.getDataset() ) _ximgsz = None _yimgsz = None for sl in range(result.numslices): filenm = result.path + '/' + '{:0>4}'.format(sl) + '.png' print filenm img = Image.open ( filenm, "r" ) if _ximgsz==None and _yimgsz==None: _ximgsz,_yimgsz = img.size imarray = np.zeros ( [result.numslices, _yimgsz, _ximgsz], dtype=np.uint16 ) else: assert _ximgsz == img.size[0] and _yimgsz == img.size[1] imarray[sl,:,:] = np.asarray ( img ) # get the size of the cube xcubedim,ycubedim,zcubedim = dbcfg.cubedim[0] # and the limits of iteration xlimit = (_ximgsz-1) / xcubedim + 1 ylimit = (_yimgsz-1) / ycubedim + 1 zlimit = (result.numslices-1) / zcubedim + 1 # open the database db = emcadb.EMCADB ( dbcfg, proj ) # get a db cursor cursor = db.conn.cursor() for z in range(zlimit): db.commit() for y in range(ylimit): for x in range(xlimit): zmin = z*zcubedim zmax = min((z+1)*zcubedim,result.numslices) zmaxrel = ((zmax-1)%zcubedim)+1 ymin = y*ycubedim ymax = min((y+1)*ycubedim,_yimgsz) ymaxrel = ((ymax-1)%ycubedim)+1 xmin = x*xcubedim xmax = min((x+1)*xcubedim,_ximgsz) xmaxrel = ((xmax-1)%xcubedim)+1 # morton key key = zindex.XYZMorton ( [x,y,z] ) # Create a channel cube cube = chancube.ChanCube ( [xcubedim,ycubedim,zcubedim] ) # data for this key cube.data[0:zmaxrel,0:ymaxrel,0:xmaxrel] = imarray[zmin:zmax,ymin:ymax,xmin:xmax] # compress the cube npz = cube.toNPZ () # add the cube to the database sql = "INSERT INTO " + proj.getTable(RESOLUTION) + "(zindex, channel, cube) VALUES (%s, %s, %s)" print sql try: cursor.execute ( sql, (key, result.channel, npz)) except MySQLdb.Error, e: raise ANNError ( "Error updating data cube: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def getAnnotations(webargs, postdata): """Get multiple annotations. Takes an HDF5 that lists ids in the post.""" [token, objectsliteral, otherargs] = webargs.split('/', 2) # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj(token) dbcfg = dbconfig.switchDataset(proj.getDataset()) db = emcadb.EMCADB(dbcfg, proj) # Read the post data HDF5 and get a list of identifiers tmpinfile = tempfile.NamedTemporaryFile() tmpinfile.write(postdata) tmpinfile.seek(0) h5in = h5py.File(tmpinfile.name) # IDENTIFIERS if not h5in.get('ANNOIDS'): logger.warning( "Requesting multiple annotations. But no HDF5 \'ANNOIDS\' field specified." ) raise ANNError( "Requesting multiple annotations. But no HDF5 \'ANNOIDS\' field specified." ) # GET the data out of the HDF5 file. Never operate on the data in place. annoids = h5in['ANNOIDS'][:] # set variables to None: need them in call to getAnnoByID, but not all paths set all corner = None dim = None resolution = None dataarg = '' # process options # Split the URL and get the args if otherargs != '': (dataarg, cutout) = otherargs.split('/', 1) if dataarg == '' or dataarg == 'nodata': dataoption = AR_NODATA elif dataarg == 'voxels': dataoption = AR_VOXELS # only arg to voxels is resolution [resstr, sym, rest] = cutout.partition('/') resolution = int(resstr) if resstr != '' else proj.getResolution() elif dataarg == 'cutout': # if blank of just resolution then a tightcutout if cutout == '' or re.match('^\d+[\/]*$', cutout): dataoption = AR_TIGHTCUTOUT [resstr, sym, rest] = cutout.partition('/') resolution = int(resstr) if resstr != '' else proj.getResolution() else: dataoption = AR_CUTOUT # Perform argument processing brargs = restargs.BrainRestArgs() brargs.cutoutArgs(cutout, dbcfg) # Extract the relevant values corner = brargs.getCorner() dim = brargs.getDim() resolution = brargs.getResolution() # RBTODO test this interface elif dataarg == 'boundingbox': # if blank of just resolution then a tightcutout if cutout == '' or re.match('^\d+[\/]*$', cutout): dataoption = AR_BOUNDINGBOX [resstr, sym, rest] = cutout.partition('/') resolution = int(resstr) if resstr != '' else proj.getResolution() else: logger.warning("In getAnnotations: Error: no such data option %s " % (dataarg)) raise ANNError("In getAnnotations: Error: no such data option %s " % (dataarg)) # Make the HDF5 output file # Create an in-memory HDF5 file tmpoutfile = tempfile.NamedTemporaryFile() h5fout = h5py.File(tmpoutfile.name) # get annotations for each identifier for annoid in annoids: # the int here is to prevent using a numpy value in an inner loop. This is a 10x performance gain. getAnnoById(int(annoid), h5fout, db, dbcfg, dataoption, resolution, corner, dim) # close temporary file h5in.close() tmpinfile.close() # Transmit back the populated HDF5 file h5fout.flush() tmpoutfile.seek(0) return tmpoutfile.read()
def getAnnotation(webargs): """Fetch a RAMON object as HDF5 by object identifier""" [token, sym, otherargs] = webargs.partition('/') # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj(token) dbcfg = dbconfig.switchDataset(proj.getDataset()) db = emcadb.EMCADB(dbcfg, proj) # Split the URL and get the args args = otherargs.split('/', 2) # Make the HDF5 file # Create an in-memory HDF5 file tmpfile = tempfile.NamedTemporaryFile() h5f = h5py.File(tmpfile.name) # if the first argument is numeric. it is an annoid if re.match('^[\d,]+$', args[0]): annoids = map(int, args[0].split(',')) for annoid in annoids: # default is no data if args[1] == '' or args[1] == 'nodata': dataoption = AR_NODATA getAnnoById(annoid, h5f, db, dbcfg, dataoption) # if you want voxels you either requested the resolution id/voxels/resolution # or you get data from the default resolution elif args[1] == 'voxels': dataoption = AR_VOXELS [resstr, sym, rest] = args[2].partition('/') resolution = int( resstr) if resstr != '' else proj.getResolution() getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution) elif args[1] == 'cutout': # if there are no args or only resolution, it's a tight cutout request if args[2] == '' or re.match('^\d+[\/]*$', args[2]): dataoption = AR_TIGHTCUTOUT [resstr, sym, rest] = args[2].partition('/') resolution = int( resstr) if resstr != '' else proj.getResolution() getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution) else: dataoption = AR_CUTOUT # Perform argument processing brargs = restargs.BrainRestArgs() brargs.cutoutArgs(args[2], dbcfg) # Extract the relevant values corner = brargs.getCorner() dim = brargs.getDim() resolution = brargs.getResolution() getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution, corner, dim) elif args[1] == 'boundingbox': dataoption = AR_BOUNDINGBOX [resstr, sym, rest] = args[2].partition('/') resolution = int( resstr) if resstr != '' else proj.getResolution() getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution) else: logger.warning( "Fetch identifier %s. Error: no such data option %s " % (annoid, args[1])) raise ANNError( "Fetch identifier %s. Error: no such data option %s " % (annoid, args[1])) # the first argument is not numeric. it is a service other than getAnnotation else: logger.warning( "Get interface %s requested. Illegal or not implemented. Args: %s" % (args[0], webargs)) raise ANNError( "Get interface %s requested. Illegal or not implemented" % (args[0])) h5f.flush() tmpfile.seek(0) return tmpfile.read()
def selectPost(webargs, dbcfg, proj, postdata): """Parse the first arg and call the right post service""" [service, sym, postargs] = webargs.partition('/') # Don't write to readonly projects if proj.getReadOnly() == 1: logger.warning("Attempt to write to read only project. %s: %s" % (proj.getDBName(), webargs)) raise ANNError("Attempt to write to read only project. %s: %s" % (proj.getDBName(), webargs)) # choose to overwrite (default), preserve, or make exception lists # when voxels conflict # Perform argument processing # Bind the annotation database db = emcadb.EMCADB(dbcfg, proj) db.startTxn() tries = 0 done = False while not done and tries < 5: try: if service == 'npvoxels': # get the resolution [entity, resolution, conflictargs] = postargs.split('/', 2) # Grab the voxel list fileobj = cStringIO.StringIO(postdata) voxlist = np.load(fileobj) conflictopt = restargs.conflictOption(conflictargs) entityid = db.annotate(int(entity), int(resolution), voxlist, conflictopt) elif service == 'npdense': # Process the arguments try: args = restargs.BrainRestArgs() args.cutoutArgs(postargs, dbcfg) except restargs.RESTArgsError, e: logger.warning("REST Arguments failed: %s" % (e)) raise ANNError(e) corner = args.getCorner() resolution = args.getResolution() # This is used for ingest only now. So, overwrite conflict option. conflictopt = restargs.conflictOption("") # get the data out of the compressed blob rawdata = zlib.decompress(postdata) fileobj = cStringIO.StringIO(rawdata) voxarray = np.load(fileobj) # Get the annotation database db = emcadb.EMCADB(dbcfg, proj) if proj.getDBType() == emcaproj.IMAGES: db.writeImageCuboid(corner, resolution, voxarray) # this is just a status entityid = 0 # Choose the verb, get the entity (as needed), and annotate # Translates the values directly else: entityid = db.annotateDense(corner, resolution, voxarray, conflictopt) db.conn.commit() else: logger.warning( "An illegal Web POST service was requested: %s. Args %s" % (service, webargs)) raise ANNError("No such Web service: %s" % service) db.commit() done = True
def putAnnotation(webargs, postdata): """Put a RAMON object as HDF5 by object identifier""" [token, sym, optionsargs] = webargs.partition('/') # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj(token) dbcfg = dbconfig.switchDataset(proj.getDataset()) db = emcadb.EMCADB(dbcfg, proj) # Don't write to readonly projects if proj.getReadOnly() == 1: logger.warning("Attempt to write to read only project. %s: %s" % (proj.getDBName(), webargs)) raise ANNError("Attempt to write to read only project. %s: %s" % (proj.getDBName(), webargs)) options = optionsargs.split('/') # return string of id values retvals = [] # Make a named temporary file for the HDF5 tmpfile = tempfile.NamedTemporaryFile() tmpfile.write(postdata) tmpfile.seek(0) h5f = h5py.File(tmpfile.name, driver='core', backing_store=False) try: for k in h5f.keys(): idgrp = h5f.get(k) # Convert HDF5 to annotation anno = h5ann.H5toAnnotation(k, idgrp) # set the identifier (separate transaction) if not ('update' in options or 'dataonly' in options or 'reduce' in options): anno.setID(db) tries = 0 done = False while not done and tries < 5: # start a transaction: get mysql out of line at a time mode db.startTxn() try: if anno.__class__ in [ annotation.AnnNeuron, annotation.AnnSeed ] and (idgrp.get('VOXELS') or idgrp.get('CUTOUT')): logger.warning("Cannot write to annotation type %s" % (anno.__class__)) raise ANNError("Cannot write to annotation type %s" % (anno.__class__)) if 'update' in options and 'dataonly' in options: logger.warning( "Illegal combination of options. Cannot use udpate and dataonly together" ) raise ANNError( "Illegal combination of options. Cannot use udpate and dataonly together" ) elif not 'dataonly' in options and not 'reduce' in options: # Put into the database db.putAnnotation(anno, options) retvals.append(anno.annid) # Is a resolution specified? or use default h5resolution = idgrp.get('RESOLUTION') if h5resolution == None: resolution = proj.getResolution() else: resolution = h5resolution[0] # Load the data associated with this annotation # Is it voxel data? voxels = idgrp.get('VOXELS') if voxels and 'reduce' not in options: if 'preserve' in options: conflictopt = 'P' elif 'exception' in options: conflictopt = 'E' else: conflictopt = 'O' # Check that the voxels have a conforming size: if voxels.shape[1] != 3: logger.warning( "Voxels data not the right shape. Must be (:,3). Shape is %s" % str(voxels.shape)) raise ANNError( "Voxels data not the right shape. Must be (:,3). Shape is %s" % str(voxels.shape)) exceptions = db.annotate(anno.annid, resolution, voxels, conflictopt) # Otherwise this is a shave operation elif voxels and 'reduce' in options: # Check that the voxels have a conforming size: if voxels.shape[1] != 3: logger.warning( "Voxels data not the right shape. Must be (:,3). Shape is %s" % str(voxels.shape)) raise ANNError( "Voxels data not the right shape. Must be (:,3). Shape is %s" % str(voxels.shape)) db.shave(anno.annid, resolution, voxels) # Is it dense data? cutout = idgrp.get('CUTOUT') h5xyzoffset = idgrp.get('XYZOFFSET') if cutout != None and h5xyzoffset != None and 'reduce' not in options: if 'preserve' in options: conflictopt = 'P' elif 'exception' in options: conflictopt = 'E' else: conflictopt = 'O' # the zstart in dbconfig is sometimes offset to make it aligned. # Probably remove the offset is the best idea. and align data # to zero regardless of where it starts. For now. corner = h5xyzoffset[:] corner[2] -= dbcfg.slicerange[0] db.annotateEntityDense(anno.annid, corner, resolution, np.array(cutout), conflictopt) elif cutout != None and h5xyzoffset != None and 'reduce' in options: corner = h5xyzoffset[:] corner[2] -= dbcfg.slicerange[0] db.shaveEntityDense(anno.annid, corner, resolution, np.array(cutout)) elif cutout != None or h5xyzoffset != None: #TODO this is a loggable error pass # Commit if there is no error db.commit() # Here with no error is successful done = True # rollback if you catch an error except MySQLdb.OperationalError, e: logger.warning("Transaction did not complete. %s" % (e)) tries += 1 db.rollback() continue except MySQLdb.Error, e: logger.warning("Put transaction rollback. %s" % (e)) db.rollback() raise