def newEMCAProj(self, token, openid, dbhost, project, dbtype, dataset, dataurl, resolution, readonly, exceptions): """Create a new emca project""" # TODO need to undo the project creation if not totally sucessful dbcfg = dbconfig.switchDataset(dataset) # Insert the project entry into the database sql = "INSERT INTO {0} VALUES (\'{1}\',\'{2}\',\'{3}\',\'{4}\',{5},\'{6}\',\'{7}\',{8},{9},{10})".format (\ emcaprivate.table, token, openid, dbhost, project, dbtype, dataset, dataurl, resolution, readonly, exceptions ) logger.info("Creating new project. Host %s. Project %s. SQL=%s" % (dbhost, project, sql)) try: cursor = self.conn.cursor() cursor.execute(sql) except MySQLdb.Error, e: logger.error( "Could not query emca projects database %d: %s. sql=%s" % (e.args[0], e.args[1], sql)) raise ANNError( "Could not query emca projects database %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def getCSV ( webargs ): """Fetch a RAMON object as CSV. Always includes bounding box. No data option.""" [ token, csvliteral, annoid, reststr ] = webargs.split ('/',3) # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj ( token ) dbcfg = dbconfig.switchDataset ( proj.getDataset() ) db = emcadb.EMCADB ( dbcfg, proj ) # Make the HDF5 file # Create an in-memory HDF5 file tmpfile = tempfile.NamedTemporaryFile() h5f = h5py.File ( tmpfile.name ) dataoption = AR_BOUNDINGBOX [resstr, sym, rest] = reststr.partition('/') resolution = int(resstr) if resstr != '' else proj.getResolution() getAnnoById ( annoid, h5f, db, dbcfg, dataoption, resolution ) # convert the HDF5 file to csv csvstr = h5ann.h5toCSV ( h5f ) return csvstr
def getCSV(webargs): """Fetch a RAMON object as CSV. Always includes bounding box. No data option.""" [token, csvliteral, annoid, reststr] = webargs.split('/', 3) # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj(token) dbcfg = dbconfig.switchDataset(proj.getDataset()) db = emcadb.EMCADB(dbcfg, proj) # Make the HDF5 file # Create an in-memory HDF5 file tmpfile = tempfile.NamedTemporaryFile() h5f = h5py.File(tmpfile.name) dataoption = AR_BOUNDINGBOX [resstr, sym, rest] = reststr.partition('/') resolution = int(resstr) if resstr != '' else proj.getResolution() getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution) # convert the HDF5 file to csv csvstr = h5ann.h5toCSV(h5f) return csvstr
def __init__(self, token): """Load the database and project""" projdb = emcaproj.EMCAProjectsDB() self.proj = projdb.getProj(token) self.dbcfg = dbconfig.switchDataset(self.proj.getDataset()) # Bind the annotation database self.imgDB = emcadb.EMCADB(self.dbcfg, self.proj)
def annopost ( webargs, postdata ): """Interface to the annotation write service Load the annotation project and invoke the appropriate dataset.""" [ token, sym, rangeargs ] = webargs.partition ('/') projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj ( token ) dbcfg = dbconfig.switchDataset ( proj.getDataset() ) return selectPost ( rangeargs, dbcfg, proj, postdata )
def annopost(webargs, postdata): """Interface to the annotation write service Load the annotation project and invoke the appropriate dataset.""" [token, sym, rangeargs] = webargs.partition('/') projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj(token) dbcfg = dbconfig.switchDataset(proj.getDataset()) return selectPost(rangeargs, dbcfg, proj, postdata)
def listAnnoObjects(webargs, postdata=None): """ Return a list of anno ids restricted by equality predicates. Equalities are alternating in field/value in the url. """ [token, dontuse, restargs] = webargs.split('/', 2) # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj(token) dbcfg = dbconfig.switchDataset(proj.getDataset()) db = emcadb.EMCADB(dbcfg, proj) # Split the URL and get the args args = restargs.split('/') predicates = dict(zip(args[::2], args[1::2])) annoids = db.getAnnoObjects(predicates) # We have a cutout as well if postdata: # RB this is a brute force implementation. This probably needs to be # optimized to use several different execution strategies based on the # cutout size and the number of objects. # Make a named temporary file for the HDF5 tmpfile = tempfile.NamedTemporaryFile() tmpfile.write(postdata) tmpfile.seek(0) h5f = h5py.File(tmpfile.name, driver='core', backing_store=False) corner = h5f['XYZOFFSET'][:] dim = h5f['CUTOUTSIZE'][:] resolution = h5f['RESOLUTION'][0] if not dbcfg.checkCube(resolution, corner[0], corner[0] + dim[0], corner[1], corner[1] + dim[1], corner[2], corner[2] + dim[2]): logger.warning("Illegal cutout corner=%s, dim=%s" % (corner, dim)) raise ANNError("Illegal cutout corner=%s, dim=%s" % (corner, dim)) # RBFIX this a hack # # the zstart in dbconfig is sometimes offset to make it aligned. # Probably remove the offset is the best idea. and align data # to zero regardless of where it starts. For now. corner[2] -= dbcfg.slicerange[0] cutout = db.cutout(corner, dim, resolution) annoids = np.intersect1d(annoids, np.unique(cutout.data)) if postdata: h5f.close() tmpfile.close() return h5ann.PackageIDs(annoids)
def __init__(self, token): """Load the annotation database and project""" projdb = emcaproj.EMCAProjectsDB() self.proj = projdb.getProj ( token ) self.dbcfg = dbconfig.switchDataset ( self.proj.getDataset() ) # Bind the annotation database self.annoDB = emcadb.EMCADB ( self.dbcfg, self.proj )
def emcaget ( webargs ): """Interface to the cutout service for annotations. Load the annotation project and invoke the appropriate dataset.""" [ token, sym, rangeargs ] = webargs.partition ('/') projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj ( token ) dbcfg = dbconfig.switchDataset ( proj.getDataset() ) return selectService ( rangeargs, dbcfg, proj )
def emcaget(webargs): """Interface to the cutout service for annotations. Load the annotation project and invoke the appropriate dataset.""" [token, sym, rangeargs] = webargs.partition('/') projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj(token) dbcfg = dbconfig.switchDataset(proj.getDataset()) return selectService(rangeargs, dbcfg, proj)
def __init__(self,token,resolution): """DB configuration and some state""" self._resolution = resolution # Get DB configuration stuff annprojdb = annproj.AnnotateProjectsDB() annoproj = annprojdb.getAnnoProj ( token ) self.dbcfg = dbconfig.switchDataset ( annoproj.getDataset() ) self.annodb = anndb.AnnotateDB ( self.dbcfg, annoproj )
def __init__(self, token, resolution): """DB configuration and some state""" self._resolution = resolution # Get DB configuration stuff annprojdb = annproj.AnnotateProjectsDB() annoproj = annprojdb.getAnnoProj(token) self.dbcfg = dbconfig.switchDataset(annoproj.getDataset()) self.annodb = anndb.AnnotateDB(self.dbcfg, annoproj)
def listAnnoObjects ( webargs, postdata=None ): """ Return a list of anno ids restricted by equality predicates. Equalities are alternating in field/value in the url. """ [ token, dontuse, restargs ] = webargs.split ('/',2) # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj ( token ) dbcfg = dbconfig.switchDataset ( proj.getDataset() ) db = emcadb.EMCADB ( dbcfg, proj ) # Split the URL and get the args args = restargs.split('/') predicates = dict(zip(args[::2], args[1::2])) annoids = db.getAnnoObjects ( predicates ) # We have a cutout as well if postdata: # RB this is a brute force implementation. This probably needs to be # optimized to use several different execution strategies based on the # cutout size and the number of objects. # Make a named temporary file for the HDF5 tmpfile = tempfile.NamedTemporaryFile ( ) tmpfile.write ( postdata ) tmpfile.seek(0) h5f = h5py.File ( tmpfile.name, driver='core', backing_store=False ) corner = h5f['XYZOFFSET'][:] dim = h5f['CUTOUTSIZE'][:] resolution = h5f['RESOLUTION'][0] if not dbcfg.checkCube( resolution, corner[0], corner[0]+dim[0], corner[1], corner[1]+dim[1], corner[2], corner[2]+dim[2] ): logger.warning ( "Illegal cutout corner=%s, dim=%s" % ( corner, dim)) raise ANNError ( "Illegal cutout corner=%s, dim=%s" % ( corner, dim)) # RBFIX this a hack # # the zstart in dbconfig is sometimes offset to make it aligned. # Probably remove the offset is the best idea. and align data # to zero regardless of where it starts. For now. corner[2] -= dbcfg.slicerange[0] cutout = db.cutout ( corner, dim, resolution ) annoids = np.intersect1d ( annoids, np.unique( cutout.data )) if postdata: h5f.close() tmpfile.close() return h5ann.PackageIDs ( annoids )
def deleteAnnotation(webargs): """Delete a RAMON object""" ## TODO add retry loop for transaction [token, sym, otherargs] = webargs.partition('/') # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj(token) dbcfg = dbconfig.switchDataset(proj.getDataset()) db = emcadb.EMCADB(dbcfg, proj) # Split the URL and get the args args = otherargs.split('/', 2) # if the first argument is numeric. it is an annoid if re.match('^[\d,]+$', args[0]): annoids = map(int, args[0].split(',')) # if not..this is not a well-formed delete request else: logger.warning( "Delete did not specify a legal object identifier = %s" % args[0]) raise ANNError( "Delete did not specify a legal object identifier = %s" % args[0]) for annoid in annoids: db.startTxn() tries = 0 done = False while not done and tries < 5: try: db.deleteAnnotation(annoid) done = True # rollback if you catch an error except MySQLdb.OperationalError, e: logger.warning("Transaction did not complete. %s" % (e)) tries += 1 db.rollback() continue except MySQLdb.Error, e: logger.warning("Put transaction rollback. %s" % (e)) db.rollback() raise except Exception, e: logger.exception( "Put transaction rollback. Unknown error. %s" % (e)) db.rollback() raise
def __init__(self, token, annoid, resolution): """DB configuration and some state""" self._annid = annoid self._resolution = resolution # Get DB configuration stuff annprojdb = annproj.AnnotateProjectsDB() annoproj = annprojdb.getAnnoProj(token) self.dbcfg = dbconfig.switchDataset(annoproj.getDataset()) self.annodb = anndb.AnnotateDB(self.dbcfg, annoproj) # Now get metadata for the old object self.seg = self.annodb.getAnnotation(self._annid) pprint(vars(self.seg))
def __init__(self,token,annoid,resolution): """DB configuration and some state""" self._annid = annoid self._resolution = resolution # Get DB configuration stuff annprojdb = annproj.AnnotateProjectsDB() annoproj = annprojdb.getAnnoProj ( token ) self.dbcfg = dbconfig.switchDataset ( annoproj.getDataset() ) self.annodb = anndb.AnnotateDB ( self.dbcfg, annoproj ) # Now get metadata for the old object self.seg = self.annodb.getAnnotation( self._annid ) pprint (vars(self.seg))
def deleteAnnotation ( webargs ): """Delete a RAMON object""" ## TODO add retry loop for transaction [ token, sym, otherargs ] = webargs.partition ('/') # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj ( token ) dbcfg = dbconfig.switchDataset ( proj.getDataset() ) db = emcadb.EMCADB ( dbcfg, proj ) # Split the URL and get the args args = otherargs.split('/', 2) # if the first argument is numeric. it is an annoid if re.match ( '^[\d,]+$', args[0] ): annoids = map(int, args[0].split(',')) # if not..this is not a well-formed delete request else: logger.warning ("Delete did not specify a legal object identifier = %s" % args[0] ) raise ANNError ("Delete did not specify a legal object identifier = %s" % args[0] ) for annoid in annoids: db.startTxn() tries = 0 done = False while not done and tries < 5: try: db.deleteAnnotation ( annoid ) done = True # rollback if you catch an error except MySQLdb.OperationalError, e: logger.warning ("Transaction did not complete. %s" % (e)) tries += 1 db.rollback() continue except MySQLdb.Error, e: logger.warning ("Put transaction rollback. %s" % (e)) db.rollback() raise except Exception, e: logger.exception ("Put transaction rollback. Unknown error. %s" % (e)) db.rollback() raise
def newEMCAProj ( self, token, openid, dbhost, project, dbtype, dataset, dataurl, resolution, readonly, exceptions ): """Create a new emca project""" # TODO need to undo the project creation if not totally sucessful dbcfg = dbconfig.switchDataset ( dataset ) # Insert the project entry into the database sql = "INSERT INTO {0} VALUES (\'{1}\',\'{2}\',\'{3}\',\'{4}\',{5},\'{6}\',\'{7}\',{8},{9},{10})".format (\ emcaprivate.table, token, openid, dbhost, project, dbtype, dataset, dataurl, resolution, readonly, exceptions ) logger.info ( "Creating new project. Host %s. Project %s. SQL=%s" % ( dbhost, project, sql )) try: cursor = self.conn.cursor() cursor.execute ( sql ) except MySQLdb.Error, e: logger.error ("Could not query emca projects database %d: %s. sql=%s" % (e.args[0], e.args[1], sql)) raise ANNError ("Could not query emca projects database %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def projInfo(webargs): """Return information about the project and database""" [token, projinfoliteral, otherargs] = webargs.split('/', 2) # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj(token) dbcfg = dbconfig.switchDataset(proj.getDataset()) # Create an in-memory HDF5 file tmpfile = tempfile.NamedTemporaryFile() h5f = h5py.File(tmpfile.name) # Populate the file with project information proj.h5Info(h5f) dbcfg.h5Info(h5f) h5f.close() tmpfile.seek(0) return tmpfile.read()
def projInfo ( webargs ): """Return information about the project and database""" [ token, projinfoliteral, otherargs ] = webargs.split ('/',2) # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj ( token ) dbcfg = dbconfig.switchDataset ( proj.getDataset() ) # Create an in-memory HDF5 file tmpfile = tempfile.NamedTemporaryFile () h5f = h5py.File ( tmpfile.name ) # Populate the file with project information proj.h5Info ( h5f ) dbcfg.h5Info ( h5f ) h5f.close() tmpfile.seek(0) return tmpfile.read()
def putAnnotation ( webargs, postdata ): """Put a RAMON object as HDF5 by object identifier""" [ token, sym, optionsargs ] = webargs.partition ('/') # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj ( token ) dbcfg = dbconfig.switchDataset ( proj.getDataset() ) db = emcadb.EMCADB ( dbcfg, proj ) # Don't write to readonly projects if proj.getReadOnly()==1: logger.warning("Attempt to write to read only project. %s: %s" % (proj.getDBName(),webargs)) raise ANNError("Attempt to write to read only project. %s: %s" % (proj.getDBName(),webargs)) options = optionsargs.split('/') # return string of id values retvals = [] # Make a named temporary file for the HDF5 tmpfile = tempfile.NamedTemporaryFile ( ) tmpfile.write ( postdata ) tmpfile.seek(0) h5f = h5py.File ( tmpfile.name, driver='core', backing_store=False ) try: for k in h5f.keys(): idgrp = h5f.get(k) # Convert HDF5 to annotation anno = h5ann.H5toAnnotation ( k, idgrp ) # set the identifier (separate transaction) if not ('update' in options or 'dataonly' in options or 'reduce' in options): anno.setID ( db ) tries = 0 done = False while not done and tries < 5: # start a transaction: get mysql out of line at a time mode db.startTxn () try: if anno.__class__ in [ annotation.AnnNeuron, annotation.AnnSeed ] and ( idgrp.get('VOXELS') or idgrp.get('CUTOUT')): logger.warning ("Cannot write to annotation type %s" % (anno.__class__)) raise ANNError ("Cannot write to annotation type %s" % (anno.__class__)) if 'update' in options and 'dataonly' in options: logger.warning ("Illegal combination of options. Cannot use udpate and dataonly together") raise ANNError ("Illegal combination of options. Cannot use udpate and dataonly together") elif not 'dataonly' in options and not 'reduce' in options: # Put into the database db.putAnnotation ( anno, options ) retvals.append(anno.annid) # Is a resolution specified? or use default h5resolution = idgrp.get('RESOLUTION') if h5resolution == None: resolution = proj.getResolution() else: resolution = h5resolution[0] # Load the data associated with this annotation # Is it voxel data? voxels = idgrp.get('VOXELS') if voxels and 'reduce' not in options: if 'preserve' in options: conflictopt = 'P' elif 'exception' in options: conflictopt = 'E' else: conflictopt = 'O' # Check that the voxels have a conforming size: if voxels.shape[1] != 3: logger.warning ("Voxels data not the right shape. Must be (:,3). Shape is %s" % str(voxels.shape)) raise ANNError ("Voxels data not the right shape. Must be (:,3). Shape is %s" % str(voxels.shape)) exceptions = db.annotate ( anno.annid, resolution, voxels, conflictopt ) # Otherwise this is a shave operation elif voxels and 'reduce' in options: # Check that the voxels have a conforming size: if voxels.shape[1] != 3: logger.warning ("Voxels data not the right shape. Must be (:,3). Shape is %s" % str(voxels.shape)) raise ANNError ("Voxels data not the right shape. Must be (:,3). Shape is %s" % str(voxels.shape)) db.shave ( anno.annid, resolution, voxels ) # Is it dense data? cutout = idgrp.get('CUTOUT') h5xyzoffset = idgrp.get('XYZOFFSET') if cutout != None and h5xyzoffset != None and 'reduce' not in options: if 'preserve' in options: conflictopt = 'P' elif 'exception' in options: conflictopt = 'E' else: conflictopt = 'O' # the zstart in dbconfig is sometimes offset to make it aligned. # Probably remove the offset is the best idea. and align data # to zero regardless of where it starts. For now. corner = h5xyzoffset[:] corner[2] -= dbcfg.slicerange[0] db.annotateEntityDense ( anno.annid, corner, resolution, np.array(cutout), conflictopt ) elif cutout != None and h5xyzoffset != None and 'reduce' in options: corner = h5xyzoffset[:] corner[2] -= dbcfg.slicerange[0] db.shaveEntityDense ( anno.annid, corner, resolution, np.array(cutout)) elif cutout != None or h5xyzoffset != None: #TODO this is a loggable error pass # Commit if there is no error db.commit() # Here with no error is successful done = True # rollback if you catch an error except MySQLdb.OperationalError, e: logger.warning ("Transaction did not complete. %s" % (e)) tries += 1 db.rollback() continue except MySQLdb.Error, e: logger.warning ("Put transaction rollback. %s" % (e)) db.rollback() raise
def mcFalseColor(webargs): """False color image of multiple channels""" [token, mcfcstr, chanstr, service, imageargs] = webargs.split('/', 4) projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj(token) dbcfg = dbconfig.switchDataset(proj.getDataset()) if proj.getDBType() != emcaproj.CHANNELS: logger.warning("Not a multiple channel project.") raise ANNError("Not a multiple channel project.") channels = chanstr.split(",") combined_img = None for i in range(len(channels)): if service == 'xy': cb = xySlice(str(channels[i]) + "/" + imageargs, dbcfg, proj) elif service == 'xz': cb = xzSlice(str(channels[i]) + "/" + imageargs, dbcfg, proj) elif service == 'yz': cb = yzSlice(str(channels[i]) + "/" + imageargs, dbcfg, proj) else: logger.warning("No such service %s. Args: %s" % (service, webargs)) raise ANNError("No such service %s" % (service)) # First channel is cyan if i == 0: data32 = np.array(cb.data * (1. / 256), dtype=np.uint32) combined_img = 0xFF000000 + np.left_shift( data32, 8) + np.left_shift(data32, 16) # Second is yellow elif i == 1: data32 = np.array(cb.data * (1. / 256), dtype=np.uint32) combined_img += np.left_shift(data32, 8) + data32 # Third is Magenta elif i == 2: data32 = np.array(cb.data * (1. / 256), dtype=np.uint32) combined_img += np.left_shift(data32, 16) + data32 # Fourth is Red elif i == 3: data32 = np.array(cb.data * (1. / 256), dtype=np.uint32) combined_img += data32 # Fifth is Green elif i == 4: data32 = np.array(cb.data * (1. / 256), dtype=np.uint32) combined_img += np.left_shift(data32, 8) # Sixth is Blue elif i == 5: data32 = np.array(cb.data * (1. / 256), dtype=np.uint32) combined_img += np.left_shift(data32, 16) else: logger.warning( "Only support six channels at a time. You requested %s " % (chanstr)) raise ANNError( "Only support six channels at a time. You requested %s " % (chanstr)) if service == 'xy': ydim, xdim = combined_img.shape[1:3] outimage = Image.frombuffer('RGBA', (xdim, ydim), combined_img[0, :, :].flatten(), 'raw', 'RGBA', 0, 1) elif service == 'xz': ydim = combined_img.shape[0] xdim = combined_img.shape[2] outimage = Image.frombuffer('RGBA', (xdim, ydim), combined_img[:, 0, :].flatten(), 'raw', 'RGBA', 0, 1) elif service == 'yz': ydim = combined_img.shape[0] xdim = combined_img.shape[1] outimage = Image.frombuffer('RGBA', (xdim, ydim), combined_img[:, :, 0].flatten(), 'raw', 'RGBA', 0, 1) # Enhance the image import ImageEnhance enhancer = ImageEnhance.Brightness(outimage) outimage = enhancer.enhance(4.0) fileobj = cStringIO.StringIO() outimage.save(fileobj, "PNG") fileobj.seek(0) return fileobj.read()
# include the paths EM_BASE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), ".." )) EM_UTIL_PATH = os.path.join(EM_BASE_PATH, "util" ) EM_EMCA_PATH = os.path.join(EM_BASE_PATH, "emca" ) EM_DBCONFIG_PATH = os.path.join(EM_BASE_PATH, "dbconfig" ) sys.path += [ EM_UTIL_PATH, EM_DBCONFIG_PATH, EM_EMCA_PATH ] import emcaproj import emcadb import dbconfig # load the project projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj ( 'kasthuri11' ) dbcfg = dbconfig.switchDataset ( proj.getDataset() ) #Load the database db = emcadb.EMCADB ( dbcfg, proj ) _ximagesz = 4096 _yimagesz = 4096 fid = open ( "/data/formisha/rasdamancutout", "w" ) for zstart in range(1,1024,16): zend = min(zstart+16,1025) corner = [1024,1024,zstart] resolution = 1
def emcacatmaid(webargs): """Interface to the cutout service for catmaid request. It does address translation.""" CM_TILESIZE = 256 token, plane, resstr, xtilestr, ytilestr, zslicestr, rest = webargs.split( '/', 7) xtile = int(xtilestr) ytile = int(ytilestr) projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj(token) dbcfg = dbconfig.switchDataset(proj.getDataset()) # datatype from the project if proj.getDBType() == emcaproj.IMAGES: datatype = np.uint8 else: datatype = np.uint32 resolution = int(resstr) # build the cutout request if plane == 'xy': # figure out the cutout (limit to max image size) xstart = xtile * CM_TILESIZE ystart = ytile * CM_TILESIZE xend = min((xtile + 1) * CM_TILESIZE, dbcfg.imagesz[resolution][0]) yend = min((ytile + 1) * CM_TILESIZE, dbcfg.imagesz[resolution][1]) # Return empty data if request is outside bounds. don't like it. if xstart >= xend or ystart >= yend: cutoutdata = np.zeros([CM_TILESIZE, CM_TILESIZE], dtype=datatype) else: imageargs = '%s/%s,%s/%s,%s/%s/' % (resstr, xstart, xend, ystart, yend, zslicestr) cb = xySlice(imageargs, dbcfg, proj) # reshape (if it's not a full cutout) if cb.data.shape != [1, CM_TILESIZE, CM_TILESIZE]: cutoutdata = np.zeros([CM_TILESIZE, CM_TILESIZE], dtype=cb.data.dtype) cutoutdata[0:cb.data.shape[1], 0:cb.data.shape[2]] = cb.data.reshape( [cb.data.shape[1], cb.data.shape[2]]) else: cutoutdata = cb.data.reshape([CM_TILESIZE, CM_TILESIZE]) elif plane == 'xz' or plane == 'yz': # The x or y plane is normal. The z plane needs some translation. # the ytilestr actually represents data in the z-plane pixelsperslice = dbcfg.zscale[int(resstr)] if plane == 'xz': # figure out the cutout (limit to max image size) xstart = xtile * CM_TILESIZE xend = min((xtile + 1) * CM_TILESIZE, dbcfg.imagesz[resolution][0]) # Now we need the ytile'th set of CM_TILESZ ystart = ytile * int( float(CM_TILESIZE) / pixelsperslice) + dbcfg.slicerange[0] # get more data so that we always have 512 pixels yend = min( (ytile + 1) * int(float(CM_TILESIZE) / pixelsperslice + 1), dbcfg.slicerange[1] - dbcfg.slicerange[0] + 1) + dbcfg.slicerange[0] # Return empty data if request is outside bounds. don't like it. if xstart == xend or ystart == yend: cutoutdata = np.zeros([CM_TILESIZE, CM_TILESIZE], dtype=datatype) else: imageargs = '%s/%s,%s/%s/%s,%s/' % (resstr, xstart, xend, zslicestr, ystart, yend) cb = xzSlice(imageargs, dbcfg, proj) if cb.data.shape != [CM_TILESIZE, 1, CM_TILESIZE]: cutoutdata = np.zeros([CM_TILESIZE, CM_TILESIZE], dtype=cb.data.dtype) cutoutdata[0:cb.data.shape[0], 0:cb.data.shape[2]] = cb.data.reshape( [cb.data.shape[0], cb.data.shape[2]]) else: # reshape cutoutdata = cb.data.reshape([CM_TILESIZE, CM_TILESIZE]) elif plane == 'yz': # figure out the cutout (limit to max image size) xtart = xtile * CM_TILESIZE xend = min((xtile + 1) * CM_TILESIZE, dbcfg.imagesz[resolution][1]) ystart = ytile * int( float(CM_TILESIZE) / pixelsperslice) + dbcfg.slicerange[0] yend = min( (ytile + 1) * int(float(CM_TILESIZE) / pixelsperslice + 1), dbcfg.slicerange[1] - dbcfg.slicerange[0] + 1) + dbcfg.slicerange[0] # Return empty data if request is outside bounds. don't like it. if xstart == xend or ystart == yend: cutoutdata = np.zeros([CM_TILESIZE, CM_TILESIZE], dtype=datatype) else: imageargs = '%s/%s/%s,%s/%s,%s/' % ( resstr, zslicestr, xtile * CM_TILESIZE, (xtile + 1) * CM_TILESIZE, ystart, yend) cb = yzSlice(imageargs, dbcfg, proj) if cb.data.shape != [CM_TILESIZE, CM_TILESIZE, 1]: cutoutdata = np.zeros([CM_TILESIZE, CM_TILESIZE], dtype=cb.data.dtype) cutoutdata[0:cb.data.shape[0], 0:cb.data.shape[1]] = cb.data.reshape( [cb.data.shape[0], cb.data.shape[1]]) else: cutoutdata = cb.data.reshape([CM_TILESIZE, CM_TILESIZE]) else: logger.warning( "No such cutout plane: %s. Must be (xy|xz|yz).. Args %s" % (service, webargs)) raise ANNError("No such cutout plane: %s. Must be (xy|xz|yz)." % plane) # Write the image to a readable stream if cutoutdata.dtype == np.uint8: outimage = Image.frombuffer('L', [CM_TILESIZE, CM_TILESIZE], cutoutdata, 'raw', 'L', 0, 1) elif cutoutdata.dtype == np.uint32: recolor_cy(cutoutdata, cutoutdata) outimage = Image.frombuffer('RGBA', [CM_TILESIZE, CM_TILESIZE], cutoutdata, 'raw', 'RGBA', 0, 1) return outimage
def main(): parser = argparse.ArgumentParser(description='Ingest a tiff stack.') parser.add_argument('token', action="store" ) parser.add_argument('channel', type=int, action="store" ) parser.add_argument('path', action="store" ) parser.add_argument('numslices', type=int, action="store" ) result = parser.parse_args() projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj ( result.token ) dbcfg = dbconfig.switchDataset ( proj.getDataset() ) _ximgsz = None _yimgsz = None for sl in range(result.numslices): filenm = result.path + '/' + '{:0>4}'.format(sl) + '.png' print filenm img = Image.open ( filenm, "r" ) if _ximgsz==None and _yimgsz==None: _ximgsz,_yimgsz = img.size imarray = np.zeros ( [result.numslices, _yimgsz, _ximgsz], dtype=np.uint16 ) else: assert _ximgsz == img.size[0] and _yimgsz == img.size[1] imarray[sl,:,:] = np.asarray ( img ) # get the size of the cube xcubedim,ycubedim,zcubedim = dbcfg.cubedim[0] # and the limits of iteration xlimit = (_ximgsz-1) / xcubedim + 1 ylimit = (_yimgsz-1) / ycubedim + 1 zlimit = (result.numslices-1) / zcubedim + 1 # open the database db = emcadb.EMCADB ( dbcfg, proj ) # get a db cursor cursor = db.conn.cursor() for z in range(zlimit): db.commit() for y in range(ylimit): for x in range(xlimit): zmin = z*zcubedim zmax = min((z+1)*zcubedim,result.numslices) zmaxrel = ((zmax-1)%zcubedim)+1 ymin = y*ycubedim ymax = min((y+1)*ycubedim,_yimgsz) ymaxrel = ((ymax-1)%ycubedim)+1 xmin = x*xcubedim xmax = min((x+1)*xcubedim,_ximgsz) xmaxrel = ((xmax-1)%xcubedim)+1 # morton key key = zindex.XYZMorton ( [x,y,z] ) # Create a channel cube cube = chancube.ChanCube ( [xcubedim,ycubedim,zcubedim] ) # data for this key cube.data[0:zmaxrel,0:ymaxrel,0:xmaxrel] = imarray[zmin:zmax,ymin:ymax,xmin:xmax] # compress the cube npz = cube.toNPZ () # add the cube to the database sql = "INSERT INTO " + proj.getTable(RESOLUTION) + "(zindex, channel, cube) VALUES (%s, %s, %s)" print sql try: cursor.execute ( sql, (key, result.channel, npz)) except MySQLdb.Error, e: raise ANNError ( "Error updating data cube: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def getAnnotations ( webargs, postdata ): """Get multiple annotations. Takes an HDF5 that lists ids in the post.""" [ token, objectsliteral, otherargs ] = webargs.split ('/',2) # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj ( token ) dbcfg = dbconfig.switchDataset ( proj.getDataset() ) db = emcadb.EMCADB ( dbcfg, proj ) # Read the post data HDF5 and get a list of identifiers tmpinfile = tempfile.NamedTemporaryFile ( ) tmpinfile.write ( postdata ) tmpinfile.seek(0) h5in = h5py.File ( tmpinfile.name ) # IDENTIFIERS if not h5in.get('ANNOIDS'): logger.warning ("Requesting multiple annotations. But no HDF5 \'ANNOIDS\' field specified.") raise ANNError ("Requesting multiple annotations. But no HDF5 \'ANNOIDS\' field specified.") # GET the data out of the HDF5 file. Never operate on the data in place. annoids = h5in['ANNOIDS'][:] # set variables to None: need them in call to getAnnoByID, but not all paths set all corner = None dim = None resolution = None dataarg = '' # process options # Split the URL and get the args if otherargs != '': ( dataarg, cutout ) = otherargs.split('/', 1) if dataarg =='' or dataarg == 'nodata': dataoption = AR_NODATA elif dataarg == 'voxels': dataoption = AR_VOXELS # only arg to voxels is resolution [resstr, sym, rest] = cutout.partition('/') resolution = int(resstr) if resstr != '' else proj.getResolution() elif dataarg == 'cutout': # if blank of just resolution then a tightcutout if cutout == '' or re.match('^\d+[\/]*$', cutout): dataoption = AR_TIGHTCUTOUT [resstr, sym, rest] = cutout.partition('/') resolution = int(resstr) if resstr != '' else proj.getResolution() else: dataoption = AR_CUTOUT # Perform argument processing brargs = restargs.BrainRestArgs (); brargs.cutoutArgs ( cutout, dbcfg ) # Extract the relevant values corner = brargs.getCorner() dim = brargs.getDim() resolution = brargs.getResolution() # RBTODO test this interface elif dataarg == 'boundingbox': # if blank of just resolution then a tightcutout if cutout == '' or re.match('^\d+[\/]*$', cutout): dataoption = AR_BOUNDINGBOX [resstr, sym, rest] = cutout.partition('/') resolution = int(resstr) if resstr != '' else proj.getResolution() else: logger.warning ("In getAnnotations: Error: no such data option %s " % ( dataarg )) raise ANNError ("In getAnnotations: Error: no such data option %s " % ( dataarg )) # Make the HDF5 output file # Create an in-memory HDF5 file tmpoutfile = tempfile.NamedTemporaryFile() h5fout = h5py.File ( tmpoutfile.name ) # get annotations for each identifier for annoid in annoids: # the int here is to prevent using a numpy value in an inner loop. This is a 10x performance gain. getAnnoById ( int(annoid), h5fout, db, dbcfg, dataoption, resolution, corner, dim ) # close temporary file h5in.close() tmpinfile.close() # Transmit back the populated HDF5 file h5fout.flush() tmpoutfile.seek(0) return tmpoutfile.read()
def getAnnotation(webargs): """Fetch a RAMON object as HDF5 by object identifier""" [token, sym, otherargs] = webargs.partition('/') # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj(token) dbcfg = dbconfig.switchDataset(proj.getDataset()) db = emcadb.EMCADB(dbcfg, proj) # Split the URL and get the args args = otherargs.split('/', 2) # Make the HDF5 file # Create an in-memory HDF5 file tmpfile = tempfile.NamedTemporaryFile() h5f = h5py.File(tmpfile.name) # if the first argument is numeric. it is an annoid if re.match('^[\d,]+$', args[0]): annoids = map(int, args[0].split(',')) for annoid in annoids: # default is no data if args[1] == '' or args[1] == 'nodata': dataoption = AR_NODATA getAnnoById(annoid, h5f, db, dbcfg, dataoption) # if you want voxels you either requested the resolution id/voxels/resolution # or you get data from the default resolution elif args[1] == 'voxels': dataoption = AR_VOXELS [resstr, sym, rest] = args[2].partition('/') resolution = int( resstr) if resstr != '' else proj.getResolution() getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution) elif args[1] == 'cutout': # if there are no args or only resolution, it's a tight cutout request if args[2] == '' or re.match('^\d+[\/]*$', args[2]): dataoption = AR_TIGHTCUTOUT [resstr, sym, rest] = args[2].partition('/') resolution = int( resstr) if resstr != '' else proj.getResolution() getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution) else: dataoption = AR_CUTOUT # Perform argument processing brargs = restargs.BrainRestArgs() brargs.cutoutArgs(args[2], dbcfg) # Extract the relevant values corner = brargs.getCorner() dim = brargs.getDim() resolution = brargs.getResolution() getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution, corner, dim) elif args[1] == 'boundingbox': dataoption = AR_BOUNDINGBOX [resstr, sym, rest] = args[2].partition('/') resolution = int( resstr) if resstr != '' else proj.getResolution() getAnnoById(annoid, h5f, db, dbcfg, dataoption, resolution) else: logger.warning( "Fetch identifier %s. Error: no such data option %s " % (annoid, args[1])) raise ANNError( "Fetch identifier %s. Error: no such data option %s " % (annoid, args[1])) # the first argument is not numeric. it is a service other than getAnnotation else: logger.warning( "Get interface %s requested. Illegal or not implemented. Args: %s" % (args[0], webargs)) raise ANNError( "Get interface %s requested. Illegal or not implemented" % (args[0])) h5f.flush() tmpfile.seek(0) return tmpfile.read()
def getAnnotation ( webargs ): """Fetch a RAMON object as HDF5 by object identifier""" [ token, sym, otherargs ] = webargs.partition ('/') # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj ( token ) dbcfg = dbconfig.switchDataset ( proj.getDataset() ) db = emcadb.EMCADB ( dbcfg, proj ) # Split the URL and get the args args = otherargs.split('/', 2) # Make the HDF5 file # Create an in-memory HDF5 file tmpfile = tempfile.NamedTemporaryFile() h5f = h5py.File ( tmpfile.name ) # if the first argument is numeric. it is an annoid if re.match ( '^[\d,]+$', args[0] ): annoids = map(int, args[0].split(',')) for annoid in annoids: # default is no data if args[1] == '' or args[1] == 'nodata': dataoption = AR_NODATA getAnnoById ( annoid, h5f, db, dbcfg, dataoption ) # if you want voxels you either requested the resolution id/voxels/resolution # or you get data from the default resolution elif args[1] == 'voxels': dataoption = AR_VOXELS [resstr, sym, rest] = args[2].partition('/') resolution = int(resstr) if resstr != '' else proj.getResolution() getAnnoById ( annoid, h5f, db, dbcfg, dataoption, resolution ) elif args[1] =='cutout': # if there are no args or only resolution, it's a tight cutout request if args[2] == '' or re.match('^\d+[\/]*$', args[2]): dataoption = AR_TIGHTCUTOUT [resstr, sym, rest] = args[2].partition('/') resolution = int(resstr) if resstr != '' else proj.getResolution() getAnnoById ( annoid, h5f, db, dbcfg, dataoption, resolution ) else: dataoption = AR_CUTOUT # Perform argument processing brargs = restargs.BrainRestArgs (); brargs.cutoutArgs ( args[2], dbcfg ) # Extract the relevant values corner = brargs.getCorner() dim = brargs.getDim() resolution = brargs.getResolution() getAnnoById ( annoid, h5f, db, dbcfg, dataoption, resolution, corner, dim ) elif args[1] == 'boundingbox': dataoption = AR_BOUNDINGBOX [resstr, sym, rest] = args[2].partition('/') resolution = int(resstr) if resstr != '' else proj.getResolution() getAnnoById ( annoid, h5f, db, dbcfg, dataoption, resolution ) else: logger.warning ("Fetch identifier %s. Error: no such data option %s " % ( annoid, args[1] )) raise ANNError ("Fetch identifier %s. Error: no such data option %s " % ( annoid, args[1] )) # the first argument is not numeric. it is a service other than getAnnotation else: logger.warning("Get interface %s requested. Illegal or not implemented. Args: %s" % ( args[0], webargs )) raise ANNError ("Get interface %s requested. Illegal or not implemented" % ( args[0] )) h5f.flush() tmpfile.seek(0) return tmpfile.read()
def emcacatmaid ( webargs ): """Interface to the cutout service for catmaid request. It does address translation.""" CM_TILESIZE=256 token, plane, resstr, xtilestr, ytilestr, zslicestr, rest = webargs.split('/',7) xtile = int(xtilestr) ytile = int(ytilestr) projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj ( token ) dbcfg = dbconfig.switchDataset ( proj.getDataset() ) # datatype from the project if proj.getDBType() == emcaproj.IMAGES: datatype = np.uint8 else: datatype = np.uint32 resolution = int(resstr) # build the cutout request if plane=='xy': # figure out the cutout (limit to max image size) xstart = xtile*CM_TILESIZE ystart = ytile*CM_TILESIZE xend = min ((xtile+1)*CM_TILESIZE,dbcfg.imagesz[resolution][0]) yend = min ((ytile+1)*CM_TILESIZE,dbcfg.imagesz[resolution][1]) # Return empty data if request is outside bounds. don't like it. if xstart>=xend or ystart>=yend: cutoutdata = np.zeros ( [CM_TILESIZE,CM_TILESIZE], dtype=datatype ) else: imageargs = '%s/%s,%s/%s,%s/%s/' % ( resstr, xstart, xend, ystart, yend, zslicestr ) cb = xySlice ( imageargs, dbcfg, proj ) # reshape (if it's not a full cutout) if cb.data.shape != [1,CM_TILESIZE,CM_TILESIZE]: cutoutdata = np.zeros ( [CM_TILESIZE,CM_TILESIZE], dtype=cb.data.dtype ) cutoutdata[0:cb.data.shape[1],0:cb.data.shape[2]] = cb.data.reshape([cb.data.shape[1],cb.data.shape[2]]) else: cutoutdata = cb.data.reshape([CM_TILESIZE,CM_TILESIZE]) elif plane=='xz' or plane=='yz': # The x or y plane is normal. The z plane needs some translation. # the ytilestr actually represents data in the z-plane pixelsperslice = dbcfg.zscale[int(resstr)] if plane=='xz': # figure out the cutout (limit to max image size) xstart = xtile*CM_TILESIZE xend = min ((xtile+1)*CM_TILESIZE,dbcfg.imagesz[resolution][0]) # Now we need the ytile'th set of CM_TILESZ ystart = ytile*int(float(CM_TILESIZE)/pixelsperslice) + dbcfg.slicerange[0] # get more data so that we always have 512 pixels yend = min((ytile+1)*int(float(CM_TILESIZE)/pixelsperslice+1),dbcfg.slicerange[1]-dbcfg.slicerange[0]+1) + dbcfg.slicerange[0] # Return empty data if request is outside bounds. don't like it. if xstart==xend or ystart==yend: cutoutdata = np.zeros ( [CM_TILESIZE,CM_TILESIZE], dtype=datatype ) else: imageargs = '%s/%s,%s/%s/%s,%s/' % ( resstr, xstart, xend, zslicestr, ystart, yend ) cb = xzSlice ( imageargs, dbcfg, proj ) if cb.data.shape != [CM_TILESIZE,1,CM_TILESIZE]: cutoutdata = np.zeros ( [CM_TILESIZE,CM_TILESIZE], dtype=cb.data.dtype ) cutoutdata[0:cb.data.shape[0],0:cb.data.shape[2]] = cb.data.reshape([cb.data.shape[0],cb.data.shape[2]]) else: # reshape cutoutdata = cb.data.reshape([CM_TILESIZE,CM_TILESIZE]) elif plane=='yz': # figure out the cutout (limit to max image size) xtart = xtile*CM_TILESIZE xend = min ((xtile+1)*CM_TILESIZE,dbcfg.imagesz[resolution][1]) ystart = ytile*int(float(CM_TILESIZE)/pixelsperslice)+ dbcfg.slicerange[0] yend = min((ytile+1)*int(float(CM_TILESIZE)/pixelsperslice+1),dbcfg.slicerange[1]-dbcfg.slicerange[0]+1)+ dbcfg.slicerange[0] # Return empty data if request is outside bounds. don't like it. if xstart==xend or ystart==yend: cutoutdata = np.zeros ( [CM_TILESIZE,CM_TILESIZE], dtype=datatype ) else: imageargs = '%s/%s/%s,%s/%s,%s/' % ( resstr, zslicestr, xtile*CM_TILESIZE, (xtile+1)*CM_TILESIZE, ystart, yend ) cb = yzSlice ( imageargs, dbcfg, proj ) if cb.data.shape != [CM_TILESIZE,CM_TILESIZE,1]: cutoutdata = np.zeros ( [CM_TILESIZE,CM_TILESIZE], dtype=cb.data.dtype) cutoutdata[0:cb.data.shape[0],0:cb.data.shape[1]] = cb.data.reshape([cb.data.shape[0],cb.data.shape[1]]) else: cutoutdata = cb.data.reshape([CM_TILESIZE,CM_TILESIZE]) else: logger.warning("No such cutout plane: %s. Must be (xy|xz|yz).. Args %s" % ( service, webargs )) raise ANNError ( "No such cutout plane: %s. Must be (xy|xz|yz)." % plane ) # Write the image to a readable stream if cutoutdata.dtype==np.uint8: outimage = Image.frombuffer ( 'L', [CM_TILESIZE,CM_TILESIZE], cutoutdata, 'raw', 'L', 0, 1 ) elif cutoutdata.dtype==np.uint32: recolor_cy (cutoutdata, cutoutdata) outimage = Image.frombuffer ( 'RGBA', [CM_TILESIZE,CM_TILESIZE], cutoutdata, 'raw', 'RGBA', 0, 1 ) return outimage
def getAnnotations(webargs, postdata): """Get multiple annotations. Takes an HDF5 that lists ids in the post.""" [token, objectsliteral, otherargs] = webargs.split('/', 2) # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj(token) dbcfg = dbconfig.switchDataset(proj.getDataset()) db = emcadb.EMCADB(dbcfg, proj) # Read the post data HDF5 and get a list of identifiers tmpinfile = tempfile.NamedTemporaryFile() tmpinfile.write(postdata) tmpinfile.seek(0) h5in = h5py.File(tmpinfile.name) # IDENTIFIERS if not h5in.get('ANNOIDS'): logger.warning( "Requesting multiple annotations. But no HDF5 \'ANNOIDS\' field specified." ) raise ANNError( "Requesting multiple annotations. But no HDF5 \'ANNOIDS\' field specified." ) # GET the data out of the HDF5 file. Never operate on the data in place. annoids = h5in['ANNOIDS'][:] # set variables to None: need them in call to getAnnoByID, but not all paths set all corner = None dim = None resolution = None dataarg = '' # process options # Split the URL and get the args if otherargs != '': (dataarg, cutout) = otherargs.split('/', 1) if dataarg == '' or dataarg == 'nodata': dataoption = AR_NODATA elif dataarg == 'voxels': dataoption = AR_VOXELS # only arg to voxels is resolution [resstr, sym, rest] = cutout.partition('/') resolution = int(resstr) if resstr != '' else proj.getResolution() elif dataarg == 'cutout': # if blank of just resolution then a tightcutout if cutout == '' or re.match('^\d+[\/]*$', cutout): dataoption = AR_TIGHTCUTOUT [resstr, sym, rest] = cutout.partition('/') resolution = int(resstr) if resstr != '' else proj.getResolution() else: dataoption = AR_CUTOUT # Perform argument processing brargs = restargs.BrainRestArgs() brargs.cutoutArgs(cutout, dbcfg) # Extract the relevant values corner = brargs.getCorner() dim = brargs.getDim() resolution = brargs.getResolution() # RBTODO test this interface elif dataarg == 'boundingbox': # if blank of just resolution then a tightcutout if cutout == '' or re.match('^\d+[\/]*$', cutout): dataoption = AR_BOUNDINGBOX [resstr, sym, rest] = cutout.partition('/') resolution = int(resstr) if resstr != '' else proj.getResolution() else: logger.warning("In getAnnotations: Error: no such data option %s " % (dataarg)) raise ANNError("In getAnnotations: Error: no such data option %s " % (dataarg)) # Make the HDF5 output file # Create an in-memory HDF5 file tmpoutfile = tempfile.NamedTemporaryFile() h5fout = h5py.File(tmpoutfile.name) # get annotations for each identifier for annoid in annoids: # the int here is to prevent using a numpy value in an inner loop. This is a 10x performance gain. getAnnoById(int(annoid), h5fout, db, dbcfg, dataoption, resolution, corner, dim) # close temporary file h5in.close() tmpinfile.close() # Transmit back the populated HDF5 file h5fout.flush() tmpoutfile.seek(0) return tmpoutfile.read()
def main(): parser = argparse.ArgumentParser(description='Ingest a tiff stack.') parser.add_argument('token', action="store" ) parser.add_argument('channel', type=int, action="store" ) parser.add_argument('path', action="store" ) parser.add_argument('numslices', type=int, action="store" ) result = parser.parse_args() projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj ( result.token ) dbcfg = dbconfig.switchDataset ( proj.getDataset() ) _ximgsz = None _yimgsz = None for sl in range(result.numslices): filenm = result.path + '/' + '{:0>4}'.format(sl) + '.png' print filenm img = Image.open ( filenm, "r" ) if _ximgsz==None and _yimgsz==None: _ximgsz,_yimgsz = img.size imarray = np.zeros ( [result.numslices, _yimgsz, _ximgsz], dtype=np.uint16 ) else: assert _ximgsz == img.size[0] and _yimgsz == img.size[1] imarray[sl,:,:] = np.asarray ( img ) # get the size of the cube xcubedim,ycubedim,zcubedim = dbcfg.cubedim[0] # and the limits of iteration xlimit = (_ximgsz-1) / xcubedim + 1 ylimit = (_yimgsz-1) / ycubedim + 1 zlimit = (result.numslices-1) / zcubedim + 1 # open the database db = emcadb.EMCADB ( dbcfg, proj ) # get a db cursor cursor = db.conn.cursor() for z in range(zlimit): db.commit() for y in range(ylimit): for x in range(xlimit): zmin = z*zcubedim zmax = min((z+1)*zcubedim,result.numslices) zmaxrel = ((zmax-1)%zcubedim)+1 ymin = y*ycubedim ymax = min((y+1)*ycubedim,_yimgsz) ymaxrel = ((ymax-1)%ycubedim)+1 xmin = x*xcubedim xmax = min((x+1)*xcubedim,_ximgsz) xmaxrel = ((xmax-1)%xcubedim)+1 # morton key key = zindex.XYZMorton ( [x,y,z] ) # Create a channel cube cube = imagecube.ImageCube16 ( [xcubedim,ycubedim,zcubedim] ) # data for this key cube.data[0:zmaxrel,0:ymaxrel,0:xmaxrel] = imarray[zmin:zmax,ymin:ymax,xmin:xmax] # compress the cube npz = cube.toNPZ () # add the cube to the database sql = "INSERT INTO " + proj.getTable(RESOLUTION) + "(channel, zindex, cube) VALUES (%s, %s, %s)" print sql try: cursor.execute ( sql, (result.channel, key, npz)) except MySQLdb.Error, e: raise ANNError ( "Error updating data cube: %d: %s. sql=%s" % (e.args[0], e.args[1], sql))
def mcFalseColor ( webargs ): """False color image of multiple channels""" [ token, mcfcstr, chanstr, service, imageargs ] = webargs.split ('/', 4) projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj ( token ) dbcfg = dbconfig.switchDataset ( proj.getDataset() ) if proj.getDBType() != emcaproj.CHANNELS: logger.warning ( "Not a multiple channel project." ) raise ANNError ( "Not a multiple channel project." ) channels = chanstr.split(",") combined_img = None for i in range(len(channels)): if service == 'xy': cb = xySlice ( str(channels[i]) + "/" + imageargs, dbcfg, proj ) elif service == 'xz': cb = xzSlice ( str(channels[i]) + "/" + imageargs, dbcfg, proj ) elif service == 'yz': cb = yzSlice ( str(channels[i]) + "/" + imageargs, dbcfg, proj ) else: logger.warning ( "No such service %s. Args: %s" % (service,webargs)) raise ANNError ( "No such service %s" % (service) ) # First channel is cyan if i == 0: data32 = np.array ( cb.data * (1./256), dtype=np.uint32 ) combined_img = 0xFF000000 + np.left_shift(data32,8) + np.left_shift(data32,16) # Second is yellow elif i == 1: data32 = np.array ( cb.data * (1./256), dtype=np.uint32 ) combined_img += np.left_shift(data32,8) + data32 # Third is Magenta elif i == 2: data32 = np.array ( cb.data * (1./256), dtype=np.uint32 ) combined_img += np.left_shift(data32,16) + data32 # Fourth is Red elif i == 3: data32 = np.array ( cb.data * (1./256), dtype=np.uint32 ) combined_img += data32 # Fifth is Green elif i == 4: data32 = np.array ( cb.data * (1./256), dtype=np.uint32 ) combined_img += np.left_shift(data32,8) # Sixth is Blue elif i == 5: data32 = np.array ( cb.data * (1./256), dtype=np.uint32 ) combined_img += np.left_shift(data32,16) else: logger.warning ( "Only support six channels at a time. You requested %s " % (chanstr)) raise ANNError ( "Only support six channels at a time. You requested %s " % (chanstr)) if service == 'xy': ydim, xdim = combined_img.shape[1:3] outimage = Image.frombuffer ( 'RGBA', (xdim,ydim), combined_img[0,:,:].flatten(), 'raw', 'RGBA', 0, 1 ) elif service == 'xz': ydim = combined_img.shape[0] xdim = combined_img.shape[2] outimage = Image.frombuffer ( 'RGBA', (xdim,ydim), combined_img[:,0,:].flatten(), 'raw', 'RGBA', 0, 1 ) elif service == 'yz': ydim = combined_img.shape[0] xdim = combined_img.shape[1] outimage = Image.frombuffer ( 'RGBA', (xdim,ydim), combined_img[:,:,0].flatten(), 'raw', 'RGBA', 0, 1 ) # Enhance the image import ImageEnhance enhancer = ImageEnhance.Brightness(outimage) outimage = enhancer.enhance(4.0) fileobj = cStringIO.StringIO ( ) outimage.save ( fileobj, "PNG" ) fileobj.seek(0) return fileobj.read()
def putAnnotation(webargs, postdata): """Put a RAMON object as HDF5 by object identifier""" [token, sym, optionsargs] = webargs.partition('/') # Get the annotation database projdb = emcaproj.EMCAProjectsDB() proj = projdb.getProj(token) dbcfg = dbconfig.switchDataset(proj.getDataset()) db = emcadb.EMCADB(dbcfg, proj) # Don't write to readonly projects if proj.getReadOnly() == 1: logger.warning("Attempt to write to read only project. %s: %s" % (proj.getDBName(), webargs)) raise ANNError("Attempt to write to read only project. %s: %s" % (proj.getDBName(), webargs)) options = optionsargs.split('/') # return string of id values retvals = [] # Make a named temporary file for the HDF5 tmpfile = tempfile.NamedTemporaryFile() tmpfile.write(postdata) tmpfile.seek(0) h5f = h5py.File(tmpfile.name, driver='core', backing_store=False) try: for k in h5f.keys(): idgrp = h5f.get(k) # Convert HDF5 to annotation anno = h5ann.H5toAnnotation(k, idgrp) # set the identifier (separate transaction) if not ('update' in options or 'dataonly' in options or 'reduce' in options): anno.setID(db) tries = 0 done = False while not done and tries < 5: # start a transaction: get mysql out of line at a time mode db.startTxn() try: if anno.__class__ in [ annotation.AnnNeuron, annotation.AnnSeed ] and (idgrp.get('VOXELS') or idgrp.get('CUTOUT')): logger.warning("Cannot write to annotation type %s" % (anno.__class__)) raise ANNError("Cannot write to annotation type %s" % (anno.__class__)) if 'update' in options and 'dataonly' in options: logger.warning( "Illegal combination of options. Cannot use udpate and dataonly together" ) raise ANNError( "Illegal combination of options. Cannot use udpate and dataonly together" ) elif not 'dataonly' in options and not 'reduce' in options: # Put into the database db.putAnnotation(anno, options) retvals.append(anno.annid) # Is a resolution specified? or use default h5resolution = idgrp.get('RESOLUTION') if h5resolution == None: resolution = proj.getResolution() else: resolution = h5resolution[0] # Load the data associated with this annotation # Is it voxel data? voxels = idgrp.get('VOXELS') if voxels and 'reduce' not in options: if 'preserve' in options: conflictopt = 'P' elif 'exception' in options: conflictopt = 'E' else: conflictopt = 'O' # Check that the voxels have a conforming size: if voxels.shape[1] != 3: logger.warning( "Voxels data not the right shape. Must be (:,3). Shape is %s" % str(voxels.shape)) raise ANNError( "Voxels data not the right shape. Must be (:,3). Shape is %s" % str(voxels.shape)) exceptions = db.annotate(anno.annid, resolution, voxels, conflictopt) # Otherwise this is a shave operation elif voxels and 'reduce' in options: # Check that the voxels have a conforming size: if voxels.shape[1] != 3: logger.warning( "Voxels data not the right shape. Must be (:,3). Shape is %s" % str(voxels.shape)) raise ANNError( "Voxels data not the right shape. Must be (:,3). Shape is %s" % str(voxels.shape)) db.shave(anno.annid, resolution, voxels) # Is it dense data? cutout = idgrp.get('CUTOUT') h5xyzoffset = idgrp.get('XYZOFFSET') if cutout != None and h5xyzoffset != None and 'reduce' not in options: if 'preserve' in options: conflictopt = 'P' elif 'exception' in options: conflictopt = 'E' else: conflictopt = 'O' # the zstart in dbconfig is sometimes offset to make it aligned. # Probably remove the offset is the best idea. and align data # to zero regardless of where it starts. For now. corner = h5xyzoffset[:] corner[2] -= dbcfg.slicerange[0] db.annotateEntityDense(anno.annid, corner, resolution, np.array(cutout), conflictopt) elif cutout != None and h5xyzoffset != None and 'reduce' in options: corner = h5xyzoffset[:] corner[2] -= dbcfg.slicerange[0] db.shaveEntityDense(anno.annid, corner, resolution, np.array(cutout)) elif cutout != None or h5xyzoffset != None: #TODO this is a loggable error pass # Commit if there is no error db.commit() # Here with no error is successful done = True # rollback if you catch an error except MySQLdb.OperationalError, e: logger.warning("Transaction did not complete. %s" % (e)) tries += 1 db.rollback() continue except MySQLdb.Error, e: logger.warning("Put transaction rollback. %s" % (e)) db.rollback() raise