def ingestCube(self, mortonidx, resolution, tilestack): print "Ingest for resolution %s morton index %s, xyz " % ( resolution, mortonidx), zindex.MortonXYZ(mortonidx) [x, y, z] = zindex.MortonXYZ(mortonidx) [xcubedim, ycubedim, zcubedim] = self.dbcfg.cubedim[resolution] bc = imagecube.ImageCube(self.dbcfg.cubedim[resolution]) corner = [x * xcubedim, y * ycubedim, z * zcubedim] bc.cubeFromFiles(corner, tilestack) return bc
def getBoundingBox(self, entityid, res): # get the size of the image and cube resolution = int(res) # all boxes in the index zidxs = self.annoIdx.getIndex(entityid, resolution) if len(zidxs) == 0: return None, None # convert to xyz coordinates xyzvals = np.array([zindex.MortonXYZ(zidx) for zidx in zidxs], dtype=np.uint32) cubedim = self.dbcfg.cubedim[resolution] # find the corners xmin = min(xyzvals[:, 0]) * cubedim[0] xmax = (max(xyzvals[:, 0]) + 1) * cubedim[0] ymin = min(xyzvals[:, 1]) * cubedim[1] ymax = (max(xyzvals[:, 1]) + 1) * cubedim[1] zmin = min(xyzvals[:, 2]) * cubedim[2] zmax = (max(xyzvals[:, 2]) + 1) * cubedim[2] corner = [xmin, ymin, zmin] dim = [xmax - xmin, ymax - ymin, zmax - zmin] return (corner, dim)
def add(self, fiber): """Add a fiber to the graph""" # Get the set of voxels in the fiber allvoxels = fiber.getVoxels() # Voxels for the big graph voxels = [] for i in allvoxels: xyz = zindex.MortonXYZ(i) # Use only the important voxels roival = self.rois.get(xyz) # if it's an roi and in the brain # if roival and self.mask.get (xyz): if roival: voxels.append(i) # Add edges to the big graph for v1, v2 in itertools.combinations((voxels), 2): if (v1 < v2): self.spedgemat[v1, v2] += 1.0 else: self.spedgemat[v2, v1] += 1.0
def add(self, fiber): """Add a fiber to the graph""" # Get the set of voxels in the fiber allvoxels = fiber.getVoxels() roilist = [] # Use only the important voxels for i in allvoxels: # this is for the small graph version xyz = zindex.MortonXYZ(i) roival = self.rois.get(xyz) # if it's an roi and in the brain # if roival and self.mask.get (xyz): if roival: roilist.append(roi.translate(roival)) roilist = set(roilist) for v1, v2 in itertools.combinations((roilist), 2): if (v1 < v2): self.spedgemat[v1, v2] += 1.0 else: self.spedgemat[v2, v1] += 1.0
def annotate(self, entityid, resolution, locations, conflictopt='O'): """Label the voxel locations or add as exceptions is the are already labeled.""" [xcubedim, ycubedim, zcubedim] = cubedim = self.dbcfg.cubedim[resolution] # An item may exist across several cubes # Convert the locations into Morton order # dictionary with the index cubeidx = defaultdict(set) # convert voxels z coordinate locations[:, 2] = locations[:, 2] - self.dbcfg.slicerange[0] cubelocs = cubeLocs_cy(np.array(locations, dtype=np.uint32), cubedim) # sort the arrary, by cubeloc cubelocs.view('u4,u4,u4,u4').sort(order=['f0'], axis=0) # get the nonzero element offsets nzdiff = np.r_[np.nonzero(np.diff(cubelocs[:, 0]))] # then turn into a set of ranges of the same element listoffsets = np.r_[0, nzdiff + 1, len(cubelocs)] for i in range(len(listoffsets) - 1): # grab the list of voxels for the first cube voxlist = cubelocs[listoffsets[i]:listoffsets[i + 1], :][:, 1:4] # and the morton key key = cubelocs[listoffsets[i], 0] cube = self.getCube(key, resolution, True) # get a voxel offset for the cube cubeoff = zindex.MortonXYZ(key) offset = [ cubeoff[0] * cubedim[0], cubeoff[1] * cubedim[1], cubeoff[2] * cubedim[2] ] # add the items exceptions = np.array(cube.annotate(entityid, offset, voxlist, conflictopt), dtype=np.uint8) # update the sparse list of exceptions if self.EXCEPT_FLAG: if len(exceptions) != 0: self.updateExceptions(key, resolution, entityid, exceptions) self.putCube(key, resolution, cube) # add this cube to the index cubeidx[entityid].add(key) # write it to the database self.annoIdx.updateIndexDense(cubeidx, resolution)
def shave(self, entityid, resolution, locations): """Label the voxel locations or add as exceptions is the are already labeled.""" [xcubedim, ycubedim, zcubedim] = cubedim = self.dbcfg.cubedim[resolution] # dictionary with the index cubeidx = defaultdict(set) # convert voxels z coordinate locations[:, 2] = locations[:, 2] - self.dbcfg.slicerange[0] cubelocs = cubeLocs_cy(np.array(locations, dtype=np.uint32), cubedim) # sort the arrary, by cubeloc cubelocs.view('u4,u4,u4,u4').sort(order=['f0'], axis=0) # get the nonzero element offsets nzdiff = np.r_[np.nonzero(np.diff(cubelocs[:, 0]))] # then turn into a set of ranges of the same element listoffsets = np.r_[0, nzdiff + 1, len(cubelocs)] for i in range(len(listoffsets) - 1): # grab the list of voxels for the first cube voxlist = cubelocs[listoffsets[i]:listoffsets[i + 1], :][:, 1:4] # and the morton key key = cubelocs[listoffsets[i], 0] cube = self.getCube(key, resolution, True) # get a voxel offset for the cube cubeoff = zindex.MortonXYZ(key) offset = [ cubeoff[0] * cubedim[0], cubeoff[1] * cubedim[1], cubeoff[2] * cubedim[2] ] # remove the items exlist, zeroed = cube.shave(entityid, offset, voxlist) # make sure that exceptions are stored as 8 bits exceptions = np.array(exlist, dtype=np.uint8) # update the sparse list of exceptions if self.EXCEPT_FLAG: if len(exceptions) != 0: self.removeExceptions(key, resolution, entityid, exceptions) # zeroed is the list that needs to get promoted here. # RBTODO self.putCube(key, resolution, cube)
def removeBB(self): """Iterate over all cubes""" # Get the source database sizes [ximagesz, yimagesz] = self.dbcfg.imagesz[self._resolution] [xcubedim, ycubedim, zcubedim] = self.dbcfg.cubedim[self._resolution] # Get the slices [startslice, endslice] = self.dbcfg.slicerange slices = endslice - startslice + 1 # Set the limits for iteration on the number of cubes in each dimension xlimit = (ximagesz - 1) / xcubedim + 1 ylimit = (yimagesz - 1) / ycubedim + 1 # Round up the zlimit to the next larger zlimit = (((slices - 1) / zcubedim + 1) * zcubedim) / zcubedim lastzindex = (zindex.XYZMorton([xlimit, ylimit, zlimit])) # call the range query # self.annodb.queryRange ( 0, lastzindex, self._resolution ); # RB restart self.annodb.queryRange(0, lastzindex, self._resolution) # get the first cube [key, cube] = self.annodb.getNextCube() count = 0 while key != None: if len(np.intersect1d(np.unique(cube.data), self.BBIDS)) != 0: print "Found bounding box data in cube ", zindex.MortonXYZ(key) # Remove annotations vector_func = np.vectorize(lambda a: np.uint32(0) if a in self.BBIDS else a) cube.data = vector_func(cube.data) assert (type(cube.data[0, 0, 0]) == np.uint32) # Put the cube self.annodb.putCube(key, self._resolution, cube) count = count + 1 else: print "No matching data in ", key # Get the next cube [key, cube] = self.annodb.getNextCube() if count == 100: self.annodb.conn.commit() count = 0 print "No more cubes"
def prefetch(self, idxbatch, cubesize): # Files are best read in the order they are laid out on disk and then transferred to to appropriate buffer # For now let's assume they are laid out in x then y order in z directories. This is the way that it works # on xfs. Don't want to have to scan the entire directory. # Enumerate all the tiles that are needed for these blocks tilelist = [] # clear the previous tilelist -- empty the hash directory self.tiledata.clear() xcubesize, ycubesize, zcubesize = cubesize for idx in idxbatch: xyz = zindex.MortonXYZ(idx) # add the tiles needed for this cube # assume that the cubes and tiles are aligned and tiles are bigger than cubes # z is not tiled for w in range(zcubesize): tilelist.append( str(xyz[2] * zcubesize + w) + ' ' + str(xyz[1] * ycubesize / self.ytilesize) + ' ' + str(xyz[0] * xcubesize / self.xtilesize)) # get the unique indices tilelist = sorted(set(tilelist)) # build the tile data dictionary for tile in tilelist: [zstr, ystr, xstr] = tile.split() tileidx = [int(xstr), int(ystr), int(zstr)] fname = self.tileFile(tileidx[0], tileidx[1], tileidx[2]) # when we run out of tiles to prefetch we're out of data, i.e. # no more z tiles or missing data. this keeps this simple, but # this means that there is no bounds checking on this routine try: tileimage = Image.open(fname, 'r') self.tiledata[str(tileidx)] = np.asarray(tileimage) print("Loaded file " + fname) except IOError: continue
def getLocations(self, entityid, res): # get the size of the image and cube resolution = int(res) voxlist = [] zidxs = self.annoIdx.getIndex(entityid, resolution) for zidx in zidxs: cb = self.getCube(zidx, resolution) # mask out the entries that do not match the annotation id vec_func = np.vectorize(lambda x: entityid if x == entityid else 0) annodata = vec_func(cb.data) # where are the entries offsets = np.nonzero(annodata) voxels = np.array(zip(offsets[2], offsets[1], offsets[0]), dtype=np.uint32) # Get cube offset information [x, y, z] = zindex.MortonXYZ(zidx) xoffset = x * self.dbcfg.cubedim[resolution][0] yoffset = y * self.dbcfg.cubedim[resolution][1] zoffset = z * self.dbcfg.cubedim[resolution][ 2] + self.dbcfg.slicerange[0] # Now add the exception voxels if self.EXCEPT_FLAG: exceptions = self.getExceptions(zidx, resolution, entityid) if exceptions != []: voxels = np.append(voxels.flatten(), exceptions.flatten()) voxels = voxels.reshape(len(voxels) / 3, 3) # Change the voxels back to image address space [ voxlist.append([a + xoffset, b + yoffset, c + zoffset]) for (a, b, c) in voxels ] return voxlist
def removeExceptions(self, key, resolution, entityid, exceptions): """Remove a list of exceptions""" curexlist = self.getExceptions(key, resolution, entityid) table = 'exc' + str(resolution) if curexlist != []: oldexlist = set([zindex.XYZMorton(trpl) for trpl in curexlist]) newexlist = set([zindex.XYZMorton(trpl) for trpl in exceptions]) exlist = oldexlist - newexlist exlist = [zindex.MortonXYZ(zidx) for zidx in exlist] sql = "UPDATE " + table + " SET exlist=(%s) WHERE zindex=%s AND id=%s" try: fileobj = cStringIO.StringIO() np.save(fileobj, exlist) self.cursor.execute( sql, (zlib.compress(fileobj.getvalue()), key, entityid)) except MySQLdb.Error, e: logger.error("Error removing exceptions %d: %s. sql=%s" % (e.args[0], e.args[1], sql)) raise
def get_coords_for_lccs(self, ncc): """Computes coordinates for each voxel in the top ncc connected components""" inlcc = (np.less_equal(self.vertexCC,ncc)*np.greater(self.vertexCC,0)).nonzero()[0] coord = np.array([zindex.MortonXYZ(v) for v in inlcc]) return np.concatenate((coord,self.vertexCC[inlcc][np.newaxis].T),axis=1)
def buildStack(self, startlevel): """Build the hierarchy of annotations""" for l in range(startlevel, len(self.dbcfg.resolutions) - 1): # Get the source database sizes [ximagesz, yimagesz] = self.dbcfg.imagesz[l] [xcubedim, ycubedim, zcubedim] = self.dbcfg.cubedim[l] # Get the slices [startslice, endslice] = self.dbcfg.slicerange slices = endslice - startslice + 1 # Set the limits for iteration on the number of cubes in each dimension xlimit = ximagesz / xcubedim ylimit = yimagesz / ycubedim # Round up the zlimit to the next larger zlimit = (((slices - 1) / zcubedim + 1) * zcubedim) / zcubedim # These constants work for all resolutions. Bigger batches are harder. # They require logic about # They also transfer entire blocks. # Create an output buffer outdata = np.zeros([zcubedim * 4, ycubedim * 2, xcubedim * 2]) # We've written to this offset already prevmortonidx = 0 # Round up to the top of the range lastzindex = (zindex.XYZMorton([xlimit, ylimit, zlimit]) / 64 + 1) * 64 # Iterate over the cubes in morton order for mortonidx in range(0, lastzindex, 64): print "Working on batch %s at %s" % ( mortonidx, zindex.MortonXYZ(mortonidx)) # call the range query self.annoDB.queryRange(mortonidx, mortonidx + 64, l) # Flag to indicate no data. No update query somedata = False # get the first cube [key, cube] = self.annoDB.getNextCube() # if there's a cube, there's data if key != None: somedata = True while key != None: xyz = zindex.MortonXYZ(key) # Compute the offset in the output data cube # we are placing 4x4x4 input blocks into a 2x2x4 cube offset = [(xyz[0] % 4) * (xcubedim / 2), (xyz[1] % 4) * (ycubedim / 2), (xyz[2] % 4) * zcubedim] print "res : zindex = ", l, ":", key, ", location", zindex.MortonXYZ( key) # add the contribution of the cube in the hierarchy #self.addData ( cube, outdata, offset ) # use the cython version addData_cy(cube, outdata, offset) # Get the next value [key, cube] = self.annoDB.getNextCube() # Now store the data if somedata == True: # Get the base location of this batch xyzout = zindex.MortonXYZ(mortonidx) outcorner = [ xyzout[0] / 2 * xcubedim, xyzout[1] / 2 * ycubedim, xyzout[2] * zcubedim ] # Data stored in z,y,x order dims in x,y,z outdim = [ outdata.shape[2], outdata.shape[1], outdata.shape[0] ] # Preserve annotations made at the specified level RBTODO fix me self.annoDB.annotateDense(outcorner, l + 1, outdata, 'O') # zero the output buffer outdata = np.zeros( [zcubedim * 4, ycubedim * 2, xcubedim * 2]) else: print "No data in this batch"
def main(): parser = argparse.ArgumentParser(description='Ingest the Rohanna data.') parser.add_argument('token', action="store", help='Token for the annotation project.') parser.add_argument('path', action="store", help='Directory with annotation TIF files.') parser.add_argument('resolution', action="store", type=int, help='Resolution') result = parser.parse_args() # convert to an argument resolution = result.resolution # load a database [db, proj, projdb] = ocpcarest.loadDBProj(result.token) # get the dataset configuration (xcubedim, ycubedim, zcubedim) = proj.datasetcfg.cubedim[resolution] (startslice, endslice) = proj.datasetcfg.slicerange batchsz = zcubedim # This doesn't work because the image size does not match exactly the cube size #(ximagesz,yimagesz)=proj.datasetcfg.imagesz[resolution] ximagesz = 5120 yimagesz = 5120 endslice = 1123 # add all of the tiles to the image for sl in range(startslice, endslice + 1, batchsz): slab = np.zeros([batchsz, yimagesz, ximagesz], dtype=np.uint32) for b in range(batchsz): if (sl + b <= endslice): # raw data filenm = result.path + '/labels_{:0>5}_ocp.tif'.format(sl + b) print "Opening filenm " + filenm img = Image.open(filenm, 'r') imgdata = np.asarray(img) slab[b, :, :] = (imgdata) # the last z offset that we ingest, if the batch ends before batchsz endz = b # Now we have a 5120x5120x16 z-aligned cube. # Send it to the database. for y in range(0, yimagesz, ycubedim): for x in range(0, ximagesz, xcubedim): mortonidx = zindex.XYZMorton([ (x + xoffsetsz) / xcubedim, (y + yoffsetsz) / ycubedim, (sl + zoffsetsz - startslice) / zcubedim ]) cubedata = np.zeros([zcubedim, ycubedim, xcubedim], dtype=np.uint32) test = zindex.MortonXYZ(mortonidx) pdb.set_trace() xmin = x ymin = y xmax = min(ximagesz, x + xcubedim) ymax = min(yimagesz, y + ycubedim) zmin = 0 zmax = min(sl + zcubedim, endslice + 1) cubedata[0:zmax - zmin, 0:ymax - ymin, 0:xmax - xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax] # check if there's anything to store if (np.count_nonzero(cubedata) == 0): continue # create the DB BLOB fileobj = cStringIO.StringIO() np.save(fileobj, cubedata) cdz = zlib.compress(fileobj.getvalue()) # insert the blob into the database cursor = db.conn.cursor() sql = "INSERT INTO res{} (zindex, cube) VALUES (%s, %s)".format( int(resolution)) # cursor.execute(sql, (mortonidx, cdz)) cursor.close() print "Commiting at x=%s, y=%s, z=%s" % ( x + xoffsetsz, y + yoffsetsz, sl + b + zoffsetsz) db.conn.commit()
(ximagesz, yimagesz) = inproj.datasetcfg.imagesz[result.resolution] (startslice, endslice) = inproj.datasetcfg.slicerange slices = endslice - startslice + 1 # Set the limits for iteration on the number of cubes in each dimension xlimit = (ximagesz - 1) / xcubedim + 1 ylimit = (yimagesz - 1) / ycubedim + 1 # Round up the zlimit to the next larger zlimit = (((slices - 1) / zcubedim + 1) * zcubedim) / zcubedim lastzindex = (zindex.XYZMorton([xlimit, ylimit, zlimit]) / 64 + 1) * 64 # iterate over the cubes in morton order for mortonidx in range(lastzindex): x, y, z = zindex.MortonXYZ(mortonidx) # only process cubes in the space if x >= xlimit or y >= ylimit or z >= zlimit: continue incube = indb.getCube(mortonidx, result.resolution) annoids = np.unique(incube.data) sql = "select * from {} where source in ({})".format( mergemap, ','.join(annoids)) # relabel the cube # outdb.putCube ( mortonidx, result.resolution )
def generateDB(self, resolution): """Generate the database from a tile stack""" [ximagesz, yimagesz] = self.dbcfg.imagesz[resolution] [xcubedim, ycubedim, zcubedim] = self.dbcfg.cubedim[resolution] [self.startslice, endslice] = self.dbcfg.slicerange self.slices = endslice - self.startslice + 1 # round up to the next largest slice if self.slices % zcubedim != 0: self.slices = (self.slices / zcubedim + 1) * zcubedim tilestack = tiles.Tiles(self.dbcfg.tilesz, self.dbcfg.inputprefix + '/' + str(resolution), self.startslice) # Set the limits on the number of cubes in each dimension xlimit = ximagesz / xcubedim ylimit = yimagesz / ycubedim zlimit = self.slices / zcubedim # list of batched indexes idxbatch = [] # This parameter needs to match to the max number of slices to be taken. # The maximum z slices that prefetching will take in this prefetch # This is useful for higher resolutions where we run out of tiles in # the x and y dimensions zmaxpf = 256 zstride = 256 # This batch size is chosen for braingraph1. # It loads 32x32x16cubes equivalent to 16 x 16 x 256 tiles @ 128x128x16 per tile and 512x512x1 per cube # this is 2^34 bytes of memory batchsize = 16384 # Ingest the slices in morton order for mortonidx in zindex.generator([xlimit, ylimit, zlimit]): xyz = zindex.MortonXYZ(mortonidx) # if this exceeds the limit on the z dimension, do the ingest if len(idxbatch) == batchsize or xyz[2] == zmaxpf: # preload the batch tilestack.prefetch(idxbatch, [xcubedim, ycubedim, zcubedim]) # ingest the batch for idx in idxbatch: bc = self.ingestCube(idx, resolution, tilestack) self.saveCube(bc, idx, resolution) # commit the batch self.conn.commit() # if we've hit our area bounds set the new limit if xyz[2] == zmaxpf: zmaxpf += zstride # Finished this batch. Start anew. idxbatch = [] #build a batch of indexes idxbatch.append(mortonidx) # preload the remaining tilestack.prefetch(idxbatch, [xcubedim, ycubedim, zcubedim]) # Ingest the remaining once the loop is over for idx in idxbatch: bc = self.ingestCube(idx, resolution, tilestack) self.saveCube(bc, idx, resolution) # commit the batch self.conn.commit()
# Round up to the top of the range if result.lastzindex == 0: lastzindex = (zindex.XYZMorton([xlimit,ylimit,zlimit])/64+1)*64 else: lastzindex = (result.lastzindex/64+1)*64 if result.firstzindex == 0: firstzindex = 0 else: firstzindex = (result.firstzindex/64-1)*64 # Iterate over the cubes in morton order for mortonidx in range(firstzindex, lastzindex, 64): # 0-63 is a 4 by 4 cube 0,0,0 to 3,3,3 x1, y1, z1 = zindex.MortonXYZ ( mortonidx ) x2, y2, z2 = zindex.MortonXYZ ( mortonidx+64-1 ) # need to round up 3,3,3 to 4,4,4 * cubedime startx = x1*xcubedim stopx = min((x2+1)*xcubedim,ximagesz) starty = y1*ycubedim stopy = min((y2+1)*ycubedim,yimagesz) startz = z1*zcubedim + startslice stopz = min((z2+1)*zcubedim + startslice,endslice+1) if startx >= ximagesz or starty >= yimagesz or startz > endslice: continue url = "http://%s/ocpca/%s/exceptions/%s/%s,%s/%s,%s/%s,%s/" % (result.baseurl,result.token,result.resolution,startx,stopx,starty,stopy,startz,stopz) try: f = urllib2.urlopen ( url )
def cutout(self, corner, dim, resolution): """Extract a cube of arbitrary size. Need not be aligned.""" [xcubedim, ycubedim, zcubedim] = self.dbcfg.cubedim[resolution] # Round to the nearest larger cube in all dimensions start = [ corner[0]/xcubedim,\ corner[1]/ycubedim,\ corner[2]/zcubedim ] numcubes = [ (corner[0]+dim[0]+xcubedim-1)/xcubedim - start[0],\ (corner[1]+dim[1]+ycubedim-1)/ycubedim - start[1],\ (corner[2]+dim[2]+zcubedim-1)/zcubedim - start[2] ] inbuf = imagecube.ImageCube(self.dbcfg.cubedim[resolution]) outbuf = imagecube.ImageCube([ numcubes[0] * xcubedim, numcubes[1] * ycubedim, numcubes[2] * zcubedim ]) # Build a list of indexes to access listofidxs = [] for z in range(numcubes[2]): for y in range(numcubes[1]): for x in range(numcubes[0]): mortonidx = zindex.XYZMorton( [x + start[0], y + start[1], z + start[2]]) listofidxs.append(mortonidx) # Sort the indexes in Morton order listofidxs.sort() # Batch query for all cubes dbname = self.dbcfg.tablebase + str(resolution) cursor = self.conn.cursor() sql = "SELECT zindex, cube from " + dbname + " where zindex in (%s)" # creats a %s for each list element in_p = ', '.join(map(lambda x: '%s', listofidxs)) # replace the single %s with the in_p string sql = sql % in_p cursor.execute(sql, listofidxs) # xyz offset stored for later use lowxyz = zindex.MortonXYZ(listofidxs[0]) # Get the objects and add to the cube for i in range(len(listofidxs)): idx, datastring = cursor.fetchone() # get the data out of the compressed blob newstr = zlib.decompress(datastring[:]) newfobj = cStringIO.StringIO(newstr) inbuf.data = np.load(newfobj) #add the query result cube to the bigger cube curxyz = zindex.MortonXYZ(int(idx)) offsetxyz = [ curxyz[0] - lowxyz[0], curxyz[1] - lowxyz[1], curxyz[2] - lowxyz[2] ] outbuf.addData(inbuf, offsetxyz) # need to trim down the array to size # only if the dimensions are not the same if dim[0] % xcubedim == 0 and\ dim[1] % ycubedim == 0 and\ dim[2] % zcubedim == 0 and\ corner[0] % xcubedim == 0 and\ corner[1] % ycubedim == 0 and\ corner[2] % zcubedim == 0: pass else: outbuf.trim ( corner[0]%xcubedim,dim[0],\ corner[1]%ycubedim,dim[1],\ corner[2]%zcubedim,dim[2] ) return outbuf
def countLocations(self, channel, resolution, threshhold): """Build the hierarchy of annotations""" # Get the source database sizes [ximagesz, yimagesz] = self.proj.datasetcfg.imagesz[resolution] [xcubedim, ycubedim, zcubedim] = self.proj.datasetcfg.cubedim[resolution] # Get the slices [startslice, endslice] = self.corner[2], self.corner[2] + self.dim[2] slices = endslice - startslice + 1 # Set the limits for iteration on the number of cubes in each dimension xstart = self.corner[0] / xcubedim ystart = self.corner[1] / ycubedim xend = (self.corner[0] + self.dim[0] - 1) / xcubedim + 1 yend = (self.corner[1] + self.dim[1] - 1) / ycubedim + 1 # Round up the zlimit to the next larger zstart = startslice / zcubedim zend = (endslice + 1 - 1) / zcubedim + 1 count = 0 for z in (zstart, zend): for y in (ystart, yend): for x in (xstart, xend): mortonidx = zindex.XYZMorton((x, y, z)) print "Working on batch %s at %s" % ( mortonidx, zindex.MortonXYZ(mortonidx)) # get the cube import pdb pdb.set_trace() cube = self.annoDB.getCube(mortonidx, resolution, channel) # Flag to indicate no data. No update query somedata = False # get the first cube [key, cube] = self.annoDB.getNextCube() # if there's a cube, there's data if key != None: somedata = True xyz = (x, y, z) # Compute the offset in the output data cube # we are placing 4x4x4 input blocks into a 2x2x4 cube offset = [ xyz[0] * xcubedim, xyz[1] * ycubedim, xyz[2] * zcubedim ] nzlocs = np.nonzero(cube.data > threshhold) count += len(nzlocs[1]) return countt
fileobj = cStringIO.StringIO() np.save(fileobj, exceptions) self.cursor.execute( sql, (key, entityid, zlib.compress(fileobj.getvalue()))) except MySQLdb.Error, e: logger.error("Error inserting exceptions %d: %s. sql=%s" % (e.args[0], e.args[1], sql)) raise # In this case we have an update query else: oldexlist = [zindex.XYZMorton(trpl) for trpl in curexlist] newexlist = [zindex.XYZMorton(trpl) for trpl in exceptions] exlist = set(newexlist + oldexlist) exlist = [zindex.MortonXYZ(zidx) for zidx in exlist] sql = "UPDATE " + table + " SET exlist=(%s) WHERE zindex=%s AND id=%s" try: fileobj = cStringIO.StringIO() np.save(fileobj, exlist) self.cursor.execute( sql, (zlib.compress(fileobj.getvalue()), key, entityid)) except MySQLdb.Error, e: logger.error("Error updating exceptions %d: %s. sql=%s" % (e.args[0], e.args[1], sql)) raise # # removeExceptions #
def findLocations(self, channel, resolution, threshhold, outfile): """Build the hierarchy of annotations""" # Get the source database sizes [ximagesz, yimagesz] = self.proj.datasetcfg.imagesz[resolution] [xcubedim, ycubedim, zcubedim] = self.proj.datasetcfg.cubedim[resolution] # Get the slices [startslice, endslice] = self.proj.datasetcfg.slicerange slices = endslice - startslice + 1 # Set the limits for iteration on the number of cubes in each dimension xlimit = ximagesz / xcubedim ylimit = yimagesz / ycubedim # Round up the zlimit to the next larger zlimit = (((slices - 1) / zcubedim + 1) * zcubedim) / zcubedim # Round up to the top of the range lastzindex = (zindex.XYZMorton([xlimit, ylimit, zlimit]) / 64 + 1) * 64 locs = [] with closing(open(outfile, 'wb')) as csvfile: csvwriter = csv.writer(csvfile) # Iterate over the cubes in morton order for mortonidx in range(0, lastzindex, 64): print "Working on batch %s at %s" % ( mortonidx, zindex.MortonXYZ(mortonidx)) # call the range query self.annoDB.queryRange(mortonidx, mortonidx + 64, resolution, channel) # Flag to indicate no data. No update query somedata = False # get the first cube [key, cube] = self.annoDB.getNextCube() # if there's a cube, there's data if key != None: somedata = True while key != None: xyz = zindex.MortonXYZ(key) # Compute the offset in the output data cube # we are placing 4x4x4 input blocks into a 2x2x4 cube offset = [ xyz[0] * xcubedim, xyz[1] * ycubedim, xyz[2] * zcubedim ] nzlocs = np.nonzero(cube.data > threshhold) if len(nzlocs[1]) != 0: print "res : zindex = ", resolution, ":", key, ", location", zindex.MortonXYZ( key) # zip together the x y z and value cubelocs = zip( nzlocs[0], nzlocs[1], nzlocs[2], cube.data[nzlocs[0], nzlocs[1], nzlocs[2]]) # translate from z,y,x,value to x,y,z,value and add global offset locs = [(pt[2] + offset[0], pt[1] + offset[1], pt[0] + offset[2], pt[3]) for pt in cubelocs] csvwriter.writerows([x for x in locs]) [key, cube] = self.annoDB.getNextCube()
# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Testing the float and uint64 issue in python""" import sys import os sys.path += [os.path.abspath('../django')] import OCP.settings os.environ['DJANGO_SETTINGS_MODULE'] = 'OCP.settings' from django.conf import settings import zindex import ocplib print zindex.MortonXYZ(831488) dims = (10234, 9703, 11009) print "DIMENSIONS:", dims key = ocplib.XYZMorton(dims) print "KEY:", key print "WHAT WE GET BACK:", ocplib.MortonXYZ(key) print "DIMENSIONS:", dims key = zindex.XYZMorton(dims) print "KEY:", key print "WHAT WE GET BACK:", zindex.MortonXYZ(key)
def cutout(self, corner, dim, resolution, channel=None): """Extract a cube of arbitrary size. Need not be aligned.""" # get the size of the image and cube [xcubedim, ycubedim, zcubedim] = cubedim = self.dbcfg.cubedim[resolution] # Round to the nearest larger cube in all dimensions zstart = corner[2] / zcubedim ystart = corner[1] / ycubedim xstart = corner[0] / xcubedim znumcubes = (corner[2] + dim[2] + zcubedim - 1) / zcubedim - zstart ynumcubes = (corner[1] + dim[1] + ycubedim - 1) / ycubedim - ystart xnumcubes = (corner[0] + dim[0] + xcubedim - 1) / xcubedim - xstart if (self.annoproj.getDBType() == emcaproj.ANNOTATIONS): # input cube is the database size incube = anncube.AnnotateCube(cubedim) # output cube is as big as was asked for and zero it. outcube = anncube.AnnotateCube ( [xnumcubes*xcubedim,\ ynumcubes*ycubedim,\ znumcubes*zcubedim] ) outcube.zeros() elif (self.annoproj.getDBType() == emcaproj.IMAGES): incube = imagecube.ImageCube(cubedim) outcube = imagecube.ImageCube ( [xnumcubes*xcubedim,\ ynumcubes*ycubedim,\ znumcubes*zcubedim] ) elif (self.annoproj.getDBType() == emcaproj.CHANNELS): incube = chancube.ChanCube(cubedim) outcube = chancube.ChanCube ( [xnumcubes*xcubedim,\ ynumcubes*ycubedim,\ znumcubes*zcubedim] ) # Build a list of indexes to access listofidxs = [] for z in range(znumcubes): for y in range(ynumcubes): for x in range(xnumcubes): mortonidx = zindex.XYZMorton( [x + xstart, y + ystart, z + zstart]) listofidxs.append(mortonidx) # Sort the indexes in Morton order listofidxs.sort() # Batch query for all cubes dbname = self.annoproj.getTable(resolution) # Customize query to the database (include channel or not) if (self.annoproj.getDBType() == emcaproj.CHANNELS): sql = "SELECT zindex, cube FROM " + dbname + " WHERE channel= " + str( channel) + " AND zindex in (%s)" else: sql = "SELECT zindex, cube FROM " + dbname + " WHERE zindex IN (%s)" # creats a %s for each list element in_p = ', '.join(map(lambda x: '%s', listofidxs)) # replace the single %s with the in_p string sql = sql % in_p rc = self.cursor.execute(sql, listofidxs) # xyz offset stored for later use lowxyz = zindex.MortonXYZ(listofidxs[0]) # Get the objects and add to the cube while (True): try: idx, datastring = self.cursor.fetchone() except: break #add the query result cube to the bigger cube curxyz = zindex.MortonXYZ(int(idx)) offset = [ curxyz[0] - lowxyz[0], curxyz[1] - lowxyz[1], curxyz[2] - lowxyz[2] ] incube.fromNPZ(datastring[:]) # add it to the output cube outcube.addData(incube, offset) # need to trim down the array to size # only if the dimensions are not the same if dim[0] % xcubedim == 0 and\ dim[1] % ycubedim == 0 and\ dim[2] % zcubedim == 0 and\ corner[0] % xcubedim == 0 and\ corner[1] % ycubedim == 0 and\ corner[2] % zcubedim == 0: pass else: outcube.trim ( corner[0]%xcubedim,dim[0],\ corner[1]%ycubedim,dim[1],\ corner[2]%zcubedim,dim[2] ) return outcube