def ingestCube(self, mortonidx, resolution, tilestack):
print "Ingest for resolution %s morton index %s, xyz " % (
resolution, mortonidx), zindex.MortonXYZ(mortonidx)
[x, y, z] = zindex.MortonXYZ(mortonidx)
[xcubedim, ycubedim, zcubedim] = self.dbcfg.cubedim[resolution]
bc = imagecube.ImageCube(self.dbcfg.cubedim[resolution])
corner = [x * xcubedim, y * ycubedim, z * zcubedim]
bc.cubeFromFiles(corner, tilestack)
return bc
def getBoundingBox(self, entityid, res):
# get the size of the image and cube
resolution = int(res)
# all boxes in the index
zidxs = self.annoIdx.getIndex(entityid, resolution)
if len(zidxs) == 0:
return None, None
# convert to xyz coordinates
xyzvals = np.array([zindex.MortonXYZ(zidx) for zidx in zidxs],
dtype=np.uint32)
cubedim = self.dbcfg.cubedim[resolution]
# find the corners
xmin = min(xyzvals[:, 0]) * cubedim[0]
xmax = (max(xyzvals[:, 0]) + 1) * cubedim[0]
ymin = min(xyzvals[:, 1]) * cubedim[1]
ymax = (max(xyzvals[:, 1]) + 1) * cubedim[1]
zmin = min(xyzvals[:, 2]) * cubedim[2]
zmax = (max(xyzvals[:, 2]) + 1) * cubedim[2]
corner = [xmin, ymin, zmin]
dim = [xmax - xmin, ymax - ymin, zmax - zmin]
return (corner, dim)
def add(self, fiber):
"""Add a fiber to the graph"""
# Get the set of voxels in the fiber
allvoxels = fiber.getVoxels()
# Voxels for the big graph
voxels = []
for i in allvoxels:
xyz = zindex.MortonXYZ(i)
# Use only the important voxels
roival = self.rois.get(xyz)
# if it's an roi and in the brain
# if roival and self.mask.get (xyz):
if roival:
voxels.append(i)
# Add edges to the big graph
for v1, v2 in itertools.combinations((voxels), 2):
if (v1 < v2):
self.spedgemat[v1, v2] += 1.0
else:
self.spedgemat[v2, v1] += 1.0
def add(self, fiber):
"""Add a fiber to the graph"""
# Get the set of voxels in the fiber
allvoxels = fiber.getVoxels()
roilist = []
# Use only the important voxels
for i in allvoxels:
# this is for the small graph version
xyz = zindex.MortonXYZ(i)
roival = self.rois.get(xyz)
# if it's an roi and in the brain
# if roival and self.mask.get (xyz):
if roival:
roilist.append(roi.translate(roival))
roilist = set(roilist)
for v1, v2 in itertools.combinations((roilist), 2):
if (v1 < v2):
self.spedgemat[v1, v2] += 1.0
else:
self.spedgemat[v2, v1] += 1.0
def annotate(self, entityid, resolution, locations, conflictopt='O'):
"""Label the voxel locations or add as exceptions is the are already labeled."""
[xcubedim, ycubedim,
zcubedim] = cubedim = self.dbcfg.cubedim[resolution]
# An item may exist across several cubes
# Convert the locations into Morton order
# dictionary with the index
cubeidx = defaultdict(set)
# convert voxels z coordinate
locations[:, 2] = locations[:, 2] - self.dbcfg.slicerange[0]
cubelocs = cubeLocs_cy(np.array(locations, dtype=np.uint32), cubedim)
# sort the arrary, by cubeloc
cubelocs.view('u4,u4,u4,u4').sort(order=['f0'], axis=0)
# get the nonzero element offsets
nzdiff = np.r_[np.nonzero(np.diff(cubelocs[:, 0]))]
# then turn into a set of ranges of the same element
listoffsets = np.r_[0, nzdiff + 1, len(cubelocs)]
for i in range(len(listoffsets) - 1):
# grab the list of voxels for the first cube
voxlist = cubelocs[listoffsets[i]:listoffsets[i + 1], :][:, 1:4]
# and the morton key
key = cubelocs[listoffsets[i], 0]
cube = self.getCube(key, resolution, True)
# get a voxel offset for the cube
cubeoff = zindex.MortonXYZ(key)
offset = [
cubeoff[0] * cubedim[0], cubeoff[1] * cubedim[1],
cubeoff[2] * cubedim[2]
]
# add the items
exceptions = np.array(cube.annotate(entityid, offset, voxlist,
conflictopt),
dtype=np.uint8)
# update the sparse list of exceptions
if self.EXCEPT_FLAG:
if len(exceptions) != 0:
self.updateExceptions(key, resolution, entityid,
exceptions)
self.putCube(key, resolution, cube)
# add this cube to the index
cubeidx[entityid].add(key)
# write it to the database
self.annoIdx.updateIndexDense(cubeidx, resolution)
def shave(self, entityid, resolution, locations):
"""Label the voxel locations or add as exceptions is the are already labeled."""
[xcubedim, ycubedim,
zcubedim] = cubedim = self.dbcfg.cubedim[resolution]
# dictionary with the index
cubeidx = defaultdict(set)
# convert voxels z coordinate
locations[:, 2] = locations[:, 2] - self.dbcfg.slicerange[0]
cubelocs = cubeLocs_cy(np.array(locations, dtype=np.uint32), cubedim)
# sort the arrary, by cubeloc
cubelocs.view('u4,u4,u4,u4').sort(order=['f0'], axis=0)
# get the nonzero element offsets
nzdiff = np.r_[np.nonzero(np.diff(cubelocs[:, 0]))]
# then turn into a set of ranges of the same element
listoffsets = np.r_[0, nzdiff + 1, len(cubelocs)]
for i in range(len(listoffsets) - 1):
# grab the list of voxels for the first cube
voxlist = cubelocs[listoffsets[i]:listoffsets[i + 1], :][:, 1:4]
# and the morton key
key = cubelocs[listoffsets[i], 0]
cube = self.getCube(key, resolution, True)
# get a voxel offset for the cube
cubeoff = zindex.MortonXYZ(key)
offset = [
cubeoff[0] * cubedim[0], cubeoff[1] * cubedim[1],
cubeoff[2] * cubedim[2]
]
# remove the items
exlist, zeroed = cube.shave(entityid, offset, voxlist)
# make sure that exceptions are stored as 8 bits
exceptions = np.array(exlist, dtype=np.uint8)
# update the sparse list of exceptions
if self.EXCEPT_FLAG:
if len(exceptions) != 0:
self.removeExceptions(key, resolution, entityid,
exceptions)
# zeroed is the list that needs to get promoted here.
# RBTODO
self.putCube(key, resolution, cube)
def removeBB(self):
"""Iterate over all cubes"""
# Get the source database sizes
[ximagesz, yimagesz] = self.dbcfg.imagesz[self._resolution]
[xcubedim, ycubedim, zcubedim] = self.dbcfg.cubedim[self._resolution]
# Get the slices
[startslice, endslice] = self.dbcfg.slicerange
slices = endslice - startslice + 1
# Set the limits for iteration on the number of cubes in each dimension
xlimit = (ximagesz - 1) / xcubedim + 1
ylimit = (yimagesz - 1) / ycubedim + 1
# Round up the zlimit to the next larger
zlimit = (((slices - 1) / zcubedim + 1) * zcubedim) / zcubedim
lastzindex = (zindex.XYZMorton([xlimit, ylimit, zlimit]))
# call the range query
# self.annodb.queryRange ( 0, lastzindex, self._resolution );
# RB restart
self.annodb.queryRange(0, lastzindex, self._resolution)
# get the first cube
[key, cube] = self.annodb.getNextCube()
count = 0
while key != None:
if len(np.intersect1d(np.unique(cube.data), self.BBIDS)) != 0:
print "Found bounding box data in cube ", zindex.MortonXYZ(key)
# Remove annotations
vector_func = np.vectorize(lambda a: np.uint32(0)
if a in self.BBIDS else a)
cube.data = vector_func(cube.data)
assert (type(cube.data[0, 0, 0]) == np.uint32)
# Put the cube
self.annodb.putCube(key, self._resolution, cube)
count = count + 1
else:
print "No matching data in ", key
# Get the next cube
[key, cube] = self.annodb.getNextCube()
if count == 100:
self.annodb.conn.commit()
count = 0
print "No more cubes"
def prefetch(self, idxbatch, cubesize):
# Files are best read in the order they are laid out on disk and then transferred to to appropriate buffer
# For now let's assume they are laid out in x then y order in z directories. This is the way that it works
# on xfs. Don't want to have to scan the entire directory.
# Enumerate all the tiles that are needed for these blocks
tilelist = []
# clear the previous tilelist -- empty the hash directory
self.tiledata.clear()
xcubesize, ycubesize, zcubesize = cubesize
for idx in idxbatch:
xyz = zindex.MortonXYZ(idx)
# add the tiles needed for this cube
# assume that the cubes and tiles are aligned and tiles are bigger than cubes
# z is not tiled
for w in range(zcubesize):
tilelist.append(
str(xyz[2] * zcubesize + w) + ' ' +
str(xyz[1] * ycubesize / self.ytilesize) + ' ' +
str(xyz[0] * xcubesize / self.xtilesize))
# get the unique indices
tilelist = sorted(set(tilelist))
# build the tile data dictionary
for tile in tilelist:
[zstr, ystr, xstr] = tile.split()
tileidx = [int(xstr), int(ystr), int(zstr)]
fname = self.tileFile(tileidx[0], tileidx[1], tileidx[2])
# when we run out of tiles to prefetch we're out of data, i.e.
# no more z tiles or missing data. this keeps this simple, but
# this means that there is no bounds checking on this routine
try:
tileimage = Image.open(fname, 'r')
self.tiledata[str(tileidx)] = np.asarray(tileimage)
print("Loaded file " + fname)
except IOError:
continue
def getLocations(self, entityid, res):
# get the size of the image and cube
resolution = int(res)
voxlist = []
zidxs = self.annoIdx.getIndex(entityid, resolution)
for zidx in zidxs:
cb = self.getCube(zidx, resolution)
# mask out the entries that do not match the annotation id
vec_func = np.vectorize(lambda x: entityid if x == entityid else 0)
annodata = vec_func(cb.data)
# where are the entries
offsets = np.nonzero(annodata)
voxels = np.array(zip(offsets[2], offsets[1], offsets[0]),
dtype=np.uint32)
# Get cube offset information
[x, y, z] = zindex.MortonXYZ(zidx)
xoffset = x * self.dbcfg.cubedim[resolution][0]
yoffset = y * self.dbcfg.cubedim[resolution][1]
zoffset = z * self.dbcfg.cubedim[resolution][
2] + self.dbcfg.slicerange[0]
# Now add the exception voxels
if self.EXCEPT_FLAG:
exceptions = self.getExceptions(zidx, resolution, entityid)
if exceptions != []:
voxels = np.append(voxels.flatten(), exceptions.flatten())
voxels = voxels.reshape(len(voxels) / 3, 3)
# Change the voxels back to image address space
[
voxlist.append([a + xoffset, b + yoffset, c + zoffset])
for (a, b, c) in voxels
]
return voxlist
def removeExceptions(self, key, resolution, entityid, exceptions):
"""Remove a list of exceptions"""
curexlist = self.getExceptions(key, resolution, entityid)
table = 'exc' + str(resolution)
if curexlist != []:
oldexlist = set([zindex.XYZMorton(trpl) for trpl in curexlist])
newexlist = set([zindex.XYZMorton(trpl) for trpl in exceptions])
exlist = oldexlist - newexlist
exlist = [zindex.MortonXYZ(zidx) for zidx in exlist]
sql = "UPDATE " + table + " SET exlist=(%s) WHERE zindex=%s AND id=%s"
try:
fileobj = cStringIO.StringIO()
np.save(fileobj, exlist)
self.cursor.execute(
sql, (zlib.compress(fileobj.getvalue()), key, entityid))
except MySQLdb.Error, e:
logger.error("Error removing exceptions %d: %s. sql=%s" %
(e.args[0], e.args[1], sql))
raise
def get_coords_for_lccs(self, ncc):
"""Computes coordinates for each voxel in the top ncc connected components"""
inlcc = (np.less_equal(self.vertexCC,ncc)*np.greater(self.vertexCC,0)).nonzero()[0]
coord = np.array([zindex.MortonXYZ(v) for v in inlcc])
return np.concatenate((coord,self.vertexCC[inlcc][np.newaxis].T),axis=1)
def buildStack(self, startlevel):
"""Build the hierarchy of annotations"""
for l in range(startlevel, len(self.dbcfg.resolutions) - 1):
# Get the source database sizes
[ximagesz, yimagesz] = self.dbcfg.imagesz[l]
[xcubedim, ycubedim, zcubedim] = self.dbcfg.cubedim[l]
# Get the slices
[startslice, endslice] = self.dbcfg.slicerange
slices = endslice - startslice + 1
# Set the limits for iteration on the number of cubes in each dimension
xlimit = ximagesz / xcubedim
ylimit = yimagesz / ycubedim
# Round up the zlimit to the next larger
zlimit = (((slices - 1) / zcubedim + 1) * zcubedim) / zcubedim
# These constants work for all resolutions. Bigger batches are harder.
# They require logic about
# They also transfer entire blocks.
# Create an output buffer
outdata = np.zeros([zcubedim * 4, ycubedim * 2, xcubedim * 2])
# We've written to this offset already
prevmortonidx = 0
# Round up to the top of the range
lastzindex = (zindex.XYZMorton([xlimit, ylimit, zlimit]) / 64 +
1) * 64
# Iterate over the cubes in morton order
for mortonidx in range(0, lastzindex, 64):
print "Working on batch %s at %s" % (
mortonidx, zindex.MortonXYZ(mortonidx))
# call the range query
self.annoDB.queryRange(mortonidx, mortonidx + 64, l)
# Flag to indicate no data. No update query
somedata = False
# get the first cube
[key, cube] = self.annoDB.getNextCube()
# if there's a cube, there's data
if key != None:
somedata = True
while key != None:
xyz = zindex.MortonXYZ(key)
# Compute the offset in the output data cube
# we are placing 4x4x4 input blocks into a 2x2x4 cube
offset = [(xyz[0] % 4) * (xcubedim / 2),
(xyz[1] % 4) * (ycubedim / 2),
(xyz[2] % 4) * zcubedim]
print "res : zindex = ", l, ":", key, ", location", zindex.MortonXYZ(
key)
# add the contribution of the cube in the hierarchy
#self.addData ( cube, outdata, offset )
# use the cython version
addData_cy(cube, outdata, offset)
# Get the next value
[key, cube] = self.annoDB.getNextCube()
# Now store the data
if somedata == True:
# Get the base location of this batch
xyzout = zindex.MortonXYZ(mortonidx)
outcorner = [
xyzout[0] / 2 * xcubedim, xyzout[1] / 2 * ycubedim,
xyzout[2] * zcubedim
]
# Data stored in z,y,x order dims in x,y,z
outdim = [
outdata.shape[2], outdata.shape[1], outdata.shape[0]
]
# Preserve annotations made at the specified level RBTODO fix me
self.annoDB.annotateDense(outcorner, l + 1, outdata, 'O')
# zero the output buffer
outdata = np.zeros(
[zcubedim * 4, ycubedim * 2, xcubedim * 2])
else:
print "No data in this batch"
def main():
parser = argparse.ArgumentParser(description='Ingest the Rohanna data.')
parser.add_argument('token',
action="store",
help='Token for the annotation project.')
parser.add_argument('path',
action="store",
help='Directory with annotation TIF files.')
parser.add_argument('resolution',
action="store",
type=int,
help='Resolution')
result = parser.parse_args()
# convert to an argument
resolution = result.resolution
# load a database
[db, proj, projdb] = ocpcarest.loadDBProj(result.token)
# get the dataset configuration
(xcubedim, ycubedim, zcubedim) = proj.datasetcfg.cubedim[resolution]
(startslice, endslice) = proj.datasetcfg.slicerange
batchsz = zcubedim
# This doesn't work because the image size does not match exactly the cube size
#(ximagesz,yimagesz)=proj.datasetcfg.imagesz[resolution]
ximagesz = 5120
yimagesz = 5120
endslice = 1123
# add all of the tiles to the image
for sl in range(startslice, endslice + 1, batchsz):
slab = np.zeros([batchsz, yimagesz, ximagesz], dtype=np.uint32)
for b in range(batchsz):
if (sl + b <= endslice):
# raw data
filenm = result.path + '/labels_{:0>5}_ocp.tif'.format(sl + b)
print "Opening filenm " + filenm
img = Image.open(filenm, 'r')
imgdata = np.asarray(img)
slab[b, :, :] = (imgdata)
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
# Now we have a 5120x5120x16 z-aligned cube.
# Send it to the database.
for y in range(0, yimagesz, ycubedim):
for x in range(0, ximagesz, xcubedim):
mortonidx = zindex.XYZMorton([
(x + xoffsetsz) / xcubedim, (y + yoffsetsz) / ycubedim,
(sl + zoffsetsz - startslice) / zcubedim
])
cubedata = np.zeros([zcubedim, ycubedim, xcubedim],
dtype=np.uint32)
test = zindex.MortonXYZ(mortonidx)
pdb.set_trace()
xmin = x
ymin = y
xmax = min(ximagesz, x + xcubedim)
ymax = min(yimagesz, y + ycubedim)
zmin = 0
zmax = min(sl + zcubedim, endslice + 1)
cubedata[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax]
# check if there's anything to store
if (np.count_nonzero(cubedata) == 0):
continue
# create the DB BLOB
fileobj = cStringIO.StringIO()
np.save(fileobj, cubedata)
cdz = zlib.compress(fileobj.getvalue())
# insert the blob into the database
cursor = db.conn.cursor()
sql = "INSERT INTO res{} (zindex, cube) VALUES (%s, %s)".format(
int(resolution))
# cursor.execute(sql, (mortonidx, cdz))
cursor.close()
print "Commiting at x=%s, y=%s, z=%s" % (
x + xoffsetsz, y + yoffsetsz, sl + b + zoffsetsz)
db.conn.commit()
(ximagesz, yimagesz) = inproj.datasetcfg.imagesz[result.resolution]
(startslice, endslice) = inproj.datasetcfg.slicerange
slices = endslice - startslice + 1
# Set the limits for iteration on the number of cubes in each dimension
xlimit = (ximagesz - 1) / xcubedim + 1
ylimit = (yimagesz - 1) / ycubedim + 1
# Round up the zlimit to the next larger
zlimit = (((slices - 1) / zcubedim + 1) * zcubedim) / zcubedim
lastzindex = (zindex.XYZMorton([xlimit, ylimit, zlimit]) / 64 + 1) * 64
# iterate over the cubes in morton order
for mortonidx in range(lastzindex):
x, y, z = zindex.MortonXYZ(mortonidx)
# only process cubes in the space
if x >= xlimit or y >= ylimit or z >= zlimit:
continue
incube = indb.getCube(mortonidx, result.resolution)
annoids = np.unique(incube.data)
sql = "select * from {} where source in ({})".format(
mergemap, ','.join(annoids))
# relabel the cube
# outdb.putCube ( mortonidx, result.resolution )
def generateDB(self, resolution):
"""Generate the database from a tile stack"""
[ximagesz, yimagesz] = self.dbcfg.imagesz[resolution]
[xcubedim, ycubedim, zcubedim] = self.dbcfg.cubedim[resolution]
[self.startslice, endslice] = self.dbcfg.slicerange
self.slices = endslice - self.startslice + 1
# round up to the next largest slice
if self.slices % zcubedim != 0:
self.slices = (self.slices / zcubedim + 1) * zcubedim
tilestack = tiles.Tiles(self.dbcfg.tilesz,
self.dbcfg.inputprefix + '/' + str(resolution),
self.startslice)
# Set the limits on the number of cubes in each dimension
xlimit = ximagesz / xcubedim
ylimit = yimagesz / ycubedim
zlimit = self.slices / zcubedim
# list of batched indexes
idxbatch = []
# This parameter needs to match to the max number of slices to be taken.
# The maximum z slices that prefetching will take in this prefetch
# This is useful for higher resolutions where we run out of tiles in
# the x and y dimensions
zmaxpf = 256
zstride = 256
# This batch size is chosen for braingraph1.
# It loads 32x32x16cubes equivalent to 16 x 16 x 256 tiles @ 128x128x16 per tile and 512x512x1 per cube
# this is 2^34 bytes of memory
batchsize = 16384
# Ingest the slices in morton order
for mortonidx in zindex.generator([xlimit, ylimit, zlimit]):
xyz = zindex.MortonXYZ(mortonidx)
# if this exceeds the limit on the z dimension, do the ingest
if len(idxbatch) == batchsize or xyz[2] == zmaxpf:
# preload the batch
tilestack.prefetch(idxbatch, [xcubedim, ycubedim, zcubedim])
# ingest the batch
for idx in idxbatch:
bc = self.ingestCube(idx, resolution, tilestack)
self.saveCube(bc, idx, resolution)
# commit the batch
self.conn.commit()
# if we've hit our area bounds set the new limit
if xyz[2] == zmaxpf:
zmaxpf += zstride
# Finished this batch. Start anew.
idxbatch = []
#build a batch of indexes
idxbatch.append(mortonidx)
# preload the remaining
tilestack.prefetch(idxbatch, [xcubedim, ycubedim, zcubedim])
# Ingest the remaining once the loop is over
for idx in idxbatch:
bc = self.ingestCube(idx, resolution, tilestack)
self.saveCube(bc, idx, resolution)
# commit the batch
self.conn.commit()
# Round up to the top of the range
if result.lastzindex == 0:
lastzindex = (zindex.XYZMorton([xlimit,ylimit,zlimit])/64+1)*64
else:
lastzindex = (result.lastzindex/64+1)*64
if result.firstzindex == 0:
firstzindex = 0
else:
firstzindex = (result.firstzindex/64-1)*64
# Iterate over the cubes in morton order
for mortonidx in range(firstzindex, lastzindex, 64):
# 0-63 is a 4 by 4 cube 0,0,0 to 3,3,3
x1, y1, z1 = zindex.MortonXYZ ( mortonidx )
x2, y2, z2 = zindex.MortonXYZ ( mortonidx+64-1 )
# need to round up 3,3,3 to 4,4,4 * cubedime
startx = x1*xcubedim
stopx = min((x2+1)*xcubedim,ximagesz)
starty = y1*ycubedim
stopy = min((y2+1)*ycubedim,yimagesz)
startz = z1*zcubedim + startslice
stopz = min((z2+1)*zcubedim + startslice,endslice+1)
if startx >= ximagesz or starty >= yimagesz or startz > endslice:
continue
url = "http://%s/ocpca/%s/exceptions/%s/%s,%s/%s,%s/%s,%s/" % (result.baseurl,result.token,result.resolution,startx,stopx,starty,stopy,startz,stopz)
try:
f = urllib2.urlopen ( url )
def cutout(self, corner, dim, resolution):
"""Extract a cube of arbitrary size. Need not be aligned."""
[xcubedim, ycubedim, zcubedim] = self.dbcfg.cubedim[resolution]
# Round to the nearest larger cube in all dimensions
start = [ corner[0]/xcubedim,\
corner[1]/ycubedim,\
corner[2]/zcubedim ]
numcubes = [ (corner[0]+dim[0]+xcubedim-1)/xcubedim - start[0],\
(corner[1]+dim[1]+ycubedim-1)/ycubedim - start[1],\
(corner[2]+dim[2]+zcubedim-1)/zcubedim - start[2] ]
inbuf = imagecube.ImageCube(self.dbcfg.cubedim[resolution])
outbuf = imagecube.ImageCube([
numcubes[0] * xcubedim, numcubes[1] * ycubedim,
numcubes[2] * zcubedim
])
# Build a list of indexes to access
listofidxs = []
for z in range(numcubes[2]):
for y in range(numcubes[1]):
for x in range(numcubes[0]):
mortonidx = zindex.XYZMorton(
[x + start[0], y + start[1], z + start[2]])
listofidxs.append(mortonidx)
# Sort the indexes in Morton order
listofidxs.sort()
# Batch query for all cubes
dbname = self.dbcfg.tablebase + str(resolution)
cursor = self.conn.cursor()
sql = "SELECT zindex, cube from " + dbname + " where zindex in (%s)"
# creats a %s for each list element
in_p = ', '.join(map(lambda x: '%s', listofidxs))
# replace the single %s with the in_p string
sql = sql % in_p
cursor.execute(sql, listofidxs)
# xyz offset stored for later use
lowxyz = zindex.MortonXYZ(listofidxs[0])
# Get the objects and add to the cube
for i in range(len(listofidxs)):
idx, datastring = cursor.fetchone()
# get the data out of the compressed blob
newstr = zlib.decompress(datastring[:])
newfobj = cStringIO.StringIO(newstr)
inbuf.data = np.load(newfobj)
#add the query result cube to the bigger cube
curxyz = zindex.MortonXYZ(int(idx))
offsetxyz = [
curxyz[0] - lowxyz[0], curxyz[1] - lowxyz[1],
curxyz[2] - lowxyz[2]
]
outbuf.addData(inbuf, offsetxyz)
# need to trim down the array to size
# only if the dimensions are not the same
if dim[0] % xcubedim == 0 and\
dim[1] % ycubedim == 0 and\
dim[2] % zcubedim == 0 and\
corner[0] % xcubedim == 0 and\
corner[1] % ycubedim == 0 and\
corner[2] % zcubedim == 0:
pass
else:
outbuf.trim ( corner[0]%xcubedim,dim[0],\
corner[1]%ycubedim,dim[1],\
corner[2]%zcubedim,dim[2] )
return outbuf
def countLocations(self, channel, resolution, threshhold):
"""Build the hierarchy of annotations"""
# Get the source database sizes
[ximagesz, yimagesz] = self.proj.datasetcfg.imagesz[resolution]
[xcubedim, ycubedim,
zcubedim] = self.proj.datasetcfg.cubedim[resolution]
# Get the slices
[startslice, endslice] = self.corner[2], self.corner[2] + self.dim[2]
slices = endslice - startslice + 1
# Set the limits for iteration on the number of cubes in each dimension
xstart = self.corner[0] / xcubedim
ystart = self.corner[1] / ycubedim
xend = (self.corner[0] + self.dim[0] - 1) / xcubedim + 1
yend = (self.corner[1] + self.dim[1] - 1) / ycubedim + 1
# Round up the zlimit to the next larger
zstart = startslice / zcubedim
zend = (endslice + 1 - 1) / zcubedim + 1
count = 0
for z in (zstart, zend):
for y in (ystart, yend):
for x in (xstart, xend):
mortonidx = zindex.XYZMorton((x, y, z))
print "Working on batch %s at %s" % (
mortonidx, zindex.MortonXYZ(mortonidx))
# get the cube
import pdb
pdb.set_trace()
cube = self.annoDB.getCube(mortonidx, resolution, channel)
# Flag to indicate no data. No update query
somedata = False
# get the first cube
[key, cube] = self.annoDB.getNextCube()
# if there's a cube, there's data
if key != None:
somedata = True
xyz = (x, y, z)
# Compute the offset in the output data cube
# we are placing 4x4x4 input blocks into a 2x2x4 cube
offset = [
xyz[0] * xcubedim, xyz[1] * ycubedim, xyz[2] * zcubedim
]
nzlocs = np.nonzero(cube.data > threshhold)
count += len(nzlocs[1])
return countt
fileobj = cStringIO.StringIO()
np.save(fileobj, exceptions)
self.cursor.execute(
sql, (key, entityid, zlib.compress(fileobj.getvalue())))
except MySQLdb.Error, e:
logger.error("Error inserting exceptions %d: %s. sql=%s" %
(e.args[0], e.args[1], sql))
raise
# In this case we have an update query
else:
oldexlist = [zindex.XYZMorton(trpl) for trpl in curexlist]
newexlist = [zindex.XYZMorton(trpl) for trpl in exceptions]
exlist = set(newexlist + oldexlist)
exlist = [zindex.MortonXYZ(zidx) for zidx in exlist]
sql = "UPDATE " + table + " SET exlist=(%s) WHERE zindex=%s AND id=%s"
try:
fileobj = cStringIO.StringIO()
np.save(fileobj, exlist)
self.cursor.execute(
sql, (zlib.compress(fileobj.getvalue()), key, entityid))
except MySQLdb.Error, e:
logger.error("Error updating exceptions %d: %s. sql=%s" %
(e.args[0], e.args[1], sql))
raise
#
# removeExceptions
#
def findLocations(self, channel, resolution, threshhold, outfile):
"""Build the hierarchy of annotations"""
# Get the source database sizes
[ximagesz, yimagesz] = self.proj.datasetcfg.imagesz[resolution]
[xcubedim, ycubedim,
zcubedim] = self.proj.datasetcfg.cubedim[resolution]
# Get the slices
[startslice, endslice] = self.proj.datasetcfg.slicerange
slices = endslice - startslice + 1
# Set the limits for iteration on the number of cubes in each dimension
xlimit = ximagesz / xcubedim
ylimit = yimagesz / ycubedim
# Round up the zlimit to the next larger
zlimit = (((slices - 1) / zcubedim + 1) * zcubedim) / zcubedim
# Round up to the top of the range
lastzindex = (zindex.XYZMorton([xlimit, ylimit, zlimit]) / 64 + 1) * 64
locs = []
with closing(open(outfile, 'wb')) as csvfile:
csvwriter = csv.writer(csvfile)
# Iterate over the cubes in morton order
for mortonidx in range(0, lastzindex, 64):
print "Working on batch %s at %s" % (
mortonidx, zindex.MortonXYZ(mortonidx))
# call the range query
self.annoDB.queryRange(mortonidx, mortonidx + 64, resolution,
channel)
# Flag to indicate no data. No update query
somedata = False
# get the first cube
[key, cube] = self.annoDB.getNextCube()
# if there's a cube, there's data
if key != None:
somedata = True
while key != None:
xyz = zindex.MortonXYZ(key)
# Compute the offset in the output data cube
# we are placing 4x4x4 input blocks into a 2x2x4 cube
offset = [
xyz[0] * xcubedim, xyz[1] * ycubedim, xyz[2] * zcubedim
]
nzlocs = np.nonzero(cube.data > threshhold)
if len(nzlocs[1]) != 0:
print "res : zindex = ", resolution, ":", key, ", location", zindex.MortonXYZ(
key)
# zip together the x y z and value
cubelocs = zip(
nzlocs[0], nzlocs[1], nzlocs[2],
cube.data[nzlocs[0], nzlocs[1], nzlocs[2]])
# translate from z,y,x,value to x,y,z,value and add global offset
locs = [(pt[2] + offset[0], pt[1] + offset[1],
pt[0] + offset[2], pt[3]) for pt in cubelocs]
csvwriter.writerows([x for x in locs])
[key, cube] = self.annoDB.getNextCube()
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Testing the float and uint64 issue in python"""
import sys
import os
sys.path += [os.path.abspath('../django')]
import OCP.settings
os.environ['DJANGO_SETTINGS_MODULE'] = 'OCP.settings'
from django.conf import settings
import zindex
import ocplib
print zindex.MortonXYZ(831488)
dims = (10234, 9703, 11009)
print "DIMENSIONS:", dims
key = ocplib.XYZMorton(dims)
print "KEY:", key
print "WHAT WE GET BACK:", ocplib.MortonXYZ(key)
print "DIMENSIONS:", dims
key = zindex.XYZMorton(dims)
print "KEY:", key
print "WHAT WE GET BACK:", zindex.MortonXYZ(key)
def cutout(self, corner, dim, resolution, channel=None):
"""Extract a cube of arbitrary size. Need not be aligned."""
# get the size of the image and cube
[xcubedim, ycubedim,
zcubedim] = cubedim = self.dbcfg.cubedim[resolution]
# Round to the nearest larger cube in all dimensions
zstart = corner[2] / zcubedim
ystart = corner[1] / ycubedim
xstart = corner[0] / xcubedim
znumcubes = (corner[2] + dim[2] + zcubedim - 1) / zcubedim - zstart
ynumcubes = (corner[1] + dim[1] + ycubedim - 1) / ycubedim - ystart
xnumcubes = (corner[0] + dim[0] + xcubedim - 1) / xcubedim - xstart
if (self.annoproj.getDBType() == emcaproj.ANNOTATIONS):
# input cube is the database size
incube = anncube.AnnotateCube(cubedim)
# output cube is as big as was asked for and zero it.
outcube = anncube.AnnotateCube ( [xnumcubes*xcubedim,\
ynumcubes*ycubedim,\
znumcubes*zcubedim] )
outcube.zeros()
elif (self.annoproj.getDBType() == emcaproj.IMAGES):
incube = imagecube.ImageCube(cubedim)
outcube = imagecube.ImageCube ( [xnumcubes*xcubedim,\
ynumcubes*ycubedim,\
znumcubes*zcubedim] )
elif (self.annoproj.getDBType() == emcaproj.CHANNELS):
incube = chancube.ChanCube(cubedim)
outcube = chancube.ChanCube ( [xnumcubes*xcubedim,\
ynumcubes*ycubedim,\
znumcubes*zcubedim] )
# Build a list of indexes to access
listofidxs = []
for z in range(znumcubes):
for y in range(ynumcubes):
for x in range(xnumcubes):
mortonidx = zindex.XYZMorton(
[x + xstart, y + ystart, z + zstart])
listofidxs.append(mortonidx)
# Sort the indexes in Morton order
listofidxs.sort()
# Batch query for all cubes
dbname = self.annoproj.getTable(resolution)
# Customize query to the database (include channel or not)
if (self.annoproj.getDBType() == emcaproj.CHANNELS):
sql = "SELECT zindex, cube FROM " + dbname + " WHERE channel= " + str(
channel) + " AND zindex in (%s)"
else:
sql = "SELECT zindex, cube FROM " + dbname + " WHERE zindex IN (%s)"
# creats a %s for each list element
in_p = ', '.join(map(lambda x: '%s', listofidxs))
# replace the single %s with the in_p string
sql = sql % in_p
rc = self.cursor.execute(sql, listofidxs)
# xyz offset stored for later use
lowxyz = zindex.MortonXYZ(listofidxs[0])
# Get the objects and add to the cube
while (True):
try:
idx, datastring = self.cursor.fetchone()
except:
break
#add the query result cube to the bigger cube
curxyz = zindex.MortonXYZ(int(idx))
offset = [
curxyz[0] - lowxyz[0], curxyz[1] - lowxyz[1],
curxyz[2] - lowxyz[2]
]
incube.fromNPZ(datastring[:])
# add it to the output cube
outcube.addData(incube, offset)
# need to trim down the array to size
# only if the dimensions are not the same
if dim[0] % xcubedim == 0 and\
dim[1] % ycubedim == 0 and\
dim[2] % zcubedim == 0 and\
corner[0] % xcubedim == 0 and\
corner[1] % ycubedim == 0 and\
corner[2] % zcubedim == 0:
pass
else:
outcube.trim ( corner[0]%xcubedim,dim[0],\
corner[1]%ycubedim,dim[1],\
corner[2]%zcubedim,dim[2] )
return outcube
