def main():
parser = argparse.ArgumentParser(
description='Build an aeropsike DB from mysql data.')
parser.add_argument('intoken',
action="store",
help='Token for the project.')
parser.add_argument('outtoken',
action="store",
help='Token for the project.')
parser.add_argument('resolution', action="store", type=int)
result = parser.parse_args()
# cassandra database
outprojdb = ocpcaproj.OCPCAProjectsDB()
outproj = outprojdb.loadProject(result.outtoken)
# mysql database
inprojdb = ocpcaproj.OCPCAProjectsDB()
inproj = inprojdb.loadProject(result.intoken)
# Bind the databases
inDB = ocpcadb.OCPCADB(inproj)
outDB = ocpcadb.OCPCADB(outproj)
# Get the source database sizes
[ximagesz, yimagesz] = inproj.datasetcfg.imagesz[result.resolution]
[xcubedim, ycubedim,
zcubedim] = cubedim = inproj.datasetcfg.cubedim[result.resolution]
# Get the slices
[startslice, endslice] = inproj.datasetcfg.slicerange
slices = endslice - startslice + 1
# Set the limits for iteration on the number of cubes in each dimension
# and the limits of iteration
xlimit = (ximagesz - 1) / xcubedim + 1
ylimit = (yimagesz - 1) / ycubedim + 1
# Round up the zlimit to the next larger
zlimit = (((slices - 1) / zcubedim + 1) * zcubedim) / zcubedim
for z in range(zlimit):
for y in range(ylimit):
for x in range(xlimit):
zidx = zindex.XYZMorton([x, y, z])
outDB.putCube(zidx, result.resolution,
inDB.getCube(zidx, result.resolution))
print "Ingesting {}".format(zidx)
def upload(self, channel, sl, imarray):
"""Transfer the array to the database"""
with closing(ocpcadb.OCPCADB(self.proj)) as self.db:
for y in range(0, self._yimgsz + 1, self.ycubedim):
for x in range(0, self._ximgsz + 1, self.xcubedim):
# zindex
key = ocplib.XYZMorton([
x / self.xcubedim, y / self.ycubedim,
(sl - self.startslice) / self.zcubedim
])
# Create a channel cube
cube = imagecube.ImageCube16(self.cubedims)
xmin = x
ymin = y
xmax = min(self._ximgsz, x + self.xcubedim)
ymax = min(self._yimgsz, y + self.ycubedim)
zmin = 0
zmax = min(sl + self.zcubedim, self.endslice + 1)
# data for this key
cube.data[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = imarray[zmin:zmax, ymin:ymax,
xmin:xmax]
print cube.data.shape
#import pdb;pdb.set_trace()
self.db.putChannelCube(key, channel, self.resolution, cube)
print " Commiting at x={}, y={}, z={}".format(x, y, sl)
self.db.conn.commit()
def ingest ( self ):
"""Read the stack and ingest"""
with closing (ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(self.token)
with closing (ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(self.channel)
# get the dataset configuration
[[ximagesz, yimagesz, zimagesz],(starttime,endtime)] = proj.datasetcfg.imageSize(self.resolution)
[xcubedim, ycubedim, zcubedim] = cubedim = proj.datasetcfg.getCubeDims()[self.resolution]
[xoffset, yoffset, zoffset] = proj.datasetcfg.getOffset()[self.resolution]
# for all specified resolutions
for resolution in range(0,1,1):
# extract parameters for iteration
numxtiles = ximagesz/self.tilesz[0]
numytiles = yimagesz/self.tilesz[1]
# Ingest in database aligned slabs in the z dimension
for slice_number in range(0, zimagesz, zcubedim):
slab = np.zeros ( [zcubedim,yimagesz,ximagesz], dtype=np.uint32 )
# over all tiles in that slice
for b in range(zcubedim):
for ytile in range(numytiles):
for xtile in range(numxtiles):
# if we are at the end of the space, quit
if slice_number+b <= zimagesz:
try:
filename = '{}{}/{}/{}/{}.png'.format(self.tilepath, resolution, slice_number+b+zoffset, ytile+17, xtile+16)
print "Opening filename {}".format(filename)
# add tile to stack
imgdata = np.asarray ( Image.open(filename, 'r').convert('RGBA') )
imgdata = np.left_shift(imgdata[:,:,3], 24, dtype=np.uint32) | np.left_shift(imgdata[:,:,2], 16, dtype=np.uint32) | np.left_shift(imgdata[:,:,1], 8, dtype=np.uint32) | np.uint32(imgdata[:,:,0])
slab [b,ytile*self.tilesz[1]:(ytile+1)*self.tilesz[1],xtile*self.tilesz[0]:(xtile+1)*self.tilesz[0]] = imgdata
except IOError, e:
print "Failed to open file {}".format(filename)
slab [b,ytile*self.tilesz[1]:(ytile+1)*self.tilesz[1],xtile*self.tilesz[0]:(xtile+1)*self.tilesz[0]] = np.zeros([self.tilesz[1], self.tilesz[0]], dtype=np.uint32)
for y in range (0, yimagesz+1, ycubedim):
for x in range (0, ximagesz+1, xcubedim):
# getting the cube id and ingesting the data one cube at a time
zidx = ocplib.XYZMorton ([x/xcubedim, y/ycubedim, (slice_number)/zcubedim])
cube = Cube.getCube(cubedim, ch.getChannelType(), ch.getDataType())
cube.zeros()
xmin, ymin = x, y
xmax = min (ximagesz, x+xcubedim)
ymax = min (yimagesz, y+ycubedim)
zmin = 0
zmax = min(slice_number+zcubedim, zimagesz+1)
cube.data[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax]
if cube.isNotZeros():
db.putCube(ch, zidx, self.resolution, cube, update=True)
def getAnnoIds(proj, ch, Xmin, Xmax, Ymin, Ymax, Zmin, Zmax):
"""Return a list of anno ids restricted by equality predicates. Equalities are alternating in field/value in the url."""
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(proj.getToken())
db = (ocpcadb.OCPCADB(proj))
resolution = ch.getResolution()
mins = (int(Xmin), int(Ymin), int(Zmin))
maxs = (int(Xmax), int(Ymax), int(Zmax))
offset = proj.datasetcfg.offset[resolution]
from operator import sub
corner = map(sub, mins, offset)
dim = map(sub, maxs, mins)
if not proj.datasetcfg.checkCube(resolution, corner, dim):
logger.warning("Illegal cutout corner={}, dim={}".format(corner, dim))
raise OCPCAError("Illegal cutout corner={}, dim={}".format(
corner, dim))
cutout = db.cutout(ch, corner, dim, resolution)
if cutout.isNotZeros():
annoids = np.unique(cutout.data)
else:
annoids = np.asarray([], dtype=np.uint32)
return annoids[1:]
def ingest(token, resolution):
""" Read the stack and ingest """
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadProject(token)
with closing(ocpcadb.OCPCADB(proj)) as db:
(xcubedim, ycubedim,
zcubedim) = cubedims = proj.datasetcfg.cubedim[resolution]
zidx = 0
cube = imagecube.ImageCube16(cubedims)
cube.zeros()
cube.data = np.array(range(xcubedim * ycubedim * zcubedim),
dtype=np.uint8).reshape(cubedims)
db.putCube(zidx, resolution, cube)
db.conn.commit()
c = db.getCube(zidx, resolution)
print c.data
cube2 = imagecube.ImageCube16(cubedims)
cube2.data = np.zeros(cubedims, dtype=np.uint8)
db.putCube(zidx, resolution, cube2, True)
db.conn.commit()
c = db.getCube(zidx, resolution)
import pdb
pdb.set_trace()
print c.data
def main():
parser = argparse.ArgumentParser(description='Build an aeropsike DB from mysql data.')
parser.add_argument('token', action="store", help='Token for the project.')
parser.add_argument('resolution', action="store", type=int)
result = parser.parse_args()
# as database
ascfg = { 'hosts': [ ('127.0.0.1', 3000) ] }
ascli = aerospike.client(ascfg).connect()
# mysql database
projdb = ocpcaproj.OCPCAProjectsDB()
proj = projdb.loadProject ( result.token )
# Bind the annotation database
imgDB = ocpcadb.OCPCADB ( proj )
# Get the source database sizes
[ximagesz, yimagesz] = proj.datasetcfg.imagesz [ result.resolution ]
[xcubedim, ycubedim, zcubedim] = cubedim = proj.datasetcfg.cubedim [ result.resolution ]
# Get the slices
[ startslice, endslice ] = proj.datasetcfg.slicerange
slices = endslice - startslice + 1
# Set the limits for iteration on the number of cubes in each dimension
# RBTODO These limits may be wrong for even (see channelingest.py)
xlimit = ximagesz / xcubedim
ylimit = yimagesz / ycubedim
# Round up the zlimit to the next larger
zlimit = (((slices-1)/zcubedim+1)*zcubedim)/zcubedim
cursor = imgDB.conn.cursor()
for z in range(zlimit):
for y in range(ylimit):
for x in range(xlimit):
mysqlcube = imgDB.cutout ( [ x*xcubedim, y*ycubedim, z*zcubedim ], cubedim, result.resolution )
zidx = zindex.XYZMorton ( [x,y,z] )
tmpfile = tempfile.NamedTemporaryFile ()
h5tocass = h5py.File ( tmpfile.name )
h5tocass.create_dataset ( "cuboid", tuple(mysqlcube.data.shape), mysqlcube.data.dtype,
compression='gzip', data=mysqlcube.data )
h5tocass.close()
tmpfile.seek(0)
askey = ("ocp",str(result.token)+":"+str(result.resolution),str(zidx))
print askey
ascli.put ( askey, { 'cuboid' : tmpfile.read().encode('hex') } )
try:
ascli.get ( askey )
except:
print "Except"
def ingest ( self ):
"""Read the stack and ingest"""
with closing ( ocpcaproj.OCPCAProjectsDB() ) as projdb:
proj = projdb.loadProject ( self.token )
with closing ( ocpcadb.OCPCADB (proj) ) as db:
(startslice, endslice) = proj.datasetcfg.slicerange
(xcubedim, ycubedim, zcubedim) = cubedims = proj.datasetcfg.cubedim[self.resolution]
(ximagesz, yimagesz) = proj.datasetcfg.imagesz[self.resolution]
batchsz = zcubedim
# Ingest in database aligned slabs in the z dimension
for sl in range( startslice, endslice, batchsz ):
slab = np.zeros ( [zcubedim, yimagesz, ximagesz], dtype=np.uint8 )
# over each slice
for b in range( batchsz ):
#if we are at the end of the space, quit
if ( sl + b <= endslice ):
filename = '{}{:0>3}____z{}.0.tif'.format(self.path, sl+b, (sl+b-1)*25)
#filename = '{}{:0>4}____z{}.0.tif'.format(self.path, sl+b, (sl+b-1)*25)
print filename
try:
img = Image.open(filename,'r')
slab [b,:,:] = np.asarray(img)
except IOError, e:
print "Failed to open file %s" % (e)
img = np.zeros((yimagesz,ximagesz), dtype=np.uint8)
slab [b,:,:] = img
for y in range ( 0, yimagesz, ycubedim ):
for x in range ( 0, ximagesz, xcubedim ):
zidx = ndlib.XYZMorton ( [ x/xcubedim, y/ycubedim, (sl-startslice)/zcubedim] )
cubedata = np.zeros ( [zcubedim, ycubedim, xcubedim], dtype=np.uint8 )
xmin = x
ymin = y
xmax = ( min(ximagesz-1, x+xcubedim-1) ) + 1
ymax = ( min(yimagesz-1, y+ycubedim-1) ) + 1
zmin = 0
zmax = min(sl+zcubedim,endslice)
cubedata[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax,ymin:ymax,xmin:xmax]
cube = imagecube.ImageCube16 ( cubedims )
cube.zeros()
cube.data = cubedata
if np.count_nonzero ( cube.data ) != 0:
print zidx, ndlib.MortonXYZ(zidx)
db.putCube ( zidx, self.resolution, cube )
print "Commiting at x=%s, y=%s, z=%s" % (x,y,sl)
db.conn.commit()
slab = None
def __init__(self, token):
"""Load the annotation database and project"""
projdb = ocpcaproj.OCPCAProjectsDB()
self.proj = projdb.loadProject(token)
# Bind the annotation database
self.annoDB = ocpcadb.OCPCADB(self.proj)
def main():
parser = argparse.ArgumentParser(description='Ingest the TIFF data')
parser.add_argument('token', action="store", type=str, help='Token for the project')
parser.add_argument('channel', action="store", type=str, help='Channel for the project')
parser.add_argument('path', action="store", type=str, help='Directory with the image files')
parser.add_argument('resolution', action="store", type=int, help='Resolution of data')
parser.add_argument('--offset', action="store", type=int, default=0, help='Offset on disk')
result = parser.parse_args()
# Load a database
with closing (ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(result.token)
with closing (ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(result.channel)
# get the dataset configuration
[[ximagesz, yimagesz, zimagesz],(starttime,endtime)] = proj.datasetcfg.imageSize(result.resolution)
[xcubedim,ycubedim,zcubedim] = cubedim = proj.datasetcfg.getCubeDims()[result.resolution]
[xoffset, yoffset, zoffset] = proj.datasetcfg.getOffset()[result.resolution]
# Get a list of the files in the directories
for slice_number in range (zoffset, zimagesz+1, zcubedim):
slab = np.zeros([zcubedim, yimagesz, ximagesz ], dtype=np.uint8)
for b in range(zcubedim):
if (slice_number + b <= zimagesz):
try:
# reading the raw data
file_name = "{}{:0>5}.tif".format(result.path, (slice_number + b))
# silvestri15
#file_name = "{}full_{:0>6}.tif".format(result.path, slice_number + b + result.offset)
print "Open filename {}".format(file_name)
slab[b,:,:] = np.asarray(Image.open(file_name, 'r'))
except IOError, e:
print e
slab[b,:,:] = np.zeros((yimagesz, ximagesz), dtype=np.uint8)
for y in range ( 0, yimagesz+1, ycubedim ):
for x in range ( 0, ximagesz+1, xcubedim ):
# Getting a Cube id and ingesting the data one cube at a time
zidx = ocplib.XYZMorton ( [x/xcubedim, y/ycubedim, (slice_number-zoffset)/zcubedim] )
cube = Cube.getCube(cubedim, ch.getChannelType(), ch.getDataType())
cube.zeros()
xmin,ymin = x,y
xmax = min ( ximagesz, x+xcubedim )
ymax = min ( yimagesz, y+ycubedim )
zmin = 0
zmax = min(slice_number+zcubedim, zimagesz+1)
cube.data[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax]
if cube.isNotZeros():
db.putCube(ch, zidx, result.resolution, cube, update=True)
slab = None
def __init__(self, token, tilesz, tilepath):
"""Load the CATMAID stack into an OCP database"""
# Get the database
self.projdb = ocpcaproj.OCPCAProjectsDB()
self.proj = self.projdb.loadProject(token)
self.db = ocpcadb.OCPCADB(self.proj)
self.tilesz = tilesz
self.prefix = tilepath
def ingest(self):
""" Read image stack and ingest """
# Load a database
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(self.token)
with closing(ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(self.channel_name)
# get the dataset configuration
[[ximagesz, yimagesz, zimagesz], (starttime, endtime)] = proj.datasetcfg.imageSize(self.resolution)
[xcubedim, ycubedim, zcubedim] = cubedim = proj.datasetcfg.getCubeDims()[self.resolution]
[xoffset, yoffset, zoffset] = proj.datasetcfg.getOffset()[self.resolution]
# Get a list of the files in the directories
for slice_number in range(zoffset, zimagesz, zcubedim):
slab = np.zeros([zcubedim, yimagesz, ximagesz], dtype=np.uint32)
for b in range(zcubedim):
if (slice_number + b <= zimagesz):
file_name = "{}{}{:0>4}.tif".format(self.path, self.token, slice_number+b)
print "Open filename {}".format(file_name)
try:
img = Image.open(file_name,'r')
slab [b,:,:] = np.asarray(img)
except IOError, e:
print "Failed to open file %s" % (e)
img = np.zeros((yimagesz,ximagesz), dtype=np.uint8)
slab [b,:,:] = img
for y in range(0, yimagesz + 1, ycubedim):
for x in range(0, ximagesz + 1, xcubedim):
# Getting a Cube id and ingesting the data one cube at a time
zidx = ocplib.XYZMorton([x / xcubedim, y / ycubedim, (slice_number - zoffset) / zcubedim])
cube = Cube.getCube(cubedim, ch.getChannelType(), ch.getDataType())
cube.zeros()
xmin = x
ymin = y
xmax = min(ximagesz, x + xcubedim)
ymax = min(yimagesz, y + ycubedim)
zmin = 0
zmax = min(slice_number + zcubedim, zimagesz + 1)
cube.data[0:zmax - zmin, 0:ymax - ymin, 0:xmax - xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax]
from operator import sub
corner = map(sub, [x,y,slice_number], [xoffset,yoffset,zoffset])
if cube.data.any():
db.annotateDense ( ch, corner, self.resolution, cube.data, 'O' )
def __init__( self, token, tilesz, tilepath, reslimit, totalprocs ):
"""Load the CATMAID stack into an OCP database"""
# Get the database
self.projdb = ocpcaproj.OCPCAProjectsDB()
self.proj = self.projdb.loadProject ( token )
self.db = ocpcadb.OCPCADB ( self.proj )
self.tilesz = tilesz
self.prefix=tilepath
self.reslimit = reslimit
self.totalprocs = totalprocs
self.token = token
def buildStack(token, channel, res):
"""Build the hierarchy of images"""
with closing (ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(token)
with closing (ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(channel)
high_res = proj.datasetcfg.scalinglevels
for cur_res in range(res, high_res+1):
# Get the source database sizes
[[ximagesz, yimagesz, zimagesz], timerange] = proj.datasetcfg.imageSize(cur_res)
[xcubedim, ycubedim, zcubedim] = cubedim = proj.datasetcfg.getCubeDims()[cur_res]
[xoffset, yoffset, zoffset] = proj.datasetcfg.getOffset()[cur_res]
biggercubedim = [xcubedim*2,ycubedim*2,zcubedim]
# Set the limits for iteration on the number of cubes in each dimension
xlimit = (ximagesz-1) / xcubedim + 1
ylimit = (yimagesz-1) / ycubedim + 1
zlimit = (zimagesz-1) / zcubedim + 1
for z in range(zlimit):
for y in range(ylimit):
for x in range(xlimit):
# cutout the data at the -1 resolution
olddata = db.cutout(ch, [ x*2*xcubedim, y*2*ycubedim, z*zcubedim], biggercubedim, cur_res-1 ).data
# target array for the new data (z,y,x) order
newdata = np.zeros([zcubedim,ycubedim,xcubedim], dtype=np.uint16)
for sl in range(zcubedim):
# Convert each slice to an image
slimage = Image.frombuffer ( 'I;16', (xcubedim*2,ycubedim*2), olddata[sl,:,:].flatten(), 'raw', 'I;16', 0, 1 )
# Resize the image
newimage = slimage.resize ( [xcubedim,ycubedim] )
# Put to a new cube
newdata[sl,:,:] = np.asarray ( newimage )
zidx = ocplib.XYZMorton ( [x,y,z] )
cube = Cube.getCube(cubedim, ch.getChannelType(), ch.getDataType())
cube.zeros()
cube.data = newdata
print "Inserting Cube {} at res {}".format(zidx, cur_res)
db.putCube(ch, zidx, cur_res, cube, update=True)
def getHist(self):
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(self.token)
with closing(ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(self.channel)
# Get the source database sizes
[[ximagesz, yimagesz, zimagesz],
timerange] = proj.datasetcfg.imageSize(self.res)
[xcubedim, ycubedim,
zcubedim] = cubedim = proj.datasetcfg.getCubeDims()[self.res]
[xoffset, yoffset, zoffset] = proj.datasetcfg.getOffset()[self.res]
# Set the limits for iteration on the number of cubes in each dimension
xlimit = (ximagesz - 1) / xcubedim + 1
ylimit = (yimagesz - 1) / ycubedim + 1
zlimit = (zimagesz - 1) / zcubedim + 1
numbins = 2**16
hist = []
bins = np.zeros(numbins + 1)
count = 0
# sum the histograms
for z in range(zlimit):
for y in range(ylimit):
for x in range(xlimit):
# cutout the data for the cube
data = db.cutout(
ch, [x * xcubedim, y * ycubedim, z * zcubedim],
cubedim, self.res).data
# compute the histogram and store it
hist.append(
np.histogram(data[data > 0],
bins=numbins,
range=(0, 2**16)))
print "Processed cube {} {} {}".format(x, y, z)
# sum the individual histograms
hist_sum = np.zeros(numbins)
bins = hist[0][1] # all bins should be the same
for i in range(len(hist)):
hist_sum += hist[i][0]
return (hist_sum, bins)
def __init__(self, token, cutout):
"""Load the annotation database and project"""
projdb = ocpcaproj.OCPCAProjectsDB()
self.proj = projdb.loadProject(token)
# Bind the annotation database
self.annoDB = ocpcadb.OCPCADB(self.proj)
# Perform argument processing
try:
args = restargs.BrainRestArgs()
args.cutoutArgs(cutout + "/", self.proj.datasetcfg)
except restargs.RESTArgsError, e:
raise OCPCAError(e.value)
def buildStack(token, channel, res, base_res):
""" build a zoom hierarchy of images """
scaling = 2**(base_res-res)
with closing (ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(token)
with closing(ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(channel)
# get db sizes
[[ximagesz, yimagesz, zimagesz], timerange] = proj.datasetcfg.imageSize(base_res)
[xcubedim, ycubedim, zcubedim] = cubedim = proj.datasetcfg.getCubeDims()[base_res]
[xoffset, yoffset, zoffset] = proj.datasetcfg.getOffset()[base_res]
newcubedim = proj.datasetcfg.getCubeDims()[res]
xlimit = (ximagesz-1) / xcubedim + 1
ylimit = (yimagesz-1) / ycubedim + 1
zlimit = (zimagesz-1) / zcubedim + 1
# iterate over the old cube
for z in range(zlimit):
for y in range(ylimit):
for x in range(xlimit):
# cutout data
old_data = db.cutout( ch, [x*xcubedim, y*ycubedim, z*zcubedim], cubedim, base_res ).data
#new_data = zoomIn(old_data, scaling)
new_data = cZoomIn(old_data, base_res-res)
newzsize = new_data.shape[0] / newcubedim[2] #old_data.shape[0]
newysize = new_data.shape[1] / newcubedim[1] #old_data.shape[1]
newxsize = new_data.shape[2] / newcubedim[0] #old_data.shape[2]
#print "sizes: {} {} {}".format(newxsize, newysize, newzsize)
for z2 in range(newzsize):
for y2 in range(newysize):
for x2 in range(newxsize):
#print "{} {} {}".format(x*newxsize+x2,y*newysize+y2,z*newzsize+z2)
zidx = ndlib.XYZMorton([x*newxsize+x2, y*newysize+y2, z*newzsize+z2])
cube = Cube.getCube(newcubedim, ch.getChannelType(), ch.getDataType())
cube.zeros()
cube.data = new_data[z2*newcubedim[2]:(z2+1)*newcubedim[2], y2*newcubedim[1]:(y2+1)*newcubedim[1], x2*newcubedim[0]:(x2+1)*newcubedim[0]]
#print "Grabbing cube from [{}:{} , {}:{}, {}:{}]".format(z2*newcubedim[2],(z2+1)*newcubedim[2], y2*newcubedim[1],(y2+1)*newcubedim[1], x2*newcubedim[0],(x2+1)*newcubedim[0])
print "Inserting Cube {} at res {}".format(zidx, res)
db.putCube(ch, zidx, res, cube, update=True)
def getTile(self, webargs):
"""Either fetch the file from mocpcache or load a new region into mocpcache by cutout"""
# parse the web args
self.token, tileszstr, self.channel, resstr, xtilestr, ytilestr, zslicestr, color, brightnessstr, rest = webargs.split(
'/', 9)
# load the database
self.loadDB()
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
self.proj = projdb.loadProject(self.token)
with closing(ocpcadb.OCPCADB(self.proj)) as self.db:
# convert args to ints
xtile = int(xtilestr)
ytile = int(ytilestr)
res = int(resstr)
# modify the zslice to the offset
zslice = int(zslicestr) - self.proj.datasetcfg.slicerange[0]
self.tilesz = int(tileszstr)
brightness = float(brightnessstr)
# memcache key
mckey = self.buildKey(res, xtile, ytile, zslice, color, brightness)
# do something to sanitize the webargs??
# if tile is in mocpcache, return it
tile = self.mc.get(mckey)
if tile != None:
fobj = cStringIO.StringIO(tile)
# load a slab into CATMAID
else:
img = self.cacheMiss(res, xtile, ytile, zslice, color,
brightness)
fobj = cStringIO.StringIO()
img.save(fobj, "PNG")
self.mc.set(mckey, fobj.getvalue())
fobj.seek(0)
return fobj
def __init__(self, token, path):
self.path = path
projdb = ocpcaproj.OCPCAProjectsDB()
self.proj = projdb.loadProject ( token )
# Bind the database
self.db = ocpcadb.OCPCADB ( self.proj )
# get spatial information
self._ximgsz = self.proj.datasetcfg.imagesz[resolution][0]
self._yimgsz = self.proj.datasetcfg.imagesz[resolution][1]
self.startslice = self.proj.datasetcfg.slicerange[0]
self.endslice = self.proj.datasetcfg.slicerange[1]
self.batchsz = self.proj.datasetcfg.cubedim[resolution][2]
# get a db cursor
self.cursor = self.db.conn.cursor()
def __init__(self, token, path, resolution, channel):
self.token = token
self.path = path
self.resolution = resolution
with closing(ocpcaproj.OCPCAProjectsDB()) as self.projdb:
self.proj = self.projdb.loadProject(token)
with closing(ocpcadb.OCPCADB(self.proj)) as self.db:
(self.xcubedim, self.ycubedim, self.zcubedim
) = self.cubedims = self.proj.datasetcfg.cubedim[resolution]
(self.startslice, self.endslice) = self.proj.datasetcfg.slicerange
self.batchsz = self.zcubedim
self.channel = channel
(self._ximgsz,
self._yimgsz) = self.proj.datasetcfg.imagesz[resolution]
def __init__(self, token, resolution, path):
self.path = path
self.resolution = resolution
self.projdb = ocpcaproj.OCPCAProjectsDB()
self.proj = self.projdb.loadProject(token)
(self._ximgsz, self._yimgsz) = self.proj.datasetcfg.imagesz[resolution]
(self.startslice, self.endslice) = self.proj.datasetcfg.slicerange
(self.ximagesz, self.yimagesz) = (9888, 7936)
self.batchsz = self.proj.datasetcfg.cubedim[resolution][2]
self.alldirs = os.listdir(path)
# open the database
self.db = ocpcadb.OCPCADB(self.proj)
# get a db cursor
self.cursor = self.db.conn.cursor()
def getHist(self):
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(self.token)
with closing(ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(self.channel)
# Get the source database sizes
[[ximagesz, yimagesz, zimagesz],
timerange] = proj.datasetcfg.imageSize(self.res)
[xcubedim, ycubedim,
zcubedim] = cubedim = proj.datasetcfg.getCubeDims()[self.res]
[xoffset, yoffset, zoffset] = proj.datasetcfg.getOffset()[self.res]
# Set the limits for iteration on the number of cubes in each dimension
xlimit = (ximagesz - 1) / xcubedim + 1
ylimit = (yimagesz - 1) / ycubedim + 1
zlimit = (zimagesz - 1) / zcubedim + 1
hist_sum = np.zeros(self.numbins, dtype=np.uint32)
# sum the histograms
for z in range(zlimit):
for y in range(ylimit):
for x in range(xlimit):
# cutout the data for the cube
data = db.cutout(
ch, [x * xcubedim, y * ycubedim, z * zcubedim],
cubedim, self.res).data
# compute the histogram and store it
(hist, bins) = np.histogram(data[data > 0],
bins=self.numbins,
range=(0, self.numbins))
hist_sum = np.add(hist_sum, hist)
print "Processed cube {} {} {}".format(x, y, z)
return (hist_sum, bins)
def clearStack(proj, ch, res=None):
""" Clear a OCP stack for a given project """
with closing(ocpcadb.OCPCADB(proj)) as db:
# pick a resolution
if res is None:
res = 1
high_res = proj.datasetcfg.scalinglevels
list_of_tables = []
sql = ""
# Creating a list of all tables to clear
list_of_tables.append(ch.getIdsTable())
for cur_res in range(res, high_res):
list_of_tables.append(ch.getTable(cur_res))
list_of_tables.append(ch.getNearIsoTable(cur_res))
list_of_tables.append(ch.getIdxTable(cur_res))
list_of_tables.append(ch.getExceptionsTable(cur_res))
for anno_type in annotation.anno_dbtables.keys():
list_of_tables.append(ch.getAnnoTable(anno_type))
# Creating the sql query to execute
for table_name in list_of_tables:
sql += "TRUNCATE table {};".format(table_name)
# Executing the query to clear the tables
try:
db.conn.cursor().execute(sql)
db.conn.commit()
except MySQLdb.Error, e:
logger.error("Error truncating the table. {}".format(e))
raise
finally:
def ingest(self):
"""Read the stack and ingest"""
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadProject(self.token)
with closing(ocpcadb.OCPCADB(proj)) as db:
(startslice, endslice) = proj.datasetcfg.slicerange
(xcubedim, ycubedim,
zcubedim) = cubedims = proj.datasetcfg.cubedim[self.resolution]
(ximagesz, yimagesz) = proj.datasetcfg.imagesz[self.resolution]
batchsz = zcubedim
numxtiles = ximagesz / self.tilesz
numytiles = yimagesz / self.tilesz
# Ingest in database aligned slabs in the z dimension
for sl in range(startslice, endslice, batchsz):
# over all tiles in that slice
for ytile in range(0, numytiles):
for xtile in range(0, numxtiles):
slab = np.zeros([zcubedim, self.tilesz, self.tilesz],
dtype=np.uint8)
# over each slice
for b in range(batchsz):
#if we are at the end of the space, quit
if (sl + b <= endslice):
filename = '{}z{:0>4}/c{:0>2}r{:0>2}.tif'.format(
self.tilepath, sl + b, xtile + 1,
ytile + 1)
#filename = '{}{}/c{:0>3}r{:0>3}.jpg'.format(self.tilepath, sl+b, xtile, ytile )
#filename = '{}{}/{}_{}_{}.jpg'.format(self.tilepath, sl+b, ytile, xtile, self.resolution )
#filename = '{}{}/{}/{}_{}.jpg'.format(self.tilepath, sl+b, self.resolution, ytile, xtile )
#filename = '{}z{:0>4}/c{:0>2}r{:0>2}.tif'.format(self.tilepath, sl+b, ytile+1, xtile+1 )
print filename
try:
# add tile to stack
img = Image.open(filename, 'r')
slab[b, :, :] = np.asarray(img)[:, :, 0]
except IOError, e:
print "Failed to open file %s" % (e)
img = np.zeros((self.tilesz, self.tilesz),
dtype=np.uint8)
slab[b, :, :] = img
for y in range(ytile * self.tilesz,
(ytile + 1) * self.tilesz, ycubedim):
for x in range(xtile * self.tilesz,
(xtile + 1) * self.tilesz,
xcubedim):
zidx = ndlib.XYZMorton([
x / xcubedim, y / ycubedim,
(sl - startslice) / zcubedim
])
cubedata = np.zeros(
[zcubedim, ycubedim, xcubedim],
dtype=np.uint8)
xmin = x % self.tilesz
ymin = y % self.tilesz
xmax = (min(ximagesz - 1, x + xcubedim - 1) %
self.tilesz) + 1
ymax = (min(yimagesz - 1, y + ycubedim - 1) %
self.tilesz) + 1
zmin = 0
zmax = min(sl + zcubedim, endslice)
cubedata[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax,
ymin:ymax,
xmin:xmax]
cube = imagecube.ImageCube16(cubedims)
cube.data = cubedata
if np.count_nonzero(cube.data) != 0:
db.putCube(zidx, self.resolution, cube)
print "Commiting at x=%s, y=%s, z=%s" % (x, y, sl)
db.conn.commit()
def ingestImageStack(self):
"""Ingest a TIF image stack"""
# Load a database
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(self.token)
with closing(ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(self.channel)
# get the dataset configuration
[[ximagesz, yimagesz, zimagesz],
(starttime, endtime)] = proj.datasetcfg.imageSize(self.resolution)
[xcubedim, ycubedim, zcubedim
] = cubedim = proj.datasetcfg.getCubeDims()[self.resolution]
[xoffset, yoffset,
zoffset] = proj.datasetcfg.getOffset()[self.resolution]
if ch.getChannelType() in TIMESERIES_CHANNELS and (
starttime == 0 and endtime == 0):
logger.error("Timeseries Data cannot have timerange (0,0)")
raise OCPCAError("Timeseries Data cannot have timerange (0,0)")
# Get a list of the files in the directories
for timestamp in range(starttime, endtime + 1):
for slice_number in range(zoffset, zimagesz, zcubedim):
slab = np.zeros([zcubedim, yimagesz, ximagesz],
dtype=OCP_dtypetonp.get(ch.getDataType()))
# fetch 16 slices at a time
if ch.getChannelType() in TIMESERIES_CHANNELS:
time_value = timestamp
else:
time_value = None
self.fetchData(
range(slice_number, slice_number +
zcubedim) if slice_number + zcubedim <= zimagesz
else range(slice_number, zimagesz),
time_value=time_value)
for b in range(zcubedim):
if (slice_number + b < zimagesz):
try:
# reading the raw data
file_name = "{}{}".format(
self.path,
self.generateFileName(slice_number + b))
print "Open filename {}".format(file_name)
logger.info(
"Open filename {}".format(file_name))
if ch.getDataType() in [
UINT8, UINT16
] and ch.getChannelType(
) in IMAGE_CHANNELS + TIMESERIES_CHANNELS:
image_data = np.asarray(
Image.open(file_name, 'r'))
slab[b, :, :] = image_data
elif ch.getDataType() in [
UINT32
] and ch.getChannelType(
) in IMAGE_CHANNELS + TIMESERIES_CHANNELS:
image_data = np.asarray(
Image.open(file_name,
'r').convert('RGBA'))
slab[b, :, :] = np.left_shift(
image_data[:, :, 3],
24,
dtype=np.uint32) | np.left_shift(
image_data[:, :, 2],
16,
dtype=np.uint32) | np.left_shift(
image_data[:, :, 1],
8,
dtype=np.uint32) | np.uint32(
image_data[:, :, 0])
elif ch.getChannelType(
) in ANNOTATION_CHANNELS:
image_data = np.asarray(
Image.open(file_name, 'r'))
slab[b, :, :] = image_data
else:
logger.error("Cannot ingest this data yet")
raise OCPCAError(
"Cannot ingest this data yet")
except IOError, e:
logger.warning("IOError {}.".format(e))
slab[b, :, :] = np.zeros((yimagesz, ximagesz),
dtype=np.uint32)
for y in range(0, yimagesz + 1, ycubedim):
for x in range(0, ximagesz + 1, xcubedim):
# Getting a Cube id and ingesting the data one cube at a time
zidx = ocplib.XYZMorton([
x / xcubedim, y / ycubedim,
(slice_number - zoffset) / zcubedim
])
cube = Cube.getCube(cubedim, ch.getChannelType(),
ch.getDataType())
cube.zeros()
xmin, ymin = x, y
xmax = min(ximagesz, x + xcubedim)
ymax = min(yimagesz, y + ycubedim)
zmin = 0
zmax = min(slice_number + zcubedim, zimagesz + 1)
cube.data[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax,
ymin:ymax,
xmin:xmax]
if cube.isNotZeros():
if ch.getChannelType() in IMAGE_CHANNELS:
db.putCube(ch,
zidx,
self.resolution,
cube,
update=False)
elif ch.getChannelType(
) in TIMESERIES_CHANNELS:
db.putTimeCube(ch,
zidx,
timestamp,
self.resolution,
cube,
update=False)
elif ch.getChannelType(
) in ANNOTATION_CHANNELS:
corner = map(sub, [x, y, slice_number],
[xoffset, yoffset, zoffset])
db.annotateDense(ch, corner,
self.resolution,
cube.data, 'O')
else:
logger.error(
"Channel type {} not supported".format(
ch.getChannelType()))
raise OCPCAError(
"Channel type {} not supported".format(
ch.getChannelType()))
# clean up the slices fetched
self.cleanData(
range(slice_number, slice_number +
zcubedim) if slice_number + zcubedim <= zimagesz
else range(slice_number, zimagesz))
def ingestCatmaidStack(self):
"""Ingest a CATMAID tile stack"""
# Load a database
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(self.token)
with closing(ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(self.channel)
# get the dataset configuration
[[ximagesz, yimagesz, zimagesz],
(starttime, endtime)] = proj.datasetcfg.imageSize(self.resolution)
[xcubedim, ycubedim, zcubedim
] = cubedim = proj.datasetcfg.getCubeDims()[self.resolution]
[xoffset, yoffset,
zoffset] = proj.datasetcfg.getOffset()[self.resolution]
if ch.getChannelType() in TIMESERIES_CHANNELS and (
starttime == 0 and endtime == 0):
pass
else:
logger.error(
"Timeseries Data cannot have timerange (0,0). Error in {}".
format(self.token))
raise OCPCAError(
"Timeseries Data cannot have timerange (0,0)".format(
self.token))
num_xtiles = ximagesz / tilesz
num_ytiles = yimagesz / tilesz
# Get a list of the files in the directories
for timestamp in range(starttime, endtime + 1):
for slice_number in range(zoffset, zimagesz + 1, zcubedim):
# over all the tiles in the slice
for ytile in range(0, num_ytiles):
for xtile in range(o, num_xtiles):
slab = np.zeros([zcubedim, tilesz, tilesz],
dtype=np.uint8)
for b in range(zcubedim):
if (slice_number + b <= zimagesz):
try:
# reading the raw data
file_name = "{}{}{:0>6}.{}".format(
self.path,
self.regex(slice_number + b),
self.file_type)
logger.info("Open filename {}".format(
file_name))
print "Open filename {}".format(
file_name)
slab[b, :, :] = np.asarray(
Image.open(file_name, 'r'))
except IOError, e:
logger.warning("IOError {}.".format(e))
slab[b, :, :] = np.zeros(
(tilesz, tilesz), dtype=np.uint32)
for y in range(ytile * tilesz,
(ytile + 1) * tilesz, ycubedim):
for x in range(xtile * tilesz,
(xtile + 1) * tilesz, xcubedim):
# Getting a Cube id and ingesting the data one cube at a time
zidx = ocplib.XYZMorton([
x / xcubedim, y / ycubedim,
(slice_number - zoffset) / zcubedim
])
cube = Cube.getCube(
cubedim, ch.getChannelType(),
ch.getDataType())
cube.zeros()
xmin = x % tilesz
ymin = y % tilesz
xmax = min(ximagesz, x + xcubedim)
ymax = min(yimagesz, y + ycubedim)
zmin = 0
zmax = min(slice_number + zcubedim,
zimagesz + 1)
cube.data[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax,
ymin:ymax,
xmin:xmax]
if cube.isNotZeros():
db.putCube(ch,
zidx,
self.resolution,
cube,
update=True)
else:
db.putTimeCube(ch,
zidx,
timestamp,
self.resolution,
cube,
update=True)
def ingest(self, channel_name):
""" Read image stack and ingest """
# Load a database
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(self.token)
with closing(ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(channel_name)
# get the dataset configuration
[[ximagesz, yimagesz, zimagesz],
(starttime, endtime)] = proj.datasetcfg.imageSize(self.resolution)
[xcubedim, ycubedim, zcubedim
] = cubedim = proj.datasetcfg.getCubeDims()[self.resolution]
[xoffset, yoffset,
zoffset] = proj.datasetcfg.getOffset()[self.resolution]
# Get a list of the files in the directories
file_name = "{}{}.tif".format(self.path, channel_name)
print "Open filename {}".format(file_name)
imgdata = tifffile.imread(file_name)
for slice_number in range(zoffset, zimagesz + 1, zcubedim):
slab = np.zeros([zcubedim, yimagesz, ximagesz],
dtype=np.uint32)
for b in range(zcubedim):
if (slice_number + b <= zimagesz):
if (slice_number + b) < zimagesz:
slab[b, :, :] = imgdata[(slice_number + b), :, :]
else:
imgdata = np.zeros((yimagesz, ximagesz),
dtype=np.uint32)
slab[b, :, :] = imgdata
for y in range(0, yimagesz + 1, ycubedim):
for x in range(0, ximagesz + 1, xcubedim):
# Getting a Cube id and ingesting the data one cube at a time
zidx = ocplib.XYZMorton([
x / xcubedim, y / ycubedim,
(slice_number - zoffset) / zcubedim
])
cube = Cube.getCube(cubedim, ch.getChannelType(),
ch.getDataType())
cube.zeros()
xmin = x
ymin = y
xmax = min(ximagesz, x + xcubedim)
ymax = min(yimagesz, y + ycubedim)
zmin = 0
zmax = min(slice_number + zcubedim, zimagesz + 1)
cube.data[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax, ymin:ymax,
xmin:xmax]
db.putCube(ch,
zidx,
self.resolution,
cube,
update=True)
def main():
parser = argparse.ArgumentParser(description='Ingest the TIFF data')
parser.add_argument('token', action="store", help='Token for the project')
parser.add_argument('resolution', action="store", type=int, help='Resolution')
parser.add_argument('path', action="store", help='Directory with the image files')
result = parser.parse_args()
#Load a database
with closing ( ocpcaproj.OCPCAProjectsDB() ) as projdb:
proj = projdb.loadProject ( result.token )
with closing ( ocpcadb.OCPCADB(proj) ) as db:
# get the dataset configuration
(xcubedim,ycubedim,zcubedim) = proj.datasetcfg.cubedim[result.resolution]
(startslice,endslice)=proj.datasetcfg.slicerange
(starttime,endtime)=proj.datasetcfg.timerange
(ximagesz,yimagesz)=proj.datasetcfg.imagesz[result.resolution]
batchsz = zcubedim
dims = (2048,1172,31)
# Set the image size to that of the actual image
ximagesz = 2048
yimagesz = 1172
endslice = 31
# Get a list of the files in the directories
for ts in range ( starttime, endtime ):
filenm = "{}TM{:0>5}_CM0_CHN00.stack".format(result.path,ts)
print "Opening file", filenm
img = open(filenm,'r')
imgdata = np.frombuffer(img.read(), dtype=np.int16, count=int(np.prod(dims))).reshape(dims, order='F')
imgdata = np.swapaxes(imgdata,0,2)
for sl in range (startslice, endslice+1, batchsz):
slab = np.zeros ( [ batchsz, yimagesz, ximagesz ], dtype=np.uint16 )
for b in range ( batchsz ):
if ( sl + b <= endslice ):
slab[b,:,:] = imgdata[b,:,:]
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
for y in range ( 0, yimagesz+1, ycubedim ):
for x in range ( 0, ximagesz+1, xcubedim ):
mortonidx = zindex.XYZMorton ( [x/xcubedim, y/ycubedim, (sl-startslice)/zcubedim] )
cubedata = np.zeros ( [zcubedim, ycubedim, xcubedim], dtype=np.uint16 )
xmin = x
ymin = y
xmax = ((min(ximagesz-1,x+xcubedim-1)))+1
ymax = ((min(yimagesz-1,y+ycubedim-1)))+1
#xmax = min ( ximagesz, x+xcubedim )
#ymax = min ( yimagesz, y+ycubedim )
zmin = 0
zmax = min(sl+zcubedim, endslice+1)
cubedata[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax]
# Create the DB Blob
fileobj = cStringIO.StringIO()
np.save ( fileobj, cubedata )
cdz = zlib.compress ( fileobj.getvalue() )
# insert the blob into the database
cursor = db.conn.cursor()
sql = "INSERT INTO res{} (zindex, timestamp, cube) VALUES (%s, %s, %s)".format(int(result.resolution))
cursor.execute(sql, (mortonidx, ts, cdz))
cursor.close()
print " Commiting at x={}, y={}, z={}".format(x, y, sl)
db.conn.commit()
slab = None
def ingestImageStack(self):
"""Ingest a TIF image stack"""
# Load a database
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(self.token)
with closing(ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(self.channel)
# get the dataset configuration
[[ximagesz, yimagesz, zimagesz],
(starttime, endtime)] = proj.datasetcfg.imageSize(self.resolution)
[xcubedim, ycubedim, zcubedim
] = cubedim = proj.datasetcfg.getCubeDims()[self.resolution]
[xoffset, yoffset,
zoffset] = proj.datasetcfg.getOffset()[self.resolution]
if ch.getChannelType() in TIMESERIES_CHANNELS and (
starttime == 0 and endtime == 0):
print "Timeseries Data cannot have timerange (0,0)"
raise
# Get a list of the files in the directories
for timestamp in range(starttime, endtime + 1):
for slice_number in range(zoffset, zimagesz + 1, zcubedim):
slab = np.zeros([zcubedim, yimagesz, ximagesz],
dtype=OCP_dtypetonp.get(ch.getDataType()))
# fetch 16 slices at a time
if ch.getChannelType() in TIMESERIES_CHANNELS:
time_value = timestamp
else:
time_value = None
self.fetchData(
range(slice_number, slice_number +
zcubedim) if slice_number + zcubedim <= zimagesz
else range(slice_number, zimagesz),
time_value=time_value)
for b in range(zcubedim):
if (slice_number + b <= zimagesz):
try:
# reading the raw data
file_name = "{}{}".format(
self.path,
self.generateFileName(slice_number + b))
print "Open filename {}".format(file_name)
slab[b, :, :] = np.asarray(
Image.open(file_name, 'r'))
except IOError, e:
print e
slab[b, :, :] = np.zeros((yimagesz, ximagesz),
dtype=np.uint32)
for y in range(0, yimagesz + 1, ycubedim):
for x in range(0, ximagesz + 1, xcubedim):
# Getting a Cube id and ingesting the data one cube at a time
zidx = ocplib.XYZMorton([
x / xcubedim, y / ycubedim,
(slice_number - zoffset) / zcubedim
])
cube = Cube.getCube(cubedim, ch.getChannelType(),
ch.getDataType())
cube.zeros()
xmin, ymin = x, y
xmax = min(ximagesz, x + xcubedim)
ymax = min(yimagesz, y + ycubedim)
zmin = 0
zmax = min(slice_number + zcubedim, zimagesz + 1)
cube.data[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax,
ymin:ymax,
xmin:xmax]
if cube.isNotZeros():
if ch.getChannelType(
) not in TIMESERIES_CHANNELS:
db.putCube(ch,
zidx,
self.resolution,
cube,
update=True)
else:
db.putTimeCube(ch,
zidx,
timestamp,
self.resolution,
cube,
update=True)
# clean up the slices fetched
self.cleanData(
range(slice_number, slice_number +
zcubedim) if slice_number + zcubedim <= zimagesz
else range(slice_number, zimagesz))
def label(self, chanid, chanstr):
""" Write the channel label/string associated with the channel identifier"""
with closing(ocpcadb.OCPCADB(self.proj)) as self.db:
self.db.putChannel(chanstr, chanid)
class SimpleCatmaid:
""" Prefetch CATMAID tiles into MocpcacheDB """
def __init__(self):
""" Bind the mocpcache """
self.proj = None
self.channel = None
self.tilesz = 512
# make the mocpcache connection
self.mc = pylibmc.Client(["127.0.0.1"],
binary=True,
behaviors={
"tcp_nodelay": True,
"ketama": True
})
def __del__(self):
pass
def buildKey(self, res, slice_type, xtile, ytile, ztile, timetile=None):
if timetile is None:
return 'simple/{}/{}/{}/{}/{}/{}/{}'.format(
self.token, self.channel, slice_type, res, xtile, ytile, ztile)
else:
return 'simple/{}/{}/{}/{}/{}/{}/{}/{}'.format(
self.token, self.channel, slice_type, res, xtile, ytile, ztile,
timetile)
def cacheMissXY(self, res, xtile, ytile, ztile, timetile=None):
"""On a miss. Cutout, return the image and load the cache in a background thread"""
# make sure that the tile size is aligned with the cubedim
if self.tilesz % self.proj.datasetcfg.cubedim[res][
0] != 0 or self.tilesz % self.proj.datasetcfg.cubedim[res][1]:
raise ("Illegal tile size. Not aligned")
# figure out the cutout (limit to max image size)
xstart = xtile * self.tilesz
ystart = ytile * self.tilesz
xend = min((xtile + 1) * self.tilesz,
self.proj.datasetcfg.imagesz[res][0])
yend = min((ytile + 1) * self.tilesz,
self.proj.datasetcfg.imagesz[res][1])
# get an xy image slice
if timetile is None:
imageargs = '{}/{}/{}/{},{}/{},{}/{}/'.format(
self.channel, 'xy', res, xstart, xend, ystart, yend, ztile)
else:
imageargs = '{}/{}/{}/{},{}/{},{}/{}/{}/'.format(
self.channel, 'xy', res, xstart, xend, ystart, yend, ztile,
timetile)
cb = ocpcarest.imgSlice(imageargs, self.proj, self.db)
if cb.data.shape != (1, self.tilesz,
self.tilesz) and cb.data.shape != (
1, 1, self.tilesz, self.tilesz):
if timetile is None:
tiledata = np.zeros((1, self.tilesz, self.tilesz),
cb.data.dtype)
tiledata[0, 0:((yend - 1) % self.tilesz + 1),
0:((xend - 1) % self.tilesz + 1)] = cb.data[0, :, :]
else:
tiledata = np.zeros((1, 1, self.tilesz, self.tilesz),
cb.data.dtype)
tiledata[0, 0, 0:((yend - 1) % self.tilesz + 1),
0:((xend - 1) % self.tilesz + 1)] = cb.data[0,
0, :, :]
cb.data = tiledata
return cb.xyImage()
def cacheMissXZ(self, res, xtile, ytile, ztile, timetile=None):
"""On a miss. Cutout, return the image and load the cache in a background thread"""
# make sure that the tile size is aligned with the cubedim
if self.tilesz % self.proj.datasetcfg.cubedim[res][
1] != 0 or self.tilesz % self.proj.datasetcfg.cubedim[res][2]:
raise ("Illegal tile size. Not aligned")
# figure out the cutout (limit to max image size)
xstart = xtile * self.tilesz
xend = min((xtile + 1) * self.tilesz,
self.proj.datasetcfg.imagesz[res][0])
# z cutouts need to get rescaled
# we'll map to the closest pixel range and tolerate one pixel error at the boundary
# Scalefactor = zvoxel / yvoxel
scalefactor = self.proj.datasetcfg.voxelres[res][
2] / self.proj.datasetcfg.voxelres[res][1]
zoffset = self.proj.datasetcfg.offset[res][2]
ztilestart = int((ztile * self.tilesz) / scalefactor) + zoffset
zstart = max(ztilestart, zoffset)
ztileend = int(math.ceil(
(ztile + 1) * self.tilesz / scalefactor)) + zoffset
zend = min(ztileend, self.proj.datasetcfg.imagesz[res][2] + 1)
# get an xz image slice
if timetile is None:
imageargs = '{}/{}/{}/{},{}/{}/{},{}/'.format(
self.channel, 'xz', res, xstart, xend, ytile, zstart, zend)
else:
imageargs = '{}/{}/{}/{},{}/{}/{},{}/{}/'.format(
self.channel, 'xz', res, xstart, xend, ytile, zstart, zend,
timetile)
cb = ocpcarest.imgSlice(imageargs, self.proj, self.db)
# scale by the appropriate amount
if cb.data.shape != (ztileend - ztilestart, 1,
self.tilesz) and cb.data.shape != (
1, ztileend - ztilestart, 1, self.tilesz):
if timetile is None:
tiledata = np.zeros((ztileend - ztilestart, 1, self.tilesz),
cb.data.dtype)
tiledata[0:zend - zstart, 0,
0:((xend - 1) % self.tilesz + 1)] = cb.data[:, 0, :]
else:
tiledata = np.zeros((1, ztileend - ztilestart, 1, self.tilesz),
cb.data.dtype)
tiledata[0, 0:zend - zstart, 0,
0:((xend - 1) % self.tilesz + 1)] = cb.data[0, :,
0, :]
cb.data = tiledata
return cb.xzImage(scalefactor)
def cacheMissYZ(self, res, xtile, ytile, ztile, timetile=None):
""" On a miss. Cutout, return the image and load the cache in a background thread """
# make sure that the tile size is aligned with the cubedim
if self.tilesz % self.proj.datasetcfg.cubedim[res][
1] != 0 or self.tilesz % self.proj.datasetcfg.cubedim[res][2]:
raise ("Illegal tile size. Not aligned")
# figure out the cutout (limit to max image size)
ystart = ytile * self.tilesz
yend = min((ytile + 1) * self.tilesz,
self.proj.datasetcfg.imagesz[res][1])
# z cutouts need to get rescaled
# we'll map to the closest pixel range and tolerate one pixel error at the boundary
# Scalefactor = zvoxel / xvoxel
scalefactor = self.proj.datasetcfg.voxelres[res][
2] / self.proj.datasetcfg.voxelres[res][0]
zoffset = self.proj.datasetcfg.offset[res][2]
ztilestart = int((ztile * self.tilesz) / scalefactor) + zoffset
zstart = max(ztilestart, zoffset)
ztileend = int(math.ceil(
(ztile + 1) * self.tilesz / scalefactor)) + zoffset
zend = min(ztileend, self.proj.datasetcfg.imagesz[res][2] + 1)
# get an yz image slice
if timetile is None:
imageargs = '{}/{}/{}/{}/{},{}/{},{}/'.format(
self.channel, 'yz', res, xtile, ystart, yend, zstart, zend)
else:
imageargs = '{}/{}/{}/{}/{},{}/{},{}/{}/'.format(
self.channel, 'yz', res, xtile, ystart, yend, zstart, zend,
timetile)
cb = ocpcarest.imgSlice(imageargs, self.proj, self.db)
# scale by the appropriate amount
if cb.data.shape != (ztileend - ztilestart, self.tilesz,
1) and cb.data.shape != (1, ztileend - ztilestart,
self.tilesz, 1):
if timetile is None:
tiledata = np.zeros((ztileend - ztilestart, self.tilesz, 1),
cb.data.dtype)
tiledata[0:zend - zstart, 0:((yend - 1) % self.tilesz + 1),
0] = cb.data[:, :, 0]
else:
tiledata = np.zeros((1, ztileend - ztilestart, self.tilesz, 1),
cb.data.dtype)
tiledata[0, 0:zend - zstart, 0:((yend - 1) % self.tilesz + 1),
0] = cb.data[0, :, :, 0]
cb.data = tiledata
return cb.yzImage(scalefactor)
def getTile(self, webargs):
"""Fetch the file from mocpcache or get a cutout from the database"""
try:
# argument of format token/channel/slice_type/z/y_x_res.png
p = re.compile(
"(\w+)/([\w+,]*?)/(xy|yz|xz|)/(\d+/)?(\d+)/(\d+)_(\d+)_(\d+).png"
)
m = p.match(webargs)
[self.token, self.channel,
slice_type] = [i for i in m.groups()[:3]]
[timetile, ztile, ytile, xtile, res] = [
int(i.strip('/')) if i is not None else None
for i in m.groups()[3:]
]
except Exception, e:
logger.warning(
"Incorrect arguments give for getTile {}. {}".format(
webargs, e))
raise OCPCAError(
"Incorrect arguments given for getTile {}. {}".format(
webargs, e))
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
self.proj = projdb.loadToken(self.token)
with closing(ocpcadb.OCPCADB(self.proj)) as self.db:
# mocpcache key
mckey = self.buildKey(res,
slice_type,
xtile,
ytile,
ztile,
timetile=timetile)
# if tile is in mocpcache, return it
tile = self.mc.get(mckey)
if tile == None:
if slice_type == 'xy':
img = self.cacheMissXY(res,
xtile,
ytile,
ztile,
timetile=timetile)
elif slice_type == 'xz':
img = self.cacheMissXZ(res,
xtile,
ytile,
ztile,
timetile=timetile)
elif slice_type == 'yz':
img = self.cacheMissYZ(res,
xtile,
ytile,
ztile,
timetile=timetile)
else:
logger.warning(
"Requested illegal image plance {}. Should be xy, xz, yz."
.format(slice_type))
raise OCPCAError(
"Requested illegal image plance {}. Should be xy, xz, yz."
.format(slice_type))
fobj = cStringIO.StringIO()
img.save(fobj, "PNG")
self.mc.set(mckey, fobj.getvalue())
else:
fobj = cStringIO.StringIO(tile)
fobj.seek(0)
return fobj