def ingest ( self ):
"""Read the stack and ingest"""
with closing (ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(self.token)
with closing (ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(self.channel)
# get the dataset configuration
[[ximagesz, yimagesz, zimagesz],(starttime,endtime)] = proj.datasetcfg.imageSize(self.resolution)
[xcubedim, ycubedim, zcubedim] = cubedim = proj.datasetcfg.getCubeDims()[self.resolution]
[xoffset, yoffset, zoffset] = proj.datasetcfg.getOffset()[self.resolution]
# for all specified resolutions
for resolution in range(0,1,1):
# extract parameters for iteration
numxtiles = ximagesz/self.tilesz[0]
numytiles = yimagesz/self.tilesz[1]
# Ingest in database aligned slabs in the z dimension
for slice_number in range(0, zimagesz, zcubedim):
slab = np.zeros ( [zcubedim,yimagesz,ximagesz], dtype=np.uint32 )
# over all tiles in that slice
for b in range(zcubedim):
for ytile in range(numytiles):
for xtile in range(numxtiles):
# if we are at the end of the space, quit
if slice_number+b <= zimagesz:
try:
filename = '{}{}/{}/{}/{}.png'.format(self.tilepath, resolution, slice_number+b+zoffset, ytile+17, xtile+16)
print "Opening filename {}".format(filename)
# add tile to stack
imgdata = np.asarray ( Image.open(filename, 'r').convert('RGBA') )
imgdata = np.left_shift(imgdata[:,:,3], 24, dtype=np.uint32) | np.left_shift(imgdata[:,:,2], 16, dtype=np.uint32) | np.left_shift(imgdata[:,:,1], 8, dtype=np.uint32) | np.uint32(imgdata[:,:,0])
slab [b,ytile*self.tilesz[1]:(ytile+1)*self.tilesz[1],xtile*self.tilesz[0]:(xtile+1)*self.tilesz[0]] = imgdata
except IOError, e:
print "Failed to open file {}".format(filename)
slab [b,ytile*self.tilesz[1]:(ytile+1)*self.tilesz[1],xtile*self.tilesz[0]:(xtile+1)*self.tilesz[0]] = np.zeros([self.tilesz[1], self.tilesz[0]], dtype=np.uint32)
for y in range (0, yimagesz+1, ycubedim):
for x in range (0, ximagesz+1, xcubedim):
# getting the cube id and ingesting the data one cube at a time
zidx = ocplib.XYZMorton ([x/xcubedim, y/ycubedim, (slice_number)/zcubedim])
cube = Cube.getCube(cubedim, ch.getChannelType(), ch.getDataType())
cube.zeros()
xmin, ymin = x, y
xmax = min (ximagesz, x+xcubedim)
ymax = min (yimagesz, y+ycubedim)
zmin = 0
zmax = min(slice_number+zcubedim, zimagesz+1)
cube.data[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax]
if cube.isNotZeros():
db.putCube(ch, zidx, self.resolution, cube, update=True)
def upload(self, channel, sl, imarray):
"""Transfer the array to the database"""
with closing(ocpcadb.OCPCADB(self.proj)) as self.db:
for y in range(0, self._yimgsz + 1, self.ycubedim):
for x in range(0, self._ximgsz + 1, self.xcubedim):
# zindex
key = ocplib.XYZMorton([
x / self.xcubedim, y / self.ycubedim,
(sl - self.startslice) / self.zcubedim
])
# Create a channel cube
cube = imagecube.ImageCube16(self.cubedims)
xmin = x
ymin = y
xmax = min(self._ximgsz, x + self.xcubedim)
ymax = min(self._yimgsz, y + self.ycubedim)
zmin = 0
zmax = min(sl + self.zcubedim, self.endslice + 1)
# data for this key
cube.data[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = imarray[zmin:zmax, ymin:ymax,
xmin:xmax]
print cube.data.shape
#import pdb;pdb.set_trace()
self.db.putChannelCube(key, channel, self.resolution, cube)
print " Commiting at x={}, y={}, z={}".format(x, y, sl)
self.db.conn.commit()
def main():
parser = argparse.ArgumentParser(description='Ingest the TIFF data')
parser.add_argument('token', action="store", type=str, help='Token for the project')
parser.add_argument('channel', action="store", type=str, help='Channel for the project')
parser.add_argument('path', action="store", type=str, help='Directory with the image files')
parser.add_argument('resolution', action="store", type=int, help='Resolution of data')
parser.add_argument('--offset', action="store", type=int, default=0, help='Offset on disk')
result = parser.parse_args()
# Load a database
with closing (ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(result.token)
with closing (ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(result.channel)
# get the dataset configuration
[[ximagesz, yimagesz, zimagesz],(starttime,endtime)] = proj.datasetcfg.imageSize(result.resolution)
[xcubedim,ycubedim,zcubedim] = cubedim = proj.datasetcfg.getCubeDims()[result.resolution]
[xoffset, yoffset, zoffset] = proj.datasetcfg.getOffset()[result.resolution]
# Get a list of the files in the directories
for slice_number in range (zoffset, zimagesz+1, zcubedim):
slab = np.zeros([zcubedim, yimagesz, ximagesz ], dtype=np.uint8)
for b in range(zcubedim):
if (slice_number + b <= zimagesz):
try:
# reading the raw data
file_name = "{}{:0>5}.tif".format(result.path, (slice_number + b))
# silvestri15
#file_name = "{}full_{:0>6}.tif".format(result.path, slice_number + b + result.offset)
print "Open filename {}".format(file_name)
slab[b,:,:] = np.asarray(Image.open(file_name, 'r'))
except IOError, e:
print e
slab[b,:,:] = np.zeros((yimagesz, ximagesz), dtype=np.uint8)
for y in range ( 0, yimagesz+1, ycubedim ):
for x in range ( 0, ximagesz+1, xcubedim ):
# Getting a Cube id and ingesting the data one cube at a time
zidx = ocplib.XYZMorton ( [x/xcubedim, y/ycubedim, (slice_number-zoffset)/zcubedim] )
cube = Cube.getCube(cubedim, ch.getChannelType(), ch.getDataType())
cube.zeros()
xmin,ymin = x,y
xmax = min ( ximagesz, x+xcubedim )
ymax = min ( yimagesz, y+ycubedim )
zmin = 0
zmax = min(slice_number+zcubedim, zimagesz+1)
cube.data[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax]
if cube.isNotZeros():
db.putCube(ch, zidx, result.resolution, cube, update=True)
slab = None
def ingest(self):
""" Read image stack and ingest """
# Load a database
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(self.token)
with closing(ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(self.channel_name)
# get the dataset configuration
[[ximagesz, yimagesz, zimagesz], (starttime, endtime)] = proj.datasetcfg.imageSize(self.resolution)
[xcubedim, ycubedim, zcubedim] = cubedim = proj.datasetcfg.getCubeDims()[self.resolution]
[xoffset, yoffset, zoffset] = proj.datasetcfg.getOffset()[self.resolution]
# Get a list of the files in the directories
for slice_number in range(zoffset, zimagesz, zcubedim):
slab = np.zeros([zcubedim, yimagesz, ximagesz], dtype=np.uint32)
for b in range(zcubedim):
if (slice_number + b <= zimagesz):
file_name = "{}{}{:0>4}.tif".format(self.path, self.token, slice_number+b)
print "Open filename {}".format(file_name)
try:
img = Image.open(file_name,'r')
slab [b,:,:] = np.asarray(img)
except IOError, e:
print "Failed to open file %s" % (e)
img = np.zeros((yimagesz,ximagesz), dtype=np.uint8)
slab [b,:,:] = img
for y in range(0, yimagesz + 1, ycubedim):
for x in range(0, ximagesz + 1, xcubedim):
# Getting a Cube id and ingesting the data one cube at a time
zidx = ocplib.XYZMorton([x / xcubedim, y / ycubedim, (slice_number - zoffset) / zcubedim])
cube = Cube.getCube(cubedim, ch.getChannelType(), ch.getDataType())
cube.zeros()
xmin = x
ymin = y
xmax = min(ximagesz, x + xcubedim)
ymax = min(yimagesz, y + ycubedim)
zmin = 0
zmax = min(slice_number + zcubedim, zimagesz + 1)
cube.data[0:zmax - zmin, 0:ymax - ymin, 0:xmax - xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax]
from operator import sub
corner = map(sub, [x,y,slice_number], [xoffset,yoffset,zoffset])
if cube.data.any():
db.annotateDense ( ch, corner, self.resolution, cube.data, 'O' )
def buildStack(token, channel, res):
"""Build the hierarchy of images"""
with closing (ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(token)
with closing (ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(channel)
high_res = proj.datasetcfg.scalinglevels
for cur_res in range(res, high_res+1):
# Get the source database sizes
[[ximagesz, yimagesz, zimagesz], timerange] = proj.datasetcfg.imageSize(cur_res)
[xcubedim, ycubedim, zcubedim] = cubedim = proj.datasetcfg.getCubeDims()[cur_res]
[xoffset, yoffset, zoffset] = proj.datasetcfg.getOffset()[cur_res]
biggercubedim = [xcubedim*2,ycubedim*2,zcubedim]
# Set the limits for iteration on the number of cubes in each dimension
xlimit = (ximagesz-1) / xcubedim + 1
ylimit = (yimagesz-1) / ycubedim + 1
zlimit = (zimagesz-1) / zcubedim + 1
for z in range(zlimit):
for y in range(ylimit):
for x in range(xlimit):
# cutout the data at the -1 resolution
olddata = db.cutout(ch, [ x*2*xcubedim, y*2*ycubedim, z*zcubedim], biggercubedim, cur_res-1 ).data
# target array for the new data (z,y,x) order
newdata = np.zeros([zcubedim,ycubedim,xcubedim], dtype=np.uint16)
for sl in range(zcubedim):
# Convert each slice to an image
slimage = Image.frombuffer ( 'I;16', (xcubedim*2,ycubedim*2), olddata[sl,:,:].flatten(), 'raw', 'I;16', 0, 1 )
# Resize the image
newimage = slimage.resize ( [xcubedim,ycubedim] )
# Put to a new cube
newdata[sl,:,:] = np.asarray ( newimage )
zidx = ocplib.XYZMorton ( [x,y,z] )
cube = Cube.getCube(cubedim, ch.getChannelType(), ch.getDataType())
cube.zeros()
cube.data = newdata
print "Inserting Cube {} at res {}".format(zidx, cur_res)
db.putCube(ch, zidx, cur_res, cube, update=True)
def ingest(self, channel_name):
""" Read image stack and ingest """
# Load a database
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(self.token)
with closing(ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(channel_name)
# get the dataset configuration
[[ximagesz, yimagesz, zimagesz],
(starttime, endtime)] = proj.datasetcfg.imageSize(self.resolution)
[xcubedim, ycubedim, zcubedim
] = cubedim = proj.datasetcfg.getCubeDims()[self.resolution]
[xoffset, yoffset,
zoffset] = proj.datasetcfg.getOffset()[self.resolution]
# Get a list of the files in the directories
file_name = "{}{}.tif".format(self.path, channel_name)
print "Open filename {}".format(file_name)
imgdata = tifffile.imread(file_name)
for slice_number in range(zoffset, zimagesz + 1, zcubedim):
slab = np.zeros([zcubedim, yimagesz, ximagesz],
dtype=np.uint32)
for b in range(zcubedim):
if (slice_number + b <= zimagesz):
if (slice_number + b) < zimagesz:
slab[b, :, :] = imgdata[(slice_number + b), :, :]
else:
imgdata = np.zeros((yimagesz, ximagesz),
dtype=np.uint32)
slab[b, :, :] = imgdata
for y in range(0, yimagesz + 1, ycubedim):
for x in range(0, ximagesz + 1, xcubedim):
# Getting a Cube id and ingesting the data one cube at a time
zidx = ocplib.XYZMorton([
x / xcubedim, y / ycubedim,
(slice_number - zoffset) / zcubedim
])
cube = Cube.getCube(cubedim, ch.getChannelType(),
ch.getDataType())
cube.zeros()
xmin = x
ymin = y
xmax = min(ximagesz, x + xcubedim)
ymax = min(yimagesz, y + ycubedim)
zmin = 0
zmax = min(slice_number + zcubedim, zimagesz + 1)
cube.data[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax, ymin:ymax,
xmin:xmax]
db.putCube(ch,
zidx,
self.resolution,
cube,
update=True)
def ingestImageStack(self):
"""Ingest a TIF image stack"""
# Load a database
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(self.token)
with closing(ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(self.channel)
# get the dataset configuration
[[ximagesz, yimagesz, zimagesz],
(starttime, endtime)] = proj.datasetcfg.imageSize(self.resolution)
[xcubedim, ycubedim, zcubedim
] = cubedim = proj.datasetcfg.getCubeDims()[self.resolution]
[xoffset, yoffset,
zoffset] = proj.datasetcfg.getOffset()[self.resolution]
if ch.getChannelType() in TIMESERIES_CHANNELS and (
starttime == 0 and endtime == 0):
print "Timeseries Data cannot have timerange (0,0)"
raise
# Get a list of the files in the directories
for timestamp in range(starttime, endtime + 1):
for slice_number in range(zoffset, zimagesz + 1, zcubedim):
slab = np.zeros([zcubedim, yimagesz, ximagesz],
dtype=OCP_dtypetonp.get(ch.getDataType()))
# fetch 16 slices at a time
if ch.getChannelType() in TIMESERIES_CHANNELS:
time_value = timestamp
else:
time_value = None
self.fetchData(
range(slice_number, slice_number +
zcubedim) if slice_number + zcubedim <= zimagesz
else range(slice_number, zimagesz),
time_value=time_value)
for b in range(zcubedim):
if (slice_number + b <= zimagesz):
try:
# reading the raw data
file_name = "{}{}".format(
self.path,
self.generateFileName(slice_number + b))
print "Open filename {}".format(file_name)
slab[b, :, :] = np.asarray(
Image.open(file_name, 'r'))
except IOError, e:
print e
slab[b, :, :] = np.zeros((yimagesz, ximagesz),
dtype=np.uint32)
for y in range(0, yimagesz + 1, ycubedim):
for x in range(0, ximagesz + 1, xcubedim):
# Getting a Cube id and ingesting the data one cube at a time
zidx = ocplib.XYZMorton([
x / xcubedim, y / ycubedim,
(slice_number - zoffset) / zcubedim
])
cube = Cube.getCube(cubedim, ch.getChannelType(),
ch.getDataType())
cube.zeros()
xmin, ymin = x, y
xmax = min(ximagesz, x + xcubedim)
ymax = min(yimagesz, y + ycubedim)
zmin = 0
zmax = min(slice_number + zcubedim, zimagesz + 1)
cube.data[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax,
ymin:ymax,
xmin:xmax]
if cube.isNotZeros():
if ch.getChannelType(
) not in TIMESERIES_CHANNELS:
db.putCube(ch,
zidx,
self.resolution,
cube,
update=True)
else:
db.putTimeCube(ch,
zidx,
timestamp,
self.resolution,
cube,
update=True)
# clean up the slices fetched
self.cleanData(
range(slice_number, slice_number +
zcubedim) if slice_number + zcubedim <= zimagesz
else range(slice_number, zimagesz))
def buildStack(token, channel, res, base_res):
""" build a zoom hierarchy of images """
scaling = 2**(base_res - res)
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(token)
with closing(ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(channel)
# get db sizes
[[ximagesz, yimagesz, zimagesz],
timerange] = proj.datasetcfg.imageSize(base_res)
[xcubedim, ycubedim,
zcubedim] = cubedim = proj.datasetcfg.getCubeDims()[base_res]
[xoffset, yoffset, zoffset] = proj.datasetcfg.getOffset()[base_res]
newcubedim = proj.datasetcfg.getCubeDims()[res]
xlimit = (ximagesz - 1) / xcubedim + 1
ylimit = (yimagesz - 1) / ycubedim + 1
zlimit = (zimagesz - 1) / zcubedim + 1
# iterate over the old cube
for z in range(zlimit):
for y in range(ylimit):
for x in range(xlimit):
# cutout data
old_data = db.cutout(
ch, [x * xcubedim, y * ycubedim, z * zcubedim],
cubedim, base_res).data
#new_data = zoomIn(old_data, scaling)
new_data = cZoomIn(old_data, base_res - res)
newzsize = new_data.shape[0] / newcubedim[
2] #old_data.shape[0]
newysize = new_data.shape[1] / newcubedim[
1] #old_data.shape[1]
newxsize = new_data.shape[2] / newcubedim[
0] #old_data.shape[2]
#print "sizes: {} {} {}".format(newxsize, newysize, newzsize)
for z2 in range(newzsize):
for y2 in range(newysize):
for x2 in range(newxsize):
#print "{} {} {}".format(x*newxsize+x2,y*newysize+y2,z*newzsize+z2)
zidx = ocplib.XYZMorton([
x * newxsize + x2, y * newysize + y2,
z * newzsize + z2
])
cube = Cube.getCube(newcubedim,
ch.getChannelType(),
ch.getDataType())
cube.zeros()
cube.data = new_data[z2 *
newcubedim[2]:(z2 + 1) *
newcubedim[2], y2 *
newcubedim[1]:(y2 + 1) *
newcubedim[1], x2 *
newcubedim[0]:(x2 + 1) *
newcubedim[0]]
#print "Grabbing cube from [{}:{} , {}:{}, {}:{}]".format(z2*newcubedim[2],(z2+1)*newcubedim[2], y2*newcubedim[1],(y2+1)*newcubedim[1], x2*newcubedim[0],(x2+1)*newcubedim[0])
print "Inserting Cube {} at res {}".format(
zidx, res)
db.putCube(ch, zidx, res, cube, update=True)
def main():
parser = argparse.ArgumentParser(description='Build a transform DB for Kwame.')
parser.add_argument('outtoken', action="store", help='Token for the Output project.')
parser.add_argument('path', action="store", help='Path to data')
parser.add_argument('resolution', action="store", type=int)
result = parser.parse_args()
with closing ( ocpcaproj.OCPCAProjectsDB() ) as outprojdb:
outproj = outprojdb.loadProject (result.outtoken)
with closing ( ocpcadb.OCPCADB(outproj) ) as outDB:
# Get the source database sizes
(ximagesz, yimagesz) = outproj.datasetcfg.imagesz [ result.resolution ]
(xcubedim, ycubedim, zcubedim) = cubedims = outproj.datasetcfg.cubedim [ result.resolution ]
(startslice, endslice) = outproj.datasetcfg.slicerange
batchsz = zcubedim
# Get the slices
slices = endslice - startslice + 1
# Set the limits for iteration on the number of cubes in each dimension and the limits of iteration
xlimit = (ximagesz-1) / xcubedim + 1
ylimit = (yimagesz-1) / ycubedim + 1
# Round up the zlimit to the next larger
zlimit = (((slices-1)/zcubedim+1)*zcubedim)/zcubedim
zscale = int(outproj.datasetcfg.zscale[result.resolution])
channel = "Grayscale"
outDB.putChannel(channel,1)
for sl in range( startslice, endslice, batchsz ):
slab = np.zeros ( (batchsz,yimagesz,ximagesz), dtype=np.uint16 )
for b in range (batchsz):
if ( sl + b <= endslice ):
filename = '{}00-164_00-152_{:0>6}.tif'.format(result.path,(sl+b)*80)
#filename = '{}00-111_000-29_{:0>6}.tif'.format(result.path,(sl+b)*50)
#filename = '{}00-199_000000_{:0>6}.tif'.format(result.path,(sl+b)*60)
#filename = '{}00-462_000000_{:0>6}.tif'.format(result.path,(sl+b)*50)
#filename = '{}00-427_000000_{:0>6}.tif'.format(result.path,(sl+b)*60)
#filename = '{}00-222_000000_{:0>6}.tif'.format(result.path,(sl+b)*50)
#filename = '{}00-415_000000_{:0>6}.tif'.format(result.path,(sl+b)*50)
#filename = '{}00-117_000000_{:0>6}.tif'.format(result.path,(sl+b)*50)
#filename = '{}00-298_000000_{:0>6}.tif'.format(result.path,(sl+b)*50)
#filename = '{}00-398_000000_{:0>6}.tif'.format(result.path,(sl+b)*60)
#filename = '{}00-532_000000_{:0>6}.tif'.format(result.path,(sl+b)*60)
#filename = '{}00-199_000000_{:0>6}.tif'.format(result.path,(sl+b)*50)
#filename = '{}00-544_000-53_{:0>6}.tif'.format(result.path,(sl+b)*50)
#imageurl = 'Grayscale/{}/{},{}/{},{}/{}/'.format(result.resolution,0,ximagesz,0,yimagesz,sl+b)
print "slice {}".format(sl+b)
try:
#imgdata = ocpcarest.cutout( imageurl, outproj, outDB )
imgdata = cv2.imread(filename,-1)
if imgdata != None:
img = Image.frombuffer( 'I;16', (imgdata.shape[::-1]), imgdata.flatten(), 'raw', 'I;16', 0, 1)
slab[b,:,:] = np.asarray(img.resize( [ximagesz,yimagesz]))
img = None
else:
slab[b,:,:] = np.zeros((yimagesz,ximagesz),dtype=np.uint16)
except IOError, e:
print "Failed to get Cutout. {}".format(e)
for y in range ( 0, yimagesz+1, ycubedim ):
for x in range ( 0, ximagesz+1, xcubedim ):
zidx = ocplib.XYZMorton ( [x/xcubedim,y/ycubedim,(sl-startslice)/zcubedim] )
cubedata = np.zeros ( (zcubedim,ycubedim,xcubedim), dtype=np.uint16 )
xmin = x
ymin = y
xmax = ((min(ximagesz-1,x+xcubedim-1)))+1
ymax = ((min(yimagesz-1,y+ycubedim-1)))+1
zmin = 0
zmax = min(sl+zcubedim,endslice+1)
cubedata[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax]
cube = imagecube.ImageCube16 ( cubedims )
cube.zeros()
cube.data = cubedata
if np.count_nonzero ( cube.data) != 0:
outDB.putChannelCube ( zidx, 1, result.resolution, cube )
print "Commiting at x:{},y:{},z{}".format(x,y,sl)
outDB.conn.commit()
def main():
parser = argparse.ArgumentParser(description='Ingest the TIFF data')
parser.add_argument('token',
action="store",
type=str,
help='Token for the project')
parser.add_argument('channel',
action="store",
type=str,
help='Channel for the project')
parser.add_argument('path',
action="store",
type=str,
help='Directory with the image files')
parser.add_argument('resolution',
action="store",
type=int,
help='Resolution of data')
result = parser.parse_args()
# Load a database
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(result.token)
with closing(ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(result.channel)
# get the dataset configuration
[[ximagesz, yimagesz, zimagesz],
(starttime, endtime)] = proj.datasetcfg.imageSize(result.resolution)
[xcubedim, ycubedim,
zcubedim] = cubedim = proj.datasetcfg.getCubeDims()[result.resolution]
[xoffset, yoffset,
zoffset] = proj.datasetcfg.getOffset()[result.resolution]
file_name = "{}".format(result.path)
tif_file = tifffile.imread(file_name)
slice_number = 0
# Get a list of the files in the directories
for iteration_number in range(starttime, endtime):
slab = np.zeros([zcubedim, yimagesz, ximagesz], dtype=np.uint16)
import pdb
pdb.set_trace()
slab[slice_number, :, :] = tif_file[iteration_number, :, :]
for y in range(0, yimagesz + 1, ycubedim):
for x in range(0, ximagesz + 1, xcubedim):
# Getting a Cube id and ingesting the data one cube at a time
zidx = ocplib.XYZMorton([
x / xcubedim, y / ycubedim,
(slice_number - zoffset) / zcubedim
])
cube = Cube.getCube(cubedim,
ch.getChannelType(),
ch.getDataType(),
timerange=[0, 1])
cube.zeros()
xmin = x
ymin = y
xmax = min(ximagesz, x + xcubedim)
ymax = min(yimagesz, y + ycubedim)
zmin = 0
zmax = min(slice_number + zcubedim, zimagesz + 1)
cube.data[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax, ymin:ymax,
xmin:xmax]
db.putTimeCube(ch,
zidx,
iteration_number,
result.resolution,
cube,
update=False)
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Testing the float and uint64 issue in python"""
import sys
import os
sys.path += [os.path.abspath('../django')]
import OCP.settings
os.environ['DJANGO_SETTINGS_MODULE'] = 'OCP.settings'
from django.conf import settings
import zindex
import ocplib
print zindex.MortonXYZ(831488)
dims = (10234, 9703, 11009)
print "DIMENSIONS:", dims
key = ocplib.XYZMorton(dims)
print "KEY:", key
print "WHAT WE GET BACK:", ocplib.MortonXYZ(key)
print "DIMENSIONS:", dims
key = zindex.XYZMorton(dims)
print "KEY:", key
print "WHAT WE GET BACK:", zindex.MortonXYZ(key)
def ingest(self):
"""Read the stack and ingest"""
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadProject(self.token)
with closing(ocpcadb.OCPCADB(proj)) as db:
(startslice, endslice) = proj.datasetcfg.slicerange
(xcubedim, ycubedim,
zcubedim) = cubedims = proj.datasetcfg.cubedim[self.resolution]
(ximagesz, yimagesz) = proj.datasetcfg.imagesz[self.resolution]
batchsz = zcubedim
# Ingest in database aligned slabs in the z dimension
for sl in range(startslice, endslice, batchsz):
slab = np.zeros([zcubedim, yimagesz, ximagesz], dtype=np.uint8)
# over each slice
for b in range(batchsz):
#if we are at the end of the space, quit
if (sl + b <= endslice):
filename = '{}dyer15_3_maskimg_{:0>4}.tif'.format(
self.path, sl + b)
#filename = '{}proj_2_{:0>5}.tif'.format(self.path, sl+b)
#filename = '{}{}_maskimg_{:0>4}.tif'.format(self.path, self.token, sl+b)
#filename = '{}xbrain_dyer15_slice{:0>4}.tif'.format(self.path, sl+b )
print filename
try:
img = Image.open(filename, 'r')
slab[b, :, :] = np.asarray(img)
except IOError, e:
print "Failed to open file %s" % (e)
img = np.zeros((yimagesz, ximagesz),
dtype=np.uint8)
slab[b, :, :] = img
for y in range(0, yimagesz, ycubedim):
for x in range(0, ximagesz, xcubedim):
zidx = ocplib.XYZMorton([
x / xcubedim, y / ycubedim,
(sl - startslice) / zcubedim
])
cubedata = np.zeros([zcubedim, ycubedim, xcubedim],
dtype=np.uint8)
xmin = x
ymin = y
xmax = (min(ximagesz - 1, x + xcubedim - 1)) + 1
ymax = (min(yimagesz - 1, y + ycubedim - 1)) + 1
zmin = 0
zmax = min(sl + zcubedim, endslice)
cubedata[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax, ymin:ymax,
xmin:xmax]
cube = imagecube.ImageCube16(cubedims)
cube.zeros()
cube.data = cubedata
if np.count_nonzero(cube.data) != 0:
print zidx, ocplib.MortonXYZ(zidx)
db.putCube(zidx, self.resolution, cube)
print "Commiting at x=%s, y=%s, z=%s" % (x, y, sl)
db.conn.commit()
def buildImageStack(proj, ch, res=None):
"""Build the hierarchy of images"""
with closing(ocpcadb.OCPCADB(proj)) as db:
# pick a resolution
if res is None:
res = 1
high_res = proj.datasetcfg.scalinglevels
scaling = proj.datasetcfg.scalingoption
for cur_res in range(res, high_res + 1):
# Get the source database sizes
[[ximagesz, yimagesz, zimagesz],
timerange] = proj.datasetcfg.imageSize(cur_res)
[xcubedim, ycubedim,
zcubedim] = cubedim = proj.datasetcfg.getCubeDims()[cur_res]
if scaling == ZSLICES:
(xscale, yscale, zscale) = (2, 2, 1)
elif scaling == ISOTROPIC:
(xscale, yscale, zscale) = (2, 2, 2)
else:
logger.error(
"Invalid scaling option in project = {}".format(scaling))
raise OCPCAError(
"Invalid scaling option in project = {}".format(scaling))
biggercubedim = [
xcubedim * xscale, ycubedim * yscale, zcubedim * zscale
]
# Set the limits for iteration on the number of cubes in each dimension
xlimit = (ximagesz - 1) / xcubedim + 1
ylimit = (yimagesz - 1) / ycubedim + 1
zlimit = (zimagesz - 1) / zcubedim + 1
# Iterating over time
for ts in range(timerange[0], timerange[1] + 1, 1):
# Iterating over zslice
for z in range(zlimit):
# Iterating over y
for y in range(ylimit):
# Iterating over x
for x in range(xlimit):
# cutout the data at the resolution
if ch.getChannelType() not in TIMESERIES_CHANNELS:
olddata = db.cutout(ch, [
x * xscale * xcubedim, y * yscale *
ycubedim, z * zscale * zcubedim
], biggercubedim, cur_res - 1).data
else:
olddata = db.timecutout(
ch, [
x * xscale * xcubedim, y * yscale *
ycubedim, z * zscale * zcubedim
], biggercubedim, cur_res - 1,
[ts, ts + 1]).data
olddata = olddata[0, :, :, :]
#olddata target array for the new data (z,y,x) order
newdata = np.zeros([zcubedim, ycubedim, xcubedim],
dtype=OCP_dtypetonp.get(
ch.getDataType()))
for sl in range(zcubedim):
if scaling == ZSLICES:
data = olddata[sl, :, :]
elif scaling == ISOTROPIC:
data = ocplib.isotropicBuild_ctype(
olddata[sl * 2, :, :],
olddata[sl * 2 + 1, :, :])
# Convert each slice to an image
# 8-bit int option
if olddata.dtype == np.uint8:
slimage = Image.frombuffer(
'L', (xcubedim * 2, ycubedim * 2),
data.flatten(), 'raw', 'L', 0, 1)
# 16-bit int option
elif olddata.dtype == np.uint16:
slimage = Image.frombuffer(
'I;16', (xcubedim * 2, ycubedim * 2),
data.flatten(), 'raw', 'I;16', 0, 1)
# 32-bit float option
elif olddata.dtype == np.float32:
slimage = Image.frombuffer(
'F', (xcubedim * 2, ycubedim * 2),
data.flatten(), 'raw', 'F', 0, 1)
# 32 bit RGBA data
elif olddata.dtype == np.uint32:
slimage = Image.fromarray(data, "RGBA")
# KL TODO Add support for 32bit and 64bit RGBA data
# Resize the image and put in the new cube array
if olddata.dtype != np.uint32:
newdata[sl, :, :] = np.asarray(
slimage.resize([xcubedim, ycubedim]))
else:
tempdata = np.asarray(
slimage.resize([xcubedim, ycubedim]))
newdata[sl, :, :] = np.left_shift(
tempdata[:, :, 3], 24,
dtype=np.uint32) | np.left_shift(
tempdata[:, :, 2],
16,
dtype=np.uint32) | np.left_shift(
tempdata[:, :, 1],
8,
dtype=np.uint32) | np.uint32(
tempdata[:, :, 0])
zidx = ocplib.XYZMorton([x, y, z])
cube = Cube.getCube(cubedim, ch.getChannelType(),
ch.getDataType())
cube.zeros()
cube.data = newdata
if ch.getChannelType() not in TIMESERIES_CHANNELS:
db.putCube(ch,
zidx,
cur_res,
cube,
update=True)
else:
db.putTimeCube(ch,
zidx,
ts,
cur_res,
cube,
update=False)
def buildAnnoStack(proj, ch, res=None):
"""Build the hierarchy for annotations"""
with closing(ocpcadb.OCPCADB(proj)) as db:
# pick a resolution
if res is None:
res = 1
high_res = proj.datasetcfg.scalinglevels
scaling = proj.datasetcfg.scalingoption
for cur_res in range(res, high_res + 1):
# Get the source database sizes
[[ximagesz, yimagesz, zimagesz],
timerange] = proj.datasetcfg.imageSize(cur_res - 1)
[xcubedim, ycubedim,
zcubedim] = cubedim = proj.datasetcfg.getCubeDims()[cur_res - 1]
# Set the limits for iteration on the number of cubes in each dimension
xlimit = (ximagesz - 1) / xcubedim + 1
ylimit = (yimagesz - 1) / ycubedim + 1
zlimit = (zimagesz - 1) / zcubedim + 1
# Choose constants that work for all resolutions. recall that cube size changes from 128x128x16 to 64*64*64
if scaling == ZSLICES:
outdata = np.zeros([zcubedim * 4, ycubedim * 2, xcubedim * 2],
dtype=OCP_dtypetonp.get(ch.getDataType()))
elif scaling == ISOTROPIC:
outdata = np.zeros([zcubedim * 2, ycubedim * 2, xcubedim * 2],
dtype=OCP_dtypetonp.get(ch.getDataType()))
else:
logger.error(
"Invalid scaling option in project = {}".format(scaling))
raise OCPCAError(
"Invalid scaling option in project = {}".format(scaling))
# Round up to the top of the range
lastzindex = (ocplib.XYZMorton([xlimit, ylimit, zlimit]) / 64 +
1) * 64
# Iterate over the cubes in morton order
for mortonidx in range(0, lastzindex, 64):
# call the range query
cuboids = db.getCubes(ch, range(mortonidx, mortonidx + 64),
cur_res - 1)
cube = Cube.getCube(cubedim, ch.getChannelType(),
ch.getDataType())
# get the first cube
for idx, datastring in cuboids:
xyz = ocplib.MortonXYZ(idx)
cube.fromNPZ(datastring)
if scaling == ZSLICES:
# Compute the offset in the output data cube
# we are placing 4x4x4 input blocks into a 2x2x4 cube
offset = [(xyz[0] % 4) * (xcubedim / 2),
(xyz[1] % 4) * (ycubedim / 2),
(xyz[2] % 4) * zcubedim]
# add the contribution of the cube in the hierarchy
ocplib.addDataToZSliceStack_ctype(
cube, outdata, offset)
elif scaling == ISOTROPIC:
# Compute the offset in the output data cube
# we are placing 4x4x4 input blocks into a 2x2x2 cube
offset = [(xyz[0] % 4) * (xcubedim / 2),
(xyz[1] % 4) * (ycubedim / 2),
(xyz[2] % 4) * (zcubedim / 2)]
# use python version for debugging
ocplib.addDataToIsotropicStack_ctype(
cube, outdata, offset)
# Get the base location of this batch
xyzout = ocplib.MortonXYZ(mortonidx)
# adjust to output corner for scale.
if scaling == ZSLICES:
outcorner = [
xyzout[0] * xcubedim / 2, xyzout[1] * ycubedim / 2,
xyzout[2] * zcubedim
]
elif scaling == ISOTROPIC:
outcorner = [
xyzout[0] * xcubedim / 2, xyzout[1] * ycubedim / 2,
xyzout[2] * zcubedim / 2
]
# Data stored in z,y,x order dims in x,y,z
outdim = outdata.shape[::-1]
# Preserve annotations made at the specified level
# KL check that the P option preserves annotations? RB changed from O
db.annotateDense(ch, outcorner, cur_res, outdata, 'O')
db.conn.commit()
# zero the output buffer
outdata = np.zeros([zcubedim * 4, ycubedim * 2, xcubedim * 2],
dtype=OCP_dtypetonp.get(ch.getDataType()))
def ingestCatmaidStack(self):
"""Ingest a CATMAID tile stack"""
# Load a database
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(self.token)
with closing(ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(self.channel)
# get the dataset configuration
[[ximagesz, yimagesz, zimagesz],
(starttime, endtime)] = proj.datasetcfg.imageSize(self.resolution)
[xcubedim, ycubedim, zcubedim
] = cubedim = proj.datasetcfg.getCubeDims()[self.resolution]
[xoffset, yoffset,
zoffset] = proj.datasetcfg.getOffset()[self.resolution]
if ch.getChannelType() in TIMESERIES_CHANNELS and (
starttime == 0 and endtime == 0):
pass
else:
logger.error(
"Timeseries Data cannot have timerange (0,0). Error in {}".
format(self.token))
raise OCPCAError(
"Timeseries Data cannot have timerange (0,0)".format(
self.token))
num_xtiles = ximagesz / tilesz
num_ytiles = yimagesz / tilesz
# Get a list of the files in the directories
for timestamp in range(starttime, endtime + 1):
for slice_number in range(zoffset, zimagesz + 1, zcubedim):
# over all the tiles in the slice
for ytile in range(0, num_ytiles):
for xtile in range(o, num_xtiles):
slab = np.zeros([zcubedim, tilesz, tilesz],
dtype=np.uint8)
for b in range(zcubedim):
if (slice_number + b <= zimagesz):
try:
# reading the raw data
file_name = "{}{}{:0>6}.{}".format(
self.path,
self.regex(slice_number + b),
self.file_type)
logger.info("Open filename {}".format(
file_name))
print "Open filename {}".format(
file_name)
slab[b, :, :] = np.asarray(
Image.open(file_name, 'r'))
except IOError, e:
logger.warning("IOError {}.".format(e))
slab[b, :, :] = np.zeros(
(tilesz, tilesz), dtype=np.uint32)
for y in range(ytile * tilesz,
(ytile + 1) * tilesz, ycubedim):
for x in range(xtile * tilesz,
(xtile + 1) * tilesz, xcubedim):
# Getting a Cube id and ingesting the data one cube at a time
zidx = ocplib.XYZMorton([
x / xcubedim, y / ycubedim,
(slice_number - zoffset) / zcubedim
])
cube = Cube.getCube(
cubedim, ch.getChannelType(),
ch.getDataType())
cube.zeros()
xmin = x % tilesz
ymin = y % tilesz
xmax = min(ximagesz, x + xcubedim)
ymax = min(yimagesz, y + ycubedim)
zmin = 0
zmax = min(slice_number + zcubedim,
zimagesz + 1)
cube.data[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax,
ymin:ymax,
xmin:xmax]
if cube.isNotZeros():
db.putCube(ch,
zidx,
self.resolution,
cube,
update=True)
else:
db.putTimeCube(ch,
zidx,
timestamp,
self.resolution,
cube,
update=True)
def ingestImageStack(self):
"""Ingest a TIF image stack"""
# Load a database
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(self.token)
with closing(ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(self.channel)
# get the dataset configuration
[[ximagesz, yimagesz, zimagesz],
(starttime, endtime)] = proj.datasetcfg.imageSize(self.resolution)
[xcubedim, ycubedim, zcubedim
] = cubedim = proj.datasetcfg.getCubeDims()[self.resolution]
[xoffset, yoffset,
zoffset] = proj.datasetcfg.getOffset()[self.resolution]
if ch.getChannelType() in TIMESERIES_CHANNELS and (
starttime == 0 and endtime == 0):
logger.error("Timeseries Data cannot have timerange (0,0)")
raise OCPCAError("Timeseries Data cannot have timerange (0,0)")
# Get a list of the files in the directories
for timestamp in range(starttime, endtime + 1):
for slice_number in range(zoffset, zimagesz, zcubedim):
slab = np.zeros([zcubedim, yimagesz, ximagesz],
dtype=OCP_dtypetonp.get(ch.getDataType()))
# fetch 16 slices at a time
if ch.getChannelType() in TIMESERIES_CHANNELS:
time_value = timestamp
else:
time_value = None
self.fetchData(
range(slice_number, slice_number +
zcubedim) if slice_number + zcubedim <= zimagesz
else range(slice_number, zimagesz),
time_value=time_value)
for b in range(zcubedim):
if (slice_number + b < zimagesz):
try:
# reading the raw data
file_name = "{}{}".format(
self.path,
self.generateFileName(slice_number + b))
print "Open filename {}".format(file_name)
logger.info(
"Open filename {}".format(file_name))
if ch.getDataType() in [
UINT8, UINT16
] and ch.getChannelType(
) in IMAGE_CHANNELS + TIMESERIES_CHANNELS:
image_data = np.asarray(
Image.open(file_name, 'r'))
slab[b, :, :] = image_data
elif ch.getDataType() in [
UINT32
] and ch.getChannelType(
) in IMAGE_CHANNELS + TIMESERIES_CHANNELS:
image_data = np.asarray(
Image.open(file_name,
'r').convert('RGBA'))
slab[b, :, :] = np.left_shift(
image_data[:, :, 3],
24,
dtype=np.uint32) | np.left_shift(
image_data[:, :, 2],
16,
dtype=np.uint32) | np.left_shift(
image_data[:, :, 1],
8,
dtype=np.uint32) | np.uint32(
image_data[:, :, 0])
elif ch.getChannelType(
) in ANNOTATION_CHANNELS:
image_data = np.asarray(
Image.open(file_name, 'r'))
slab[b, :, :] = image_data
else:
logger.error("Cannot ingest this data yet")
raise OCPCAError(
"Cannot ingest this data yet")
except IOError, e:
logger.warning("IOError {}.".format(e))
slab[b, :, :] = np.zeros((yimagesz, ximagesz),
dtype=np.uint32)
for y in range(0, yimagesz + 1, ycubedim):
for x in range(0, ximagesz + 1, xcubedim):
# Getting a Cube id and ingesting the data one cube at a time
zidx = ocplib.XYZMorton([
x / xcubedim, y / ycubedim,
(slice_number - zoffset) / zcubedim
])
cube = Cube.getCube(cubedim, ch.getChannelType(),
ch.getDataType())
cube.zeros()
xmin, ymin = x, y
xmax = min(ximagesz, x + xcubedim)
ymax = min(yimagesz, y + ycubedim)
zmin = 0
zmax = min(slice_number + zcubedim, zimagesz + 1)
cube.data[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax,
ymin:ymax,
xmin:xmax]
if cube.isNotZeros():
if ch.getChannelType() in IMAGE_CHANNELS:
db.putCube(ch,
zidx,
self.resolution,
cube,
update=False)
elif ch.getChannelType(
) in TIMESERIES_CHANNELS:
db.putTimeCube(ch,
zidx,
timestamp,
self.resolution,
cube,
update=False)
elif ch.getChannelType(
) in ANNOTATION_CHANNELS:
corner = map(sub, [x, y, slice_number],
[xoffset, yoffset, zoffset])
db.annotateDense(ch, corner,
self.resolution,
cube.data, 'O')
else:
logger.error(
"Channel type {} not supported".format(
ch.getChannelType()))
raise OCPCAError(
"Channel type {} not supported".format(
ch.getChannelType()))
# clean up the slices fetched
self.cleanData(
range(slice_number, slice_number +
zcubedim) if slice_number + zcubedim <= zimagesz
else range(slice_number, zimagesz))
def ingest ( self ):
"""Read the stack and ingest"""
with closing ( ocpcaproj.OCPCAProjectsDB() ) as projdb:
proj = projdb.loadProject ( self.token )
with closing ( ocpcadb.OCPCADB (proj) ) as db:
(startslice, endslice) = proj.datasetcfg.slicerange
(xcubedim, ycubedim, zcubedim) = cubedims = proj.datasetcfg.cubedim[self.resolution]
(ximagesz, yimagesz) = proj.datasetcfg.imagesz[self.resolution]
batchsz = zcubedim
numxtiles = ximagesz / self.tilesz
numytiles = yimagesz / self.tilesz
# Ingest in database aligned slabs in the z dimension
for sl in range( startslice, endslice, batchsz ):
# over all tiles in that slice
for ytile in range( 0, numytiles ):
for xtile in range( 0, numxtiles ):
slab = np.zeros ( [zcubedim, self.tilesz, self.tilesz], dtype=np.uint8 )
# over each slice
for b in range( batchsz ):
#if we are at the end of the space, quit
if ( sl + b <= endslice ):
filename = '{}z{:0>4}/c{:0>2}r{:0>2}.tif'.format(self.tilepath, sl+b, xtile+1, ytile+1 )
#filename = '{}{}/c{:0>3}r{:0>3}.jpg'.format(self.tilepath, sl+b, xtile, ytile )
#filename = '{}{}/{}_{}_{}.jpg'.format(self.tilepath, sl+b, ytile, xtile, self.resolution )
#filename = '{}{}/{}/{}_{}.jpg'.format(self.tilepath, sl+b, self.resolution, ytile, xtile )
#filename = '{}z{:0>4}/c{:0>2}r{:0>2}.tif'.format(self.tilepath, sl+b, ytile+1, xtile+1 )
print filename
try:
# add tile to stack
img = Image.open ( filename, 'r' )
slab [b,:,:] = np.asarray ( img )[:,:,0]
except IOError, e:
print "Failed to open file %s" % (e)
img = np.zeros((self.tilesz,self.tilesz), dtype=np.uint8)
slab [b,:,:] = img
for y in range ( ytile*self.tilesz, (ytile+1)*self.tilesz, ycubedim ):
for x in range ( xtile*self.tilesz, (xtile+1)*self.tilesz, xcubedim ):
zidx = ocplib.XYZMorton ( [ x/xcubedim, y/ycubedim, (sl-startslice)/zcubedim] )
cubedata = np.zeros ( [zcubedim, ycubedim, xcubedim], dtype=np.uint8 )
xmin = x % self.tilesz
ymin = y % self.tilesz
xmax = ( min(ximagesz-1, x+xcubedim-1)%self.tilesz ) + 1
ymax = ( min(yimagesz-1, y+ycubedim-1)%self.tilesz ) + 1
zmin = 0
zmax = min(sl+zcubedim,endslice)
cubedata[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax,ymin:ymax,xmin:xmax]
cube = imagecube.ImageCube16 ( cubedims )
cube.data = cubedata
if np.count_nonzero ( cube.data ) != 0:
db.putCube ( zidx, self.resolution, cube )
print "Commiting at x=%s, y=%s, z=%s" % (x,y,sl)
db.conn.commit()
