def ingest(token, resolution):
""" Read the stack and ingest """
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadProject(token)
with closing(ocpcadb.OCPCADB(proj)) as db:
(xcubedim, ycubedim,
zcubedim) = cubedims = proj.datasetcfg.cubedim[resolution]
zidx = 0
cube = imagecube.ImageCube16(cubedims)
cube.zeros()
cube.data = np.array(range(xcubedim * ycubedim * zcubedim),
dtype=np.uint8).reshape(cubedims)
db.putCube(zidx, resolution, cube)
db.conn.commit()
c = db.getCube(zidx, resolution)
print c.data
cube2 = imagecube.ImageCube16(cubedims)
cube2.data = np.zeros(cubedims, dtype=np.uint8)
db.putCube(zidx, resolution, cube2, True)
db.conn.commit()
c = db.getCube(zidx, resolution)
import pdb
pdb.set_trace()
print c.data
def upload(self, channel, sl, imarray):
"""Transfer the array to the database"""
with closing(ocpcadb.OCPCADB(self.proj)) as self.db:
for y in range(0, self._yimgsz + 1, self.ycubedim):
for x in range(0, self._ximgsz + 1, self.xcubedim):
# zindex
key = ocplib.XYZMorton([
x / self.xcubedim, y / self.ycubedim,
(sl - self.startslice) / self.zcubedim
])
# Create a channel cube
cube = imagecube.ImageCube16(self.cubedims)
xmin = x
ymin = y
xmax = min(self._ximgsz, x + self.xcubedim)
ymax = min(self._yimgsz, y + self.ycubedim)
zmin = 0
zmax = min(sl + self.zcubedim, self.endslice + 1)
# data for this key
cube.data[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = imarray[zmin:zmax, ymin:ymax,
xmin:xmax]
print cube.data.shape
#import pdb;pdb.set_trace()
self.db.putChannelCube(key, channel, self.resolution, cube)
print " Commiting at x={}, y={}, z={}".format(x, y, sl)
self.db.conn.commit()
def ingest ( self ):
"""Read the stack and ingest"""
with closing ( ocpcaproj.OCPCAProjectsDB() ) as projdb:
proj = projdb.loadProject ( self.token )
with closing ( ocpcadb.OCPCADB (proj) ) as db:
(startslice, endslice) = proj.datasetcfg.slicerange
(xcubedim, ycubedim, zcubedim) = cubedims = proj.datasetcfg.cubedim[self.resolution]
(ximagesz, yimagesz) = proj.datasetcfg.imagesz[self.resolution]
batchsz = zcubedim
# Ingest in database aligned slabs in the z dimension
for sl in range( startslice, endslice, batchsz ):
slab = np.zeros ( [zcubedim, yimagesz, ximagesz], dtype=np.uint8 )
# over each slice
for b in range( batchsz ):
#if we are at the end of the space, quit
if ( sl + b <= endslice ):
filename = '{}{:0>3}____z{}.0.tif'.format(self.path, sl+b, (sl+b-1)*25)
#filename = '{}{:0>4}____z{}.0.tif'.format(self.path, sl+b, (sl+b-1)*25)
print filename
try:
img = Image.open(filename,'r')
slab [b,:,:] = np.asarray(img)
except IOError, e:
print "Failed to open file %s" % (e)
img = np.zeros((yimagesz,ximagesz), dtype=np.uint8)
slab [b,:,:] = img
for y in range ( 0, yimagesz, ycubedim ):
for x in range ( 0, ximagesz, xcubedim ):
zidx = ndlib.XYZMorton ( [ x/xcubedim, y/ycubedim, (sl-startslice)/zcubedim] )
cubedata = np.zeros ( [zcubedim, ycubedim, xcubedim], dtype=np.uint8 )
xmin = x
ymin = y
xmax = ( min(ximagesz-1, x+xcubedim-1) ) + 1
ymax = ( min(yimagesz-1, y+ycubedim-1) ) + 1
zmin = 0
zmax = min(sl+zcubedim,endslice)
cubedata[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax,ymin:ymax,xmin:xmax]
cube = imagecube.ImageCube16 ( cubedims )
cube.zeros()
cube.data = cubedata
if np.count_nonzero ( cube.data ) != 0:
print zidx, ndlib.MortonXYZ(zidx)
db.putCube ( zidx, self.resolution, cube )
print "Commiting at x=%s, y=%s, z=%s" % (x,y,sl)
db.conn.commit()
slab = None
def upload(self, channel, sl, imarray):
"""Transfer the array to the database"""
# get the size of the cube
xcubedim, ycubedim, zcubedim = self.proj.datasetcfg.cubedim[
self.resolution]
# and the limits of iteration
xlimit = (self._ximgsz - 1) / xcubedim + 1
ylimit = (self._yimgsz - 1) / ycubedim + 1
for y in range(ylimit):
for x in range(xlimit):
# each batch is the last slice in a cube
z = sl / zcubedim
# zindex
key = zindex.XYZMorton([x, y, z])
# Create a channel cube
cube = imagecube.ImageCube16([xcubedim, ycubedim, zcubedim])
# data for this key
cube.data = imarray[:, y * ycubedim:(y + 1) * ycubedim,
x * xcubedim:(x + 1) * xcubedim]
# compress the cube
npz = cube.toNPZ()
# add the cube to the database
sql = "INSERT INTO " + self.proj.getTable(
self.resolution
) + "(channel, zindex, cube) VALUES (%s, %s, %s)"
try:
self.cursor.execute(sql, (channel, key, npz))
except MySQLdb.Error, e:
print("Error updating data cube: %d: %s. sql=%s" %
(e.args[0], e.args[1], sql))
raise
def ingest(self):
"""Read the stack and ingest"""
with closing(ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadProject(self.token)
with closing(ocpcadb.OCPCADB(proj)) as db:
(startslice, endslice) = proj.datasetcfg.slicerange
(xcubedim, ycubedim,
zcubedim) = cubedims = proj.datasetcfg.cubedim[self.resolution]
(ximagesz, yimagesz) = proj.datasetcfg.imagesz[self.resolution]
batchsz = zcubedim
numxtiles = ximagesz / self.tilesz
numytiles = yimagesz / self.tilesz
# Ingest in database aligned slabs in the z dimension
for sl in range(startslice, endslice, batchsz):
# over all tiles in that slice
for ytile in range(0, numytiles):
for xtile in range(0, numxtiles):
slab = np.zeros([zcubedim, self.tilesz, self.tilesz],
dtype=np.uint8)
# over each slice
for b in range(batchsz):
#if we are at the end of the space, quit
if (sl + b <= endslice):
filename = '{}z{:0>4}/c{:0>2}r{:0>2}.tif'.format(
self.tilepath, sl + b, xtile + 1,
ytile + 1)
#filename = '{}{}/c{:0>3}r{:0>3}.jpg'.format(self.tilepath, sl+b, xtile, ytile )
#filename = '{}{}/{}_{}_{}.jpg'.format(self.tilepath, sl+b, ytile, xtile, self.resolution )
#filename = '{}{}/{}/{}_{}.jpg'.format(self.tilepath, sl+b, self.resolution, ytile, xtile )
#filename = '{}z{:0>4}/c{:0>2}r{:0>2}.tif'.format(self.tilepath, sl+b, ytile+1, xtile+1 )
print filename
try:
# add tile to stack
img = Image.open(filename, 'r')
slab[b, :, :] = np.asarray(img)[:, :, 0]
except IOError, e:
print "Failed to open file %s" % (e)
img = np.zeros((self.tilesz, self.tilesz),
dtype=np.uint8)
slab[b, :, :] = img
for y in range(ytile * self.tilesz,
(ytile + 1) * self.tilesz, ycubedim):
for x in range(xtile * self.tilesz,
(xtile + 1) * self.tilesz,
xcubedim):
zidx = ndlib.XYZMorton([
x / xcubedim, y / ycubedim,
(sl - startslice) / zcubedim
])
cubedata = np.zeros(
[zcubedim, ycubedim, xcubedim],
dtype=np.uint8)
xmin = x % self.tilesz
ymin = y % self.tilesz
xmax = (min(ximagesz - 1, x + xcubedim - 1) %
self.tilesz) + 1
ymax = (min(yimagesz - 1, y + ycubedim - 1) %
self.tilesz) + 1
zmin = 0
zmax = min(sl + zcubedim, endslice)
cubedata[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax,
ymin:ymax,
xmin:xmax]
cube = imagecube.ImageCube16(cubedims)
cube.data = cubedata
if np.count_nonzero(cube.data) != 0:
db.putCube(zidx, self.resolution, cube)
print "Commiting at x=%s, y=%s, z=%s" % (x, y, sl)
db.conn.commit()
def main():
parser = argparse.ArgumentParser(description='Build a transform DB for Kwame.')
parser.add_argument('outtoken', action="store", help='Token for the Output project.')
parser.add_argument('path', action="store", help='Path to data')
parser.add_argument('resolution', action="store", type=int)
result = parser.parse_args()
with closing ( ocpcaproj.OCPCAProjectsDB() ) as outprojdb:
outproj = outprojdb.loadProject (result.outtoken)
with closing ( ocpcadb.OCPCADB(outproj) ) as outDB:
# Get the source database sizes
(ximagesz, yimagesz) = outproj.datasetcfg.imagesz [ result.resolution ]
(xcubedim, ycubedim, zcubedim) = cubedims = outproj.datasetcfg.cubedim [ result.resolution ]
(startslice, endslice) = outproj.datasetcfg.slicerange
batchsz = zcubedim
# Get the slices
slices = endslice - startslice + 1
# Set the limits for iteration on the number of cubes in each dimension and the limits of iteration
xlimit = (ximagesz-1) / xcubedim + 1
ylimit = (yimagesz-1) / ycubedim + 1
# Round up the zlimit to the next larger
zlimit = (((slices-1)/zcubedim+1)*zcubedim)/zcubedim
zscale = int(outproj.datasetcfg.zscale[result.resolution])
channel = "Grayscale"
outDB.putChannel(channel,1)
for sl in range( startslice, endslice, batchsz ):
slab = np.zeros ( (batchsz,yimagesz,ximagesz), dtype=np.uint16 )
for b in range (batchsz):
if ( sl + b <= endslice ):
filename = '{}00-164_00-152_{:0>6}.tif'.format(result.path,(sl+b)*80)
#filename = '{}00-111_000-29_{:0>6}.tif'.format(result.path,(sl+b)*50)
#filename = '{}00-199_000000_{:0>6}.tif'.format(result.path,(sl+b)*60)
#filename = '{}00-462_000000_{:0>6}.tif'.format(result.path,(sl+b)*50)
#filename = '{}00-427_000000_{:0>6}.tif'.format(result.path,(sl+b)*60)
#filename = '{}00-222_000000_{:0>6}.tif'.format(result.path,(sl+b)*50)
#filename = '{}00-415_000000_{:0>6}.tif'.format(result.path,(sl+b)*50)
#filename = '{}00-117_000000_{:0>6}.tif'.format(result.path,(sl+b)*50)
#filename = '{}00-298_000000_{:0>6}.tif'.format(result.path,(sl+b)*50)
#filename = '{}00-398_000000_{:0>6}.tif'.format(result.path,(sl+b)*60)
#filename = '{}00-532_000000_{:0>6}.tif'.format(result.path,(sl+b)*60)
#filename = '{}00-199_000000_{:0>6}.tif'.format(result.path,(sl+b)*50)
#filename = '{}00-544_000-53_{:0>6}.tif'.format(result.path,(sl+b)*50)
#imageurl = 'Grayscale/{}/{},{}/{},{}/{}/'.format(result.resolution,0,ximagesz,0,yimagesz,sl+b)
print "slice {}".format(sl+b)
try:
#imgdata = ocpcarest.cutout( imageurl, outproj, outDB )
imgdata = cv2.imread(filename,-1)
if imgdata != None:
img = Image.frombuffer( 'I;16', (imgdata.shape[::-1]), imgdata.flatten(), 'raw', 'I;16', 0, 1)
slab[b,:,:] = np.asarray(img.resize( [ximagesz,yimagesz]))
img = None
else:
slab[b,:,:] = np.zeros((yimagesz,ximagesz),dtype=np.uint16)
except IOError, e:
print "Failed to get Cutout. {}".format(e)
for y in range ( 0, yimagesz+1, ycubedim ):
for x in range ( 0, ximagesz+1, xcubedim ):
zidx = ocplib.XYZMorton ( [x/xcubedim,y/ycubedim,(sl-startslice)/zcubedim] )
cubedata = np.zeros ( (zcubedim,ycubedim,xcubedim), dtype=np.uint16 )
xmin = x
ymin = y
xmax = ((min(ximagesz-1,x+xcubedim-1)))+1
ymax = ((min(yimagesz-1,y+ycubedim-1)))+1
zmin = 0
zmax = min(sl+zcubedim,endslice+1)
cubedata[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax]
cube = imagecube.ImageCube16 ( cubedims )
cube.zeros()
cube.data = cubedata
if np.count_nonzero ( cube.data) != 0:
outDB.putChannelCube ( zidx, 1, result.resolution, cube )
print "Commiting at x:{},y:{},z{}".format(x,y,sl)
outDB.conn.commit()
def main():
parser = argparse.ArgumentParser(description='Ingest a tiff stack.')
parser.add_argument('token', action="store")
parser.add_argument('channel', type=int, action="store")
parser.add_argument('path', action="store")
parser.add_argument('numslices', type=int, action="store")
result = parser.parse_args()
projdb = emcaproj.EMCAProjectsDB()
proj = projdb.getProj(result.token)
dbcfg = dbconfig.switchDataset(proj.getDataset())
_ximgsz = None
_yimgsz = None
for sl in range(result.numslices):
filenm = result.path + '/' + '{:0>4}'.format(sl) + '.png'
print filenm
img = Image.open(filenm, "r")
if _ximgsz == None and _yimgsz == None:
_ximgsz, _yimgsz = img.size
imarray = np.zeros([result.numslices, _yimgsz, _ximgsz],
dtype=np.uint16)
else:
assert _ximgsz == img.size[0] and _yimgsz == img.size[1]
imarray[sl, :, :] = np.asarray(img)
# get the size of the cube
xcubedim, ycubedim, zcubedim = dbcfg.cubedim[0]
# and the limits of iteration
xlimit = (_ximgsz - 1) / xcubedim + 1
ylimit = (_yimgsz - 1) / ycubedim + 1
zlimit = (result.numslices - 1) / zcubedim + 1
# open the database
db = emcadb.EMCADB(dbcfg, proj)
# get a db cursor
cursor = db.conn.cursor()
for z in range(zlimit):
db.commit()
for y in range(ylimit):
for x in range(xlimit):
zmin = z * zcubedim
zmax = min((z + 1) * zcubedim, result.numslices)
zmaxrel = ((zmax - 1) % zcubedim) + 1
ymin = y * ycubedim
ymax = min((y + 1) * ycubedim, _yimgsz)
ymaxrel = ((ymax - 1) % ycubedim) + 1
xmin = x * xcubedim
xmax = min((x + 1) * xcubedim, _ximgsz)
xmaxrel = ((xmax - 1) % xcubedim) + 1
# morton key
key = zindex.XYZMorton([x, y, z])
# Create a channel cube
cube = imagecube.ImageCube16([xcubedim, ycubedim, zcubedim])
# data for this key
cube.data[0:zmaxrel, 0:ymaxrel,
0:xmaxrel] = imarray[zmin:zmax, ymin:ymax, xmin:xmax]
# compress the cube
npz = cube.toNPZ()
# add the cube to the database
sql = "INSERT INTO " + proj.getTable(
RESOLUTION) + "(channel, zindex, cube) VALUES (%s, %s, %s)"
print sql
try:
cursor.execute(sql, (result.channel, key, npz))
except MySQLdb.Error, e:
raise ANNError("Error updating data cube: %d: %s. sql=%s" %
(e.args[0], e.args[1], sql))