def main():
parser = argparse.ArgumentParser(
description='Read a single cube and write images')
parser.add_argument('token',
action="store",
help='Token for the annotation project.')
parser.add_argument('zindex', type=int, action="store", help='')
parser.add_argument('resolution', type=int, action="store", help='')
result = parser.parse_args()
# load a database
[db, proj, projdb] = ocpcarest.loadDBProj(result.token)
# get the dataset configuration
(xcubedim, ycubedim,
zcubedim) = cubedim = proj.datasetcfg.cubedim[result.resolution]
(startslice, endslice) = proj.datasetcfg.slicerange
import pdb
pdb.set_trace()
sql = "SELECT zindex, cube FROM res{} WHERE zindex={}".format(
result.resolution, result.zindex)
db.cursor.execute(sql)
idx, datastring = db.cursor.fetchone()
incube = imagecube.ImageCube8(cubedim)
incube.fromNPZ(datastring[:])
def loadDB(self):
"""Load the database if it hasn't been load."""
if self.db == None:
# load the database/token
[self.db, self.proj,
self.projdb] = ocpcarest.loadDBProj(self.token)
def getTile ( self, webargs ):
"""Either fetch the file from mocpcache or get a mcfc image"""
# parse the web args
self.token, tileszstr, self.channels, resstr, xtilestr, ytilestr, zslicestr, rest = webargs.split('/',8)
[ self.db, self.proj, projdb ] = ocpcarest.loadDBProj ( self.token )
# convert args to ints
xtile = int(xtilestr)
ytile = int(ytilestr)
res = int(resstr)
# modify the zslice to the offset
zslice = int(zslicestr)-self.proj.datasetcfg.slicerange[0]
self.tilesz = int(tileszstr)
# mocpcache key
mckey = self.buildKey(res,xtile,ytile,zslice)
# do something to sanitize the webargs??
# if tile is in mocpcache, return it
tile = self.mc.get(mckey)
if tile == None:
img=self.cacheMiss(res,xtile,ytile,zslice)
fobj = cStringIO.StringIO ( )
img.save ( fobj, "PNG" )
self.mc.set(mckey,fobj.getvalue())
else:
fobj = cStringIO.StringIO(tile)
fobj.seek(0)
return fobj
def main():
parser = argparse.ArgumentParser(description='Create a mergemap file.')
parser.add_argument('baseurl', action="store", help='Baseurl for OCP service')
parser.add_argument('token', action="store", help='Database token to access')
parser.add_argument('csvfile', action="store", help='File containing csv list of merges.')
# customize for kasthuri11
resolution = 1
result = parser.parse_args()
# Convert the csv file into a dictionary
f = open ( result.csvfile )
# default dictionary of integers
d = defaultdict(int)
csvr = csv.reader(f, delimiter=',')
for r in csvr:
for s in r[1:]:
d[int(s)]=int(r[0])
# load the in database
[ db, proj, projdb ] = ocpcarest.loadDBProj ( result.token )
# get the dataset configuration
(xcubedim,ycubedim,zcubedim) = proj.datasetcfg.cubedim[resolution]
(ximagesz,yimagesz) = proj.datasetcfg.imagesz[resolution]
(startslice,endslice) = proj.datasetcfg.slicerange
slices = endslice - startslice + 1
# iterate over the defined set of cubes in the database
sql = "SELECT zindex from res1;"
cursor = db.conn.cursor()
cursor.execute(sql)
zindexes = np.array([item[0] for item in cursor.fetchall()])
i=0
# iterate over the cubes in morton order
for mortonidx in zindexes:
print "Working on cube {}".format(i)
i = i+1
cuboid = db.getCube ( mortonidx, resolution )
annoids = np.unique ( cuboid.data )
for id in annoids:
if d[id]!=0:
print "Rewrite id {} to {} for cube {}".format(id,d[id],mortonidx)
vec_func = np.vectorize ( lambda x: x if x != id else d[id] )
cuboid.data = vec_func ( cuboid.data )
else:
if id != 0:
print "Id {} is one of the target identifiers".format(id)
db.putCube ( mortonidx, resolution, cuboid )
db.commit()
def main():
parser = argparse.ArgumentParser( description='Check the zscale for the dataset')
parser.add_argument('dataset', action="store", help='Dataset name for the project')
result = parser.parse_args()
[ db, proj, projdb ] = ocpcarest.loadDBProj ( result.dataset )
( xcubedim, ycubedim, zcubedim ) = proj.datasetcfg.cubedim [ 0 ]
for res in proj.datasetcfg.resolutions:
print "Resolution ", res, "Zscale ", proj.datasetcfg.zscale[res], "Dims ", proj.datasetcfg.cubedim[res]
def main():
parser = argparse.ArgumentParser(
description='Check the zscale for the dataset')
parser.add_argument('dataset',
action="store",
help='Dataset name for the project')
result = parser.parse_args()
[db, proj, projdb] = ocpcarest.loadDBProj(result.dataset)
(xcubedim, ycubedim, zcubedim) = proj.datasetcfg.cubedim[0]
for res in proj.datasetcfg.resolutions:
print "Resolution ", res, "Zscale ", proj.datasetcfg.zscale[
res], "Dims ", proj.datasetcfg.cubedim[res]
def __init__(self, token, path, resolution, channel):
self.path = path
self.resolution = resolution
[ self.db, self.proj, self.projdb ] = ocpcarest.loadDBProj ( token )
(self.xcubedim, self.ycubedim, self.zcubedim) = self.proj.datasetcfg.cubedim[ resolution ]
(self.startslice, self.endslice) = self.proj.datasetcfg.slicerange
self.batchsz = self.zcubedim
self.channel = channel
self._ximgsz = self.proj.datasetcfg.imagesz[resolution][0]
self._yimgsz = self.proj.datasetcfg.imagesz[resolution][1]
self.cursor = self.db.conn.cursor()
def __init__(self, token, path, resolution, channel):
self.path = path
self.resolution = resolution
[ self.db, self.proj, self.projdb ] = ocpcarest.loadDBProj ( token )
(self.xcubedim, self.ycubedim, self.zcubedim) = self.proj.datasetcfg.cubedim[ resolution ]
(self.startslice, self.endslice) = self.proj.datasetcfg.slicerange
self.batchsz = self.zcubedim
self.channel = channel
self._ximgsz = self.proj.datasetcfg.imagesz[resolution][0]
self._yimgsz = self.proj.datasetcfg.imagesz[resolution][1]
self.cursor = self.db.conn.cursor()
def __init__ (self, token, channels, centroid):
# arguments
self.token = token
self.channels = channels
self.centroid = centroid
# parameter defaults. set be accessors.
self.sog_width = 200
self.sog_frame = 20
self.width = 11
self.normalize = True
self.normalize2 = False
self.resolution = 0
self.refchannels = []
self.emchannels = []
self.enhance = None
[ self.db, self.proj, self.projdb ] = ocpcarest.loadDBProj ( self.token )
def main():
parser = argparse.ArgumentParser(description='Read a single cube and write images')
parser.add_argument('token', action="store", help='Token for the annotation project.')
parser.add_argument('zindex', type=int, action="store", help='')
parser.add_argument('resolution', type=int, action="store", help='')
result = parser.parse_args()
# load a database
[ db, proj, projdb ] = ocpcarest.loadDBProj ( result.token )
# get the dataset configuration
(xcubedim,ycubedim,zcubedim)=cubedim=proj.datasetcfg.cubedim[result.resolution]
(startslice,endslice)=proj.datasetcfg.slicerange
import pdb; pdb.set_trace()
sql = "SELECT zindex, cube FROM res{} WHERE zindex={}".format(result.resolution,result.zindex)
db.cursor.execute(sql)
idx, datastring = db.cursor.fetchone()
incube = imagecube.ImageCube8 ( cubedim )
incube.fromNPZ ( datastring[:] )
def main():
parser = argparse.ArgumentParser(description='Ingest the TIFF data')
parser.add_argument('token', action="store", help='Token for the project')
parser.add_argument('path', action="store", help='Directory with the image files')
result = parser.parse_args()
resolution = 0
#Load a database
[ db, proj, projdb ] = ocpcarest.loadDBProj ( result.token )
# get the dataset configuration
(xcubedim,ycubedim,zcubedim) = proj.datasetcfg.cubedim[resolution]
(startslice,endslice)=proj.datasetcfg.slicerange
batchsz = zcubedim
batchsz = 16
(ximagesz,yimagesz)=proj.datasetcfg.imagesz[resolution]
# Set the image size to that of the actual image
ximagesz = 6144
yimagesz = 6144
tilesz = 6144
filelist = glob.glob( "{}*.tif".format(result.path) )
# Get a list of the files in the directories
for sl in range (startslice, endslice+1, batchsz):
slab = np.zeros ( [ batchsz, yimagesz, ximagesz ], dtype=np.uint8 )
for b in range ( batchsz ):
if ( sl + b <= endslice ):
# raw data
#filenm = result.path + '{:0>4}_r1-c1_w01_s01_sec01_'.format(sl+b) + '{*}.tif'
print "Opening filenm" + filelist[sl+b-1]
img = Image.open (filelist[sl+b-1], 'r')
imgdata = np.asarray ( img )
slab[b,:,:] = imgdata
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
for y in range ( 0, yimagesz+1, ycubedim ):
for x in range ( 0, ximagesz+1, xcubedim ):
mortonidx = zindex.XYZMorton ( [x/xcubedim, y/ycubedim, (sl-startslice)/zcubedim] )
cubedata = np.zeros ( [zcubedim, ycubedim, xcubedim], dtype=np.uint8 )
xmin = x
ymin = y
xmax = ((min(ximagesz-1,x+xcubedim-1))%tilesz)+1
ymax = ((min(yimagesz-1,y+ycubedim-1))%tilesz)+1
#xmax = min ( ximagesz, x+xcubedim )
#ymax = min ( yimagesz, y+ycubedim )
zmin = 0
zmax = min(sl+zcubedim, endslice+1)
cubedata[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax]
# Create the DB Blob
fileobj = cStringIO.StringIO()
np.save ( fileobj, cubedata )
cdz = zlib.compress ( fileobj.getvalue() )
# insert the blob into the database
cursor = db.conn.cursor()
sql = "INSERT INTO res{} (zindex, cube) VALUES (%s, %s)".format(int(resolution))
cursor.execute(sql, (mortonidx, cdz))
cursor.close()
print " Commiting at x={}, y={}, z={}".format(x, y, sl)
db.conn.commit()
slab = None
def main():
parser = argparse.ArgumentParser(
description='Ingest the FlyEM image data.')
parser.add_argument('token',
action="store",
help='Token for the annotation project.')
parser.add_argument('path',
action="store",
help='Directory with annotation PNG files.')
result = parser.parse_args()
# convert to an argument
resolution = 0
# load a database
[db, proj, projdb] = ocpcarest.loadDBProj(result.token)
# get the dataset configuration
(xcubedim, ycubedim, zcubedim) = proj.datasetcfg.cubedim[resolution]
(startslice, endslice) = proj.datasetcfg.slicerange
batchsz = zcubedim
# This doesn't work because the image size does not match exactly the cube size
#(ximagesz,yimagesz)=proj.datasetcfg.imagesz[resolution]
ximagesz = 49152
yimagesz = 32768
# add all of the tiles to the image
for sl in range(startslice, endslice + 1, batchsz):
for ytile in range(ytiles):
for xtile in range(xtiles):
slab = np.zeros([batchsz, tilesz, tilesz], dtype=np.uint32)
for b in range(batchsz):
if (sl + b <= endslice):
# raw data
filenm = result.path + '/S1Column_Localcellbodies_97-classified_export_s{:0>3}_Y{}_X{}.png'.format(
sl + b, ytile, xtile)
print "Opening filenm " + filenm
img = Image.open(filenm, 'r')
imgdata = np.asarray(img)
slab[b, :, :] = kanno_cy.pngto32(imgdata)
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
# Now we have a 8192x8192x16 z-aligned cube.
# Send it to the database.
for y in range(ytile * tilesz, (ytile + 1) * tilesz, ycubedim):
for x in range(xtile * tilesz, (xtile + 1) * tilesz,
xcubedim):
mortonidx = zindex.XYZMorton([
x / xcubedim, y / ycubedim,
(sl - startslice) / zcubedim
])
cubedata = np.zeros([zcubedim, ycubedim, xcubedim],
dtype=np.uint32)
xmin = x % tilesz
ymin = y % tilesz
xmax = (
(min(ximagesz - 1, x + xcubedim - 1)) % tilesz) + 1
ymax = (
(min(yimagesz - 1, y + ycubedim - 1)) % tilesz) + 1
zmin = 0
zmax = min(sl + zcubedim, endslice + 1)
cubedata[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax, ymin:ymax,
xmin:xmax]
# check if there's anything to store
if (np.count_nonzero(cubedata) == 0):
continue
# create the DB BLOB
fileobj = cStringIO.StringIO()
np.save(fileobj, cubedata)
cdz = zlib.compress(fileobj.getvalue())
# insert the blob into the database
cursor = db.conn.cursor()
sql = "INSERT INTO res{} (zindex, cube) VALUES (%s, %s)".format(
int(resolution))
print mortonidx
cursor.execute(sql, (mortonidx, cdz))
cursor.close()
print "Commiting at x=%s, y=%s, z=%s" % (x, y, sl)
db.conn.commit()
def main():
parser = argparse.ArgumentParser(description='Ingest the FlyEM image data.')
parser.add_argument('token', action="store", help='Token for the annotation project.')
parser.add_argument('path', action="store", help='Directory with annotation PNG files.')
result = parser.parse_args()
# convert to an argument
resolution = 0
# load a database
[ db, proj, projdb ] = ocpcarest.loadDBProj ( result.token )
# get the dataset configuration
(xcubedim,ycubedim,zcubedim)=proj.datasetcfg.cubedim[resolution]
(startslice,endslice)=proj.datasetcfg.slicerange
batchsz=zcubedim
batchsz=1
# This doesn't work because the image size does not match exactly the cube size
#(ximagesz,yimagesz)=proj.datasetcfg.imagesz[resolution]
ximagesz = 49152
yimagesz = 32768
startslice = 100
# add all of the tiles to the image
for sl in range (startslice,endslice+1,batchsz):
for ytile in range(ytiles):
for xtile in range(xtiles):
slab = np.zeros ( [ batchsz, tilesz, tilesz ], dtype=np.uint32 )
for b in range ( batchsz ):
if ( sl + b <= endslice ):
# raw data
filenm = result.path + '/S1Column_Localcellbodies_97-classified_export_s{:0>3}_Y{}_X{}.png'.format(sl+b,ytile,xtile)
print "Opening filenm " + filenm
img = Image.open ( filenm, 'r' )
imgdata = np.asarray ( img )
slab[b,:,:] = kanno_cy.pngto32 ( imgdata )
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
# Now we have a 8192x8192x16 z-aligned cube.
# Send it to the database.
for y in range ( ytile*tilesz, (ytile+1)*tilesz, ycubedim ):
for x in range ( xtile*tilesz, (xtile+1)*tilesz, xcubedim ):
mortonidx = zindex.XYZMorton ( [ x/xcubedim, y/ycubedim, (sl-startslice)/zcubedim] )
cubedata = np.zeros ( [zcubedim, ycubedim, xcubedim], dtype=np.uint32 )
xmin = x%tilesz
ymin = y%tilesz
xmax = ((min(ximagesz-1,x+xcubedim-1))%tilesz)+1
ymax = ((min(yimagesz-1,y+ycubedim-1))%tilesz)+1
zmin = 0
zmax = min(sl+zcubedim,endslice+1)
cubedata[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax,ymin:ymax,xmin:xmax]
if y == 8064:
print x,y,xmin,xmax,ymin,ymax,zmin,zmax
# check if there's anything to store
if ( np.count_nonzero(cubedata) == 0 ):
continue
# create the DB BLOB
fileobj = cStringIO.StringIO ()
np.save ( fileobj, cubedata )
cdz = zlib.compress (fileobj.getvalue())
# insert the blob into the database
cursor = db.conn.cursor()
sql = "INSERT INTO res{} (zindex, cube) VALUES (%s, %s)".format(int(resolution))
cursor.execute(sql, (mortonidx, cdz))
cursor.close()
print "Commiting at x=%s, y=%s, z=%s" % (x,y,sl)
db.conn.commit()
# Connection info
try:
mmconn = MySQLdb.connect(host='localhost',
user=result.dbuser,
passwd=result.dbpass,
db=result.dbname)
except MySQLdb.Error, e:
print("Failed to connect to database: %s" % (result.dbname))
raise
# get a cursor of the mergemap
mmcursor = mmconn.cursor()
# load the in database
[indb, inproj, inprojdb] = ocpcarest.loadDBProj(result.intoken)
# load the out database
[outdb, outproj, outprojdb] = ocpcarest.loadDBProj(result.outtoken)
# get the dataset configuration
(xcubedim, ycubedim,
zcubedim) = inproj.datasetcfg.cubedim[result.resolution]
(ximagesz, yimagesz) = inproj.datasetcfg.imagesz[result.resolution]
(startslice, endslice) = inproj.datasetcfg.slicerange
slices = endslice - startslice + 1
# Set the limits for iteration on the number of cubes in each dimension
xlimit = (ximagesz - 1) / xcubedim + 1
ylimit = (yimagesz - 1) / ycubedim + 1
# Round up the zlimit to the next larger
def main():
parser = argparse.ArgumentParser(description='Ingest the Rohanna data.')
parser.add_argument('token',
action="store",
help='Token for the annotation project.')
parser.add_argument('path',
action="store",
help='Directory with annotation TIF files.')
parser.add_argument('resolution',
action="store",
type=int,
help='Resolution')
result = parser.parse_args()
# convert to an argument
resolution = result.resolution
# load a database
[db, proj, projdb] = ocpcarest.loadDBProj(result.token)
# get the dataset configuration
(xcubedim, ycubedim, zcubedim) = proj.datasetcfg.cubedim[resolution]
(startslice, endslice) = proj.datasetcfg.slicerange
batchsz = zcubedim
# This doesn't work because the image size does not match exactly the cube size
#(ximagesz,yimagesz)=proj.datasetcfg.imagesz[resolution]
ximagesz = 5120
yimagesz = 5120
endslice = 1123
# add all of the tiles to the image
for sl in range(startslice, endslice + 1, batchsz):
slab = np.zeros([batchsz, yimagesz, ximagesz], dtype=np.uint32)
for b in range(batchsz):
if (sl + b <= endslice):
# raw data
filenm = result.path + '/labels_{:0>5}_ocp.tif'.format(sl + b)
print "Opening filenm " + filenm
img = Image.open(filenm, 'r')
imgdata = np.asarray(img)
slab[b, :, :] = (imgdata)
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
# Now we have a 5120x5120x16 z-aligned cube.
# Send it to the database.
for y in range(0, yimagesz, ycubedim):
for x in range(0, ximagesz, xcubedim):
mortonidx = zindex.XYZMorton([
(x + xoffsetsz) / xcubedim, (y + yoffsetsz) / ycubedim,
(sl + zoffsetsz - startslice) / zcubedim
])
cubedata = np.zeros([zcubedim, ycubedim, xcubedim],
dtype=np.uint32)
test = zindex.MortonXYZ(mortonidx)
pdb.set_trace()
xmin = x
ymin = y
xmax = min(ximagesz, x + xcubedim)
ymax = min(yimagesz, y + ycubedim)
zmin = 0
zmax = min(sl + zcubedim, endslice + 1)
cubedata[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax]
# check if there's anything to store
if (np.count_nonzero(cubedata) == 0):
continue
# create the DB BLOB
fileobj = cStringIO.StringIO()
np.save(fileobj, cubedata)
cdz = zlib.compress(fileobj.getvalue())
# insert the blob into the database
cursor = db.conn.cursor()
sql = "INSERT INTO res{} (zindex, cube) VALUES (%s, %s)".format(
int(resolution))
# cursor.execute(sql, (mortonidx, cdz))
cursor.close()
print "Commiting at x=%s, y=%s, z=%s" % (
x + xoffsetsz, y + yoffsetsz, sl + b + zoffsetsz)
db.conn.commit()
def main():
parser = argparse.ArgumentParser(description='Ingest the JP@ data')
parser.add_argument('baseurl', action="store", help='Base URL to of ocp service no http://, e.g. openconnecto.me')
parser.add_argument('token', action="store", help='Token for the annotation project.')
parser.add_argument('path', action="store", help='Directory with annotation TIF files.')
parser.add_argument('resolution', action="store", type=int, help="Resolution for the project")
result = parser.parse_args()
#Load a database
[ db, proj, projdb ] = ocpcarest.loadDBProj ( result.token )
# get the dataset configuration
(xcubedim,ycubedim,zcubedim) = proj.datasetcfg.cubedim[result.resolution]
(startslice,endslice)=proj.datasetcfg.slicerange
batchsz = 1
(ximagesz,yimagesz)=proj.datasetcfg.imagesz[result.resolution]
yimagesz = 18000
ximagesz = 24000
ytilesz = 3600
xtilesz = 4800
startslice = 174
# Get a list of the files in the directories
for sl in range (startslice,endslice+1,batchsz):
for b in range ( batchsz ):
slab = np.zeros ( [ batchsz, ytilesz, xtilesz ], dtype=np.uint64 )
if ( sl + b <= endslice ):
# raw data
try:
filenm = result.path + '{:0>4}'.format(sl+b) + '.tiff'
print "Opening filenm" + filenm
imgdata = cv2.imread( filenm, -1 )
if imgdata != None:
newimgdata = np.left_shift(65535, 48, dtype=np.uint64) | np.left_shift(imgdata[:,:, 0], 32, dtype=np.uint64) | np.left_shift(imgdata[:,:,1], 16, dtype=np.uint64) | np.uint64(imgdata[:,:,2])
else:
newimgdata = np.zeros( [ yimagesz, ximagesz ], dtype=np.uint64 )
except IOError, e:
print e
newimgdata = np.zeros( [ yimagesz, ximagesz ], dtype=np.uint64 )
newytilesz = 0
newxtilesz = 0
for tile in range(0,25):
if tile%5==0 and tile!=0:
newytilesz = newytilesz + ytilesz
newxtilesz = 0
elif tile!=0:
# Updating the value
newxtilesz = newxtilesz + xtilesz
if newimgdata == None:
print "Skipping Slice {} as it does not exist".format(sl+b)
continue
else:
slab[b,:,:] = newimgdata[newytilesz:(tile/5+1)*ytilesz,newxtilesz:(tile%5+1)*xtilesz]
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
# Now we have a 3600x4800 tile to the server
# Construct the URL
url = 'http://{}/ocp/ca/{}/npz/{}/{},{}/{},{}/{},{}/'.format(result.baseurl, result.token,result.resolution, newxtilesz, (tile%5+1)*xtilesz, newytilesz, (tile/5+1)*ytilesz, sl+zoffset, sl+batchsz)
print url
# Encode the voxelist an pickle
fileobj = cStringIO.StringIO ()
np.save ( fileobj, slab )
cdz = zlib.compress (fileobj.getvalue())
print " Sending URL"
# Build the post request
try:
req = urllib2.Request(url = url, data = cdz)
response = urllib2.urlopen(req)
the_page = response.read()
except Exception, e:
print "Failed ", e
def main():
parser = argparse.ArgumentParser(description='Ingest the JP@ data')
parser.add_argument(
'baseurl',
action="store",
help='Base URL to of ocp service no http://, e.g. openconnecto.me')
parser.add_argument('token',
action="store",
help='Token for the annotation project.')
parser.add_argument('path',
action="store",
help='Directory with annotation TIF files.')
parser.add_argument('resolution',
action="store",
type=int,
help="Resolution for the project")
result = parser.parse_args()
#Load a database
[db, proj, projdb] = ocpcarest.loadDBProj(result.token)
# get the dataset configuration
(xcubedim, ycubedim, zcubedim) = proj.datasetcfg.cubedim[result.resolution]
(startslice, endslice) = proj.datasetcfg.slicerange
batchsz = 1
(ximagesz, yimagesz) = proj.datasetcfg.imagesz[result.resolution]
yimagesz = 18000
ximagesz = 24000
ytilesz = 3600
xtilesz = 4800
startslice = 174
# Get a list of the files in the directories
for sl in range(startslice, endslice + 1, batchsz):
for b in range(batchsz):
slab = np.zeros([batchsz, ytilesz, xtilesz], dtype=np.uint64)
if (sl + b <= endslice):
# raw data
try:
filenm = result.path + '{:0>4}'.format(sl + b) + '.tiff'
print "Opening filenm" + filenm
imgdata = cv2.imread(filenm, -1)
if imgdata != None:
newimgdata = np.left_shift(
65535, 48, dtype=np.uint64) | np.left_shift(
imgdata[:, :, 0], 32,
dtype=np.uint64) | np.left_shift(
imgdata[:, :, 1], 16,
dtype=np.uint64) | np.uint64(imgdata[:, :,
2])
else:
newimgdata = np.zeros([yimagesz, ximagesz],
dtype=np.uint64)
except IOError, e:
print e
newimgdata = np.zeros([yimagesz, ximagesz],
dtype=np.uint64)
newytilesz = 0
newxtilesz = 0
for tile in range(0, 25):
if tile % 5 == 0 and tile != 0:
newytilesz = newytilesz + ytilesz
newxtilesz = 0
elif tile != 0:
# Updating the value
newxtilesz = newxtilesz + xtilesz
if newimgdata == None:
print "Skipping Slice {} as it does not exist".format(
sl + b)
continue
else:
slab[b, :, :] = newimgdata[newytilesz:(tile / 5 + 1) *
ytilesz,
newxtilesz:(tile % 5 + 1) *
xtilesz]
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
# Now we have a 3600x4800 tile to the server
# Construct the URL
url = 'http://{}/ocp/ca/{}/npz/{}/{},{}/{},{}/{},{}/'.format(
result.baseurl, result.token, result.resolution,
newxtilesz, (tile % 5 + 1) * xtilesz, newytilesz,
(tile / 5 + 1) * ytilesz, sl + zoffset, sl + batchsz)
print url
# Encode the voxelist an pickle
fileobj = cStringIO.StringIO()
np.save(fileobj, slab)
cdz = zlib.compress(fileobj.getvalue())
print " Sending URL"
# Build the post request
try:
req = urllib2.Request(url=url, data=cdz)
response = urllib2.urlopen(req)
the_page = response.read()
except Exception, e:
print "Failed ", e
def main():
parser = argparse.ArgumentParser(description='Ingest the TIFF data')
parser.add_argument('token', action="store", help='Token for the project')
parser.add_argument('path', action="store", help='Directory with the image files')
parser.add_argument('resolution', action="store", type=int, help='Resolution of data')
result = parser.parse_args()
#Load a database
[ db, proj, projdb ] = ocpcarest.loadDBProj ( result.token )
# get the dataset configuration
(xcubedim,ycubedim,zcubedim) = proj.datasetcfg.cubedim[result.resolution]
(startslice,endslice)=proj.datasetcfg.slicerange
batchsz = zcubedim
batchsz = 16
(ximagesz,yimagesz)=proj.datasetcfg.imagesz[result.resolution]
yimagesz = 18000
ximagesz = 24000
# Get a list of the files in the directories
for sl in range ( startslice, endslice+1, batchsz):
slab = np.zeros ( [ batchsz, yimagesz, ximagesz ], dtype=np.uint32 )
for b in range ( batchsz ):
if ( sl + b <= endslice ):
# raw data
try:
filenm = result.path + '{:0>4}'.format(sl+b) + '.tiff'
print "Opening filenm" + filenm
img = Image.open (filenm, 'r').convert("RGBA")
imgdata = np.asarray ( img )
slab[b,:,:] = np.left_shift(imgdata[:,:,3], 24, dtype=np.uint32) | np.left_shift(imgdata[:,:,2], 16, dtype=np.uint32) | np.left_shift(imgdata[:,:,1], 8, dtype=np.uint32) | np.uint32(imgdata[:,:,0])
except IOError, e:
print e
imgdata = np.zeros((yimagesz, ximagesz), dtype=np.uint32)
slab[b,:,:] = imgdata
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
for y in range ( 0, yimagesz+1, ycubedim ):
for x in range ( 0, ximagesz+1, xcubedim ):
# Getting a Cube id and ingesting the data one cube at a time
mortonidx = zindex.XYZMorton ( [x/xcubedim, y/ycubedim, (sl-startslice)/zcubedim] )
cubedata = np.zeros ( [zcubedim, ycubedim, xcubedim], dtype=np.uint32 )
xmin = x
ymin = y
xmax = min ( ximagesz, x+xcubedim )
ymax = min ( yimagesz, y+ycubedim )
zmin = 0
zmax = min(sl+zcubedim, endslice+1)
cubedata[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax]
# Create the DB Blob
fileobj = cStringIO.StringIO()
np.save ( fileobj, cubedata )
cdz = zlib.compress ( fileobj.getvalue() )
# insert the blob into the database
cursor = db.conn.cursor()
sql = "INSERT INTO res{} (zindex, cube) VALUES (%s, %s)".format(int(result.resolution))
cursor.execute(sql, (mortonidx, cdz))
cursor.close()
print " Commiting at x={}, y={}, z={}".format(x, y, sl)
db.conn.commit()
# Freeing memory
slab = None
def main():
parser = argparse.ArgumentParser(description='Ingest the Rohanna data.')
parser.add_argument('token', action="store", help='Token for the annotation project.')
parser.add_argument('path', action="store", help='Directory with annotation TIF files.')
parser.add_argument('resolution', action="store", type=int, help='Resolution')
result = parser.parse_args()
# convert to an argument
resolution = result.resolution
# load a database
[ db, proj, projdb ] = ocpcarest.loadDBProj ( result.token )
# get the dataset configuration
(xcubedim,ycubedim,zcubedim)=proj.datasetcfg.cubedim[resolution]
(startslice,endslice)=proj.datasetcfg.slicerange
batchsz=zcubedim
# This doesn't work because the image size does not match exactly the cube size
#(ximagesz,yimagesz)=proj.datasetcfg.imagesz[resolution]
ximagesz = 10748
yimagesz = 12896
# add all of the tiles to the image
for sl in range (993,endslice+1,batchsz):
slab = np.zeros ( [ batchsz, yimagesz, ximagesz ], dtype=np.uint32 )
for b in range ( batchsz ):
if ( sl + b <= endslice ):
# raw data
filenm = result.path + 'RCvesiclescleaned_s{:0>4}.png'.format(sl+b-1)
print "Opening filenm " + filenm
try:
img = Image.open ( filenm, 'r' )
imgdata = np.asarray ( img )
imgdata = np.left_shift(imgdata[:,:,0], 16, dtype=np.uint32) | np.left_shift(imgdata[:,:,1], 16, dtype=np.uint32) | np.uint32(imgdata[:,:,2])
except IOError, e:
print e
imgdata = np.zeros((yimagesz,ximagesz), dtype=np.uint32)
slab[b,:,:] = ( imgdata )
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
# Now we have a 5120x5120x16 z-aligned cube.
# Send it to the database.
for y in range ( 0, yimagesz, ycubedim ):
for x in range ( 0, ximagesz, xcubedim ):
mortonidx = zindex.XYZMorton ( [ (x)/xcubedim, (y)/ycubedim, (sl-startslice)/zcubedim] )
cubedata = np.zeros ( [zcubedim, ycubedim, xcubedim], dtype=np.uint32 )
test = zindex.MortonXYZ (mortonidx )
xmin = x
ymin = y
xmax = min ( ximagesz, x+xcubedim )
ymax = min ( yimagesz, y+ycubedim )
zmin = 0
zmax = min(sl+zcubedim,endslice+1)
cubedata[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax,ymin:ymax,xmin:xmax]
# check if there's anything to store
if ( np.count_nonzero(cubedata) == 0 ):
continue
# create the DB BLOB
fileobj = cStringIO.StringIO ()
np.save ( fileobj, cubedata )
cdz = zlib.compress (fileobj.getvalue())
# insert the blob into the database
cursor = db.conn.cursor()
sql = "INSERT INTO res{} (zindex, cube) VALUES (%s, %s)".format(int(resolution))
cursor.execute(sql, (mortonidx, cdz))
cursor.close()
print "Commiting at x=%s, y=%s, z=%s" % (x,y,sl+b)
db.conn.commit()
def main():
parser = argparse.ArgumentParser(
description='Ingest the FlyEM image data.')
parser.add_argument(
'baseurl',
action="store",
help='Base URL to of ocp service no http://, e. g. neurodata.io')
parser.add_argument('token',
action="store",
help='Token for the annotation project.')
parser.add_argument('path',
action="store",
help='Directory with annotation PNG files.')
result = parser.parse_args()
# convert to an argument
resolution = 0
# load a database
[db, proj, projdb] = ocpcarest.loadDBProj(result.token)
# get the dataset configuration
(xcubedim, ycubedim, zcubedim) = proj.datasetcfg.cubedim[resolution]
(realstartslice, realendslice) = proj.datasetcfg.slicerange
batchsz = zcubedim
# This doesn't work because the image size does not match exactly the cube size
#(ximagesz,yimagesz)=proj.datasetcfg.imagesz[resolution]
ximagesz = 12000
yimagesz = 12000
# Adding the startslice and endslice manually
startslice = 979
endslice = 1000
batchsz = 1
# Accessing the dict in dbm
anydb = anydbm.open('bodydict', 'r')
# Get a list of the files in the directories
for sl in range(startslice, endslice + 1, batchsz):
slab = np.zeros([batchsz, yimagesz, ximagesz], dtype=np.uint32)
for b in range(batchsz):
if (sl + b <= endslice):
# Reading the raw data
filenm = result.path + '/superpixel.' + '{:0>5}'.format(
sl + b) + '.png'
print "Opening filenm " + filenm
img = Image.open(filenm, 'r')
imgdata = np.asarray(img)
superpixelarray = imgdata[:, :, 0] + (
np.uint32(imgdata[:, :, 1]) << 8)
uniqueidlist = np.unique(superpixelarray)
smalldict = {}
# Creating a small dict of all the ids in the given slice
for i in uniqueidlist:
smalldict[i] = int(anydb.get(str(sl) + ',' + str(i)))
# Looking the in the small dictionary for the body ids
for i in range(superpixelarray.shape[0]):
for j in range(superpixelarray.shape[1]):
superpixelarray[i,
j] = smalldict.get(superpixelarray[i,
j])
slab[b, :, :] = superpixelarray
print "end on slice:", sl
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
# Now we have a 1024x1024x16 z-aligned cube.
# Send it to the database.
for y in range(0, yimagesz, ycubedim):
for x in range(0, ximagesz, xcubedim):
mortonidx = zindex.XYZMorton(
[x / xcubedim, y / ycubedim, (sl - startslice) / zcubedim])
cubedata = np.zeros([1, ycubedim, xcubedim], dtype=np.uint32)
#cubedata = np.zeros ( [zcubedim, ycubedim, xcubedim], dtype=np.uint32 )
xmin = x
ymin = y
xmax = min(ximagesz, x + xcubedim)
ymax = min(yimagesz, y + ycubedim)
zmin = 0
zmax = min(sl + zcubedim, endslice + 1)
cubedata[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax]
# insert the blob into the database
db.annotateDense((x, y, sl - realstartslice), resolution,
cubedata, 'O')
print "Commiting at x=%s, y=%s, z=%s" % (x, y,
sl - realstartslice)
db.conn.commit()
# Connection info
try:
mmconn = MySQLdb.connect (host = 'localhost',
user = result.dbuser,
passwd = result.dbpass,
db = result.dbname )
except MySQLdb.Error, e:
print("Failed to connect to database: %s" % (result.dbname))
raise
# get a cursor of the mergemap
mmcursor = mmconn.cursor()
# load the in database
[ indb, inproj, inprojdb ] = ocpcarest.loadDBProj ( result.intoken )
# load the out database
[ outdb, outproj, outprojdb ] = ocpcarest.loadDBProj ( result.outtoken )
# get the dataset configuration
(xcubedim,ycubedim,zcubedim) = inproj.datasetcfg.cubedim[result.resolution]
(ximagesz,yimagesz) = inproj.datasetcfg.imagesz[result.resolution]
(startslice,endslice) = inproj.datasetcfg.slicerange
slices = endslice - startslice + 1
# Set the limits for iteration on the number of cubes in each dimension
xlimit = (ximagesz-1)/xcubedim+1
ylimit = (yimagesz-1)/ycubedim+1
# Round up the zlimit to the next larger
zlimit = (((slices-1)/zcubedim+1)*zcubedim)/zcubedim
def main():
parser = argparse.ArgumentParser(
description='Ingest the FlyEM image data.')
parser.add_argument('token',
action="store",
help='Token for the annotation project.')
parser.add_argument('path',
action="store",
help='Directory with annotation PNG files.')
result = parser.parse_args()
# convert to an argument
resolution = 0
# load a database
[db, proj, projdb] = ocpcarest.loadDBProj(result.token)
# get the dataset configuration
(xcubedim, ycubedim, zcubedim) = proj.datasetcfg.cubedim[resolution]
(startslice, endslice) = proj.datasetcfg.slicerange
batchsz = zcubedim
# This doesn't work because the image size does not match exactly the cube size
#(ximagesz,yimagesz)=proj.datasetcfg.imagesz[resolution]
ximagesz = 49152
yimagesz = 32768
xtilesz = 8192
ytilesz = 8192
numxtiles = (6 + 1)
numytiles = (4 + 1)
# Get a list of the files in the directories
for sl in range(startslice, endslice + 1, batchsz):
for ytile in range(1, numytiles):
for xtile in range(1, numxtiles):
slab = np.zeros([batchsz, ytilesz, xtilesz], dtype=np.uint8)
for b in range(batchsz):
if (sl + b <= endslice):
# raw data
filenm = result.path + 'S1Column_affine_s{:0>3}_x{:0>2}_y{:0>2}.png'.format(
sl + b + 1, xtile, ytile)
print "Opening filenm " + filenm
img = Image.open(filenm, 'r')
imgdata = np.asarray(img)
slab[b, :, :] = imgdata
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
# Now we have a 1024x1024x16 z-aligned cube.
# Send it to the database.
for y in range((ytile - 1) * ytilesz, (ytile) * ytilesz,
ycubedim):
for x in range((xtile - 1) * xtilesz, (xtile) * xtilesz,
xcubedim):
mortonidx = zindex.XYZMorton([
x / xcubedim, y / ycubedim,
(sl - startslice) / zcubedim
])
cubedata = np.zeros([zcubedim, ycubedim, xcubedim],
dtype=np.uint8)
xmin = x % xtilesz
ymin = y % ytilesz
xmax = (min(ximagesz - 1, x + xcubedim - 1) %
xtilesz) + 1
ymax = (min(yimagesz - 1, y + ycubedim - 1) %
ytilesz) + 1
zmin = 0
zmax = min(sl + zcubedim, endslice + 1)
cubedata[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax, ymin:ymax,
xmin:xmax]
# Dont' ingest empty cubes
if len(np.unique(cubedata)) == 1:
continue
# create the DB BLOB
fileobj = cStringIO.StringIO()
np.save(fileobj, cubedata)
cdz = zlib.compress(fileobj.getvalue())
# insert the blob into the database
cursor = db.conn.cursor()
sql = "INSERT INTO res{} (zindex, cube) VALUES (%s, %s)".format(
int(resolution))
cursor.execute(sql, (mortonidx, cdz))
cursor.close()
print "Commiting at x=%s, y=%s, z=%s" % (x, y, sl)
db.conn.commit()
slab = None
import gc
gc.collect()
def loadDB ( self ):
"""Load the database if it hasn't been load."""
if self.db == None:
# load the database/token
[ self.db, self.proj, self.projdb ] = ocpcarest.loadDBProj ( self.token )
# Stuff we make take from a config or the command line in the future
#ximagesz = 12000
#yimagesz = 12000
parser = argparse.ArgumentParser(description='Ingest the FlyEM image data.')
parser.add_argument('baseurl', action="store", help='Base URL to of ocp service no http://, e. g. openconnecto.me')
parser.add_argument('token', action="store", help='Token for the annotation project.')
parser.add_argument('path', action="store", help='Directory with annotation PNG files.')
parser.add_argument('process', action="store", help='Number of processes.')
result = parser.parse_args()
# convert to an argument
resolution = 0
# load a database
[ db, proj, projdb ] = ocpcarest.loadDBProj ( result.token )
# get the dataset configuration
(xcubedim,ycubedim,zcubedim)=proj.datasetcfg.cubedim[resolution]
(startslice,endslice)=proj.datasetcfg.slicerange
batchsz=zcubedim
# This doesn't work because the image size does not match exactly the cube size
#(ximagesz,yimagesz)=proj.datasetcfg.imagesz[resolution]
ximagesz = 12000
yimagesz = 12000
batchsz=16
totalslices = range(startslice,endslice,16)
totalprocs = int(result.process)
#global anydb
def main():
parser = argparse.ArgumentParser(description='Ingest the FlyEM image data.')
parser.add_argument('baseurl', action="store", help='Base URL to of ocp service no http://, e.g. neurodata.io')
parser.add_argument('token', action="store", help='Token for the annotation project.')
parser.add_argument('path', action="store", help='Directory with annotation PNG files.')
result = parser.parse_args()
# convert to an argument
resolution = 0
# load a database
[ db, proj, projdb ] = ocpcarest.loadDBProj ( result.token )
# get the dataset configuration
(xcubedim,ycubedim,zcubedim)=proj.datasetcfg.cubedim[resolution]
(startslice,endslice)=proj.datasetcfg.slicerange
batchsz=zcubedim
# This doesn't work because the image size does not match exactly the cube size
#(ximagesz,yimagesz)=proj.datasetcfg.imagesz[resolution]
ximagesz = 12000
yimagesz = 12000
# Get a list of the files in the directories
for sl in range (startslice,endslice+1,batchsz):
slab = np.zeros ( [ batchsz, yimagesz, ximagesz ], dtype=np.uint8 )
for b in range ( batchsz ):
if ( sl + b <= endslice ):
# raw data
filenm = result.path + '/grayscale.' + '{:0>5}'.format(sl+b) + '.png'
print "Opening filenm " + filenm
img = Image.open ( filenm, 'r' )
imgdata = np.asarray ( img )
slab[b,:,:] = imgdata
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
# Now we have a 1024x1024x16 z-aligned cube.
# Send it to the database.
for y in range ( 0, yimagesz, ycubedim ):
for x in range ( 0, ximagesz, xcubedim ):
mortonidx = zindex.XYZMorton ( [ x/xcubedim, y/ycubedim, (sl-startslice)/zcubedim] )
cubedata = np.zeros ( [zcubedim, ycubedim, xcubedim], dtype=np.uint8 )
xmin = x
ymin = y
xmax = min ( ximagesz, x+xcubedim )
ymax = min ( yimagesz, y+ycubedim )
zmin = 0
zmax = min(sl+zcubedim,endslice+1)
cubedata[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax,ymin:ymax,xmin:xmax]
# create the DB BLOB
fileobj = cStringIO.StringIO ()
np.save ( fileobj, cubedata )
cdz = zlib.compress (fileobj.getvalue())
# insert the blob into the database
cursor = db.conn.cursor()
sql = "INSERT INTO res{} (zindex, cube) VALUES (%s, %s)".format(int(resolution))
cursor.execute(sql, (mortonidx, cdz))
cursor.close()
print "Commiting at x=%s, y=%s, z=%s" % (x,y,sl)
db.conn.commit()
def parallelwrite(slicenumber):
# Accessing the dict in dbm
#anydb = anydbm.open('bodydict','r')
[ db, proj, projdb ] = ocpcarest.loadDBProj ( result.token )
#print slicenumber
startslice = slicenumber
endslice = startslice+16
# Get a list of the files in the directories
for sl in range (startslice, endslice+1, batchsz):
slab = np.zeros ( [ batchsz, yimagesz, ximagesz ], dtype=np.uint32 )
for b in range ( batchsz ):
if ( sl + b <= endslice and sl + b<=1460 ):
# raw data
filenm = result.path + '/superpixel.' + '{:0>5}'.format(sl+b) + '.png'
#print "Opening filenm " + filenm
img = Image.open ( filenm, 'r' )
imgdata = np.asarray ( img )
#Adding new lines
anydb = anydbm.open('bodydict2','r')
superpixelarray = imgdata[:,:,0] + (np.uint32(imgdata[:,:,1])<<8)
newdata = np.zeros([superpixelarray.shape[0],superpixelarray.shape[1]], dtype=np.uint32)
#print "slice",sl+b,"batch",sl
print sl+b,multiprocessing.current_process()
for i in range(superpixelarray.shape[0]):
for j in range(superpixelarray.shape[1]):
key = str(sl)+','+str(superpixelarray[i,j])
if( key not in anydb):
f = open('missing_keys', 'a')
f.write(key+'\n')
f.close()
print "Error Detected Writing to File"
dictvalue = '0'
else:
dictvalue = anydb.get( key )
newdata[i,j] = int(dictvalue)
slab[b,:,:] = newdata
print "end of slice:",sl+b
anydb.close()
print "Entering commit phase"
# Now we have a 1024x1024x16 z-aligned cube.
# Send it to the database.
for y in range ( 0, yimagesz, ycubedim ):
for x in range ( 0, ximagesz, xcubedim ):
mortonidx = zindex.XYZMorton ( [ x/xcubedim, y/ycubedim, (sl-startslice)/zcubedim] )
cubedata = np.zeros ( [zcubedim, ycubedim, xcubedim], dtype=np.uint32 )
xmin = x
ymin = y
xmax = min ( ximagesz, x+xcubedim )
ymax = min ( yimagesz, y+ycubedim )
zmin = 0
zmax = min(sl+zcubedim,endslice+1)
cubedata[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax,ymin:ymax,xmin:xmax]
# insert the blob into the database
db.annotateDense ((x,y,sl-startslice), resolution, cubedata, 'O')
print "Commiting at x=%s, y=%s, z=%s" % (x,y,sl)
db.conn.commit()
return None
def main():
parser = argparse.ArgumentParser(description='Ingest the TIFF data')
parser.add_argument('token', action="store", help='Token for the project')
parser.add_argument('path', action="store", help='Directory with the image files')
parser.add_argument('resolution', action="store", type=int, help='Resolution of data')
result = parser.parse_args()
#Load a database
[ db, proj, projdb ] = ocpcarest.loadDBProj ( result.token )
# get the dataset configuration
(xcubedim,ycubedim,zcubedim) = proj.datasetcfg.cubedim[result.resolution]
(startslice,endslice)=proj.datasetcfg.slicerange
batchsz = zcubedim
batchsz = 16
(ximagesz,yimagesz)=proj.datasetcfg.imagesz[result.resolution]
yimagesz = 18000
ximagesz = 24000
# Get a list of the files in the directories
for sl in range (startslice, endslice+1, batchsz):
slab = np.zeros ( [ batchsz, yimagesz, ximagesz ], dtype=np.uint64 )
for b in range ( batchsz ):
if ( sl + b <= endslice ):
# raw data
try:
filenm = result.path + '{:0>4}'.format(sl+b) + '.tiff'
print "Opening filenm" + filenm
# Returns the image in BGR order. IN 8-bit script PIL returns it in correct order.
imgdata = cv2.imread( filenm, -1 )
if imgdata != None:
slab[b,:,:] = np.left_shift(65535, 48, dtype=np.uint64) | np.left_shift(imgdata[:,:,0], 32, dtype=np.uint64) | np.left_shift(imgdata[:,:,1], 16, dtype=np.uint64) | np.uint64(imgdata[:,:,2])
else:
slab[b,:,:] = np.zeros( [ yimagesz, ximagesz ], dtype=np.uint64)
except IOError, e:
slab[b,:,:] = np.zeros( [ yimagesz, ximagesz ], dtype=np.uint64)
print e
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
for y in range ( 0, yimagesz+1, ycubedim ):
for x in range ( 0, ximagesz+1, xcubedim ):
mortonidx = zindex.XYZMorton ( [x/xcubedim, y/ycubedim, (sl-startslice)/zcubedim] )
cubedata = np.zeros ( [zcubedim, ycubedim, xcubedim], dtype=np.uint64 )
xmin = x
ymin = y
xmax = min ( ximagesz, x+xcubedim )
ymax = min ( yimagesz, y+ycubedim )
zmin = 0
zmax = min(sl+zcubedim, endslice+1)
cubedata[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax]
# Create the DB Blob
fileobj = cStringIO.StringIO()
np.save ( fileobj, cubedata )
cdz = zlib.compress ( fileobj.getvalue() )
# insert the blob into the database
cursor = db.conn.cursor()
sql = "INSERT INTO res{} (zindex, cube) VALUES (%s, %s)".format(int(result.resolution))
cursor.execute(sql, (mortonidx, cdz))
cursor.close()
print " Commiting at x={}, y={}, z={}".format(x, y, sl)
db.conn.commit()
# Freeing memory
slab = None
def main():
parser = argparse.ArgumentParser(description='Ingest the TIFF data')
parser.add_argument('token', action="store", help='Token for the project')
parser.add_argument('path', action="store", help='Directory with the image files')
parser.add_argument('resolution', action="store", type=int, help='Resolution of data')
result = parser.parse_args()
#Load a database
[ db, proj, projdb ] = ocpcarest.loadDBProj ( result.token )
# get the dataset configuration
(xcubedim,ycubedim,zcubedim) = proj.datasetcfg.cubedim[result.resolution]
(startslice,endslice)=proj.datasetcfg.slicerange
batchsz = zcubedim
batchsz = 16
(ximagesz,yimagesz)=proj.datasetcfg.imagesz[result.resolution]
ximagesz = 3992
yimagesz = 4144
pdb.set_trace()
# Get a list of the files in the directories
for sl in range (startslice, endslice+1, batchsz):
slab = np.zeros ( [ batchsz, yimagesz, ximagesz ], dtype=np.uint16 )
for b in range ( batchsz ):
if ( sl + b <= endslice ):
# raw data
try:
filenm = result.path + 'x0.25_unspmask3-0.6_s_{:0>4}'.format(sl+b-1) + '.tif'
print "Opening filenm" + filenm
img = Image.open (filenm, 'r')
imgdata = np.asarray ( img )
slab[b,:,:] = imgdata
except IOError, e:
print e
imgdata = np.zeros((yimagesz, ximagesz), dtype=np.uint16)
slab[b,:,:] = imgdata
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
for y in range ( 0, yimagesz+1, ycubedim ):
for x in range ( 0, ximagesz+1, xcubedim ):
# Getting a Cube id and ingesting the data one cube at a time
mortonidx = zindex.XYZMorton ( [x/xcubedim, y/ycubedim, (sl-startslice)/zcubedim] )
cubedata = np.zeros ( [zcubedim, ycubedim, xcubedim], dtype=np.uint8 )
xmin = x
ymin = y
xmax = min ( ximagesz, x+xcubedim )
ymax = min ( yimagesz, y+ycubedim )
zmin = 0
zmax = min(sl+zcubedim, endslice+1)
cubedata[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax]
# Create the DB Blob
fileobj = cStringIO.StringIO()
np.save ( fileobj, cubedata )
cdz = zlib.compress ( fileobj.getvalue() )
# insert the blob into the database
cursor = db.conn.cursor()
sql = "INSERT INTO res{} (zindex, cube) VALUES (%s, %s)".format(int(result.resolution))
cursor.execute(sql, (mortonidx, cdz))
cursor.close()
print " Commiting at x={}, y={}, z={}".format(x, y, sl)
db.conn.commit()
# Freeing memory
slab = None
def main():
parser = argparse.ArgumentParser(description="Ingest the FlyEM image data.")
parser.add_argument("token", action="store", help="Token for the annotation project.")
parser.add_argument("path", action="store", help="Directory with annotation PNG files.")
result = parser.parse_args()
# convert to an argument
resolution = 0
# load a database
[db, proj, projdb] = ocpcarest.loadDBProj(result.token)
# get the dataset configuration
(xcubedim, ycubedim, zcubedim) = proj.datasetcfg.cubedim[resolution]
(startslice, endslice) = proj.datasetcfg.slicerange
batchsz = zcubedim
# This doesn't work because the image size does not match exactly the cube size
# (ximagesz,yimagesz)=proj.datasetcfg.imagesz[resolution]
ximagesz = 49152
yimagesz = 32768
xtilesz = 8192
ytilesz = 8192
numxtiles = 6 + 1
numytiles = 4 + 1
# Get a list of the files in the directories
for sl in range(startslice, endslice + 1, batchsz):
for ytile in range(1, numytiles):
for xtile in range(1, numxtiles):
slab = np.zeros([batchsz, ytilesz, xtilesz], dtype=np.uint8)
for b in range(batchsz):
if sl + b <= endslice:
# raw data
filenm = result.path + "S1Column_affine_s{:0>3}_x{:0>2}_y{:0>2}.png".format(
sl + b + 1, xtile, ytile
)
print "Opening filenm " + filenm
img = Image.open(filenm, "r")
imgdata = np.asarray(img)
slab[b, :, :] = imgdata
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
# Now we have a 1024x1024x16 z-aligned cube.
# Send it to the database.
for y in range((ytile - 1) * ytilesz, (ytile) * ytilesz, ycubedim):
for x in range((xtile - 1) * xtilesz, (xtile) * xtilesz, xcubedim):
mortonidx = zindex.XYZMorton([x / xcubedim, y / ycubedim, (sl - startslice) / zcubedim])
cubedata = np.zeros([zcubedim, ycubedim, xcubedim], dtype=np.uint8)
xmin = x % xtilesz
ymin = y % ytilesz
xmax = (min(ximagesz - 1, x + xcubedim - 1) % xtilesz) + 1
ymax = (min(yimagesz - 1, y + ycubedim - 1) % ytilesz) + 1
zmin = 0
zmax = min(sl + zcubedim, endslice + 1)
cubedata[0 : zmax - zmin, 0 : ymax - ymin, 0 : xmax - xmin] = slab[
zmin:zmax, ymin:ymax, xmin:xmax
]
# Dont' ingest empty cubes
if len(np.unique(cubedata)) == 1:
continue
# create the DB BLOB
fileobj = cStringIO.StringIO()
np.save(fileobj, cubedata)
cdz = zlib.compress(fileobj.getvalue())
# insert the blob into the database
cursor = db.conn.cursor()
sql = "INSERT INTO res{} (zindex, cube) VALUES (%s, %s)".format(int(resolution))
cursor.execute(sql, (mortonidx, cdz))
cursor.close()
print "Commiting at x=%s, y=%s, z=%s" % (x, y, sl)
db.conn.commit()
slab = None
import gc
gc.collect()
def main():
parser = argparse.ArgumentParser(
description='Ingest the FlyEM image data.')
parser.add_argument(
'baseurl',
action="store",
help='Base URL to of ocp service no http://, e.g. openconnecto.me')
parser.add_argument('token',
action="store",
help='Token for the annotation project.')
parser.add_argument('path',
action="store",
help='Directory with annotation PNG files.')
result = parser.parse_args()
# convert to an argument
resolution = 0
# load a database
[db, proj, projdb] = ocpcarest.loadDBProj(result.token)
# get the dataset configuration
(xcubedim, ycubedim, zcubedim) = proj.datasetcfg.cubedim[resolution]
(startslice, endslice) = proj.datasetcfg.slicerange
batchsz = zcubedim
# This doesn't work because the image size does not match exactly the cube size
#(ximagesz,yimagesz)=proj.datasetcfg.imagesz[resolution]
ximagesz = 12000
yimagesz = 12000
# Get a list of the files in the directories
for sl in range(startslice, endslice + 1, batchsz):
slab = np.zeros([batchsz, yimagesz, ximagesz], dtype=np.uint8)
for b in range(batchsz):
if (sl + b <= endslice):
# raw data
filenm = result.path + '/grayscale.' + '{:0>5}'.format(
sl + b) + '.png'
print "Opening filenm " + filenm
img = Image.open(filenm, 'r')
imgdata = np.asarray(img)
slab[b, :, :] = imgdata
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
# Now we have a 1024x1024x16 z-aligned cube.
# Send it to the database.
for y in range(0, yimagesz, ycubedim):
for x in range(0, ximagesz, xcubedim):
mortonidx = zindex.XYZMorton(
[x / xcubedim, y / ycubedim, (sl - startslice) / zcubedim])
cubedata = np.zeros([zcubedim, ycubedim, xcubedim],
dtype=np.uint8)
xmin = x
ymin = y
xmax = min(ximagesz, x + xcubedim)
ymax = min(yimagesz, y + ycubedim)
zmin = 0
zmax = min(sl + zcubedim, endslice + 1)
cubedata[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax]
# create the DB BLOB
fileobj = cStringIO.StringIO()
np.save(fileobj, cubedata)
cdz = zlib.compress(fileobj.getvalue())
# insert the blob into the database
cursor = db.conn.cursor()
sql = "INSERT INTO res{} (zindex, cube) VALUES (%s, %s)".format(
int(resolution))
cursor.execute(sql, (mortonidx, cdz))
cursor.close()
print "Commiting at x=%s, y=%s, z=%s" % (x, y, sl)
db.conn.commit()
def main():
parser = argparse.ArgumentParser(description='Ingest the FlyEM image data.')
parser.add_argument('baseurl', action="store", help='Base URL to of ocp service no http://, e. g. neurodata.io')
parser.add_argument('token', action="store", help='Token for the annotation project.')
parser.add_argument('path', action="store", help='Directory with annotation PNG files.')
result = parser.parse_args()
# convert to an argument
resolution = 0
# load a database
[ db, proj, projdb ] = ocpcarest.loadDBProj ( result.token )
# get the dataset configuration
(xcubedim,ycubedim,zcubedim)=proj.datasetcfg.cubedim[resolution]
(realstartslice,realendslice)=proj.datasetcfg.slicerange
batchsz=zcubedim
# This doesn't work because the image size does not match exactly the cube size
#(ximagesz,yimagesz)=proj.datasetcfg.imagesz[resolution]
ximagesz = 12000
yimagesz = 12000
# Adding the startslice and endslice manually
startslice = 979
endslice = 1000
batchsz=1
# Accessing the dict in dbm
anydb = anydbm.open('bodydict','r')
# Get a list of the files in the directories
for sl in range (startslice,endslice+1,batchsz):
slab = np.zeros ( [ batchsz, yimagesz, ximagesz ], dtype=np.uint32 )
for b in range ( batchsz ):
if ( sl + b <= endslice ):
# Reading the raw data
filenm = result.path + '/superpixel.' + '{:0>5}'.format(sl+b) + '.png'
print "Opening filenm " + filenm
img = Image.open ( filenm, 'r' )
imgdata = np.asarray ( img )
superpixelarray = imgdata[:,:,0] + (np.uint32(imgdata[:,:,1])<<8)
uniqueidlist = np.unique( superpixelarray)
smalldict = {}
# Creating a small dict of all the ids in the given slice
for i in uniqueidlist:
smalldict[i] = int( anydb.get( str(sl)+','+str(i) ) )
# Looking the in the small dictionary for the body ids
for i in range(superpixelarray.shape[0]):
for j in range(superpixelarray.shape[1]):
superpixelarray[i,j] = smalldict.get(superpixelarray[i,j])
slab[b,:,:] = superpixelarray
print "end on slice:",sl
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
# Now we have a 1024x1024x16 z-aligned cube.
# Send it to the database.
for y in range ( 0, yimagesz, ycubedim ):
for x in range ( 0, ximagesz, xcubedim ):
mortonidx = zindex.XYZMorton ( [ x/xcubedim, y/ycubedim, (sl-startslice)/zcubedim] )
cubedata = np.zeros ( [1, ycubedim, xcubedim], dtype=np.uint32 )
#cubedata = np.zeros ( [zcubedim, ycubedim, xcubedim], dtype=np.uint32 )
xmin = x
ymin = y
xmax = min ( ximagesz, x+xcubedim )
ymax = min ( yimagesz, y+ycubedim )
zmin = 0
zmax = min(sl+zcubedim,endslice+1)
cubedata[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax,ymin:ymax,xmin:xmax]
# insert the blob into the database
db.annotateDense ((x,y,sl-realstartslice),resolution,cubedata,'O')
print "Commiting at x=%s, y=%s, z=%s" % (x,y,sl-realstartslice)
db.conn.commit()
