def main():
parser = argparse.ArgumentParser(description='Ingest the kasthuri11 dataset annotations.')
parser.add_argument('token', action="store", help='Token for the annotation project.')
parser.add_argument('resolution', action="store", help='Resolution of the ingest data.')
parser.add_argument('path', action="store", help='Directory with annotation PNG files.')
result = parser.parse_args()
# Get a list of the files in the directories
for sl in range (_startslice,_endslice+1,_batchsz):
newdata = np.zeros ( [ _batchsz, _ytilesz, _xtilesz ], dtype=np.uint32 )
for b in range ( _batchsz ):
if ( sl + b <= _endslice ):
filenm = result.path + '/' + _prefix + '{:0>4}'.format(sl+b) + '.png'
print filenm
tileimage = Image.open ( filenm, 'r' )
imgdata = np.asarray ( tileimage )
newdata[b,:,:] = kanno_cy.pngto32 ( imgdata )
# the last z offset that we ingest, if the batch ends before _batchsz
endz = b
zlow = sl+1
zhigh = sl+endz+2
ylow = 0
yhigh = _ytilesz
xlow = 0
xhigh = _xtilesz
# Send a cube at a time to the database
for z in range ( zlow, zhigh, _zingestsz ):
for y in range ( ylow, yhigh, _yingestsz ):
for x in range ( xlow, xhigh, _xingestsz ):
# cutout the data
data = newdata[ z-zlow:min(zhigh,z+_zingestsz)-zlow,\
y-ylow:min(yhigh,y+_yingestsz)-ylow,\
x-xlow:min(xhigh,x+_xingestsz)-xlow]
# check if there's anything to store
if ( np.count_nonzero(data) != 0 ):
url = 'http://localhost/ocp/ca/%s/npz/%s/%s,%s/%s,%s/%s,%s/' % ( result.token, result.resolution, x, min(xhigh,x+_xingestsz), y, min(yhigh,y+_yingestsz), z, min(zhigh,z+_zingestsz ))
print url
# Encode the voxelist an pickle
fileobj = cStringIO.StringIO ()
np.save ( fileobj, data )
cdz = zlib.compress (fileobj.getvalue())
# Build the post request
req = urllib2.Request(url, cdz)
response = urllib2.urlopen(req)
the_page = response.read()
def main():
parser = argparse.ArgumentParser(description='Ingest the FlyEM image data.')
parser.add_argument('token', action="store", help='Token for the annotation project.')
parser.add_argument('path', action="store", help='Directory with annotation PNG files.')
result = parser.parse_args()
# convert to an argument
resolution = 0
# load a database
[ db, proj, projdb ] = ocpcarest.loadDBProj ( result.token )
# get the dataset configuration
(xcubedim,ycubedim,zcubedim)=proj.datasetcfg.cubedim[resolution]
(startslice,endslice)=proj.datasetcfg.slicerange
batchsz=zcubedim
batchsz=1
# This doesn't work because the image size does not match exactly the cube size
#(ximagesz,yimagesz)=proj.datasetcfg.imagesz[resolution]
ximagesz = 49152
yimagesz = 32768
startslice = 100
# add all of the tiles to the image
for sl in range (startslice,endslice+1,batchsz):
for ytile in range(ytiles):
for xtile in range(xtiles):
slab = np.zeros ( [ batchsz, tilesz, tilesz ], dtype=np.uint32 )
for b in range ( batchsz ):
if ( sl + b <= endslice ):
# raw data
filenm = result.path + '/S1Column_Localcellbodies_97-classified_export_s{:0>3}_Y{}_X{}.png'.format(sl+b,ytile,xtile)
print "Opening filenm " + filenm
img = Image.open ( filenm, 'r' )
imgdata = np.asarray ( img )
slab[b,:,:] = kanno_cy.pngto32 ( imgdata )
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
# Now we have a 8192x8192x16 z-aligned cube.
# Send it to the database.
for y in range ( ytile*tilesz, (ytile+1)*tilesz, ycubedim ):
for x in range ( xtile*tilesz, (xtile+1)*tilesz, xcubedim ):
mortonidx = zindex.XYZMorton ( [ x/xcubedim, y/ycubedim, (sl-startslice)/zcubedim] )
cubedata = np.zeros ( [zcubedim, ycubedim, xcubedim], dtype=np.uint32 )
xmin = x%tilesz
ymin = y%tilesz
xmax = ((min(ximagesz-1,x+xcubedim-1))%tilesz)+1
ymax = ((min(yimagesz-1,y+ycubedim-1))%tilesz)+1
zmin = 0
zmax = min(sl+zcubedim,endslice+1)
cubedata[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax,ymin:ymax,xmin:xmax]
if y == 8064:
print x,y,xmin,xmax,ymin,ymax,zmin,zmax
# check if there's anything to store
if ( np.count_nonzero(cubedata) == 0 ):
continue
# create the DB BLOB
fileobj = cStringIO.StringIO ()
np.save ( fileobj, cubedata )
cdz = zlib.compress (fileobj.getvalue())
# insert the blob into the database
cursor = db.conn.cursor()
sql = "INSERT INTO res{} (zindex, cube) VALUES (%s, %s)".format(int(resolution))
cursor.execute(sql, (mortonidx, cdz))
cursor.close()
print "Commiting at x=%s, y=%s, z=%s" % (x,y,sl)
db.conn.commit()
def main():
parser = argparse.ArgumentParser(
description='Ingest the FlyEM image data.')
parser.add_argument('token',
action="store",
help='Token for the annotation project.')
parser.add_argument('path',
action="store",
help='Directory with annotation PNG files.')
result = parser.parse_args()
# convert to an argument
resolution = 0
# load a database
[db, proj, projdb] = ocpcarest.loadDBProj(result.token)
# get the dataset configuration
(xcubedim, ycubedim, zcubedim) = proj.datasetcfg.cubedim[resolution]
(startslice, endslice) = proj.datasetcfg.slicerange
batchsz = zcubedim
# This doesn't work because the image size does not match exactly the cube size
#(ximagesz,yimagesz)=proj.datasetcfg.imagesz[resolution]
ximagesz = 49152
yimagesz = 32768
# add all of the tiles to the image
for sl in range(startslice, endslice + 1, batchsz):
for ytile in range(ytiles):
for xtile in range(xtiles):
slab = np.zeros([batchsz, tilesz, tilesz], dtype=np.uint32)
for b in range(batchsz):
if (sl + b <= endslice):
# raw data
filenm = result.path + '/S1Column_Localcellbodies_97-classified_export_s{:0>3}_Y{}_X{}.png'.format(
sl + b, ytile, xtile)
print "Opening filenm " + filenm
img = Image.open(filenm, 'r')
imgdata = np.asarray(img)
slab[b, :, :] = kanno_cy.pngto32(imgdata)
# the last z offset that we ingest, if the batch ends before batchsz
endz = b
# Now we have a 8192x8192x16 z-aligned cube.
# Send it to the database.
for y in range(ytile * tilesz, (ytile + 1) * tilesz, ycubedim):
for x in range(xtile * tilesz, (xtile + 1) * tilesz,
xcubedim):
mortonidx = zindex.XYZMorton([
x / xcubedim, y / ycubedim,
(sl - startslice) / zcubedim
])
cubedata = np.zeros([zcubedim, ycubedim, xcubedim],
dtype=np.uint32)
xmin = x % tilesz
ymin = y % tilesz
xmax = (
(min(ximagesz - 1, x + xcubedim - 1)) % tilesz) + 1
ymax = (
(min(yimagesz - 1, y + ycubedim - 1)) % tilesz) + 1
zmin = 0
zmax = min(sl + zcubedim, endslice + 1)
cubedata[0:zmax - zmin, 0:ymax - ymin,
0:xmax - xmin] = slab[zmin:zmax, ymin:ymax,
xmin:xmax]
# check if there's anything to store
if (np.count_nonzero(cubedata) == 0):
continue
# create the DB BLOB
fileobj = cStringIO.StringIO()
np.save(fileobj, cubedata)
cdz = zlib.compress(fileobj.getvalue())
# insert the blob into the database
cursor = db.conn.cursor()
sql = "INSERT INTO res{} (zindex, cube) VALUES (%s, %s)".format(
int(resolution))
print mortonidx
cursor.execute(sql, (mortonidx, cdz))
cursor.close()
print "Commiting at x=%s, y=%s, z=%s" % (x, y, sl)
db.conn.commit()
