def main(): parser = argparse.ArgumentParser(description='Ingest the kasthuri11 dataset annotations.') parser.add_argument('token', action="store", help='Token for the annotation project.') parser.add_argument('resolution', action="store", help='Resolution of the ingest data.') parser.add_argument('path', action="store", help='Directory with annotation PNG files.') result = parser.parse_args() # Get a list of the files in the directories for sl in range (_startslice,_endslice+1,_batchsz): newdata = np.zeros ( [ _batchsz, _ytilesz, _xtilesz ], dtype=np.uint32 ) for b in range ( _batchsz ): if ( sl + b <= _endslice ): filenm = result.path + '/' + _prefix + '{:0>4}'.format(sl+b) + '.png' print filenm tileimage = Image.open ( filenm, 'r' ) imgdata = np.asarray ( tileimage ) newdata[b,:,:] = kanno_cy.pngto32 ( imgdata ) # the last z offset that we ingest, if the batch ends before _batchsz endz = b zlow = sl+1 zhigh = sl+endz+2 ylow = 0 yhigh = _ytilesz xlow = 0 xhigh = _xtilesz # Send a cube at a time to the database for z in range ( zlow, zhigh, _zingestsz ): for y in range ( ylow, yhigh, _yingestsz ): for x in range ( xlow, xhigh, _xingestsz ): # cutout the data data = newdata[ z-zlow:min(zhigh,z+_zingestsz)-zlow,\ y-ylow:min(yhigh,y+_yingestsz)-ylow,\ x-xlow:min(xhigh,x+_xingestsz)-xlow] # check if there's anything to store if ( np.count_nonzero(data) != 0 ): url = 'http://localhost/ocp/ca/%s/npz/%s/%s,%s/%s,%s/%s,%s/' % ( result.token, result.resolution, x, min(xhigh,x+_xingestsz), y, min(yhigh,y+_yingestsz), z, min(zhigh,z+_zingestsz )) print url # Encode the voxelist an pickle fileobj = cStringIO.StringIO () np.save ( fileobj, data ) cdz = zlib.compress (fileobj.getvalue()) # Build the post request req = urllib2.Request(url, cdz) response = urllib2.urlopen(req) the_page = response.read()
def main(): parser = argparse.ArgumentParser(description='Ingest the FlyEM image data.') parser.add_argument('token', action="store", help='Token for the annotation project.') parser.add_argument('path', action="store", help='Directory with annotation PNG files.') result = parser.parse_args() # convert to an argument resolution = 0 # load a database [ db, proj, projdb ] = ocpcarest.loadDBProj ( result.token ) # get the dataset configuration (xcubedim,ycubedim,zcubedim)=proj.datasetcfg.cubedim[resolution] (startslice,endslice)=proj.datasetcfg.slicerange batchsz=zcubedim batchsz=1 # This doesn't work because the image size does not match exactly the cube size #(ximagesz,yimagesz)=proj.datasetcfg.imagesz[resolution] ximagesz = 49152 yimagesz = 32768 startslice = 100 # add all of the tiles to the image for sl in range (startslice,endslice+1,batchsz): for ytile in range(ytiles): for xtile in range(xtiles): slab = np.zeros ( [ batchsz, tilesz, tilesz ], dtype=np.uint32 ) for b in range ( batchsz ): if ( sl + b <= endslice ): # raw data filenm = result.path + '/S1Column_Localcellbodies_97-classified_export_s{:0>3}_Y{}_X{}.png'.format(sl+b,ytile,xtile) print "Opening filenm " + filenm img = Image.open ( filenm, 'r' ) imgdata = np.asarray ( img ) slab[b,:,:] = kanno_cy.pngto32 ( imgdata ) # the last z offset that we ingest, if the batch ends before batchsz endz = b # Now we have a 8192x8192x16 z-aligned cube. # Send it to the database. for y in range ( ytile*tilesz, (ytile+1)*tilesz, ycubedim ): for x in range ( xtile*tilesz, (xtile+1)*tilesz, xcubedim ): mortonidx = zindex.XYZMorton ( [ x/xcubedim, y/ycubedim, (sl-startslice)/zcubedim] ) cubedata = np.zeros ( [zcubedim, ycubedim, xcubedim], dtype=np.uint32 ) xmin = x%tilesz ymin = y%tilesz xmax = ((min(ximagesz-1,x+xcubedim-1))%tilesz)+1 ymax = ((min(yimagesz-1,y+ycubedim-1))%tilesz)+1 zmin = 0 zmax = min(sl+zcubedim,endslice+1) cubedata[0:zmax-zmin,0:ymax-ymin,0:xmax-xmin] = slab[zmin:zmax,ymin:ymax,xmin:xmax] if y == 8064: print x,y,xmin,xmax,ymin,ymax,zmin,zmax # check if there's anything to store if ( np.count_nonzero(cubedata) == 0 ): continue # create the DB BLOB fileobj = cStringIO.StringIO () np.save ( fileobj, cubedata ) cdz = zlib.compress (fileobj.getvalue()) # insert the blob into the database cursor = db.conn.cursor() sql = "INSERT INTO res{} (zindex, cube) VALUES (%s, %s)".format(int(resolution)) cursor.execute(sql, (mortonidx, cdz)) cursor.close() print "Commiting at x=%s, y=%s, z=%s" % (x,y,sl) db.conn.commit()
def main(): parser = argparse.ArgumentParser( description='Ingest the FlyEM image data.') parser.add_argument('token', action="store", help='Token for the annotation project.') parser.add_argument('path', action="store", help='Directory with annotation PNG files.') result = parser.parse_args() # convert to an argument resolution = 0 # load a database [db, proj, projdb] = ocpcarest.loadDBProj(result.token) # get the dataset configuration (xcubedim, ycubedim, zcubedim) = proj.datasetcfg.cubedim[resolution] (startslice, endslice) = proj.datasetcfg.slicerange batchsz = zcubedim # This doesn't work because the image size does not match exactly the cube size #(ximagesz,yimagesz)=proj.datasetcfg.imagesz[resolution] ximagesz = 49152 yimagesz = 32768 # add all of the tiles to the image for sl in range(startslice, endslice + 1, batchsz): for ytile in range(ytiles): for xtile in range(xtiles): slab = np.zeros([batchsz, tilesz, tilesz], dtype=np.uint32) for b in range(batchsz): if (sl + b <= endslice): # raw data filenm = result.path + '/S1Column_Localcellbodies_97-classified_export_s{:0>3}_Y{}_X{}.png'.format( sl + b, ytile, xtile) print "Opening filenm " + filenm img = Image.open(filenm, 'r') imgdata = np.asarray(img) slab[b, :, :] = kanno_cy.pngto32(imgdata) # the last z offset that we ingest, if the batch ends before batchsz endz = b # Now we have a 8192x8192x16 z-aligned cube. # Send it to the database. for y in range(ytile * tilesz, (ytile + 1) * tilesz, ycubedim): for x in range(xtile * tilesz, (xtile + 1) * tilesz, xcubedim): mortonidx = zindex.XYZMorton([ x / xcubedim, y / ycubedim, (sl - startslice) / zcubedim ]) cubedata = np.zeros([zcubedim, ycubedim, xcubedim], dtype=np.uint32) xmin = x % tilesz ymin = y % tilesz xmax = ( (min(ximagesz - 1, x + xcubedim - 1)) % tilesz) + 1 ymax = ( (min(yimagesz - 1, y + ycubedim - 1)) % tilesz) + 1 zmin = 0 zmax = min(sl + zcubedim, endslice + 1) cubedata[0:zmax - zmin, 0:ymax - ymin, 0:xmax - xmin] = slab[zmin:zmax, ymin:ymax, xmin:xmax] # check if there's anything to store if (np.count_nonzero(cubedata) == 0): continue # create the DB BLOB fileobj = cStringIO.StringIO() np.save(fileobj, cubedata) cdz = zlib.compress(fileobj.getvalue()) # insert the blob into the database cursor = db.conn.cursor() sql = "INSERT INTO res{} (zindex, cube) VALUES (%s, %s)".format( int(resolution)) print mortonidx cursor.execute(sql, (mortonidx, cdz)) cursor.close() print "Commiting at x=%s, y=%s, z=%s" % (x, y, sl) db.conn.commit()