def main():
''' run with
mpiexex -n 16 mesh_generator $LABEL_PATH
'''
parser = argparse.ArgumentParser()
parser.add_argument('labels',
default=None,
help="path to precomputed labels")
parser.add_argument('--verbose',
default=False,
help="wether to use progressbar")
parser.add_argument('--dim_size',
default='64,64,64',
help="mesh chunksize")
args = parser.parse_args()
if rank == 0:
in_path = 'file://' + args.labels
mip = 0
dim_size = tuple(int(d) for d in args.dim_size.split(','))
print(dim_size)
print("Making meshes...")
mtq = mpiTaskQueue()
tc.create_meshing_tasks(mtq, in_path, mip, shape=Vec(*dim_size))
L = len(mtq._queue)
#print('total', rank,size, L)
all_range = np.arange(L)
sub_ranges = np.array_split(all_range, size)
#print(sub_ranges)
else:
sub_ranges = None
mtq = None
sub_ranges = comm.bcast(sub_ranges, root=0)
mtq = comm.bcast(mtq, root=0)
mtq.run(sub_ranges[rank], args.verbose)
comm.barrier()
if rank == 0:
mtq.clean(all_range)
print("Cleaned", len(mtq._queue))
print("Updating metadata...")
tc.create_mesh_manifest_tasks(mtq, in_path)
print(len(mtq._queue))
all_range = np.arange(L)
sub_ranges = np.array_split(all_range, size)
else:
sub_ranges = None
mtq = None
sub_ranges = comm.bcast(sub_ranges, root=0)
mtq = comm.bcast(mtq, root=0)
mtq.run(sub_ranges[rank], args.verbose)
#mtq.run(sub_ranges[rank])
comm.barrier()
if rank == 0:
command = r'gunzip {}/mesh/*.gz'.format(args.labels)
print(command)
subprocess.call(command, shell=True)
print("Done!")
def create_mesh(animal, mip, mse):
fileLocationManager = FileLocationManager(animal)
channel_outdir = 'color_mesh_v2'
OUTPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data,
channel_outdir)
if not os.path.exists(OUTPUT_DIR):
print(f'DIR {OUTPUT_DIR} does not exist, exiting.')
sys.exit()
cloudpath = f"file://{OUTPUT_DIR}"
cv = CloudVolume(cloudpath, mip)
workers, _ = get_cpus()
tq = LocalTaskQueue(parallel=workers)
mesh_dir = f'mesh_mip_{mip}_err_{mse}'
cv.info['mesh'] = mesh_dir
cv.commit_info()
tasks = tc.create_meshing_tasks(cv.layer_cloudpath,
mip=mip,
mesh_dir=mesh_dir,
max_simplification_error=mse)
tq.insert(tasks)
tq.execute()
tasks = tc.create_mesh_manifest_tasks(cv.layer_cloudpath,
mesh_dir=mesh_dir)
tq.insert(tasks)
tq.execute()
print("Done!")
def make_demo_mesh():
# Mesh on 8 cores, use True to use all cores
cloudpath = 'file:///home/ahoag/ngdemo/demo_bucket/atlas/allenatlas_2017'
with LocalTaskQueue(parallel=8) as tq:
tasks = tc.create_meshing_tasks(cloudpath, mip=0, shape=(256, 256, 256))
tq.insert_all(tasks)
tasks = tc.create_mesh_manifest_tasks(cloudpath)
tq.insert_all(tasks)
print("Done!")
def make_mesh(vol, cores=8):
# Mesh on 8 cores, use parallel=True to use all cores
cloudpath = vol.cloudpath
with LocalTaskQueue(parallel=cores) as tq:
tasks = tc.create_meshing_tasks(cloudpath,
mip=0,
shape=(256, 256, 256))
tq.insert_all(tasks)
tasks = tc.create_mesh_manifest_tasks(cloudpath)
tq.insert_all(tasks)
print("Done!")
def mesh(opt):
gs_path = opt.gs_output
# Mesh
if opt.mesh:
assert opt.vol_type == 'segmentation'
# Create mesh
with LocalTaskQueue(parallel=opt.parallel) as tq:
tasks = tc.create_meshing_tasks(gs_path, mip=opt.mesh_mip)
tq.insert_all(tasks)
# Manifest
with MockTaskQueue() as tq:
tasks = tc.create_mesh_manifest_tasks(gs_path)
tq.insert_all(tasks)
def mesh_merge(ctx, path, queue, magnitude, dir):
"""
(2) Merge the mesh pieces produced from the forging step.
The per-cutout mesh fragments are then assembled and
merged. However, this process occurs by compiling
a list of fragment files and uploading a "mesh manifest"
file that is an index for locating the fragments.
"""
tasks = tc.create_mesh_manifest_tasks(
path, magnitude=magnitude, mesh_dir=dir
)
parallel = int(ctx.obj.get("parallel", 1))
tq = TaskQueue(normalize_path(queue))
tq.insert(tasks, parallel=parallel)
def add_segmentation_mesh(self, shape=[448, 448, 448], mip=0):
if self.precomputed_vol is None:
raise NotImplementedError(
'You have to call init_precomputed before calling this function.'
)
_, cpus = get_cpus()
tq = LocalTaskQueue(parallel=cpus)
tasks = tc.create_meshing_tasks(
self.precomputed_vol.layer_cloudpath,
mip=mip,
max_simplification_error=40,
shape=shape,
compress=True) # The first phase of creating mesh
tq.insert(tasks)
tq.execute()
# It should be able to incoporated to above tasks, but it will give a weird bug. Don't know the reason
tasks = tc.create_mesh_manifest_tasks(
self.precomputed_vol.layer_cloudpath
) # The second phase of creating mesh
tq.insert(tasks)
tq.execute()
vol[bbx] = (h5_data[bbx.to_slices()[::-1]].T).astype(np_type)
print("KNeuroViz pre-processing DONE!")
if (h5_class.ImageType == 'segmentation'):
seg_mesh_path = "file://" + h5_class.Destination_path
with LocalTaskQueue(parallel=8) as tq:
tasks = tc.create_meshing_tasks(seg_mesh_path,
mip=0,
shape=Vec(h5_class.shape_x // 2,
h5_class.shape_x // 2,
h5_class.shape_x // 2 // 8))
tq.insert_all(tasks)
tasks = tc.create_mesh_manifest_tasks(seg_mesh_path, magnitude=2)
tq.insert_all(tasks)
print("Mesh manifest processing DONE!")
else:
print('channel is not running for 3D mesh manifest work')
sys.exit(1)
'''
seg_mesh_path = "file://"+h5_class.Destination_path
with LocalTaskQueue(parallel=8) as tq:
tasks = tc.create_meshing_tasks(seg_mesh_path, mip=0, shape=Vec(h5_class.shape_x // 2, h5_class.shape_x // 2, h5_class.shape_x // 2 //8))
tq.insert_all(tasks)
tasks = tc.create_mesh_manifest_tasks(seg_mesh_path, magnitude=2)
tq.insert_all(tasks)
def segment(args):
"""Run segmentation on contiguous block of affinities from CV
Args:
args: ArgParse object from main
"""
bbox_start = Vec(*args.bbox_start)
bbox_size = Vec(*args.bbox_size)
chunk_size = Vec(*args.chunk_size)
bbox = Bbox(bbox_start, bbox_start + bbox_size)
src_cv = CloudVolume(args.src_path,
fill_missing=True,
parallel=args.parallel)
info = CloudVolume.create_new_info(
num_channels=1,
layer_type='segmentation',
data_type='uint64',
encoding='raw',
resolution=src_cv.info['scales'][args.mip]['resolution'],
voxel_offset=bbox_start,
chunk_size=chunk_size,
volume_size=bbox_size,
mesh='mesh_mip_{}_err_{}'.format(args.mip,
args.max_simplification_error))
dst_cv = CloudVolume(args.dst_path, info=info, parallel=args.parallel)
dst_cv.provenance.description = 'ws+agg using waterz'
dst_cv.provenance.processing.append({
'method': {
'task': 'watershed+agglomeration',
'src_path': args.src_path,
'dst_path': args.dst_path,
'mip': args.mip,
'shape': bbox_size.tolist(),
'bounds': [
bbox.minpt.tolist(),
bbox.maxpt.tolist(),
],
},
'by': args.owner,
'date': strftime('%Y-%m-%d%H:%M %Z'),
})
dst_cv.provenance.owners = [args.owner]
dst_cv.commit_info()
dst_cv.commit_provenance()
if args.segment:
print('Downloading affinities')
aff = src_cv[bbox.to_slices()]
aff = np.transpose(aff, (3, 0, 1, 2))
aff = np.ascontiguousarray(aff, dtype=np.float32)
thresholds = [args.threshold]
print('Starting ws+agg')
seg_gen = waterz.agglomerate(aff, thresholds)
seg = next(seg_gen)
print('Deleting affinities')
del aff
print('Uploading segmentation')
dst_cv[bbox.to_slices()] = seg
if args.mesh:
print('Starting meshing')
with LocalTaskQueue(parallel=args.parallel) as tq:
tasks = tc.create_meshing_tasks(
layer_path=args.dst_path,
mip=args.mip,
shape=args.chunk_size,
simplification=True,
max_simplification_error=args.max_simplification_error,
progress=True)
tq.insert_all(tasks)
tasks = tc.create_mesh_manifest_tasks(layer_path=args.dst_path,
magnitude=args.magnitude)
tq.insert_all(tasks)
print("Meshing complete")
def create_mesh(animal, limit, mse):
#chunks = calculate_chunks('full', -1)
chunks = [64,64,1]
scales = (10400, 10400, 20000)
fileLocationManager = FileLocationManager(animal)
INPUT = "/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/DK52/preps/CH3/shapes"
OUTPUT1_DIR = os.path.join(fileLocationManager.neuroglancer_data, 'mesh_input')
OUTPUT2_DIR = os.path.join(fileLocationManager.neuroglancer_data, 'mesh')
if 'ultraman' in get_hostname():
if os.path.exists(OUTPUT1_DIR):
shutil.rmtree(OUTPUT1_DIR)
if os.path.exists(OUTPUT2_DIR):
shutil.rmtree(OUTPUT2_DIR)
files = sorted(os.listdir(INPUT))
os.makedirs(OUTPUT1_DIR, exist_ok=True)
os.makedirs(OUTPUT2_DIR, exist_ok=True)
len_files = len(files)
midpoint = len_files // 2
midfilepath = os.path.join(INPUT, files[midpoint])
midfile = io.imread(midfilepath)
data_type = midfile.dtype
#image = np.load('/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/structures/allen/allen.npy')
#ids = np.unique(image)
ids = {'infrahypoglossal': 200, 'perifacial': 210, 'suprahypoglossal': 220}
height, width = midfile.shape
volume_size = (width, height, len(files)) # neuroglancer is width, height
print('volume size', volume_size)
ng = NumpyToNeuroglancer(animal, None, scales, layer_type='segmentation',
data_type=data_type, chunk_size=chunks)
ng.init_precomputed(OUTPUT1_DIR, volume_size, progress_id=1)
file_keys = []
for i,f in enumerate(tqdm(files)):
infile = os.path.join(INPUT, f)
file_keys.append([i, infile])
#ng.process_image([i, infile])
#sys.exit()
start = timer()
workers, cpus = get_cpus()
print(f'Working on {len(file_keys)} files with {workers} cpus')
with ProcessPoolExecutor(max_workers=workers) as executor:
executor.map(ng.process_image, sorted(file_keys), chunksize=1)
executor.shutdown(wait=True)
ng.precomputed_vol.cache.flush()
end = timer()
print(f'Create volume method took {end - start} seconds')
##### rechunk
cloudpath1 = f"file://{OUTPUT1_DIR}"
cv1 = CloudVolume(cloudpath1, 0)
_, workers = get_cpus()
tq = LocalTaskQueue(parallel=workers)
cloudpath2 = f'file://{OUTPUT2_DIR}'
tasks = tc.create_transfer_tasks(cloudpath1, dest_layer_path=cloudpath2,
chunk_size=[64,64,64], mip=0, skip_downsamples=True)
tq.insert(tasks)
tq.execute()
##### add segment properties
cv2 = CloudVolume(cloudpath2, 0)
cv2.info['segment_properties'] = 'names'
cv2.commit_info()
segment_properties_path = os.path.join(cloudpath2.replace('file://', ''), 'names')
os.makedirs(segment_properties_path, exist_ok=True)
info = {
"@type": "neuroglancer_segment_properties",
"inline": {
"ids": [str(value) for key, value in ids.items()],
"properties": [{
"id": "label",
"type": "label",
"values": [str(key) for key, value in ids.items()]
}]
}
}
with open(os.path.join(segment_properties_path, 'info'), 'w') as file:
json.dump(info, file, indent=2)
##### first mesh task, create meshing tasks
workers, _ = get_cpus()
tq = LocalTaskQueue(parallel=workers)
mesh_dir = f'mesh_mip_0_err_{mse}'
cv2.info['mesh'] = mesh_dir
cv2.commit_info()
tasks = tc.create_meshing_tasks(cv2.layer_cloudpath, mip=0, mesh_dir=mesh_dir, max_simplification_error=mse)
tq.insert(tasks)
tq.execute()
##### 2nd mesh task, create manifest
tasks = tc.create_mesh_manifest_tasks(cv2.layer_cloudpath, mesh_dir=mesh_dir)
tq.insert(tasks)
tq.execute()
print("Done!")
def generate_mesh(labels, dim_size=(64, 64, 64), quiet=False):
"""Generate a mesh of label data.
Create a mesh from a CloudVolume segmentation layer and save the result in
the correct place for visualization. Mesh generation is parallelized over
MPI ranks.
Parameters
----------
labels : str
Path to a CloudVolume layer of segmentation labels.
dim_size : tuple of int
The size of the subvolume assigned to each task.
quiet : bool
If True, suppress stdout logging output.
Notes
-----
To take advantage of MPI parallelism, this script must be run as
``mpiexec -n <n_ranks> python mesh_generator_v3.py <...args>``.
"""
args = parse_args()
# Disable logging if requested.
if args.quiet:
LOGGER.logger.removeHandler(syslog)
noop = logging.NullHandler()
LOGGER.logger.addHandler(noop)
LOGGER.info('Starting on host {}'.format(HOST))
# Set up the meshing task queue on rank 0.
if RANK == 0:
# Load in the layer data.
LOGGER.info('Loading CloudVolume layer at {}'.format(args.labels))
if os.path.isdir(
args.labels) and not re.search(r'^file://', args.labels):
in_path = 'file://' + args.labels
mip = 0
LOGGER.info('Meshing volume with dimensions {}'.format(dim_size))
# Create a queue of meshing tasks over subvolumes of the layer.
LOGGER.info("Setting up meshing task queue.")
mtq = MPITaskQueue()
tasks = tc.create_meshing_tasks(layer_path=in_path,
mip=mip,
shape=Vec(*dim_size),
mesh_dir='mesh')
for t in tasks:
mtq.insert(t)
# Compute the tasks for each rank to complete.
LOGGER.info('{} tasks created.'.format(len(mtq._queue)))
L = len(mtq._queue)
all_range = np.arange(L)
sub_ranges = np.array_split(all_range, SIZE)
else:
sub_ranges = None
mtq = None
# Synchronize and broadast the task queue and assigned tasks to all ranks.
sub_ranges = COMM.bcast(sub_ranges, root=0)
mtq = COMM.bcast(mtq, root=0)
# Run the tasks assigned to this rank, then wait for all to finish.
LOGGER.info('Running task queue 1.')
mtq.run(sub_ranges[RANK])
LOGGER.info('Finished task queue 1.')
COMM.barrier()
# Set up the metadata update queue
if RANK == 0:
LOGGER.info('Cleaning {} tasks.'.format(len(mtq._queue)))
mtq.clean(all_range)
# Create a queue of `igneous` metadata update tasks.
LOGGER.info('Setting up manifest update task queue.')
mtq2 = MPITaskQueue()
tasks = tc.create_mesh_manifest_tasks(in_path, magnitude=3)
for t in tasks:
mtq2.insert(t)
# Compute the tasks for each rank to complete.
LOGGER.info('Created {} mesh manifest tasks.'.format(len(mtq2._queue)))
L2 = len(mtq2._queue)
all_range = np.arange(L2)
sub_ranges = np.array_split(all_range, SIZE)
else:
sub_ranges = None
mtq2 = None
# Synchronize and broadcast the metadata update task queue and assignments.
sub_ranges = COMM.bcast(sub_ranges, root=0)
mtq2 = COMM.bcast(mtq2, root=0)
# Run the metadata update queue.
LOGGER.info('Running task queue 2.')
mtq2.run(sub_ranges[RANK])
LOGGER.info('Finished task queue 2.')
COMM.barrier()
LOGGER.info('Done')
def upload_seg(
meta: PreviewMeta,
data: ndarray,
slack_response: SlackResponse,
transpose: bool = False,
):
from numpy import transpose as np_transpose
em = CloudVolume(meta.em_layer, mip=meta.dst_mip)
output_layer = f"{environ['GT_BUCKET_PATH']}/{meta.author}/preview/{token_hex(8)}"
info = CloudVolume.create_new_info(
num_channels=1,
layer_type="segmentation",
data_type="uint32",
encoding="raw",
resolution=em.resolution,
voxel_offset=meta.dst_bbox.minpt,
volume_size=meta.dst_bbox.size3(),
mesh=f"mesh_mip_{meta.dst_mip}_err_0",
chunk_size=(64, 64, 8),
)
dst_cv = CloudVolume(output_layer, info=info, mip=0, cdn_cache=False)
dst_cv.provenance.description = "Image directory ingest"
dst_cv.provenance.processing.append({
"method": {
"task": "ingest",
"image_path": meta.em_layer,
},
"date": str(datetime.today()),
"script": "cloud_bot",
})
dst_cv.provenance.owners = [meta.author]
dst_cv.commit_info()
dst_cv.commit_provenance()
checkpoint_notify("Processing data.", slack_response)
crop_bbox = meta.dst_bbox - meta.src_bbox.minpt
data = data[crop_bbox.to_slices()]
dst_cv[meta.dst_bbox.to_slices()] = (np_transpose(data, (1, 0, 2))
if transpose else data)
with LocalTaskQueue(parallel=16) as tq:
tasks = tc.create_downsampling_tasks(output_layer,
mip=0,
fill_missing=True,
preserve_chunk_size=True)
tq.insert_all(tasks)
checkpoint_notify("Creating meshing tasks.", slack_response)
tasks = tc.create_meshing_tasks(
output_layer,
mip=meta.dst_mip,
simplification=False,
shape=(320, 320, 40),
max_simplification_error=0,
)
tq.insert_all(tasks)
tasks = tc.create_mesh_manifest_tasks(output_layer, magnitude=1)
tq.insert_all(tasks)
return output_layer