def __iter__(self):
for boundstr, volume_id in volume_map.items():
bbox = Bbox.from_filename(boundstr)
bbox.minpt = Vec.clamp(bbox.minpt, vol.bounds.minpt, vol.bounds.maxpt)
bbox.maxpt = Vec.clamp(bbox.maxpt, vol.bounds.minpt, vol.bounds.maxpt)
yield HyperSquareConsensusTask(
src_path=src_path,
dest_path=dest_path,
ew_volume_id=int(volume_id),
consensus_map_path=consensus_map_path,
shape=bbox.size3(),
offset=bbox.minpt.clone(),
)
def generate_chunks(meta, img, offset, mip):
shape = Vec(*img.shape)[:3]
offset = Vec(*offset)[:3]
bounds = Bbox(offset, shape + offset)
alignment_check = bounds.round_to_chunk_size(meta.chunk_size(mip),
meta.voxel_offset(mip))
if not np.all(alignment_check.minpt == bounds.minpt):
raise AlignmentError("""
Only chunk aligned writes are supported by this function.
Got: {}
Volume Offset: {}
Nearest Aligned: {}
""".format(bounds, meta.voxel_offset(mip), alignment_check))
bounds = Bbox.clamp(bounds, meta.bounds(mip))
img_offset = bounds.minpt - offset
img_end = Vec.clamp(bounds.size3() + img_offset, Vec(0, 0, 0), shape)
for startpt in xyzrange(img_offset, img_end, meta.chunk_size(mip)):
startpt = startpt.clone()
endpt = min2(startpt + meta.chunk_size(mip), shape)
spt = (startpt + bounds.minpt).astype(int)
ept = (endpt + bounds.minpt).astype(int)
yield (startpt, endpt, spt, ept)
def create_hypersquare_consensus_tasks(task_queue, src_path, dest_path,
volume_map_file, consensus_map_path):
"""
Transfer an Eyewire consensus into neuroglancer. This first requires
importing the raw segmentation via a hypersquare ingest task. However,
this can probably be streamlined at some point.
The volume map file should be JSON encoded and
look like { "X-X_Y-Y_Z-Z": EW_VOLUME_ID }
The consensus map file should look like:
{ VOLUMEID: { CELLID: [segids] } }
"""
with open(volume_map_file, 'r') as f:
volume_map = json.loads(f.read())
vol = CloudVolume(dest_path)
for boundstr, volume_id in tqdm(
volume_map.items(),
desc="Inserting HyperSquare Consensus Remap Tasks"):
bbox = Bbox.from_filename(boundstr)
bbox.minpt = Vec.clamp(bbox.minpt, vol.bounds.minpt, vol.bounds.maxpt)
bbox.maxpt = Vec.clamp(bbox.maxpt, vol.bounds.minpt, vol.bounds.maxpt)
task = HyperSquareConsensusTask(
src_path=src_path,
dest_path=dest_path,
ew_volume_id=int(volume_id),
consensus_map_path=consensus_map_path,
shape=bbox.size3(),
offset=bbox.minpt.clone(),
)
task_queue.insert(task)
task_queue.wait()
def generate_chunks(meta, img, offset, mip):
shape = Vec(*img.shape)[:3]
offset = Vec(*offset)[:3]
bounds = Bbox(offset, shape + offset)
alignment_check = bounds.round_to_chunk_size(meta.chunk_size(mip),
meta.voxel_offset(mip))
if not np.all(alignment_check.minpt == bounds.minpt):
raise AlignmentError(f"""
Only chunk aligned writes are supported by this function.
Got: {bounds}
Volume Offset: {meta.voxel_offset(mip)}
Nearest Aligned: {alignment_check}
""")
bounds = Bbox.clamp(bounds, meta.bounds(mip))
img_offset = bounds.minpt - offset
img_end = Vec.clamp(bounds.size3() + img_offset, Vec(0, 0, 0), shape)
class ChunkIterator():
def __len__(self):
csize = meta.chunk_size(mip)
bbox = Bbox(img_offset, img_end)
# round up and avoid conversion to float
n_chunks = (bbox.dx + csize[0] - 1) // csize[0]
n_chunks *= (bbox.dy + csize[1] - 1) // csize[1]
n_chunks *= (bbox.dz + csize[2] - 1) // csize[2]
return n_chunks
def __iter__(self):
for startpt in xyzrange(img_offset, img_end, meta.chunk_size(mip)):
startpt = startpt.clone()
endpt = min2(startpt + meta.chunk_size(mip), shape)
spt = (startpt + bounds.minpt).astype(int)
ept = (endpt + bounds.minpt).astype(int)
yield (startpt, endpt, spt, ept)
return ChunkIterator()
def ingest(args):
"""
Ingest an HDF file to a CloudVolume bucket
"""
if args.local_hdf_path:
hdf_file = h5py.File(args.local_hdf_path, "r")
else:
with Storage(args.cloud_src_path) as storage:
hdf_file = h5py.File(storage.get_file(args.cloud_hdf_filename),
"r")
cur_hdf_group = hdf_file
for group_name in args.hdf_keys_to_dataset:
cur_hdf_group = cur_hdf_group[group_name]
hdf_dataset = cur_hdf_group
if args.zyx:
dataset_shape = np.array(
[hdf_dataset.shape[2], hdf_dataset.shape[1], hdf_dataset.shape[0]])
else:
dataset_shape = np.array([*hdf_dataset.shape])
if args.layer_type == "image":
data_type = "uint8"
else:
data_type = "uint64"
voxel_offset = args.voxel_offset
info = CloudVolume.create_new_info(
num_channels=1,
layer_type=args.layer_type,
data_type=data_type,
encoding="raw",
resolution=args.resolution,
voxel_offset=voxel_offset,
chunk_size=args.chunk_size,
volume_size=dataset_shape,
)
provenance = {
"description": args.provenance_description,
"owners": [args.owner]
}
vol = CloudVolume(args.dst_path, info=info, provenance=provenance)
vol.commit_info()
vol.commit_provenance()
all_files = set()
for x in np.arange(voxel_offset[0], voxel_offset[0] + dataset_shape[0],
args.chunk_size[0]):
for y in np.arange(voxel_offset[1], voxel_offset[1] + dataset_shape[1],
args.chunk_size[1]):
for z in np.arange(voxel_offset[2],
voxel_offset[2] + dataset_shape[2],
args.chunk_size[2]):
all_files.add(tuple((x, y, z)))
progress_dir = mkdir(
"progress/") # unlike os.mkdir doesn't crash on prexisting
done_files = set()
for done_file in os.listdir(progress_dir):
done_files.add(tuple(done_file.split(",")))
to_upload = all_files.difference(done_files)
for chunk_start_tuple in to_upload:
chunk_start = np.array(list(chunk_start_tuple))
end_of_dataset = np.array(voxel_offset) + dataset_shape
chunk_end = chunk_start + np.array(args.chunk_size)
chunk_end = Vec(*chunk_end)
chunk_end = Vec.clamp(chunk_end, Vec(0, 0, 0), end_of_dataset)
chunk_hdf_start = chunk_start - voxel_offset
chunk_hdf_end = chunk_end - voxel_offset
if args.zyx:
chunk = hdf_dataset[chunk_hdf_start[2]:chunk_hdf_end[2],
chunk_hdf_start[1]:chunk_hdf_end[1],
chunk_hdf_start[0]:chunk_hdf_end[0], ]
chunk = chunk.T
else:
chunk = hdf_dataset[chunk_hdf_start[0]:chunk_hdf_end[0],
chunk_hdf_start[1]:chunk_hdf_end[1],
chunk_hdf_start[2]:chunk_hdf_end[2], ]
print("Processing ", chunk_start_tuple)
array = np.array(chunk, dtype=np.dtype(data_type), order="F")
vol[chunk_start[0]:chunk_end[0], chunk_start[1]:chunk_end[1],
chunk_start[2]:chunk_end[2], ] = array
touch(os.path.join(progress_dir, str(chunk_start_tuple)))
def create_contrast_normalization_tasks(src_path,
dest_path,
levels_path=None,
shape=None,
mip=0,
clip_fraction=0.01,
fill_missing=False,
translate=(0, 0, 0),
minval=None,
maxval=None,
bounds=None):
srcvol = CloudVolume(src_path, mip=mip)
try:
dvol = CloudVolume(dest_path, mip=mip)
except Exception: # no info file
info = copy.deepcopy(srcvol.info)
dvol = CloudVolume(dest_path, mip=mip, info=info)
dvol.info['scales'] = dvol.info['scales'][:mip + 1]
dvol.commit_info()
if shape == None:
shape = Bbox((0, 0, 0), (2048, 2048, 64))
shape = shape.shrink_to_chunk_size(dvol.underlying).size3()
shape = Vec.clamp(shape, (1, 1, 1), bounds.size3())
shape = Vec(*shape)
create_downsample_scales(dest_path,
mip=mip,
ds_shape=shape,
preserve_chunk_size=True)
dvol.refresh_info()
bounds = get_bounds(srcvol, bounds, shape, mip)
class ContrastNormalizationTaskIterator(object):
def __len__(self):
return int(reduce(operator.mul, np.ceil(bounds.size3() / shape)))
def __iter__(self):
for startpt in xyzrange(bounds.minpt, bounds.maxpt, shape):
task_shape = min2(shape.clone(), srcvol.bounds.maxpt - startpt)
yield ContrastNormalizationTask(
src_path=src_path,
dest_path=dest_path,
levels_path=levels_path,
shape=task_shape,
offset=startpt.clone(),
clip_fraction=clip_fraction,
mip=mip,
fill_missing=fill_missing,
translate=translate,
minval=minval,
maxval=maxval,
)
dvol.provenance.processing.append({
'method': {
'task': 'ContrastNormalizationTask',
'src_path': src_path,
'dest_path': dest_path,
'shape': Vec(*shape).tolist(),
'clip_fraction': clip_fraction,
'mip': mip,
'translate': Vec(*translate).tolist(),
'minval': minval,
'maxval': maxval,
'bounds': [bounds.minpt.tolist(),
bounds.maxpt.tolist()],
},
'by':
OPERATOR_CONTACT,
'date':
strftime('%Y-%m-%d %H:%M %Z'),
})
dvol.commit_provenance()
return ContrastNormalizationTaskIterator()
def create_contrast_normalization_tasks(src_path,
dest_path,
levels_path=None,
shape=None,
mip=0,
clip_fraction=0.01,
fill_missing=False,
translate=(0, 0, 0),
minval=None,
maxval=None,
bounds=None,
bounds_mip=0):
"""
Use the output of luminence levels to contrast
correct the image by stretching the histogram
to cover the full range of the data type.
"""
srcvol = CloudVolume(src_path, mip=mip)
try:
dvol = CloudVolume(dest_path, mip=mip)
except Exception: # no info file
info = copy.deepcopy(srcvol.info)
dvol = CloudVolume(dest_path, mip=mip, info=info)
dvol.info['scales'] = dvol.info['scales'][:mip + 1]
dvol.commit_info()
if bounds is None:
bounds = srcvol.bounds.clone()
if shape is None:
shape = Bbox((0, 0, 0), (2048, 2048, 64))
shape = shape.shrink_to_chunk_size(dvol.underlying).size3()
shape = Vec.clamp(shape, (1, 1, 1), bounds.size3())
shape = Vec(*shape)
downsample_scales.create_downsample_scales(dest_path,
mip=mip,
ds_shape=shape,
preserve_chunk_size=True)
dvol.refresh_info()
bounds = get_bounds(srcvol, bounds, mip, bounds_mip=bounds_mip)
class ContrastNormalizationTaskIterator(FinelyDividedTaskIterator):
def task(self, shape, offset):
return ContrastNormalizationTask(
src_path=src_path,
dest_path=dest_path,
levels_path=levels_path,
shape=shape.clone(),
offset=offset.clone(),
clip_fraction=clip_fraction,
mip=mip,
fill_missing=fill_missing,
translate=translate,
minval=minval,
maxval=maxval,
)
def on_finish(self):
dvol.provenance.processing.append({
'method': {
'task': 'ContrastNormalizationTask',
'src_path': src_path,
'dest_path': dest_path,
'shape': Vec(*shape).tolist(),
'clip_fraction': clip_fraction,
'mip': mip,
'translate': Vec(*translate).tolist(),
'minval': minval,
'maxval': maxval,
'bounds': [bounds.minpt.tolist(),
bounds.maxpt.tolist()],
},
'by':
operator_contact(),
'date':
strftime('%Y-%m-%d %H:%M %Z'),
})
dvol.commit_provenance()
return ContrastNormalizationTaskIterator(bounds, shape)
