def export_sparsevol(server, uuid, instance, neurons_df, scale=5, format='tiff', output_dir='.'):
import os
import vigra
import numpy as np
from neuclease.util import round_box, tqdm_proxy
from neuclease.dvid import fetch_sparsevol, resolve_ref, fetch_volume_box, box_to_slicing
uuid = resolve_ref(server, uuid)
# Determine the segmentation bounding box at the given scale,
# which is used as the mask shape.
seg = (server, uuid, instance)
box = round_box(fetch_volume_box(*seg), 64, 'out')
box[0] = (0,0,0)
box_scaled = box // 2**scale
# How many digits will we need in each slice file name?
digits = int(np.ceil(np.log10(box_scaled[1, 0])))
# Export a mask stack for each group.
groups = neurons_df.groupby('group', sort=False)
num_groups = neurons_df['group'].nunique()
group_prog = tqdm_proxy(groups, total=num_groups)
for group, df in group_prog:
group_prog.write(f'Group "{group}": Assembling mask')
group_mask = np.zeros(box_scaled[1], dtype=bool)
group_mask = vigra.taggedView(group_mask, 'zyx')
# Overlay each body mask in the current group
for body in tqdm_proxy(df['body'], leave=False):
body_mask, mask_box = fetch_sparsevol(*seg, body, scale=scale, format='mask')
group_mask[box_to_slicing(*mask_box)] |= body_mask
# Write out the slice files
group_prog.write(f'Group "{group}": Writing slices')
d = f'{output_dir}/{group}.stack'
os.makedirs(d, exist_ok=True)
for z in tqdm_proxy(range(group_mask.shape[0]), leave=False):
p = ('{d}/{z:' + f'0{digits}' + 'd}.{f}').format(d=d, z=z, f=format)
vigra.impex.writeImage(group_mask[z].astype(np.uint8), p)
def _generate_and_store_mesh():
try:
dvid = request.args['dvid']
body = request.args['body']
except KeyError as ex:
return Response(f"Missing required parameter: {ex.args[0]}", 400)
segmentation = request.args.get('segmentation', 'segmentation')
mesh_kv = request.args.get('mesh_kv', f'{segmentation}_meshes')
uuid = request.args.get('uuid') or find_master(dvid)
if not uuid:
uuid = find_master(dvid)
scale = request.args.get('scale')
if scale is not None:
scale = int(scale)
smoothing = int(request.args.get('smoothing', 2))
# Note: This is just the effective desired decimation assuming scale-1 data.
# If we're forced to select a higher scale than scale-1, then we'll increase
# this number to compensate.
decimation = float(request.args.get('decimation', 0.1))
user = request.args.get('u')
user = user or request.args.get('user', "UNKNOWN")
# TODO: The global cache of DVID sessions should store authentication info
# and use it as part of the key lookup, to avoid creating a new dvid
# session for every single cloud call!
dvid_session = default_dvid_session('cloud-meshgen', user)
auth = request.headers.get('Authorization')
if auth:
dvid_session = copy.deepcopy(dvid_session)
dvid_session.headers['Authorization'] = auth
with Timer(f"Body {body}: Fetching coarse sparsevol"):
svc_ranges = fetch_sparsevol_coarse(dvid,
uuid,
segmentation,
body,
format='ranges',
session=dvid_session)
#svc_mask, _svc_box = fetch_sparsevol_coarse(dvid, uuid, segmentation, body, format='mask', session=dvid_session)
#np.save(f'mask-{body}-svc.npy', svc_mask)
box_s6 = rle_ranges_box(svc_ranges)
box_s0 = box_s6 * (2**6)
logger.info(f"Body {body}: Bounding box: {box_s0[:, ::-1].tolist()}")
if scale is None:
# Use scale 1 if possible or a higher scale
# if necessary due to bounding-box RAM usage.
scale = max(1, select_scale(box_s0))
if scale > 1:
# If we chose a low-res scale, then we
# can reduce the decimation as needed.
decimation = min(1.0, decimation * 4**(scale - 1))
with Timer(f"Body {body}: Fetching scale-{scale} sparsevol"):
mask, mask_box = fetch_sparsevol(dvid,
uuid,
segmentation,
body,
scale=scale,
format='mask',
session=dvid_session)
# np.save(f'mask-{body}-s{scale}.npy', mask)
# Pad with a thin halo of zeros to avoid holes in the mesh at the box boundary
mask = np.pad(mask, 1)
mask_box += [(-1, -1, -1), (1, 1, 1)]
with Timer(f"Body {body}: Computing mesh"):
# The 'ilastik' marching cubes implementation supports smoothing during mesh construction.
mesh = Mesh.from_binary_vol(mask,
mask_box * VOXEL_NM * (2**scale),
smoothing_rounds=smoothing)
logger.info(f"Body {body}: Decimating mesh at fraction {decimation}")
mesh.simplify(decimation)
logger.info(f"Body {body}: Preparing ngmesh")
mesh_bytes = mesh.serialize(fmt='ngmesh')
if scale > 2:
logger.info(f"Body {body}: Not storing to dvid (scale > 2)")
else:
with Timer(
f"Body {body}: Storing {body}.ngmesh in DVID ({len(mesh_bytes)/MB:.1f} MB)"
):
try:
post_key(dvid,
uuid,
mesh_kv,
f"{body}.ngmesh",
mesh_bytes,
session=dvid_session)
except HTTPError as ex:
err = ex.response.content.decode('utf-8')
if 'locked node' in err:
logger.info(
"Body {body}: Not storing to dvid (uuid {uuid[:4]} is locked)."
)
else:
logger.warning("Mesh could not be cached to dvid:\n{err}")
r = make_response(mesh_bytes)
r.headers.set('Content-Type', 'application/octet-stream')
return r
def compute_mesh_and_write(body):
with Timer() as timer:
# Fetch the sparsevol to determine the bounding-box size (in scale-0 voxels)
try:
with mgr_client.access_context(server, True, 1, 0):
# sparsevol-coarse is at scale-6
coords_s6 = fetch_sparsevol_coarse(
server, uuid, instance, body, is_supervoxels)
except:
return (body, 0, 0, 0, 0.0, timer.seconds,
'error-sparsevol-coarse')
box_s6 = np.array(
[coords_s6.min(axis=0), 1 + coords_s6.max(axis=0)])
box_s0 = (2**6) * box_s6
shape_s0 = (box_s0[1] - box_s0[0])
box_voxels_s0 = np.prod(shape_s0.astype(float))
# Determine the scale we'll use.
# Solve for 'scale' in the following relationship:
#
# box_voxels_s0/((2^scale)^3) <= max_box_voxels
#
scale = log2(pow(box_voxels_s0 / max_box_voxels, 1 / 3))
scale = max(ceil(scale), min_scale)
if scale > max_scale:
raise RuntimeError(
f"Can't compute mesh for body {body}. Bounding box is {box_s0[:, ::-1].tolist()}, "
f"which is too large to fit in desired RAM, even at scale {max_scale}"
)
try:
with mgr_client.access_context(server, True, 1, 0):
coords = fetch_sparsevol(server,
uuid,
instance,
body,
is_supervoxels,
scale,
dtype=np.int16)
except:
return (body, 0, 0, 0, 0.0, timer.seconds,
'error-sparsevol')
box = box_s0 // (2**scale)
coords -= box[0]
num_voxels = len(coords)
shape = box[1] - box[0]
vol = np.zeros(shape, np.uint8)
vol[(*coords.transpose(), )] = 1
del coords
try:
mesh = Mesh.from_binary_vol(vol, box_s0)
except:
return (body, scale, num_voxels, 0, 0.0, timer.seconds,
'error-construction')
del vol
try:
mesh.laplacian_smooth(smoothing_iterations)
except:
return (body, scale, num_voxels, 0.0,
len(mesh.vertices_zyx), timer.seconds,
'error-smoothing')
fraction = decimation_fraction
if scale > min_scale:
# Since we're starting from a lower resolution than the user requested,
# Reduce the decimation we're applying accordingly.
# Since meshes are 2D surfaces, we approximate the difference in
# vertexes as the SQUARE of the difference in resolution.
fraction *= (2**(scale - min_scale))**2
fraction = min(fraction, 1.0)
try:
mesh.simplify(fraction, in_memory=True)
except:
return (body, scale, num_voxels, 0.0,
len(mesh.vertices_zyx), timer.seconds,
'error-decimation')
output_path = f'{options["output-directory"]}/{body}.{options["format"]}'
mesh.serialize(output_path)
return (body, scale, num_voxels, fraction,
len(mesh.vertices_zyx), timer.seconds, 'success')
def autogen_points(input_seg,
count,
roi,
body,
tbars,
use_skeleton,
random_seed=None,
minimum_distance=0):
"""
Generate a list of points within the input segmentation, based on the given criteria.
See the main help text below for details.
"""
if tbars and not body:
sys.exit(
"If you want to auto-generate tbar points, please specify a body.")
if not tbars and not count:
sys.exit(
"You must supply a --count unless you are generating all tbars of a body."
)
if use_skeleton:
if not body:
sys.exit(
"You must supply a body ID if you want to use a skeleton.")
if tbars:
sys.exit(
"You can't select both tbar points and skeleton points. Pick one or the other."
)
if not count and minimum_distance > 0:
sys.exit(
"You must supply a --count if you want skeleton point samples to respect the minimum distance."
)
if not count and not roi and minimum_distance == 0:
logger.warning(
"You are using all nodes of a skeleton without any ROI filter! Is that what you meant?"
)
rng = default_rng(random_seed)
if tbars:
logger.info(f"Fetching synapses for body {body}")
syn_df = fetch_annotation_label(*input_seg[:2],
'synapses',
body,
format='pandas')
tbars = syn_df.query('kind == "PreSyn"')[[*'zyx']]
if roi:
logger.info(f"Filtering tbars for roi {roi}")
determine_point_rois(*input_seg[:2], [roi], tbars)
tbars = tbars.query('roi == @roi')[[*'zyx']]
if minimum_distance:
logger.info(
f"Pruning close points from {len(tbars)} total tbar points")
tbars = prune_close_pairs(tbars, minimum_distance, rng)
logger.info(f"After pruning, {len(tbars)} tbars remain.")
if count:
count = min(count, len(tbars))
logger.info(f"Sampling {count} tbars")
choices = rng.choice(tbars.index, size=count, replace=False)
tbars = tbars.loc[choices]
logger.info(f"Returning {len(tbars)} tbar points")
return tbars
elif use_skeleton:
assert body
logger.info(f"Fetching skeleton for body {body}")
skeleton_instance = f'{input_seg[2]}_skeletons'
swc = fetch_key(*input_seg[:2], skeleton_instance, f'{body}_swc')
skeleton_df = swc_to_dataframe(swc)
skeleton_df['x'] = skeleton_df['x'].astype(int)
skeleton_df['y'] = skeleton_df['y'].astype(int)
skeleton_df['z'] = skeleton_df['z'].astype(int)
if roi:
logger.info(f"Filtering skeleton for roi {roi}")
determine_point_rois(*input_seg[:2], [roi], skeleton_df)
skeleton_df = skeleton_df.query('roi == @roi')[[*'zyx']]
if minimum_distance:
assert count
# Distance-pruning is very expensive on a huge number of close points.
# If skeleton is large, first reduce the workload by pre-selecting a
# random sample of skeleton points, and prune more from there.
if len(skeleton_df) > 10_000:
# FIXME: random_state can't use rng until I upgrade to pandas 1.0
skeleton_df = skeleton_df.sample(min(4 * count,
len(skeleton_df)),
random_state=None)
logger.info(
f"Pruning close points from {len(skeleton_df)} skeleton points"
)
prune_close_pairs(skeleton_df, minimum_distance, rng)
logger.info(
f"After pruning, {len(skeleton_df)} skeleton points remain.")
if count:
count = min(count, len(skeleton_df))
logger.info(f"Sampling {count} skeleton points")
choices = rng.choice(skeleton_df.index, size=count, replace=False)
skeleton_df = skeleton_df.loc[choices]
logger.info(f"Returning {len(skeleton_df)} skeleton points")
return skeleton_df
elif body:
assert count
if roi:
# TODO: intersect the ranges with the ROI.
raise NotImplementedError(
"Sorry, I haven't yet implemented support for "
"body+roi filtering. Pick one or the other, "
"or ask Stuart to fix this.")
logger.info(f"Fetching sparsevol for body {body}")
ranges = fetch_sparsevol(*input_seg, body, format='ranges')
logger.info("Sampling from sparsevol")
if minimum_distance > 0:
# Sample 4x extra so we still have enough after pruning.
points = sample_points_from_ranges(ranges, 4 * count, rng)
else:
points = sample_points_from_ranges(ranges, count, rng)
points = pd.DataFrame(points, columns=[*'zyx'])
if minimum_distance > 0:
logger.info(f"Pruning close points from {len(points)} body points")
prune_close_pairs(points, minimum_distance, rng)
logger.info(f"After pruning, {len(points)} body points remain")
points = points.iloc[:count]
logger.info(f"Returning {len(points)} body points")
return points
elif roi:
assert count
logger.info(f"Fetching roi {roi}")
roi_ranges = fetch_roi_roi(*input_seg[:2], roi, format='ranges')
logger.info("Sampling from ranges")
if minimum_distance > 0:
# Sample 4x extra so we can prune some out if necessary.
points_s5 = sample_points_from_ranges(roi_ranges, 4 * count, rng)
else:
points_s5 = sample_points_from_ranges(roi_ranges, count, rng)
corners_s0 = points_s5 * (2**5)
points_s0 = rng.integers(corners_s0, corners_s0 + (2**5))
points = pd.DataFrame(points_s0, columns=[*'zyx'])
if minimum_distance > 0:
logger.info(f"Pruning close points from {len(points)} roi points")
prune_close_pairs(points, minimum_distance, rng)
logger.info(
f"After pruning, points from {len(points)} roi points remain")
points = points.iloc[:count]
logger.info(f"Returning {len(points)} roi points")
return points
else:
# No body or roi specified, just sample from the whole non-zero segmentation area
assert count
logger.info("Sampling random points from entire input segmentation")
logger.info("Fetching low-res input volume")
box_s6 = round_box(fetch_volume_box(*input_seg), 2**6, 'out') // 2**6
seg_s6 = fetch_labelmap_voxels(*input_seg, box_s6, scale=6)
mask_s6 = seg_s6.astype(bool)
logger.info("Encoding segmentation as ranges")
seg_ranges = runlength_encode_mask_to_ranges(mask_s6, box_s6)
logger.info("Sampling from ranges")
if minimum_distance > 0:
# Sample 4x extra so we can prune some out if necessary.
points_s6 = sample_points_from_ranges(seg_ranges, 4 * count, rng)
else:
points_s6 = sample_points_from_ranges(seg_ranges, count, rng)
corners_s0 = points_s6 * (2**6)
points_s0 = rng.integers(corners_s0, corners_s0 + (2**6))
points = pd.DataFrame(points_s0, columns=[*'zyx'])
if minimum_distance > 0:
logger.info(
f"Pruning close points from {len(points)} segmentation points")
prune_close_pairs(points, minimum_distance, rng)
logger.info(
f"After pruning, points from {len(points)} segmentation points remain"
)
points = points.iloc[:count]
logger.info(f"Returning {len(points)} segmentation points")
return points