def main():
# Hard-coded parameters
prod = 'emdata4:8900'
master = (prod, find_master(prod))
master_seg = (*master, 'segmentation')
# I accidentally corrupted the labelindex of bodies in this region
patch_box = 20480 + np.array([[0, 0, 0], [1024, 1024, 1024]])
with Timer("Fetching supervoxels", logger):
boxes = boxes_from_grid(patch_box, Grid((64, 64, 6400)), clipped=True)
sv_sets = compute_parallel(partial(_fetch_svs, master_seg),
boxes,
processes=32,
ordered=False,
leave_progress=True)
svs = set(chain(*sv_sets)) - set([0])
bodies = set(fetch_mapping(*master_seg, svs))
with Timer(f"Repairing {len(bodies)} labelindexes", logger):
compute_parallel(partial(_repair_index, master_seg),
bodies,
processes=32,
ordered=False,
leave_progress=True)
print("DONE.")
def copy_vnc_subvolume(box_zyx,
copy_grayscale=True,
copy_segmentation=True,
chunk_shape=(64, 64, 2048)):
assert not (box_zyx % 64).any(), \
"Only 64px block-aligned volumes can be copied."
import numpy as np
from neuclease.util import boxes_from_grid, tqdm_proxy, round_box
from neuclease.dvid import find_master, fetch_raw, post_raw, fetch_subvol, post_labelmap_voxels
vnc_master = ('emdata4:8200', find_master('emdata4:8200'))
NUM_SCALES = 8
num_voxels = np.prod(box_zyx[1] - box_zyx[0])
if copy_grayscale:
logger.info(
f"Copying grayscale from box {box_zyx[:,::-1].tolist()} ({num_voxels/1e6:.1f} Mvox) for {NUM_SCALES} scales"
)
for scale in tqdm_proxy(range(NUM_SCALES)):
if scale == 0:
input_name = 'grayscalejpeg'
output_name = 'local-grayscalejpeg'
else:
input_name = f'grayscalejpeg_{scale}'
output_name = f'local-grayscalejpeg_{scale}'
scaled_box_zyx = np.maximum(box_zyx // 2**scale, 1)
scaled_box_zyx = round_box(scaled_box_zyx, 64, 'out')
for chunk_box in tqdm_proxy(boxes_from_grid(scaled_box_zyx,
chunk_shape,
clipped=True),
leave=False):
chunk = fetch_subvol(*vnc_master,
input_name,
chunk_box,
progress=False)
post_raw(*vnc_master, output_name, chunk_box[0], chunk)
if copy_segmentation:
logger.info(
f"Copying segmentation from box {box_zyx[:,::-1].tolist()} ({num_voxels/1e6:.2f} Mvox)"
)
for chunk_box in tqdm_proxy(
boxes_from_grid(box_zyx, chunk_shape, clipped=True)):
chunk = fetch_raw(*vnc_master,
'segmentation',
chunk_box,
dtype=np.uint64)
post_labelmap_voxels(*vnc_master,
'local-segmentation',
chunk_box[0],
chunk,
downres=True)
# TODO: Update label indexes?
logger.info("DONE")
def _generate_and_store_mesh():
try:
dvid = request.args['dvid']
body = request.args['body']
except KeyError as ex:
return Response(f"Missing required parameter: {ex.args[0]}", 400)
segmentation = request.args.get('segmentation', 'segmentation')
mesh_kv = request.args.get('mesh_kv', f'{segmentation}_meshes')
uuid = request.args.get('uuid') or find_master(dvid)
if not uuid:
uuid = find_master(dvid)
scale = request.args.get('scale')
if scale is not None:
scale = int(scale)
smoothing = int(request.args.get('smoothing', 2))
# Note: This is just the effective desired decimation assuming scale-1 data.
# If we're forced to select a higher scale than scale-1, then we'll increase
# this number to compensate.
decimation = float(request.args.get('decimation', 0.1))
user = request.args.get('u')
user = user or request.args.get('user', "UNKNOWN")
# TODO: The global cache of DVID sessions should store authentication info
# and use it as part of the key lookup, to avoid creating a new dvid
# session for every single cloud call!
dvid_session = default_dvid_session('cloud-meshgen', user)
auth = request.headers.get('Authorization')
if auth:
dvid_session = copy.deepcopy(dvid_session)
dvid_session.headers['Authorization'] = auth
with Timer(f"Body {body}: Fetching coarse sparsevol"):
svc_ranges = fetch_sparsevol_coarse(dvid,
uuid,
segmentation,
body,
format='ranges',
session=dvid_session)
#svc_mask, _svc_box = fetch_sparsevol_coarse(dvid, uuid, segmentation, body, format='mask', session=dvid_session)
#np.save(f'mask-{body}-svc.npy', svc_mask)
box_s6 = rle_ranges_box(svc_ranges)
box_s0 = box_s6 * (2**6)
logger.info(f"Body {body}: Bounding box: {box_s0[:, ::-1].tolist()}")
if scale is None:
# Use scale 1 if possible or a higher scale
# if necessary due to bounding-box RAM usage.
scale = max(1, select_scale(box_s0))
if scale > 1:
# If we chose a low-res scale, then we
# can reduce the decimation as needed.
decimation = min(1.0, decimation * 4**(scale - 1))
with Timer(f"Body {body}: Fetching scale-{scale} sparsevol"):
mask, mask_box = fetch_sparsevol(dvid,
uuid,
segmentation,
body,
scale=scale,
format='mask',
session=dvid_session)
# np.save(f'mask-{body}-s{scale}.npy', mask)
# Pad with a thin halo of zeros to avoid holes in the mesh at the box boundary
mask = np.pad(mask, 1)
mask_box += [(-1, -1, -1), (1, 1, 1)]
with Timer(f"Body {body}: Computing mesh"):
# The 'ilastik' marching cubes implementation supports smoothing during mesh construction.
mesh = Mesh.from_binary_vol(mask,
mask_box * VOXEL_NM * (2**scale),
smoothing_rounds=smoothing)
logger.info(f"Body {body}: Decimating mesh at fraction {decimation}")
mesh.simplify(decimation)
logger.info(f"Body {body}: Preparing ngmesh")
mesh_bytes = mesh.serialize(fmt='ngmesh')
if scale > 2:
logger.info(f"Body {body}: Not storing to dvid (scale > 2)")
else:
with Timer(
f"Body {body}: Storing {body}.ngmesh in DVID ({len(mesh_bytes)/MB:.1f} MB)"
):
try:
post_key(dvid,
uuid,
mesh_kv,
f"{body}.ngmesh",
mesh_bytes,
session=dvid_session)
except HTTPError as ex:
err = ex.response.content.decode('utf-8')
if 'locked node' in err:
logger.info(
"Body {body}: Not storing to dvid (uuid {uuid[:4]} is locked)."
)
else:
logger.warning("Mesh could not be cached to dvid:\n{err}")
r = make_response(mesh_bytes)
r.headers.set('Content-Type', 'application/octet-stream')
return r
def main():
configure_default_logging()
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument(
'--use-mapping',
action='store_true',
help=
'Use in-memory map + /exists instead of /missing, as described in the general help text above.'
)
parser.add_argument(
'--output',
'-o',
default='missing-from-tsv.csv',
help='Where to write the output CSV (default: missing-from-tsv.csv)')
parser.add_argument(
'--kafka-timestamp',
'-k',
type=str,
help='Alternative to providing your own bodies list.\n'
'Use the kafka log automatically determine the list of bodies that have changed after the given timestamp.\n'
'Examples: -k="2018-11-22" -k="2018-11-22 17:34:00"')
parser.add_argument('server', help='dvid server, e.g. emdata3:8900')
parser.add_argument(
'uuid',
help=
'dvid node to analyze or "master" for the latest master branch uuid')
parser.add_argument(
'tsv_instance',
help="Name of a tarsupervoxels instance, e.g. segmentation_sv_meshes.\n"
"Must be sync'd to a labelmap (segmentation) instance.")
parser.add_argument(
'bodies_csv',
nargs='?',
help='CSV containing a column named "body", which will be read.\n'
'If no "body" column exists, the first column is used, regardless of the name.\n'
'(Omit this arg if you are using --kafka-timestamp)')
args = parser.parse_args()
if not (bool(args.kafka_timestamp) ^ bool(args.bodies_csv)):
print(
"You must provide either --kafka-timestamp or a bodies list (not both)",
file=sys.stderr)
sys.exit(1)
if args.uuid == "master":
args.uuid = find_master(args.server)
# Determine segmentation instance
info = fetch_instance_info(args.server, args.uuid, args.tsv_instance)
seg_instance = info["Base"]["Syncs"][0]
kafka_msgs = None
if args.bodies_csv:
if 'body' in read_csv_header(args.bodies_csv):
bodies = pd.read_csv(args.bodies_csv)['body'].drop_duplicates()
else:
# Just read the first column, no matter what it's named
bodies = read_csv_col(args.bodies_csv, 0,
np.uint64).drop_duplicates()
elif args.kafka_timestamp:
# Validate timestamp format before fetching kafka log, which takes a while.
parse_timestamp(args.kafka_timestamp)
kafka_msgs = read_kafka_messages(args.server, args.uuid, seg_instance)
filtered_kafka_msgs = filter_kafka_msgs_by_timerange(
kafka_msgs, min_timestamp=args.kafka_timestamp)
new_bodies, changed_bodies, _removed_bodies, new_supervoxels, _deleted_svs = compute_affected_bodies(
filtered_kafka_msgs)
sv_split_bodies = set(
fetch_mapping(args.server, args.uuid, seg_instance,
new_supervoxels)) - set([0])
bodies = set(chain(new_bodies, changed_bodies, sv_split_bodies))
bodies = np.fromiter(bodies, np.uint64)
bodies.sort()
else:
raise AssertionError("Shouldn't get here.")
if args.use_mapping:
missing_entries = check_tarsupervoxels_status_via_exists(
args.server,
args.uuid,
args.tsv_instance,
bodies,
seg_instance,
kafka_msgs=kafka_msgs)
else:
missing_entries = check_tarsupervoxels_status_via_missing(
args.server, args.uuid, args.tsv_instance, bodies)
logger.info(f"Writing to {args.output}")
missing_entries.to_csv(args.output, index=True, header=True)
logging.info("DONE")
from cloudvolume.lib import Bbox
from neuclease import dvid
from chunkflow.chunk import Chunk
# these example parameters are from the example of [neuclease](https://github.com/janelia-flyem/neuclease)
supervoxels = False
server = "http://hemibrain-dvid.janelia.org:80"
server = server.strip('/')
if ':' not in server[-5:]:
# add a default port
server += ':8000'
instance = "segmentation"
uuid = dvid.find_master(server)
# uuid = "20631f"
print('server: ', server)
print('instance: ', instance)
print('uuid: ', uuid)
def execute(bbox: Bbox):
print('bounding box: ', bbox)
box = [tuple(bbox.minpt), tuple(bbox.maxpt)]
subvol = dvid.fetch_labelmap_voxels(server,
uuid,
instance,
box,
scale=0,
supervoxels=supervoxels)
# print('cutout volume: \n', subvol)
def locate_body():
"""
Locate a body and return an arbitrary point that lies within it.
Downloads the coarse sparsevol DVID, selects the "middle" (in the
scan-order sense) block within the sparsevol representation,
and downloads that block of segmentation at scale-0.
Then selects the "middle" coordinate within the voxels of that
block which match your body ID.
Note: The returned point corresponds to the middle of the
body's sparse representation, but is otherwise arbitrary.
It is not the center of mass, nor is necessarily near the center
of the body's bounding box.
Pass DVID Authorization token via the 'Authorization' header,
which will be forwarded to dvid requests.
All other parameters should be passed via the query string.
Parameters
----------
dvid:
dvid server. Required.
uuid:
Dvid uuid to read from and write to.
segmentation:
Name of the labelmap segmentation instance to read from.
Default: 'segmentation'
body:
Body ID.
user:
The user name associated with this request.
Will be forwarded to dvid requests.
Returns:
JSON [x,y,z]
"""
try:
dvid = request.args['dvid']
body = request.args['body']
except KeyError as ex:
return Response(f"Missing required parameter: {ex.args[0]}", 400)
body = int(body)
uuid = request.args.get('uuid') or find_master(dvid)
segmentation = request.args.get('segmentation', 'segmentation')
try:
supervoxels = request.args.get('segmentation', 'false').lower()
supervoxels = {'true': True, 'false': False}[supervoxels]
except KeyError:
return Response(
f"Invalid argument for 'supervoxels' parameter: '{supervoxels}'")
user = request.args.get('u') or request.args.get('user', "UNKNOWN")
# TODO: The global cache of DVID sessions should store authentication info
# and use it as part of the key lookup, to avoid creating a new dvid
# session for every single cloud call!
dvid_session = default_dvid_session('cloud-meshgen', user)
auth = request.headers.get('Authorization')
if auth:
dvid_session = copy.deepcopy(dvid_session)
dvid_session.headers['Authorization'] = auth
coord_zyx = generate_sample_coordinate(dvid,
uuid,
segmentation,
body,
supervoxels,
session=dvid_session)
coord_xyz = coord_zyx.tolist()[::-1]
return jsonify(coord_xyz)
def tag_soma_anchors(server,
uuid,
soma_instance,
seg_instance="segmentation",
ann_instance=None,
dry_run=False):
"""
For every body with a soma, upgrade its body annotation to
'Soma Anchor' unless it already has that status or better.
Args:
server:
dvid server
uuid:
uuid to read and write to.
The string "master" can be used as a shortcut for the most recent node on the master branch.
soma_instance:
keyvalue instance containing soma line annotations
seg_instance:
labelmap instance
ann_instance:
body annotation keyvalue instance
dry_run:
If True, determine which bodies to upgrade, but don't write the updates.
"""
from neuclease.dvid import find_master, fetch_sphere_annotations, fetch_body_annotations, post_keyvalues, DEFAULT_BODY_STATUS_CATEGORIES
if uuid == "master":
uuid = find_master(server)
if ann_instance is None:
ann_instance = f"{seg_instance}_annotations"
logger.info("Fetching soma annotations")
soma_df = fetch_sphere_annotations(server, uuid, soma_instance,
seg_instance)
logger.info("Fetching body annotations")
ann_df = fetch_body_annotations(server, uuid, ann_instance,
soma_df['body'])
soma_df = soma_df.query('body != 0')
soma_df = soma_df.merge(ann_df['status'],
'left',
left_on='body',
right_index=True)
soma_df['status'].fillna("", inplace=True)
all_statuses = DEFAULT_BODY_STATUS_CATEGORIES
keep_statuses = all_statuses[all_statuses.index('Soma Anchor'):]
upgrade_df = soma_df.query('status not in @keep_statuses')
new_data = {}
for body in upgrade_df['body'].tolist():
try:
body_dict = copy.copy(ann_df.loc[body, 'json'])
except KeyError:
body_dict = {"body ID": body}
body_dict["status"] = "Soma Anchor"
new_data[str(body)] = body_dict
status_counts = upgrade_df['status'].value_counts().rename(
"count").rename_axis("status").to_frame().query("count > 0")
logger.info(f"Upgrading {len(upgrade_df)} statuses from:\n" +
str(status_counts))
logger.info(f"Bodies: {' '.join(new_data.keys())}")
if not dry_run:
post_keyvalues(server, uuid, ann_instance, new_data)
logger.info("DONE")
upgrade_df[['body', 'status', *'xyz']].to_csv('upgraded-soma-bodies.csv',
header=True,
index=False)
return upgrade_df