def create_mesh(animal, mip, mse):
fileLocationManager = FileLocationManager(animal)
channel_outdir = 'color_mesh_v2'
OUTPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data,
channel_outdir)
if not os.path.exists(OUTPUT_DIR):
print(f'DIR {OUTPUT_DIR} does not exist, exiting.')
sys.exit()
cloudpath = f"file://{OUTPUT_DIR}"
cv = CloudVolume(cloudpath, mip)
workers, _ = get_cpus()
tq = LocalTaskQueue(parallel=workers)
mesh_dir = f'mesh_mip_{mip}_err_{mse}'
cv.info['mesh'] = mesh_dir
cv.commit_info()
tasks = tc.create_meshing_tasks(cv.layer_cloudpath,
mip=mip,
mesh_dir=mesh_dir,
max_simplification_error=mse)
tq.insert(tasks)
tq.execute()
tasks = tc.create_mesh_manifest_tasks(cv.layer_cloudpath,
mesh_dir=mesh_dir)
tq.insert(tasks)
tq.execute()
print("Done!")
def create_downsamples(animal, channel, suffix, downsample):
fileLocationManager = FileLocationManager(animal)
channel_outdir = f'C{channel}'
first_chunk = calculate_chunks(downsample, 0)
mips = [0, 1, 2, 3, 4, 5, 6, 7]
if downsample:
channel_outdir += 'T'
mips = [0, 1]
outpath = os.path.join(fileLocationManager.neuroglancer_data,
f'{channel_outdir}')
outpath = f'file://{outpath}'
if suffix is not None:
outpath += suffix
channel_outdir += "_rechunkme"
INPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data,
f'{channel_outdir}')
if not os.path.exists(INPUT_DIR):
print(f'DIR {INPUT_DIR} does not exist, exiting.')
sys.exit()
cloudpath = f"file://{INPUT_DIR}"
_, workers = get_cpus()
tq = LocalTaskQueue(parallel=workers)
tasks = tc.create_transfer_tasks(cloudpath,
dest_layer_path=outpath,
chunk_size=first_chunk,
mip=0,
skip_downsamples=True)
tq.insert(tasks)
tq.execute()
#mips = 7 shows good results in neuroglancer
for mip in mips:
cv = CloudVolume(outpath, mip)
chunks = calculate_chunks(downsample, mip)
factors = calculate_factors(downsample, mip)
tasks = tc.create_downsampling_tasks(cv.layer_cloudpath,
mip=mip,
num_mips=1,
factor=factors,
preserve_chunk_size=False,
compress=True,
chunk_size=chunks)
tq.insert(tasks)
tq.execute()
print("Done!")
def downsample_dataset(dataset_name, from_mip=-1, num_mips=1, local=False,
n_download_workers=1, n_threads=32):
if dataset_name == "pinky":
ws_path = "gs://neuroglancer/svenmd/pinky40_v11/watershed/"
elif dataset_name == "basil":
ws_path = "gs://neuroglancer/svenmd/basil_4k_oldnet_cg/watershed/"
elif dataset_name == "pinky100":
ws_path = "gs://neuroglancer/nkem/pinky100_v0/ws/lost_no-random/bbox1_0/"
else:
raise Exception("Dataset unknown")
if local:
if n_threads == 1:
with MockTaskQueue() as task_queue:
tc.create_downsampling_tasks(task_queue, ws_path, mip=from_mip,
fill_missing=True, num_mips=num_mips,
n_download_workers=n_download_workers,
preserve_chunk_size=True)
else:
with LocalTaskQueue(parallel=n_threads) as task_queue:
tc.create_downsampling_tasks(task_queue, ws_path, mip=from_mip,
fill_missing=True, num_mips=num_mips,
n_download_workers=n_download_workers,
preserve_chunk_size=True)
else:
with TaskQueue(queue_server='sqs',
qurl="https://sqs.us-east-1.amazonaws.com/098703261575/nkem-igneous") as task_queue:
tc.create_downsampling_tasks(task_queue, ws_path, mip=from_mip,
fill_missing=True, num_mips=num_mips,
n_download_workers=n_download_workers,
preserve_chunk_size=True)
def make_demo_mesh():
# Mesh on 8 cores, use True to use all cores
cloudpath = 'file:///home/ahoag/ngdemo/demo_bucket/atlas/allenatlas_2017'
with LocalTaskQueue(parallel=8) as tq:
tasks = tc.create_meshing_tasks(cloudpath, mip=0, shape=(256, 256, 256))
tq.insert_all(tasks)
tasks = tc.create_mesh_manifest_tasks(cloudpath)
tq.insert_all(tasks)
print("Done!")
def downsample(opt):
gs_path = opt.gs_output
# Downsample
if opt.downsample:
with LocalTaskQueue(parallel=opt.parallel) as tq:
tasks = tc.create_downsampling_tasks(gs_path,
mip=0,
fill_missing=True)
tq.insert_all(tasks)
def make_mesh(vol, cores=8):
# Mesh on 8 cores, use parallel=True to use all cores
cloudpath = vol.cloudpath
with LocalTaskQueue(parallel=cores) as tq:
tasks = tc.create_meshing_tasks(cloudpath,
mip=0,
shape=(256, 256, 256))
tq.insert_all(tasks)
tasks = tc.create_mesh_manifest_tasks(cloudpath)
tq.insert_all(tasks)
print("Done!")
def main():
parser = argparse.ArgumentParser()
parser.add_argument('labels', default=None,
help="path to precomputed labels")
parser.add_argument('--verbose', '-v', action="store_true",
help="wether to use progressbar")
args = parser.parse_args()
in_path = 'file://'+args.labels
mip = 0
# First Pass: Generate Skeletons
with LocalTaskQueue(parallel=True) as tq:
tasks = tc.create_skeletonizing_tasks(
in_path, mip,
shape=(512, 512, 512),
# see Kimimaro's documentation for the below parameters
# teasar_params={ ... }, # controls skeletonization
teasar_params={
'scale': 4,
'const': 500, # physical units
'pdrf_exponent': 4,
'pdrf_scale': 100000,
'soma_detection_threshold': 1100, # physical units
'soma_acceptance_threshold': 3500, # physical units
'soma_invalidation_scale': 1.0,
'soma_invalidation_const': 300, # physical units
'max_paths': 15, # default None
},
object_ids=None, # object id mask if applicable
progress=args.verbose
)
tq.insert_all(tasks)
# Second Pass: Fuse Skeletons
tasks = tc.create_skeleton_merge_tasks(
in_path, mip,
crop=0, # in voxels
magnitude=3, # same as mesh manifests
dust_threshold=4000, # in nm
tick_threshold=6000, # in nm
delete_fragments=False
)
tq.insert_all(tasks)
skel_info_path = os.path.join(args.labels, 'skeletons_mip_0/info')
with open(skel_info_path, 'w') as f:
json.dump(
{
"@type": "neuroglancer_skeletons",
"transform": [1,0,0,0,0,1,0,0,0,0,1,0],
"vertex_attributes": []
},
f)
def add_segmentation_mesh(self, shape=[448, 448, 448], mip=0):
if self.precomputed_vol is None:
raise NotImplementedError(
'You have to call init_precomputed before calling this function.'
)
_, cpus = get_cpus()
tq = LocalTaskQueue(parallel=cpus)
tasks = tc.create_meshing_tasks(
self.precomputed_vol.layer_cloudpath,
mip=mip,
max_simplification_error=40,
shape=shape,
compress=True) # The first phase of creating mesh
tq.insert(tasks)
tq.execute()
# It should be able to incoporated to above tasks, but it will give a weird bug. Don't know the reason
tasks = tc.create_mesh_manifest_tasks(
self.precomputed_vol.layer_cloudpath
) # The second phase of creating mesh
tq.insert(tasks)
tq.execute()
def make_demo_downsample(type_vol="647", mip_start=0, num_mips=3):
cloudpath = "file://"+home_dir+"/"+brain+"/"+type_vol
with LocalTaskQueue(parallel=8) as tq:
tasks = tc.create_downsampling_tasks(
cloudpath,
mip=mip_start, # Start downsampling from this mip level (writes to next level up)
fill_missing=False, # Ignore missing chunks and fill them with black
axis="z",
num_mips=num_mips, # number of downsamples to produce. Downloaded shape is chunk_size * 2^num_mip
chunk_size=[ 128, 128, 32 ], # manually set chunk size of next scales, overrides preserve_chunk_size
preserve_chunk_size=True, # use existing chunk size, don"t halve to get more downsamples
)
tq.insert_all(tasks)
print("Done!")
def mesh(opt):
gs_path = opt.gs_output
# Mesh
if opt.mesh:
assert opt.vol_type == 'segmentation'
# Create mesh
with LocalTaskQueue(parallel=opt.parallel) as tq:
tasks = tc.create_meshing_tasks(gs_path, mip=opt.mesh_mip)
tq.insert_all(tasks)
# Manifest
with MockTaskQueue() as tq:
tasks = tc.create_mesh_manifest_tasks(gs_path)
tq.insert_all(tasks)
def add_downsampled_volumes(self, chunk_size=[128, 128, 64], num_mips=4):
if self.precomputed_vol is None:
raise NotImplementedError(
'You have to call init_precomputed before calling this function.'
)
_, cpus = get_cpus()
tq = LocalTaskQueue(parallel=cpus)
tasks = tc.create_downsampling_tasks(
self.precomputed_vol.layer_cloudpath,
preserve_chunk_size=False,
num_mips=num_mips,
chunk_size=chunk_size,
compress=True)
tq.insert(tasks)
tq.execute()
def add_rechunking(self, outpath, downsample, chunks=None):
if self.precomputed_vol is None:
raise NotImplementedError(
'You have to call init_precomputed before calling this function.'
)
cpus, _ = get_cpus()
tq = LocalTaskQueue(parallel=cpus)
outpath = f'file://{outpath}'
if chunks is None:
chunks = calculate_chunks(downsample, 0)
tasks = tc.create_transfer_tasks(self.precomputed_vol.layer_cloudpath,
dest_layer_path=outpath,
chunk_size=chunks,
skip_downsamples=True)
tq.insert(tasks)
tq.execute()
def test_local_taskqueue():
tq = LocalTaskQueue(parallel=True, progress=False)
tasks = (MockTask(arg=i) for i in range(20000))
tq.insert(tasks)
tq = LocalTaskQueue(parallel=1, progress=False)
tasks = ((ExecutePrintTask(), [i], {'wow2': 4}) for i in range(200))
tq.insert(tasks)
tq = LocalTaskQueue(parallel=True, progress=False)
tasks = ((ExecutePrintTask(), [i], {'wow2': 4}) for i in range(200))
tq.insert(tasks)
tq = LocalTaskQueue(parallel=True, progress=False)
epts = [PrintTask(i) for i in range(200)]
tq.insert(epts)
def create_structures(animal):
"""
This is the important method called from main. This does all the work.
Args:
animal: string to identify the animal/stack
Returns:
Nothing, creates a directory of the precomputed volume. Copy this directory somewhere apache can read it. e.g.,
/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/
"""
sqlController = SqlController(animal)
fileLocationManager = FileLocationManager(animal)
# Set all relevant directories
THUMBNAIL_PATH = os.path.join(fileLocationManager.prep, 'CH1', 'thumbnail')
CSV_PATH = '/net/birdstore/Active_Atlas_Data/data_root/atlas_data/foundation_brain_annotations'
CLEANED = os.path.join(fileLocationManager.prep, 'CH1', 'thumbnail_cleaned')
PRECOMPUTE_PATH = f'/net/birdstore/Active_Atlas_Data/data_root/atlas_data/foundation_brain_annotations/{animal}'
width = sqlController.scan_run.width
height = sqlController.scan_run.height
width = int(width * SCALING_FACTOR)
height = int(height * SCALING_FACTOR)
aligned_shape = np.array((width, height))
THUMBNAILS = sorted(os.listdir(THUMBNAIL_PATH))
num_section = len(THUMBNAILS)
structure_dict = sqlController.get_structures_dict()
csvfile = os.path.join(CSV_PATH, f'{animal}_annotation.csv')
hand_annotations = pd.read_csv(csvfile)
hand_annotations['vertices'] = hand_annotations['vertices'] \
.apply(lambda x: x.replace(' ', ','))\
.apply(lambda x: x.replace('\n',','))\
.apply(lambda x: x.replace(',]',']'))\
.apply(lambda x: x.replace(',,', ','))\
.apply(lambda x: x.replace(',,', ','))\
.apply(lambda x: x.replace(',,', ',')).apply(lambda x: x.replace(',,', ','))
hand_annotations['vertices'] = hand_annotations['vertices'].apply(lambda x: ast.literal_eval(x))
structures = list(hand_annotations['name'].unique())
section_structure_vertices = defaultdict(dict)
for structure in tqdm(structures):
contour_annotations, first_sec, last_sec = get_contours_from_annotations(animal, structure, hand_annotations, densify=4)
for section in contour_annotations:
section_structure_vertices[section][structure] = contour_annotations[section][structure][1]
##### Reproduce create_clean transform
section_offset = {}
for file_name in tqdm(THUMBNAILS):
filepath = os.path.join(THUMBNAIL_PATH, file_name)
img = io.imread(filepath)
section = int(file_name.split('.')[0])
section_offset[section] = (aligned_shape - img.shape[:2][::-1]) // 2
##### Reproduce create_alignment transform
image_name_list = sorted(os.listdir(CLEANED))
anchor_idx = len(image_name_list) // 2
transformation_to_previous_sec = {}
for i in range(1, len(image_name_list)):
fixed_fn = os.path.splitext(image_name_list[i - 1])[0]
moving_fn = os.path.splitext(image_name_list[i])[0]
transformation_to_previous_sec[i] = load_consecutive_section_transform(animal, moving_fn, fixed_fn)
transformation_to_anchor_sec = {}
# Converts every transformation
for moving_idx in range(len(image_name_list)):
if moving_idx == anchor_idx:
transformation_to_anchor_sec[image_name_list[moving_idx]] = np.eye(3)
elif moving_idx < anchor_idx:
T_composed = np.eye(3)
for i in range(anchor_idx, moving_idx, -1):
T_composed = np.dot(np.linalg.inv(transformation_to_previous_sec[i]), T_composed)
transformation_to_anchor_sec[image_name_list[moving_idx]] = T_composed
else:
T_composed = np.eye(3)
for i in range(anchor_idx + 1, moving_idx + 1):
T_composed = np.dot(transformation_to_previous_sec[i], T_composed)
transformation_to_anchor_sec[image_name_list[moving_idx]] = T_composed
warp_transforms = create_warp_transforms(animal, transformation_to_anchor_sec, 'thumbnail', 'thumbnail')
ordered_transforms = sorted(warp_transforms.items())
section_transform = {}
for section, transform in ordered_transforms:
section_num = int(section.split('.')[0])
transform = np.linalg.inv(transform)
section_transform[section_num] = transform
##### Alignment of annotation coordinates
keys = [k for k in structure_dict.keys()]
# This missing_sections will need to be manually built up from Beth's spreadsheet
missing_sections = {k: [117] for k in keys}
fill_sections = defaultdict(dict)
pr5_sections = []
other_structures = set()
volume = np.zeros((aligned_shape[1], aligned_shape[0], num_section), dtype=np.uint8)
for section in section_structure_vertices:
template = np.zeros((aligned_shape[1], aligned_shape[0]), dtype=np.uint8)
for structure in section_structure_vertices[section]:
points = np.array(section_structure_vertices[section][structure])
points = points // 32
points = points + section_offset[section] # create_clean offset
points = transform_create_alignment(points, section_transform[section]) # create_alignment transform
points = points.astype(np.int32)
try:
missing_list = missing_sections[structure]
except:
missing_list = []
if section in missing_list:
fill_sections[structure][section] = points
if 'pr5' in structure.lower():
pr5_sections.append(section)
try:
# color = colors[structure.upper()]
color = structure_dict[structure][1] # structure dict returns a list of [description, color]
# for each key
except:
color = 255
other_structures.add(structure)
cv2.polylines(template, [points], True, color, 2, lineType=cv2.LINE_AA)
volume[:, :, section - 1] = template
# fill up missing sections
template = np.zeros((aligned_shape[1], aligned_shape[0]), dtype=np.uint8)
for structure, v in fill_sections.items():
color = structure_dict[structure][1]
for section, points in v.items():
cv2.polylines(template, [points], True, color, 2, lineType=cv2.LINE_AA)
volume[:, :, section] = template
volume_filepath = os.path.join(CSV_PATH, f'{animal}_annotations.npy')
volume = np.swapaxes(volume, 0, 1)
print('Saving:', volume_filepath, 'with shape', volume.shape)
with open(volume_filepath, 'wb') as file:
np.save(file, volume)
# now use 9-1 notebook to convert to a precomputed.
# Voxel resolution in nanometer (how much nanometer each element in numpy array represent)
resol = (14464, 14464, 20000)
# Voxel offset
offset = (0, 0, 0)
# Layer type
layer_type = 'segmentation'
# number of channels
num_channels = 1
# segmentation properties in the format of [(number1, label1), (number2, label2) ...]
# where number is an integer that is in the volume and label is a string that describes that segmenetation
segmentation_properties = [(number, f'{structure}: {label}') for structure, (label, number) in structure_dict.items()]
extra_structures = ['Pr5', 'VTg', 'DRD', 'IF', 'MPB', 'Op', 'RPC', 'LSO', 'MVe', 'CnF',
'pc', 'DTgC', 'LPB', 'Pr5DM', 'DTgP', 'RMC', 'VTA', 'IPC', 'DRI', 'LDTg',
'IPA', 'PTg', 'DTg', 'IPL', 'SuVe', 'Sol', 'IPR', '8n', 'Dk', 'IO',
'Cb', 'Pr5VL', 'APT', 'Gr', 'RR', 'InC', 'X', 'EW']
segmentation_properties += [(len(structure_dict) + index + 1, structure) for index, structure in enumerate(extra_structures)]
cloudpath = f'file://{PRECOMPUTE_PATH}'
info = CloudVolume.create_new_info(
num_channels = num_channels,
layer_type = layer_type,
data_type = str(volume.dtype), # Channel images might be 'uint8'
encoding = 'raw', # raw, jpeg, compressed_segmentation, fpzip, kempressed
resolution = resol, # Voxel scaling, units are in nanometers
voxel_offset = offset, # x,y,z offset in voxels from the origin
chunk_size = [64, 64, 64], # units are voxels
volume_size = volume.shape, # e.g. a cubic millimeter dataset
)
vol = CloudVolume(cloudpath, mip=0, info=info, compress=False)
vol.commit_info()
vol[:, :, :] = volume[:, :, :]
vol.info['segment_properties'] = 'names'
vol.commit_info()
segment_properties_path = os.path.join(PRECOMPUTE_PATH, 'names')
os.makedirs(segment_properties_path, exist_ok=True)
info = {
"@type": "neuroglancer_segment_properties",
"inline": {
"ids": [str(number) for number, label in segmentation_properties],
"properties": [{
"id": "label",
"description": "Name of structures",
"type": "label",
"values": [str(label) for number, label in segmentation_properties]
}]
}
}
print('Creating names in', segment_properties_path)
with open(os.path.join(segment_properties_path, 'info'), 'w') as file:
json.dump(info, file, indent=2)
# Setting parallel to a number > 1 hangs the script. It still runs fast with parallel=1
tq = LocalTaskQueue(parallel=1)
tasks = tc.create_downsampling_tasks(cloudpath, compress=False) # Downsample the volumes
tq.insert(tasks)
tq.execute()
print('Finished')
done_files = set([int(z) for z in os.listdir(progress_dir)])
all_files = set(range(vol.bounds.minpt.z, vol.bounds.maxpt.z))
to_upload = [int(z) for z in list(all_files.difference(done_files))]
to_upload.sort()
print(f"Have {len(to_upload)} planes to upload")
with ProcessPoolExecutor(max_workers=cpus) as executor:
for job in executor.map(process_slice, to_upload):
try:
print(job)
except Exception as exc:
print(f'generated an exception: {exc}')
elif step == 'step2': # transfer tasks
orig_vol = CloudVolume(f'file://{orig_layer_dir}')
first_chunk = calculate_chunks(downsample='full', mip=0)
tq = LocalTaskQueue(parallel=cpus)
tasks = tc.create_transfer_tasks(orig_vol.cloudpath,
dest_layer_path=rechunked_cloudpath,
chunk_size=first_chunk,
mip=0,
skip_downsamples=True)
print(len(tasks))
tq.insert(tasks)
tq.execute()
elif step == 'step3': # downsampling
print("step 3, downsampling")
tq = LocalTaskQueue(parallel=cpus)
downsample = "full"
mips = [0, 1, 2, 3, 4]
cell_map = np.zeros((z_dim,y_dim,x_dim)).astype('uint16')
#fill volume
for x,y,z in xyz:
try:
cell_map[z-1:z+2,y,x] = 5000 # z dilation of a single plane
except Exception as e:
# Some cells will fall outside the volume - just how clearmap works
print(e)
#apply x y dilation
r = 2
selem = ball(r)[int(r/2)]
cell_map = np.asarray([cv2.dilate(cell_map[i], selem, iterations = 1) for i in range(cell_map.shape[0])])
done_files = set([ int(z) for z in os.listdir(progress_dir) ])
all_files = set(range(vol.bounds.minpt.z, vol.bounds.maxpt.z + 1))
to_upload = [ int(z) for z in list(all_files.difference(done_files)) ]
to_upload.sort()
print(f"Have {len(to_upload)} planes to upload")
with ProcessPoolExecutor(max_workers=16) as executor:
executor.map(process_slice, to_upload)
elif step == 'step2': # downsampling
print("step 2")
vol = CloudVolume(f'file://{layer_dir}')
tasks = make_downsample_tasks(vol,mip_start=0,num_mips=2)
with LocalTaskQueue(parallel=12) as tq:
tq.insert_all(tasks)
def ingest(data, opt, tag=None):
# Neuroglancer format
data = py_utils.to_tensor(data)
data = data.transpose((3,2,1,0))
num_channels = data.shape[-1]
shape = data.shape[:-1]
# Offset
if opt.offset is None:
opt.offset = opt.begin
# MIP level correction
if opt.gs_input and opt.in_mip > 0:
o = opt.offset
p = pow(2,opt.in_mip)
offset = (o[0]//p, o[1]//p, o[2])
else:
offset = opt.offset
# Patch offset correction (when output patch is smaller than input patch)
patch_offset = (np.array(opt.inputsz) - np.array(opt.outputsz)) // 2
offset = tuple(np.array(offset) + np.flip(patch_offset, 0))
# Create info
info = make_info(num_channels, 'image', str(data.dtype), shape,
opt.resolution, offset=offset, chunk_size=opt.chunk_size)
print(info)
gs_path = opt.gs_output
if '{}' in opt.gs_output:
if opt.keywords:
gs_path = gs_path.format(*opt.keywords)
else:
if opt.center is not None:
coord = "x{}_y{}_z{}".format(*opt.center)
coord += "_s{}-{}-{}".format(*opt.size)
else:
coord = '_'.join(['{}-{}'.format(b,e) for b,e in zip(opt.begin,opt.end)])
gs_path = gs_path.format(coord)
# Tagging
if tag is not None:
if gs_path[-1] == '/':
gs_path += tag
else:
gs_path += ('/' + tag)
print("gs_output:\n{}".format(gs_path))
cvol = cv.CloudVolume(gs_path, mip=0, info=info,
parallel=opt.parallel)
cvol[:,:,:,:] = data
cvol.commit_info()
# Downsample
if opt.downsample:
import igneous
from igneous.task_creation import create_downsampling_tasks
with LocalTaskQueue(parallel=opt.parallel) as tq:
# create_downsampling_tasks(tq, gs_path, mip=0, fill_missing=True)
tasks = create_downsampling_tasks(gs_path, mip=0, fill_missing=True)
tq.insert_all(tasks)
def create_layer(animal, id, start, debug):
"""
This is the important method called from main. This does all the work.
Args:
animal: string to identify the animal/stack
Returns:
Nothing, creates a directory of the precomputed volume. Copy this directory somewhere apache can read it. e.g.,
/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/
"""
# Set all relevant directories
INPUT = '/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/DK52/preps/CH3/thumbnail_aligned'
OUTPUT = '/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/DK52/preps/CH3/shapes'
PRECOMPUTE_PATH = f'/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/{animal}/neuroglancer_data/shapes'
ATLAS_DIR = '/net/birdstore/Active_Atlas_Data/data_root/atlas_data'
outpath = os.path.join(ATLAS_DIR, 'shapes', animal)
os.makedirs(OUTPUT, exist_ok=True)
os.makedirs(outpath, exist_ok=True)
files = os.listdir(INPUT)
num_sections = len(files)
midpoint = num_sections // 2
midfilepath = os.path.join(INPUT, files[midpoint])
midfile = io.imread(midfilepath, img_num=0)
height = midfile.shape[0]
width = midfile.shape[1]
structures = set()
colors = {'infrahypoglossal': 200, 'perifacial': 210, 'suprahypoglossal': 220}
aligned_shape = np.array((width, height))
section_structure_vertices = defaultdict(dict)
with connection.cursor() as cursor:
sql = """select el.frame + %s as section, el.points, elab.name
from engine_labeledshape el
inner join engine_job ej on el.job_id = ej.id
inner join engine_label elab on el.label_id = elab.id
where elab.task_id = %s
order by elab.name, el.frame"""
cursor.execute(sql, [start, id])
rows = cursor.fetchall()
for row in rows:
section = row[0]
pts = row[1]
structure = row[2]
structures.add(structure)
pts = np.array([tuple(map(float, x.split())) for x in pts.strip().split(',')])
vertices = pts.reshape(pts.shape[0]//2, 2).astype(np.float64)
addme = vertices[0].reshape(1,2)
vertices = np.concatenate((vertices,addme), axis=0)
lp = vertices.shape[0]
if lp > 2:
new_len = max(lp, 100)
vertices = interpolate(vertices, new_len)
section_structure_vertices[section][structure] = vertices
##### Alignment of annotation coordinates
volume = np.zeros((aligned_shape[1], aligned_shape[0], num_sections), dtype=np.uint8)
#for section in section_structure_vertices:
for section, file in enumerate(files):
template = np.zeros((aligned_shape[1], aligned_shape[0]), dtype=np.uint8)
for structure in section_structure_vertices[section]:
points = section_structure_vertices[section][structure]
print(section, structure, points.shape, np.amax(points), np.amin(points))
cv2.fillPoly(template, [points.astype(np.int32)], colors[structure])
outfile = str(section).zfill(3) + ".tif"
imgpath = os.path.join(OUTPUT, outfile)
cv2.imwrite(imgpath, template)
volume[:, :, section - 1] = template
print(colors)
sys.exit()
volume_filepath = os.path.join(outpath, f'{animal}_shapes.npy')
volume = np.swapaxes(volume, 0, 1)
print('Saving:', volume_filepath, 'with shape', volume.shape)
#with open(volume_filepath, 'wb') as file:
# np.save(file, volume)
# now use 9-1 notebook to convert to a precomputed.
# Voxel resolution in nanometer (how much nanometer each element in numpy array represent)
resol = (14464, 14464, 20000)
# Voxel offset
offset = (0, 0, 0)
# Layer type
layer_type = 'segmentation'
# number of channels
num_channels = 1
# segmentation properties in the format of [(number1, label1), (number2, label2) ...]
# where number is an integer that is in the volume and label is a string that describes that segmenetation
segmentation_properties = [(len(structures) + index + 1, structure) for index, structure in enumerate(structures)]
cloudpath = f'file://{PRECOMPUTE_PATH}'
info = CloudVolume.create_new_info(
num_channels = num_channels,
layer_type = layer_type,
data_type = str(volume.dtype), # Channel images might be 'uint8'
encoding = 'raw', # raw, jpeg, compressed_segmentation, fpzip, kempressed
resolution = resol, # Voxel scaling, units are in nanometers
voxel_offset = offset, # x,y,z offset in voxels from the origin
chunk_size = [64, 64, 64], # units are voxels
volume_size = volume.shape, # e.g. a cubic millimeter dataset
)
vol = CloudVolume(cloudpath, mip=0, info=info, compress=True)
vol.commit_info()
vol[:, :, :] = volume[:, :, :]
vol.info['segment_properties'] = 'names'
vol.commit_info()
segment_properties_path = os.path.join(PRECOMPUTE_PATH, 'names')
os.makedirs(segment_properties_path, exist_ok=True)
info = {
"@type": "neuroglancer_segment_properties",
"inline": {
"ids": [str(number) for number, label in segmentation_properties],
"properties": [{
"id": "label",
"description": "Name of structures",
"type": "label",
"values": [str(label) for number, label in segmentation_properties]
}]
}
}
print('Creating names in', segment_properties_path)
with open(os.path.join(segment_properties_path, 'info'), 'w') as file:
json.dump(info, file, indent=2)
# Setting parallel to a number > 1 hangs the script. It still runs fast with parallel=1
tq = LocalTaskQueue(parallel=1)
tasks = tc.create_downsampling_tasks(cloudpath, compress=True) # Downsample the volumes
tq.insert(tasks)
tq.execute()
print('Finished')
def segment(args):
"""Run segmentation on contiguous block of affinities from CV
Args:
args: ArgParse object from main
"""
bbox_start = Vec(*args.bbox_start)
bbox_size = Vec(*args.bbox_size)
chunk_size = Vec(*args.chunk_size)
bbox = Bbox(bbox_start, bbox_start + bbox_size)
src_cv = CloudVolume(args.src_path,
fill_missing=True,
parallel=args.parallel)
info = CloudVolume.create_new_info(
num_channels=1,
layer_type='segmentation',
data_type='uint64',
encoding='raw',
resolution=src_cv.info['scales'][args.mip]['resolution'],
voxel_offset=bbox_start,
chunk_size=chunk_size,
volume_size=bbox_size,
mesh='mesh_mip_{}_err_{}'.format(args.mip,
args.max_simplification_error))
dst_cv = CloudVolume(args.dst_path, info=info, parallel=args.parallel)
dst_cv.provenance.description = 'ws+agg using waterz'
dst_cv.provenance.processing.append({
'method': {
'task': 'watershed+agglomeration',
'src_path': args.src_path,
'dst_path': args.dst_path,
'mip': args.mip,
'shape': bbox_size.tolist(),
'bounds': [
bbox.minpt.tolist(),
bbox.maxpt.tolist(),
],
},
'by': args.owner,
'date': strftime('%Y-%m-%d%H:%M %Z'),
})
dst_cv.provenance.owners = [args.owner]
dst_cv.commit_info()
dst_cv.commit_provenance()
if args.segment:
print('Downloading affinities')
aff = src_cv[bbox.to_slices()]
aff = np.transpose(aff, (3, 0, 1, 2))
aff = np.ascontiguousarray(aff, dtype=np.float32)
thresholds = [args.threshold]
print('Starting ws+agg')
seg_gen = waterz.agglomerate(aff, thresholds)
seg = next(seg_gen)
print('Deleting affinities')
del aff
print('Uploading segmentation')
dst_cv[bbox.to_slices()] = seg
if args.mesh:
print('Starting meshing')
with LocalTaskQueue(parallel=args.parallel) as tq:
tasks = tc.create_meshing_tasks(
layer_path=args.dst_path,
mip=args.mip,
shape=args.chunk_size,
simplification=True,
max_simplification_error=args.max_simplification_error,
progress=True)
tq.insert_all(tasks)
tasks = tc.create_mesh_manifest_tasks(layer_path=args.dst_path,
magnitude=args.magnitude)
tq.insert_all(tasks)
print("Meshing complete")
def test_local_taskqueue():
with LocalTaskQueue(parallel=True, progress=False) as tq:
for i in range(20000):
tq.insert(
MockTask(arg=i)
)
def upload_seg(
meta: PreviewMeta,
data: ndarray,
slack_response: SlackResponse,
transpose: bool = False,
):
from numpy import transpose as np_transpose
em = CloudVolume(meta.em_layer, mip=meta.dst_mip)
output_layer = f"{environ['GT_BUCKET_PATH']}/{meta.author}/preview/{token_hex(8)}"
info = CloudVolume.create_new_info(
num_channels=1,
layer_type="segmentation",
data_type="uint32",
encoding="raw",
resolution=em.resolution,
voxel_offset=meta.dst_bbox.minpt,
volume_size=meta.dst_bbox.size3(),
mesh=f"mesh_mip_{meta.dst_mip}_err_0",
chunk_size=(64, 64, 8),
)
dst_cv = CloudVolume(output_layer, info=info, mip=0, cdn_cache=False)
dst_cv.provenance.description = "Image directory ingest"
dst_cv.provenance.processing.append({
"method": {
"task": "ingest",
"image_path": meta.em_layer,
},
"date": str(datetime.today()),
"script": "cloud_bot",
})
dst_cv.provenance.owners = [meta.author]
dst_cv.commit_info()
dst_cv.commit_provenance()
checkpoint_notify("Processing data.", slack_response)
crop_bbox = meta.dst_bbox - meta.src_bbox.minpt
data = data[crop_bbox.to_slices()]
dst_cv[meta.dst_bbox.to_slices()] = (np_transpose(data, (1, 0, 2))
if transpose else data)
with LocalTaskQueue(parallel=16) as tq:
tasks = tc.create_downsampling_tasks(output_layer,
mip=0,
fill_missing=True,
preserve_chunk_size=True)
tq.insert_all(tasks)
checkpoint_notify("Creating meshing tasks.", slack_response)
tasks = tc.create_meshing_tasks(
output_layer,
mip=meta.dst_mip,
simplification=False,
shape=(320, 320, 40),
max_simplification_error=0,
)
tq.insert_all(tasks)
tasks = tc.create_mesh_manifest_tasks(output_layer, magnitude=1)
tq.insert_all(tasks)
return output_layer