def create_mesh(animal, mip, mse):
fileLocationManager = FileLocationManager(animal)
channel_outdir = 'color_mesh_v2'
OUTPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data,
channel_outdir)
if not os.path.exists(OUTPUT_DIR):
print(f'DIR {OUTPUT_DIR} does not exist, exiting.')
sys.exit()
cloudpath = f"file://{OUTPUT_DIR}"
cv = CloudVolume(cloudpath, mip)
workers, _ = get_cpus()
tq = LocalTaskQueue(parallel=workers)
mesh_dir = f'mesh_mip_{mip}_err_{mse}'
cv.info['mesh'] = mesh_dir
cv.commit_info()
tasks = tc.create_meshing_tasks(cv.layer_cloudpath,
mip=mip,
mesh_dir=mesh_dir,
max_simplification_error=mse)
tq.insert(tasks)
tq.execute()
tasks = tc.create_mesh_manifest_tasks(cv.layer_cloudpath,
mesh_dir=mesh_dir)
tq.insert(tasks)
tq.execute()
print("Done!")
def create_downsamples(animal, channel, suffix, downsample):
fileLocationManager = FileLocationManager(animal)
channel_outdir = f'C{channel}'
first_chunk = calculate_chunks(downsample, 0)
mips = [0, 1, 2, 3, 4, 5, 6, 7]
if downsample:
channel_outdir += 'T'
mips = [0, 1]
outpath = os.path.join(fileLocationManager.neuroglancer_data,
f'{channel_outdir}')
outpath = f'file://{outpath}'
if suffix is not None:
outpath += suffix
channel_outdir += "_rechunkme"
INPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data,
f'{channel_outdir}')
if not os.path.exists(INPUT_DIR):
print(f'DIR {INPUT_DIR} does not exist, exiting.')
sys.exit()
cloudpath = f"file://{INPUT_DIR}"
_, workers = get_cpus()
tq = LocalTaskQueue(parallel=workers)
tasks = tc.create_transfer_tasks(cloudpath,
dest_layer_path=outpath,
chunk_size=first_chunk,
mip=0,
skip_downsamples=True)
tq.insert(tasks)
tq.execute()
#mips = 7 shows good results in neuroglancer
for mip in mips:
cv = CloudVolume(outpath, mip)
chunks = calculate_chunks(downsample, mip)
factors = calculate_factors(downsample, mip)
tasks = tc.create_downsampling_tasks(cv.layer_cloudpath,
mip=mip,
num_mips=1,
factor=factors,
preserve_chunk_size=False,
compress=True,
chunk_size=chunks)
tq.insert(tasks)
tq.execute()
print("Done!")
def add_downsampled_volumes(self, chunk_size=[128, 128, 64], num_mips=4):
if self.precomputed_vol is None:
raise NotImplementedError(
'You have to call init_precomputed before calling this function.'
)
_, cpus = get_cpus()
tq = LocalTaskQueue(parallel=cpus)
tasks = tc.create_downsampling_tasks(
self.precomputed_vol.layer_cloudpath,
preserve_chunk_size=False,
num_mips=num_mips,
chunk_size=chunk_size,
compress=True)
tq.insert(tasks)
tq.execute()
def test_local_taskqueue():
tq = LocalTaskQueue(parallel=True, progress=False)
tasks = (MockTask(arg=i) for i in range(20000))
assert tq.insert(tasks) == 20000
tq = LocalTaskQueue(parallel=1, progress=False)
tasks = ((ExecutePrintTask(), [i], {'wow2': 4}) for i in range(200))
assert tq.insert(tasks) == 200
tq = LocalTaskQueue(parallel=True, progress=False)
tasks = ((ExecutePrintTask(), [i], {'wow2': 4}) for i in range(200))
assert tq.insert(tasks) == 200
tq = LocalTaskQueue(parallel=True, progress=False)
epts = [PrintTask(i) for i in range(200)]
assert tq.insert(epts) == 200
def add_rechunking(self, outpath, downsample, chunks=None):
if self.precomputed_vol is None:
raise NotImplementedError(
'You have to call init_precomputed before calling this function.'
)
cpus, _ = get_cpus()
tq = LocalTaskQueue(parallel=cpus)
outpath = f'file://{outpath}'
if chunks is None:
chunks = calculate_chunks(downsample, 0)
tasks = tc.create_transfer_tasks(self.precomputed_vol.layer_cloudpath,
dest_layer_path=outpath,
chunk_size=chunks,
skip_downsamples=True)
tq.insert(tasks)
tq.execute()
def add_segmentation_mesh(self, shape=[448, 448, 448], mip=0):
if self.precomputed_vol is None:
raise NotImplementedError(
'You have to call init_precomputed before calling this function.'
)
_, cpus = get_cpus()
tq = LocalTaskQueue(parallel=cpus)
tasks = tc.create_meshing_tasks(
self.precomputed_vol.layer_cloudpath,
mip=mip,
max_simplification_error=40,
shape=shape,
compress=True) # The first phase of creating mesh
tq.insert(tasks)
tq.execute()
# It should be able to incoporated to above tasks, but it will give a weird bug. Don't know the reason
tasks = tc.create_mesh_manifest_tasks(
self.precomputed_vol.layer_cloudpath
) # The second phase of creating mesh
tq.insert(tasks)
tq.execute()
def create_structures(animal):
"""
This is the important method called from main. This does all the work.
Args:
animal: string to identify the animal/stack
Returns:
Nothing, creates a directory of the precomputed volume. Copy this directory somewhere apache can read it. e.g.,
/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/
"""
sqlController = SqlController(animal)
fileLocationManager = FileLocationManager(animal)
# Set all relevant directories
THUMBNAIL_PATH = os.path.join(fileLocationManager.prep, 'CH1', 'thumbnail')
CSV_PATH = '/net/birdstore/Active_Atlas_Data/data_root/atlas_data/foundation_brain_annotations'
CLEANED = os.path.join(fileLocationManager.prep, 'CH1', 'thumbnail_cleaned')
PRECOMPUTE_PATH = f'/net/birdstore/Active_Atlas_Data/data_root/atlas_data/foundation_brain_annotations/{animal}'
width = sqlController.scan_run.width
height = sqlController.scan_run.height
width = int(width * SCALING_FACTOR)
height = int(height * SCALING_FACTOR)
aligned_shape = np.array((width, height))
THUMBNAILS = sorted(os.listdir(THUMBNAIL_PATH))
num_section = len(THUMBNAILS)
structure_dict = sqlController.get_structures_dict()
csvfile = os.path.join(CSV_PATH, f'{animal}_annotation.csv')
hand_annotations = pd.read_csv(csvfile)
hand_annotations['vertices'] = hand_annotations['vertices'] \
.apply(lambda x: x.replace(' ', ','))\
.apply(lambda x: x.replace('\n',','))\
.apply(lambda x: x.replace(',]',']'))\
.apply(lambda x: x.replace(',,', ','))\
.apply(lambda x: x.replace(',,', ','))\
.apply(lambda x: x.replace(',,', ',')).apply(lambda x: x.replace(',,', ','))
hand_annotations['vertices'] = hand_annotations['vertices'].apply(lambda x: ast.literal_eval(x))
structures = list(hand_annotations['name'].unique())
section_structure_vertices = defaultdict(dict)
for structure in tqdm(structures):
contour_annotations, first_sec, last_sec = get_contours_from_annotations(animal, structure, hand_annotations, densify=4)
for section in contour_annotations:
section_structure_vertices[section][structure] = contour_annotations[section][structure][1]
##### Reproduce create_clean transform
section_offset = {}
for file_name in tqdm(THUMBNAILS):
filepath = os.path.join(THUMBNAIL_PATH, file_name)
img = io.imread(filepath)
section = int(file_name.split('.')[0])
section_offset[section] = (aligned_shape - img.shape[:2][::-1]) // 2
##### Reproduce create_alignment transform
image_name_list = sorted(os.listdir(CLEANED))
anchor_idx = len(image_name_list) // 2
transformation_to_previous_sec = {}
for i in range(1, len(image_name_list)):
fixed_fn = os.path.splitext(image_name_list[i - 1])[0]
moving_fn = os.path.splitext(image_name_list[i])[0]
transformation_to_previous_sec[i] = load_consecutive_section_transform(animal, moving_fn, fixed_fn)
transformation_to_anchor_sec = {}
# Converts every transformation
for moving_idx in range(len(image_name_list)):
if moving_idx == anchor_idx:
transformation_to_anchor_sec[image_name_list[moving_idx]] = np.eye(3)
elif moving_idx < anchor_idx:
T_composed = np.eye(3)
for i in range(anchor_idx, moving_idx, -1):
T_composed = np.dot(np.linalg.inv(transformation_to_previous_sec[i]), T_composed)
transformation_to_anchor_sec[image_name_list[moving_idx]] = T_composed
else:
T_composed = np.eye(3)
for i in range(anchor_idx + 1, moving_idx + 1):
T_composed = np.dot(transformation_to_previous_sec[i], T_composed)
transformation_to_anchor_sec[image_name_list[moving_idx]] = T_composed
warp_transforms = create_warp_transforms(animal, transformation_to_anchor_sec, 'thumbnail', 'thumbnail')
ordered_transforms = sorted(warp_transforms.items())
section_transform = {}
for section, transform in ordered_transforms:
section_num = int(section.split('.')[0])
transform = np.linalg.inv(transform)
section_transform[section_num] = transform
##### Alignment of annotation coordinates
keys = [k for k in structure_dict.keys()]
# This missing_sections will need to be manually built up from Beth's spreadsheet
missing_sections = {k: [117] for k in keys}
fill_sections = defaultdict(dict)
pr5_sections = []
other_structures = set()
volume = np.zeros((aligned_shape[1], aligned_shape[0], num_section), dtype=np.uint8)
for section in section_structure_vertices:
template = np.zeros((aligned_shape[1], aligned_shape[0]), dtype=np.uint8)
for structure in section_structure_vertices[section]:
points = np.array(section_structure_vertices[section][structure])
points = points // 32
points = points + section_offset[section] # create_clean offset
points = transform_create_alignment(points, section_transform[section]) # create_alignment transform
points = points.astype(np.int32)
try:
missing_list = missing_sections[structure]
except:
missing_list = []
if section in missing_list:
fill_sections[structure][section] = points
if 'pr5' in structure.lower():
pr5_sections.append(section)
try:
# color = colors[structure.upper()]
color = structure_dict[structure][1] # structure dict returns a list of [description, color]
# for each key
except:
color = 255
other_structures.add(structure)
cv2.polylines(template, [points], True, color, 2, lineType=cv2.LINE_AA)
volume[:, :, section - 1] = template
# fill up missing sections
template = np.zeros((aligned_shape[1], aligned_shape[0]), dtype=np.uint8)
for structure, v in fill_sections.items():
color = structure_dict[structure][1]
for section, points in v.items():
cv2.polylines(template, [points], True, color, 2, lineType=cv2.LINE_AA)
volume[:, :, section] = template
volume_filepath = os.path.join(CSV_PATH, f'{animal}_annotations.npy')
volume = np.swapaxes(volume, 0, 1)
print('Saving:', volume_filepath, 'with shape', volume.shape)
with open(volume_filepath, 'wb') as file:
np.save(file, volume)
# now use 9-1 notebook to convert to a precomputed.
# Voxel resolution in nanometer (how much nanometer each element in numpy array represent)
resol = (14464, 14464, 20000)
# Voxel offset
offset = (0, 0, 0)
# Layer type
layer_type = 'segmentation'
# number of channels
num_channels = 1
# segmentation properties in the format of [(number1, label1), (number2, label2) ...]
# where number is an integer that is in the volume and label is a string that describes that segmenetation
segmentation_properties = [(number, f'{structure}: {label}') for structure, (label, number) in structure_dict.items()]
extra_structures = ['Pr5', 'VTg', 'DRD', 'IF', 'MPB', 'Op', 'RPC', 'LSO', 'MVe', 'CnF',
'pc', 'DTgC', 'LPB', 'Pr5DM', 'DTgP', 'RMC', 'VTA', 'IPC', 'DRI', 'LDTg',
'IPA', 'PTg', 'DTg', 'IPL', 'SuVe', 'Sol', 'IPR', '8n', 'Dk', 'IO',
'Cb', 'Pr5VL', 'APT', 'Gr', 'RR', 'InC', 'X', 'EW']
segmentation_properties += [(len(structure_dict) + index + 1, structure) for index, structure in enumerate(extra_structures)]
cloudpath = f'file://{PRECOMPUTE_PATH}'
info = CloudVolume.create_new_info(
num_channels = num_channels,
layer_type = layer_type,
data_type = str(volume.dtype), # Channel images might be 'uint8'
encoding = 'raw', # raw, jpeg, compressed_segmentation, fpzip, kempressed
resolution = resol, # Voxel scaling, units are in nanometers
voxel_offset = offset, # x,y,z offset in voxels from the origin
chunk_size = [64, 64, 64], # units are voxels
volume_size = volume.shape, # e.g. a cubic millimeter dataset
)
vol = CloudVolume(cloudpath, mip=0, info=info, compress=False)
vol.commit_info()
vol[:, :, :] = volume[:, :, :]
vol.info['segment_properties'] = 'names'
vol.commit_info()
segment_properties_path = os.path.join(PRECOMPUTE_PATH, 'names')
os.makedirs(segment_properties_path, exist_ok=True)
info = {
"@type": "neuroglancer_segment_properties",
"inline": {
"ids": [str(number) for number, label in segmentation_properties],
"properties": [{
"id": "label",
"description": "Name of structures",
"type": "label",
"values": [str(label) for number, label in segmentation_properties]
}]
}
}
print('Creating names in', segment_properties_path)
with open(os.path.join(segment_properties_path, 'info'), 'w') as file:
json.dump(info, file, indent=2)
# Setting parallel to a number > 1 hangs the script. It still runs fast with parallel=1
tq = LocalTaskQueue(parallel=1)
tasks = tc.create_downsampling_tasks(cloudpath, compress=False) # Downsample the volumes
tq.insert(tasks)
tq.execute()
print('Finished')
try:
print(job)
except Exception as exc:
print(f'generated an exception: {exc}')
elif step == 'step2': # transfer tasks
orig_vol = CloudVolume(f'file://{orig_layer_dir}')
first_chunk = calculate_chunks(downsample='full', mip=0)
tq = LocalTaskQueue(parallel=cpus)
tasks = tc.create_transfer_tasks(orig_vol.cloudpath,
dest_layer_path=rechunked_cloudpath,
chunk_size=first_chunk,
mip=0,
skip_downsamples=True)
print(len(tasks))
tq.insert(tasks)
tq.execute()
elif step == 'step3': # downsampling
print("step 3, downsampling")
tq = LocalTaskQueue(parallel=cpus)
downsample = "full"
mips = [0, 1, 2, 3, 4]
for mip in mips:
print(f"Mip: {mip}")
cv = CloudVolume(rechunked_cloudpath, mip)
chunks = calculate_chunks(downsample, mip)
factors = calculate_factors(downsample, mip)
print(f"Chunk size: {chunks}")
print(f"Downsample factors: {factors}")
tasks = tc.create_downsampling_tasks(cv.layer_cloudpath,
def create_layer(animal, id, start, debug):
"""
This is the important method called from main. This does all the work.
Args:
animal: string to identify the animal/stack
Returns:
Nothing, creates a directory of the precomputed volume. Copy this directory somewhere apache can read it. e.g.,
/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/
"""
# Set all relevant directories
INPUT = '/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/DK52/preps/CH3/thumbnail_aligned'
OUTPUT = '/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/DK52/preps/CH3/shapes'
PRECOMPUTE_PATH = f'/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/{animal}/neuroglancer_data/shapes'
ATLAS_DIR = '/net/birdstore/Active_Atlas_Data/data_root/atlas_data'
outpath = os.path.join(ATLAS_DIR, 'shapes', animal)
os.makedirs(OUTPUT, exist_ok=True)
os.makedirs(outpath, exist_ok=True)
files = os.listdir(INPUT)
num_sections = len(files)
midpoint = num_sections // 2
midfilepath = os.path.join(INPUT, files[midpoint])
midfile = io.imread(midfilepath, img_num=0)
height = midfile.shape[0]
width = midfile.shape[1]
structures = set()
colors = {'infrahypoglossal': 200, 'perifacial': 210, 'suprahypoglossal': 220}
aligned_shape = np.array((width, height))
section_structure_vertices = defaultdict(dict)
with connection.cursor() as cursor:
sql = """select el.frame + %s as section, el.points, elab.name
from engine_labeledshape el
inner join engine_job ej on el.job_id = ej.id
inner join engine_label elab on el.label_id = elab.id
where elab.task_id = %s
order by elab.name, el.frame"""
cursor.execute(sql, [start, id])
rows = cursor.fetchall()
for row in rows:
section = row[0]
pts = row[1]
structure = row[2]
structures.add(structure)
pts = np.array([tuple(map(float, x.split())) for x in pts.strip().split(',')])
vertices = pts.reshape(pts.shape[0]//2, 2).astype(np.float64)
addme = vertices[0].reshape(1,2)
vertices = np.concatenate((vertices,addme), axis=0)
lp = vertices.shape[0]
if lp > 2:
new_len = max(lp, 100)
vertices = interpolate(vertices, new_len)
section_structure_vertices[section][structure] = vertices
##### Alignment of annotation coordinates
volume = np.zeros((aligned_shape[1], aligned_shape[0], num_sections), dtype=np.uint8)
#for section in section_structure_vertices:
for section, file in enumerate(files):
template = np.zeros((aligned_shape[1], aligned_shape[0]), dtype=np.uint8)
for structure in section_structure_vertices[section]:
points = section_structure_vertices[section][structure]
print(section, structure, points.shape, np.amax(points), np.amin(points))
cv2.fillPoly(template, [points.astype(np.int32)], colors[structure])
outfile = str(section).zfill(3) + ".tif"
imgpath = os.path.join(OUTPUT, outfile)
cv2.imwrite(imgpath, template)
volume[:, :, section - 1] = template
print(colors)
sys.exit()
volume_filepath = os.path.join(outpath, f'{animal}_shapes.npy')
volume = np.swapaxes(volume, 0, 1)
print('Saving:', volume_filepath, 'with shape', volume.shape)
#with open(volume_filepath, 'wb') as file:
# np.save(file, volume)
# now use 9-1 notebook to convert to a precomputed.
# Voxel resolution in nanometer (how much nanometer each element in numpy array represent)
resol = (14464, 14464, 20000)
# Voxel offset
offset = (0, 0, 0)
# Layer type
layer_type = 'segmentation'
# number of channels
num_channels = 1
# segmentation properties in the format of [(number1, label1), (number2, label2) ...]
# where number is an integer that is in the volume and label is a string that describes that segmenetation
segmentation_properties = [(len(structures) + index + 1, structure) for index, structure in enumerate(structures)]
cloudpath = f'file://{PRECOMPUTE_PATH}'
info = CloudVolume.create_new_info(
num_channels = num_channels,
layer_type = layer_type,
data_type = str(volume.dtype), # Channel images might be 'uint8'
encoding = 'raw', # raw, jpeg, compressed_segmentation, fpzip, kempressed
resolution = resol, # Voxel scaling, units are in nanometers
voxel_offset = offset, # x,y,z offset in voxels from the origin
chunk_size = [64, 64, 64], # units are voxels
volume_size = volume.shape, # e.g. a cubic millimeter dataset
)
vol = CloudVolume(cloudpath, mip=0, info=info, compress=True)
vol.commit_info()
vol[:, :, :] = volume[:, :, :]
vol.info['segment_properties'] = 'names'
vol.commit_info()
segment_properties_path = os.path.join(PRECOMPUTE_PATH, 'names')
os.makedirs(segment_properties_path, exist_ok=True)
info = {
"@type": "neuroglancer_segment_properties",
"inline": {
"ids": [str(number) for number, label in segmentation_properties],
"properties": [{
"id": "label",
"description": "Name of structures",
"type": "label",
"values": [str(label) for number, label in segmentation_properties]
}]
}
}
print('Creating names in', segment_properties_path)
with open(os.path.join(segment_properties_path, 'info'), 'w') as file:
json.dump(info, file, indent=2)
# Setting parallel to a number > 1 hangs the script. It still runs fast with parallel=1
tq = LocalTaskQueue(parallel=1)
tasks = tc.create_downsampling_tasks(cloudpath, compress=True) # Downsample the volumes
tq.insert(tasks)
tq.execute()
print('Finished')