def get_db_structure_infos():
sqlController = SqlController('MD589')
db_structures = sqlController.get_structures_dict()
structures = {}
for structure, v in db_structures.items():
if '_' in structure:
structure = structure[0:-2]
structures[structure] = v
return structures
def create_structures(animal):
"""
This is the important method called from main. This does all the work.
Args:
animal: string to identify the animal/stack
Returns:
Nothing, creates a directory of the precomputed volume. Copy this directory somewhere apache can read it. e.g.,
/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/
"""
sqlController = SqlController(animal)
fileLocationManager = FileLocationManager(animal)
# Set all relevant directories
THUMBNAIL_PATH = os.path.join(fileLocationManager.prep, 'CH1', 'thumbnail')
CSV_PATH = '/net/birdstore/Active_Atlas_Data/data_root/atlas_data/foundation_brain_annotations'
CLEANED = os.path.join(fileLocationManager.prep, 'CH1', 'thumbnail_cleaned')
PRECOMPUTE_PATH = f'/net/birdstore/Active_Atlas_Data/data_root/atlas_data/foundation_brain_annotations/{animal}'
width = sqlController.scan_run.width
height = sqlController.scan_run.height
width = int(width * SCALING_FACTOR)
height = int(height * SCALING_FACTOR)
aligned_shape = np.array((width, height))
THUMBNAILS = sorted(os.listdir(THUMBNAIL_PATH))
num_section = len(THUMBNAILS)
structure_dict = sqlController.get_structures_dict()
csvfile = os.path.join(CSV_PATH, f'{animal}_annotation.csv')
hand_annotations = pd.read_csv(csvfile)
hand_annotations['vertices'] = hand_annotations['vertices'] \
.apply(lambda x: x.replace(' ', ','))\
.apply(lambda x: x.replace('\n',','))\
.apply(lambda x: x.replace(',]',']'))\
.apply(lambda x: x.replace(',,', ','))\
.apply(lambda x: x.replace(',,', ','))\
.apply(lambda x: x.replace(',,', ',')).apply(lambda x: x.replace(',,', ','))
hand_annotations['vertices'] = hand_annotations['vertices'].apply(lambda x: ast.literal_eval(x))
structures = list(hand_annotations['name'].unique())
section_structure_vertices = defaultdict(dict)
for structure in tqdm(structures):
contour_annotations, first_sec, last_sec = get_contours_from_annotations(animal, structure, hand_annotations, densify=4)
for section in contour_annotations:
section_structure_vertices[section][structure] = contour_annotations[section][structure][1]
##### Reproduce create_clean transform
section_offset = {}
for file_name in tqdm(THUMBNAILS):
filepath = os.path.join(THUMBNAIL_PATH, file_name)
img = io.imread(filepath)
section = int(file_name.split('.')[0])
section_offset[section] = (aligned_shape - img.shape[:2][::-1]) // 2
##### Reproduce create_alignment transform
image_name_list = sorted(os.listdir(CLEANED))
anchor_idx = len(image_name_list) // 2
transformation_to_previous_sec = {}
for i in range(1, len(image_name_list)):
fixed_fn = os.path.splitext(image_name_list[i - 1])[0]
moving_fn = os.path.splitext(image_name_list[i])[0]
transformation_to_previous_sec[i] = load_consecutive_section_transform(animal, moving_fn, fixed_fn)
transformation_to_anchor_sec = {}
# Converts every transformation
for moving_idx in range(len(image_name_list)):
if moving_idx == anchor_idx:
transformation_to_anchor_sec[image_name_list[moving_idx]] = np.eye(3)
elif moving_idx < anchor_idx:
T_composed = np.eye(3)
for i in range(anchor_idx, moving_idx, -1):
T_composed = np.dot(np.linalg.inv(transformation_to_previous_sec[i]), T_composed)
transformation_to_anchor_sec[image_name_list[moving_idx]] = T_composed
else:
T_composed = np.eye(3)
for i in range(anchor_idx + 1, moving_idx + 1):
T_composed = np.dot(transformation_to_previous_sec[i], T_composed)
transformation_to_anchor_sec[image_name_list[moving_idx]] = T_composed
warp_transforms = create_warp_transforms(animal, transformation_to_anchor_sec, 'thumbnail', 'thumbnail')
ordered_transforms = sorted(warp_transforms.items())
section_transform = {}
for section, transform in ordered_transforms:
section_num = int(section.split('.')[0])
transform = np.linalg.inv(transform)
section_transform[section_num] = transform
##### Alignment of annotation coordinates
keys = [k for k in structure_dict.keys()]
# This missing_sections will need to be manually built up from Beth's spreadsheet
missing_sections = {k: [117] for k in keys}
fill_sections = defaultdict(dict)
pr5_sections = []
other_structures = set()
volume = np.zeros((aligned_shape[1], aligned_shape[0], num_section), dtype=np.uint8)
for section in section_structure_vertices:
template = np.zeros((aligned_shape[1], aligned_shape[0]), dtype=np.uint8)
for structure in section_structure_vertices[section]:
points = np.array(section_structure_vertices[section][structure])
points = points // 32
points = points + section_offset[section] # create_clean offset
points = transform_create_alignment(points, section_transform[section]) # create_alignment transform
points = points.astype(np.int32)
try:
missing_list = missing_sections[structure]
except:
missing_list = []
if section in missing_list:
fill_sections[structure][section] = points
if 'pr5' in structure.lower():
pr5_sections.append(section)
try:
# color = colors[structure.upper()]
color = structure_dict[structure][1] # structure dict returns a list of [description, color]
# for each key
except:
color = 255
other_structures.add(structure)
cv2.polylines(template, [points], True, color, 2, lineType=cv2.LINE_AA)
volume[:, :, section - 1] = template
# fill up missing sections
template = np.zeros((aligned_shape[1], aligned_shape[0]), dtype=np.uint8)
for structure, v in fill_sections.items():
color = structure_dict[structure][1]
for section, points in v.items():
cv2.polylines(template, [points], True, color, 2, lineType=cv2.LINE_AA)
volume[:, :, section] = template
volume_filepath = os.path.join(CSV_PATH, f'{animal}_annotations.npy')
volume = np.swapaxes(volume, 0, 1)
print('Saving:', volume_filepath, 'with shape', volume.shape)
with open(volume_filepath, 'wb') as file:
np.save(file, volume)
# now use 9-1 notebook to convert to a precomputed.
# Voxel resolution in nanometer (how much nanometer each element in numpy array represent)
resol = (14464, 14464, 20000)
# Voxel offset
offset = (0, 0, 0)
# Layer type
layer_type = 'segmentation'
# number of channels
num_channels = 1
# segmentation properties in the format of [(number1, label1), (number2, label2) ...]
# where number is an integer that is in the volume and label is a string that describes that segmenetation
segmentation_properties = [(number, f'{structure}: {label}') for structure, (label, number) in structure_dict.items()]
extra_structures = ['Pr5', 'VTg', 'DRD', 'IF', 'MPB', 'Op', 'RPC', 'LSO', 'MVe', 'CnF',
'pc', 'DTgC', 'LPB', 'Pr5DM', 'DTgP', 'RMC', 'VTA', 'IPC', 'DRI', 'LDTg',
'IPA', 'PTg', 'DTg', 'IPL', 'SuVe', 'Sol', 'IPR', '8n', 'Dk', 'IO',
'Cb', 'Pr5VL', 'APT', 'Gr', 'RR', 'InC', 'X', 'EW']
segmentation_properties += [(len(structure_dict) + index + 1, structure) for index, structure in enumerate(extra_structures)]
cloudpath = f'file://{PRECOMPUTE_PATH}'
info = CloudVolume.create_new_info(
num_channels = num_channels,
layer_type = layer_type,
data_type = str(volume.dtype), # Channel images might be 'uint8'
encoding = 'raw', # raw, jpeg, compressed_segmentation, fpzip, kempressed
resolution = resol, # Voxel scaling, units are in nanometers
voxel_offset = offset, # x,y,z offset in voxels from the origin
chunk_size = [64, 64, 64], # units are voxels
volume_size = volume.shape, # e.g. a cubic millimeter dataset
)
vol = CloudVolume(cloudpath, mip=0, info=info, compress=False)
vol.commit_info()
vol[:, :, :] = volume[:, :, :]
vol.info['segment_properties'] = 'names'
vol.commit_info()
segment_properties_path = os.path.join(PRECOMPUTE_PATH, 'names')
os.makedirs(segment_properties_path, exist_ok=True)
info = {
"@type": "neuroglancer_segment_properties",
"inline": {
"ids": [str(number) for number, label in segmentation_properties],
"properties": [{
"id": "label",
"description": "Name of structures",
"type": "label",
"values": [str(label) for number, label in segmentation_properties]
}]
}
}
print('Creating names in', segment_properties_path)
with open(os.path.join(segment_properties_path, 'info'), 'w') as file:
json.dump(info, file, indent=2)
# Setting parallel to a number > 1 hangs the script. It still runs fast with parallel=1
tq = LocalTaskQueue(parallel=1)
tasks = tc.create_downsampling_tasks(cloudpath, compress=False) # Downsample the volumes
tq.insert(tasks)
tq.execute()
print('Finished')
def create_volumes(animal, create):
sqlController = SqlController(animal)
section_structure_vertices = defaultdict(dict)
csvfile = os.path.join(DATA_PATH,
'atlas_data/foundation_brain_annotations',
f'{animal}_annotation.csv')
hand_annotations = pd.read_csv(csvfile)
hand_annotations['vertices'] = hand_annotations['vertices'] \
.apply(lambda x: x.replace(' ', ',')) \
.apply(lambda x: x.replace('\n', ',')) \
.apply(lambda x: x.replace(',]', ']')) \
.apply(lambda x: x.replace(',,', ',')) \
.apply(lambda x: x.replace(',,', ',')) \
.apply(lambda x: x.replace(',,', ',')).apply(lambda x: x.replace(',,', ','))
hand_annotations['vertices'] = hand_annotations['vertices'].apply(
lambda x: ast.literal_eval(x))
structures = sqlController.get_structures_dict()
for structure, values in structures.items():
contour_annotations, first_sec, last_sec = get_contours_from_annotations(
animal, structure, hand_annotations, densify=4)
for section in contour_annotations:
section_structure_vertices[section][
structure] = contour_annotations[section][structure]
section_offset = create_clean_transform(animal)
transforms = parse_elastix(animal)
warp_transforms = create_warp_transforms(transforms, DOWNSAMPLE_FACTOR)
ordered_transforms = sorted(warp_transforms.items())
section_transform = {}
for section, transform in ordered_transforms:
section_num = int(section.split('.')[0])
transform = np.linalg.inv(transform)
section_transform[section_num] = transform
aligned_section_structure_polygons = defaultdict(dict)
for section in section_structure_vertices:
for structure in section_structure_vertices[section]:
points = np.array(section_structure_vertices[section]
[structure]) // DOWNSAMPLE_FACTOR
points = points + section_offset[section] # create_clean offset
points = transform_create_alignment(
points,
section_transform[section]) # create_alignment transform
aligned_section_structure_polygons[section][structure] = points
structure_section_vertices = defaultdict(dict)
for section, v in aligned_section_structure_polygons.items():
for structure, vertices in v.items():
structure_section_vertices[structure][section] = vertices
for structure, values in structures.items():
color = values[1]
try:
volume, xyz_offsets = create_volume(
structure_section_vertices[structure], structure, color)
except Exception as e:
print(structure, e)
continue
x, y, z = xyz_offsets
#x = xy_neuroglancer2atlas(x)
#y = xy_neuroglancer2atlas(y)
#z = section_neuroglancer2atlas(z)
print(animal, structure, volume.shape, x * 32, y * 32, z)
if create:
save_volume_origin(ATLAS_NAME, animal, structure, volume,
xyz_offsets)
CSV_DIR_PATH = '/net/birdstore/Active_Atlas_Data/data_root/atlas_data/foundation_brain_annotations'
IMAGE_DIR_PATH = f'/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/{animal}/preps/CH1/full'
sqlController = SqlController(animal)
resolution = sqlController.scan_run.resolution
aligned_shape = np.array(
(sqlController.scan_run.width, sqlController.scan_run.height))
num_section = len(os.listdir(IMAGE_DIR_PATH))
downsampled_aligned_shape = np.round(aligned_shape /
downsample_factor).astype(int)
scales = np.array([
resolution * downsample_factor, resolution * downsample_factor, 20
]) * 1000
db_structures = sqlController.get_structures_dict()
##### Get the annotation points
def get_contours_from_annotations(stack, target_structure, hand_annotations):
MD585_ng_section_min = 83
num_annotations = len(hand_annotations)
str_contours_annotation = {}
for i in range(num_annotations):
structure = hand_annotations['name'][i]
section = hand_annotations['section'][i]
first_sec = 0
last_sec = 0
