def run_offsets(animal, transforms, channel, downsample, masks, create_csv,
allen):
"""
This gets the dictionary from the above method, and uses the coordinates
to feed into the Imagemagick convert program. This method also uses a Pool to spawn multiple processes.
Args:
animal: the animal
transforms: the dictionary of file, coordinates
limit: number of jobs
Returns: nothing
"""
fileLocationManager = FileLocationManager(animal)
sqlController = SqlController(animal)
channel_dir = 'CH{}'.format(channel)
INPUT = os.path.join(fileLocationManager.prep, channel_dir,
'thumbnail_cleaned')
OUTPUT = os.path.join(fileLocationManager.prep, channel_dir,
'thumbnail_aligned')
if not downsample:
INPUT = os.path.join(fileLocationManager.prep, channel_dir,
'full_cleaned')
OUTPUT = os.path.join(fileLocationManager.prep, channel_dir,
'full_aligned')
error = test_dir(animal, INPUT, downsample=downsample, same_size=True)
if len(error) > 0 and not create_csv:
print(error)
sys.exit()
if masks:
INPUT = os.path.join(fileLocationManager.prep, 'rotated_masked')
error = test_dir(animal, INPUT, full=False, same_size=True)
if len(error) > 0:
print(error)
sys.exit()
OUTPUT = os.path.join(fileLocationManager.prep,
'rotated_aligned_masked')
os.makedirs(OUTPUT, exist_ok=True)
progress_id = sqlController.get_progress_id(downsample, channel, 'ALIGN')
sqlController.set_task(animal, progress_id)
warp_transforms = create_warp_transforms(animal, transforms, 'thumbnail',
downsample)
ordered_transforms = OrderedDict(sorted(warp_transforms.items()))
file_keys = []
r90 = np.array([[0, -1, 0], [1, 0, 0], [0, 0, 1]])
for i, (file, T) in enumerate(ordered_transforms.items()):
if allen:
ROT_DIR = os.path.join(fileLocationManager.root, animal,
'rotations')
rotfile = file.replace('tif', 'txt')
rotfile = os.path.join(ROT_DIR, rotfile)
R_cshl = np.loadtxt(rotfile)
R_cshl[0, 2] = R_cshl[0, 2] / 32
R_cshl[1, 2] = R_cshl[1, 2] / 32
R_cshl = R_cshl @ r90
R_cshl = np.linalg.inv(R_cshl)
R = T @ R_cshl
infile = os.path.join(INPUT, file)
outfile = os.path.join(OUTPUT, file)
if os.path.exists(outfile) and not create_csv:
continue
file_keys.append([i, infile, outfile, T])
if create_csv:
create_csv_data(animal, file_keys)
else:
start = timer()
workers, _ = get_cpus()
print(f'Working on {len(file_keys)} files with {workers} cpus')
with ProcessPoolExecutor(max_workers=workers) as executor:
executor.map(process_image, sorted(file_keys))
end = timer()
print(f'Create cleaned files took {end - start} seconds total',
end="\t")
print(f' { (end - start)/len(file_keys)} per file')
print('Finished')
def create_structures(animal, create):
"""
This is the important method called from main. This does all the work.
Args:
animal: string to identify the animal/stack
Returns:
Nothing, creates a directory of the precomputed volume under the neuroglancer_data dir
"""
sqlController = SqlController(animal)
fileLocationManager = FileLocationManager(animal)
# Set all relevant directories
THUMBNAIL_PATH = os.path.join(fileLocationManager.prep, 'CH1', 'thumbnail')
CSV_PATH = '/net/birdstore/Active_Atlas_Data/data_root/atlas_data/foundation_brain_annotations'
CLEANED = os.path.join(fileLocationManager.prep, 'CH1',
'thumbnail_cleaned')
OUTPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data,
'structures')
if os.path.exists(OUTPUT_DIR) and create:
shutil.rmtree(OUTPUT_DIR)
os.makedirs(OUTPUT_DIR, exist_ok=True)
resolution = sqlController.scan_run.resolution
SCALE = (10 / resolution)
resolution = int(resolution * 1000 * SCALE)
print('Resolution at', resolution)
width = sqlController.scan_run.width
height = sqlController.scan_run.height
width = int(width * SCALING_FACTOR)
height = int(height * SCALING_FACTOR)
aligned_shape = np.array((width, height))
THUMBNAILS = sorted(os.listdir(THUMBNAIL_PATH))
num_section = len(THUMBNAILS)
structure_dict = sqlController.get_structures_dict()
csvfile = os.path.join(CSV_PATH, f'{animal}_annotation.csv')
hand_annotations = pd.read_csv(csvfile)
hand_annotations['vertices'] = hand_annotations['vertices'] \
.apply(lambda x: x.replace(' ', ','))\
.apply(lambda x: x.replace('\n',','))\
.apply(lambda x: x.replace(',]',']'))\
.apply(lambda x: x.replace(',,', ','))\
.apply(lambda x: x.replace(',,', ','))\
.apply(lambda x: x.replace(',,', ',')).apply(lambda x: x.replace(',,', ','))
hand_annotations['vertices'] = hand_annotations['vertices'].apply(
lambda x: ast.literal_eval(x))
structures = list(hand_annotations['name'].unique())
section_structure_vertices = defaultdict(dict)
for structure in tqdm(structures):
contour_annotations, first_sec, last_sec = get_contours_from_annotations(
animal, structure, hand_annotations, densify=4)
for section in contour_annotations:
section_structure_vertices[section][
structure] = contour_annotations[section][structure][1]
##### Reproduce create_clean transform
section_offset = {}
for file_name in tqdm(THUMBNAILS):
filepath = os.path.join(THUMBNAIL_PATH, file_name)
img = io.imread(filepath)
section = int(file_name.split('.')[0])
section_offset[section] = (aligned_shape - img.shape[:2][::-1]) // 2
##### Reproduce create_alignment transform
image_name_list = sorted(os.listdir(CLEANED))
anchor_idx = len(image_name_list) // 2
transformation_to_previous_sec = {}
for i in range(1, len(image_name_list)):
fixed_fn = os.path.splitext(image_name_list[i - 1])[0]
moving_fn = os.path.splitext(image_name_list[i])[0]
transformation_to_previous_sec[i] = load_consecutive_section_transform(
animal, moving_fn, fixed_fn)
transformation_to_anchor_sec = {}
# Converts every transformation
for moving_idx in range(len(image_name_list)):
if moving_idx == anchor_idx:
transformation_to_anchor_sec[image_name_list[moving_idx]] = np.eye(
3)
elif moving_idx < anchor_idx:
T_composed = np.eye(3)
for i in range(anchor_idx, moving_idx, -1):
T_composed = np.dot(
np.linalg.inv(transformation_to_previous_sec[i]),
T_composed)
transformation_to_anchor_sec[
image_name_list[moving_idx]] = T_composed
else:
T_composed = np.eye(3)
for i in range(anchor_idx + 1, moving_idx + 1):
T_composed = np.dot(transformation_to_previous_sec[i],
T_composed)
transformation_to_anchor_sec[
image_name_list[moving_idx]] = T_composed
warp_transforms = create_warp_transforms(animal,
transformation_to_anchor_sec,
'thumbnail', 'thumbnail')
ordered_transforms = sorted(warp_transforms.items())
section_transform = {}
for section, transform in ordered_transforms:
section_num = int(section.split('.')[0])
transform = np.linalg.inv(transform)
section_transform[section_num] = transform
##### Alignment of annotation coordinates
keys = [k for k in structure_dict.keys()]
# This missing_sections will need to be manually built up from Beth's spreadsheet
missing_sections = {k: [117] for k in keys}
fill_sections = defaultdict(dict)
pr5_sections = []
other_structures = set()
volume = np.zeros((aligned_shape[1], aligned_shape[0], num_section),
dtype=np.uint8)
for section in tqdm(section_structure_vertices):
template = np.zeros((aligned_shape[1], aligned_shape[0]),
dtype=np.uint8)
for structure in section_structure_vertices[section]:
points = np.array(section_structure_vertices[section][structure])
points = points / 32
points = points + section_offset[section] # create_clean offset
points = transform_create_alignment(
points,
section_transform[section]) # create_alignment transform
points = points.astype(np.int32)
try:
missing_list = missing_sections[structure]
except:
missing_list = []
if section in missing_list:
fill_sections[structure][section] = points
if 'pr5' in structure.lower():
pr5_sections.append(section)
try:
# color = colors[structure.upper()]
color = structure_dict[structure][
1] # structure dict returns a list of [description, color]
# for each key
except:
color = 255
other_structures.add(structure)
cv2.polylines(template, [points],
True,
color,
2,
lineType=cv2.LINE_AA)
volume[:, :, section - 1] = template
if len(pr5_sections) > 0:
print('Pr5 sections')
for s in sorted(pr5_sections):
print(s)
# fill up missing sections
template = np.zeros((aligned_shape[1], aligned_shape[0]), dtype=np.uint8)
for structure, v in fill_sections.items():
color = structure_dict[structure][1]
for section, points in v.items():
cv2.polylines(template, [points],
True,
color,
2,
lineType=cv2.LINE_AA)
volume[:, :, section] = template
volume_filepath = os.path.join(CSV_PATH, f'{animal}_annotations.npy')
volume = np.swapaxes(volume, 0, 1)
if create:
print('Saving:', volume_filepath, 'with shape', volume.shape)
with open(volume_filepath, 'wb') as file:
np.save(file, volume)
# now use 9-1 notebook to convert to a precomputed.
# Voxel resolution in nanometer (how much nanometer each element in numpy array represent)
#resolution = (14464, 14464, 20000)
# Voxel offset
offset = (0, 0, 0)
# Layer type
layer_type = 'segmentation'
# number of channels
num_channels = 1
# segmentation properties in the format of [(number1, label1), (number2, label2) ...]
# where number is an integer that is in the volume and label is a string that describes that segmenetation
segmentation_properties = [
(number, f'{structure}: {label}')
for structure, (label, number) in structure_dict.items()
]
extra_structures = [
'Pr5', 'VTg', 'DRD', 'IF', 'MPB', 'Op', 'RPC', 'LSO', 'MVe', 'CnF',
'pc', 'DTgC', 'LPB', 'Pr5DM', 'DTgP', 'RMC', 'VTA', 'IPC', 'DRI',
'LDTg', 'IPA', 'PTg', 'DTg', 'IPL', 'SuVe', 'Sol', 'IPR', '8n', 'Dk',
'IO', 'Cb', 'Pr5VL', 'APT', 'Gr', 'RR', 'InC', 'X', 'EW'
]
segmentation_properties += [
(len(structure_dict) + index + 1, structure)
for index, structure in enumerate(extra_structures)
]
if create:
cloudpath = f'file://{OUTPUT_DIR}'
info = CloudVolume.create_new_info(
num_channels=num_channels,
layer_type=layer_type,
data_type=str(volume.dtype), # Channel images might be 'uint8'
encoding=
'raw', # raw, jpeg, compressed_segmentation, fpzip, kempressed
resolution=[resolution, resolution,
20000], # Voxel scaling, units are in nanometers
voxel_offset=offset, # x,y,z offset in voxels from the origin
chunk_size=[64, 64, 64], # units are voxels
volume_size=volume.shape, # e.g. a cubic millimeter dataset
)
vol = CloudVolume(cloudpath, mip=0, info=info, compress=False)
vol.commit_info()
vol[:, :, :] = volume[:, :, :]
vol.info['segment_properties'] = 'names'
vol.commit_info()
segment_properties_path = os.path.join(OUTPUT_DIR, 'names')
os.makedirs(segment_properties_path, exist_ok=True)
info = {
"@type": "neuroglancer_segment_properties",
"inline": {
"ids":
[str(number) for number, label in segmentation_properties],
"properties": [{
"id":
"label",
"description":
"Name of structures",
"type":
"label",
"values":
[str(label) for number, label in segmentation_properties]
}]
}
}
print('Creating names in', segment_properties_path)
with open(os.path.join(segment_properties_path, 'info'), 'w') as file:
json.dump(info, file, indent=2)
# Setting parallel to a number > 1 hangs the script. It still runs fast with parallel=1
tq = LocalTaskQueue(parallel=1)
tasks = tc.create_downsampling_tasks(
cloudpath, compress=False) # Downsample the volumes
tq.insert(tasks)
tq.execute()
print('Finished')
import os, sys
import numpy as np
import cv2
sys.path.append("utilities")
from utilities.data_manager_v2 import DataManager
from utilities.metadata import all_known_structures_sided, ROOT_DIR
from utilities.sqlcontroller import SqlController
sqlController = SqlController()
script_list = [
'initial setup gui', # 0
'a_script_preprocess_setup', # 1
'a_script_preprocess_1', # 2
'a_script_preprocess_2', # 3
'a_script_preprocess_3', # 4
'a_script_preprocess_4', # 5
'a_script_preprocess_5', # 6
'a_script_preprocess_6', # 7
'a_script_preprocess_7', # 8
'a_script_processing_setup', # 9
'a_script_processing'
] # 10
# necessary_image_files_by_script_ntb
necessary_files_by_script = {}
necessary_files_by_script['ntb'] = {}
necessary_files_by_script['thionin'] = {}
def create_atlas(animal, create):
fileLocationManager = FileLocationManager(animal)
atlas_name = 'atlasV7'
DATA_PATH = '/net/birdstore/Active_Atlas_Data/data_root'
ROOT_DIR = '/net/birdstore/Active_Atlas_Data/data_root/pipeline_data'
THUMBNAIL_DIR = os.path.join(ROOT_DIR, animal, 'preps', 'CH1', 'thumbnail')
ATLAS_PATH = os.path.join(DATA_PATH, 'atlas_data', atlas_name)
ORIGIN_PATH = os.path.join(ATLAS_PATH, 'origin')
VOLUME_PATH = os.path.join(ATLAS_PATH, 'structure')
OUTPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data, 'atlas')
if os.path.exists(OUTPUT_DIR):
shutil.rmtree(OUTPUT_DIR)
os.makedirs(OUTPUT_DIR, exist_ok=True)
origin_files = sorted(os.listdir(ORIGIN_PATH))
volume_files = sorted(os.listdir(VOLUME_PATH))
sqlController = SqlController(animal)
resolution = sqlController.scan_run.resolution
surface_threshold = 0.8
SCALE = (10 / resolution)
structure_volume_origin = {}
for volume_filename, origin_filename in zip(volume_files, origin_files):
structure = os.path.splitext(volume_filename)[0]
if structure not in origin_filename:
print(structure, origin_filename)
break
color = get_structure_number(
structure.replace('_L', '').replace('_R', ''))
origin = np.loadtxt(os.path.join(ORIGIN_PATH, origin_filename))
volume = np.load(os.path.join(VOLUME_PATH, volume_filename))
volume = np.rot90(volume, axes=(0, 1))
volume = np.flip(volume, axis=0)
volume[volume > surface_threshold] = color
volume = volume.astype(np.uint8)
structure_volume_origin[structure] = (volume, origin)
col_length = sqlController.scan_run.width / SCALE
row_length = sqlController.scan_run.height / SCALE
z_length = len(os.listdir(THUMBNAIL_DIR))
col_length = 1000
row_length = 1000
z_length = 300
atlasV7_volume = np.zeros((int(row_length), int(col_length), z_length),
dtype=np.uint8)
print('atlas volume shape', atlasV7_volume.shape)
for structure, (volume, origin) in sorted(structure_volume_origin.items()):
print(str(structure).ljust(7), end=": ")
x, y, z = origin
x_start = int(round(x + col_length / 2))
y_start = int(round(y + row_length / 2))
z_start = int(z) // 2 + z_length // 2
x_end = x_start + volume.shape[0]
y_end = y_start + volume.shape[1]
z_end = z_start + (volume.shape[2] + 1) // 2
print('Row range',
str(y_start).rjust(4),
str(y_end).rjust(4),
'col range',
str(x_start).rjust(4),
str(x_end).rjust(4),
'z range',
str(z_start).rjust(4),
str(z_end).rjust(4),
end=" ")
z_indices = [z for z in range(volume.shape[2]) if z % 2 == 0]
volume = volume[:, :, z_indices]
#volume = np.swapaxes(volume, 0, 1)
try:
atlasV7_volume[x_start:x_end, y_start:y_end,
z_start:z_end] += volume
except ValueError as ve:
print(ve, end=" ")
print()
resolution = int(resolution * 1000 * SCALE)
print('Shape of downsampled atlas volume', atlasV7_volume.shape)
print('Resolution at', resolution)
if create:
#offset = [21959.308659539533, 6238.690939678455, 66.74432595997823]
#atlasV7_volume = np.rot90(atlasV7_volume, axes=(0, 1))
#atlasV7_volume = np.fliplr(atlasV7_volume)
#atlasV7_volume = np.flipud(atlasV7_volume)
#atlasV7_volume = np.fliplr(atlasV7_volume)
offset = [0, 0, 0]
ng = NumpyToNeuroglancer(atlasV7_volume,
[resolution, resolution, 20000],
offset=offset)
ng.init_precomputed(OUTPUT_DIR)
ng.add_segment_properties(get_segment_properties())
ng.add_downsampled_volumes()
ng.add_segmentation_mesh()
#outpath = os.path.join(ATLAS_PATH, f'{atlas_name}.npz')
#np.savez(outpath, atlasV7_volume.astype(np.uint8))
end = timer()
print(f'Finito! Program took {end - start} seconds')
sys.path.append(PATH)
from utilities.file_location import FileLocationManager
from utilities.sqlcontroller import SqlController
#from utilities.utilities_cvat_neuroglancer import get_structure_number
animal = 'MD589'
downsample_factor = 16
all_structures = False
# OUTPUT_DIR_PATH = os.path.join(os.path.expanduser('~'))
OUTPUT_DIR_PATH = os.path.join('./')
CSV_DIR_PATH = '/net/birdstore/Active_Atlas_Data/data_root/atlas_data/foundation_brain_annotations'
IMAGE_DIR_PATH = f'/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/{animal}/preps/CH1/full'
sqlController = SqlController(animal)
resolution = sqlController.scan_run.resolution
aligned_shape = np.array(
(sqlController.scan_run.width, sqlController.scan_run.height))
num_section = len(os.listdir(IMAGE_DIR_PATH))
downsampled_aligned_shape = np.round(aligned_shape /
downsample_factor).astype(int)
scales = np.array([
resolution * downsample_factor, resolution * downsample_factor, 20
]) * 1000
db_structures = sqlController.get_structures_dict()
##### Get the annotation points
type=int,
help="The downsampled factor of the brain images")
parser.add_argument("cvat_file",
type=str,
help="Path to cvat exported file")
parser.add_argument("precomputed_path",
type=str,
help="Path to Neuroglancer Precomputed dir")
parser.add_argument("draw_lines",
type=bool,
help="Whether to draw lines or dots for Neuroglancer",
default=True)
args = parser.parse_args()
sqlController = SqlController(args.animal)
resolution = sqlController.scan_run.resolution
aligned_shape = np.array(
(sqlController.scan_run.width, sqlController.scan_run.height))
# num_section = len(os.listdir(IMAGE_DIR_PATH))
num_section = 452
downsampled_aligned_shape = np.round(aligned_shape /
args.downsample_factor).astype(int)
scales = np.array([
resolution * args.downsample_factor,
resolution * args.downsample_factor, 20
]) * 1000
contours = read_from_cvat(args.cvat_file)
volume = draw_numpy(contours,
downsampled_aligned_shape,
def make_histogram(animal, channel):
"""
This method creates an individual histogram for each tif file by channel.
Args:
animal: the prep id of the animal
channel: the channel of the stack to process {1,2,3}
Returns:
nothing
"""
logger = get_logger(animal)
fileLocationManager = FileLocationManager(animal)
sqlController = SqlController(animal)
INPUT = os.path.join(fileLocationManager.prep, f'CH{channel}', 'thumbnail')
MASK_INPUT = fileLocationManager.thumbnail_masked
tifs = sqlController.get_sections(animal, channel)
error = test_dir(animal, INPUT, downsample=True, same_size=False)
if len(tifs) == 0:
error += " No sections in the database"
if len(error) > 0:
print(error)
sys.exit()
ch_dir = f'CH{channel}'
OUTPUT = os.path.join(fileLocationManager.histogram, ch_dir)
os.makedirs(OUTPUT, exist_ok=True)
progress_id = sqlController.get_progress_id(True, channel, 'HISTOGRAM')
sqlController.set_task(animal, progress_id)
for i, tif in enumerate(tqdm(tifs)):
filename = str(i).zfill(3) + '.tif'
input_path = os.path.join(INPUT, filename)
mask_path = os.path.join(MASK_INPUT, filename)
output_path = os.path.join(OUTPUT,
os.path.splitext(tif.file_name)[0] + '.png')
if not os.path.exists(input_path):
print('Input tif does not exist', input_path)
continue
if os.path.exists(output_path):
continue
try:
img = io.imread(input_path)
except:
logger.warning(f'Could not open {input_path}')
continue
try:
mask = io.imread(mask_path)
except:
logger.warning(f'Could not open {mask_path}')
continue
img = cv2.bitwise_and(img, img, mask=mask)
if img.shape[0] * img.shape[1] > 1000000000:
scale = 1 / float(2)
img = img[::int(1. / scale), ::int(1. / scale)]
try:
flat = img.flatten()
except:
logger.warning(f'Could not flat {input_path}')
continue
fig = plt.figure()
plt.rcParams['figure.figsize'] = [10, 6]
plt.hist(flat, flat.max(), [0, 10000], color=COLORS[channel])
plt.style.use('ggplot')
plt.yscale('log')
plt.grid(axis='y', alpha=0.75)
plt.xlabel('Value')
plt.ylabel('Frequency')
plt.title(f'{tif.file_name} @16bit')
plt.close()
fig.savefig(output_path, bbox_inches='tight')