def crop_image(op, resol, inverse=False, return_str=False, stack=None):
fileLocationManager = FileLocationManager(stack)
fileLocationManager = FileLocationManager(stack)
image_name_list = sorted(os.listdir(fileLocationManager.masked))
cropbox_resol = op['resolution']
if 'cropboxes_csv' in op: # each image has a different cropbox
cropbox_csv = os.path.join(fileLocationManager.brain_info, 'cropbox.csv')
cropboxes_all = csv_to_dict(cropbox_csv)
cropboxes = {}
for img_name in image_name_list:
arr_xxyy = convert_cropbox_fmt(data=cropboxes_all[img_name], in_fmt='arr_xywh', out_fmt='arr_xxyy')
if inverse:
arr_xxyy = np.array([-arr_xxyy[0], arr_xxyy[1], -arr_xxyy[2], arr_xxyy[3]])
cropboxes[img_name] = convert_cropbox_fmt(data=arr_xxyy, in_fmt='arr_xxyy',
out_fmt='str_xywh' if return_str else 'arr_xywh',
in_resol=cropbox_resol, out_resol=resol, stack=stack)
else: # a single cropbox for all images
arr_xxyy = convert_cropbox_fmt(data=op, in_fmt='dict', out_fmt='arr_xxyy', in_resol=cropbox_resol,
out_resol=resol, stack=stack)
if inverse:
arr_xxyy = np.array([-arr_xxyy[0], arr_xxyy[1], -arr_xxyy[2], arr_xxyy[3]])
cropbox = convert_cropbox_fmt(data=arr_xxyy, in_fmt='arr_xxyy',
out_fmt='str_xywh' if return_str else 'arr_xywh', stack=stack)
cropboxes = {img_name: cropbox for img_name in image_name_list}
return cropboxes
def make_pages(animal, fetch, layer):
templateLoader = jinja2.FileSystemLoader(searchpath="./templates")
templateEnv = jinja2.Environment(loader=templateLoader)
if 'sql' in fetch:
sqlController = SqlController(animal)
valid_sections = sqlController.get_sections(animal, 1)
container = "container.html"
else:
fileLocationManager = FileLocationManager(animal)
valid_sections = sorted(os.listdir(fileLocationManager.thumbnail_web))
container = "list_dir.html"
if layer is not None:
fileLocationManager = FileLocationManager(animal)
INPUT = os.path.join(fileLocationManager.thumbnail_web, 'points',
layer)
valid_sections = sorted(os.listdir(INPUT))
container = "list_points.html"
template = templateEnv.get_template(container)
lfiles = len(valid_sections)
outputText = template.render(animal=animal,
valid_sections=valid_sections,
lfiles=lfiles,
layer=layer)
else:
template = templateEnv.get_template(container)
lfiles = len(valid_sections)
outputText = template.render(
animal=animal, valid_sections=valid_sections, lfiles=lfiles
) # this is where to put args to the template renderer
# to save the results
filename = '{}.thumbnails.html'.format(animal)
with open(filename, "w") as fh:
fh.write(outputText)
def update_tifs(animal, channel):
"""
Args:
animal: the prep id of the animal
channel: the channel of the stack to process
njobs: number of jobs for parallel computing
Returns:
nothing
"""
fileLocationManager = FileLocationManager(animal)
sqlController = SqlController(animal)
tifs = sqlController.get_distinct_section_filenames(animal, channel)
INPUT = os.path.join(fileLocationManager.prep, 'CH1', 'full')
# Update TIFs' size
try:
os.listdir(INPUT)
except OSError as e:
print(e)
sys.exit()
for i, tif in enumerate(tqdm(tifs)):
print(tif.file_name)
input_path = os.path.join(INPUT, str(i).zfill(3) + '.tif')
if os.path.exists(input_path):
print(input_path)
tif.file_size = os.path.getsize(input_path)
sqlController.update_row(tif)
def padder(animal, bgcolor):
fileLocationManager = FileLocationManager(animal)
sqlController = SqlController()
sqlController.get_animal_info(animal)
INPUT = os.path.join(fileLocationManager.prep, 'thumbnail_masked')
OUTPUT = '/net/birdstore/Active_Atlas_Data/data_root/brains_info/masks/{}/prealigned'.format(animal)
files = sorted(os.listdir(INPUT))
width = sqlController.scan_run.width
height = sqlController.scan_run.height
max_width = int(width * SCALING_FACTOR)
max_height = int(height * SCALING_FACTOR)
for i, file in enumerate(tqdm(files)):
infile = os.path.join(INPUT, file)
try:
img = io.imread(infile)
except:
print('Could not open', infile)
continue
fixed = place_image(img, file, max_width, max_height, bgcolor)
outpath = os.path.join(OUTPUT, file)
cv2.imwrite(outpath, fixed.astype('uint8'))
print('Finished')
def create_mesh(animal, mip, mse):
fileLocationManager = FileLocationManager(animal)
channel_outdir = 'color_mesh_v2'
OUTPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data,
channel_outdir)
if not os.path.exists(OUTPUT_DIR):
print(f'DIR {OUTPUT_DIR} does not exist, exiting.')
sys.exit()
cloudpath = f"file://{OUTPUT_DIR}"
cv = CloudVolume(cloudpath, mip)
workers, _ = get_cpus()
tq = LocalTaskQueue(parallel=workers)
mesh_dir = f'mesh_mip_{mip}_err_{mse}'
cv.info['mesh'] = mesh_dir
cv.commit_info()
tasks = tc.create_meshing_tasks(cv.layer_cloudpath,
mip=mip,
mesh_dir=mesh_dir,
max_simplification_error=mse)
tq.insert(tasks)
tq.execute()
tasks = tc.create_mesh_manifest_tasks(cv.layer_cloudpath,
mesh_dir=mesh_dir)
tq.insert(tasks)
tq.execute()
print("Done!")
def __init__(self, stack, parent=None):
super(GUISortedFilenames, self).__init__(parent)
self.stack = stack
self.fileLocationManager = FileLocationManager(self.stack)
self.sqlController = SqlController()
self.sqlController.get_animal_info(self.stack)
self.valid_sections = self.sqlController.get_valid_sections(stack)
self.valid_section_keys = sorted(list(self.valid_sections))
self.curr_section_index = 0
self.curr_section = None
self.init_ui()
self.b_rotate_left.clicked.connect(
lambda: self.click_button(self.b_rotate_left))
self.b_rotate_right.clicked.connect(
lambda: self.click_button(self.b_rotate_right))
self.b_flip_vertical.clicked.connect(
lambda: self.click_button(self.b_flip_vertical))
self.b_flip_horozontal.clicked.connect(
lambda: self.click_button(self.b_flip_horozontal))
self.b_move_left.clicked.connect(
lambda: self.click_button(self.b_move_left))
self.b_move_right.clicked.connect(
lambda: self.click_button(self.b_move_right))
self.b_quality.currentIndexChanged.connect(
lambda: self.click_button(self.b_quality))
self.b_remove.clicked.connect(lambda: self.click_button(self.b_remove))
self.b_help.clicked.connect(lambda: self.click_button(self.b_help))
self.b_done.clicked.connect(lambda: self.click_button(self.b_done))
self.set_curr_section(self.curr_section_index)
def __init__(self, stack, parent=None):
super(GUISetupMain, self).__init__(parent)
# Stack specific info, determined from dropdown menu selection
self.stack = stack
self.fileLocationManager = FileLocationManager(self.stack)
self.sqlController = SqlController()
self.sqlController.get_animal_info(self.stack)
self.stain = self.sqlController.histology.counterstain
self.curr_step = self.sqlController.get_current_step_from_progress_ini(
self.stack)
# Init UI
self.init_ui()
# Set buttons functionality
self.b_1_4.clicked.connect(lambda: self.click_button(self.b_1_4))
self.b_1_6.clicked.connect(lambda: self.click_button(self.b_1_6))
self.b_exit.clicked.connect(lambda: self.click_button(self.b_exit))
# Update buttons
self.update_buttons()
# Center the GUI
self.center()
def make_web_thumbnails(animal):
"""
This was originally getting the thumbnails from the preps/thumbnail dir but they aren't usuable.
The ones in the preps/CH1/thumbnail_aligned are much better
But we need to test if there ane aligned files, if not use the cleaned ones.
Thumbnails are always created from CH1
Args:
animal: the prep id of the animal
njobs: number of jobs for parallel computing
Returns:
nothing
"""
fileLocationManager = FileLocationManager(animal)
sqlController = SqlController(animal)
INPUT = '/home/eodonnell/DK39/cshl/jpg'
OUTPUT = os.path.join(fileLocationManager.root, animal, 'cshl/jpg')
tifs = sqlController.get_sections(animal, 1)
for i, tif in enumerate(tqdm(tifs)):
input_path = os.path.join(INPUT,
os.path.splitext(tif.file_name)[0] + '.jpg')
output_path = os.path.join(OUTPUT, str(i).zfill(3) + '.jpg')
if not os.path.exists(input_path):
continue
if os.path.exists(output_path):
continue
os.makedirs(os.path.dirname(output_path), exist_ok=True)
cmd = "convert {} {}".format(input_path, output_path)
subprocess.run(cmd, shell=True)
def create_histogram(animal, channel, section):
fileLocationManager = FileLocationManager(animal)
channel_dir = f'CH{channel}/thumbnail_aligned'
filename = str(section).zfill(3) + '.tif'
filepath = os.path.join(fileLocationManager.prep, channel_dir, filename)
#maskpath = os.path.join(fileLocationManager.prep, 'thumbnail_masked', filename)
outputpath = os.path.join(HOME, filename)
img = io.imread(filepath, img_num=0)
#mask = io.imread(maskpath)
#img = cv2.bitwise_and(img, img, mask=mask)
flat = img.flatten()
fig = plt.figure()
plt.rcParams['figure.figsize'] = [10, 6]
plt.hist(flat, flat.max(), [0, 65535], color=COLORS[channel])
plt.style.use('ggplot')
plt.yscale('log')
plt.grid(axis='y', alpha=0.75)
plt.xlabel('Value')
plt.ylabel('Frequency')
plt.title(f'CH{channel} {filename} @{img.dtype}')
plt.close()
fig.savefig(outputpath, bbox_inches='tight')
def test_czi(animal, czi_file):
"""
Scans the czi dir to extract the meta information for each tif file
Args:
animal: the animal as primary key
Returns: nothing
"""
fileLocationManager = FileLocationManager(animal)
# Get metadata from the czi file
czi_file_path = os.path.join(fileLocationManager.czi, czi_file)
metadata_dict = get_czi_metadata(czi_file_path)
print(f'All data for {czi_file}')
pprint(metadata_dict)
print()
series = get_fullres_series_indices(metadata_dict)
for j, series_index in enumerate(series):
scene_number = j + 1
channels = range(metadata_dict[series_index]['channels'])
#print('channels range and dict', channels,metadata_dict[series_index]['channels'])
channel_counter = 0
width = metadata_dict[series_index]['width']
height = metadata_dict[series_index]['height']
for channel in channels:
channel_counter += 1
newtif = '{}_S{}_C{}.tif'.format(czi_file, scene_number,
channel_counter)
newtif = newtif.replace('.czi', '').replace('__', '_')
print(
f'INDEX={series_index}, file:{newtif}, height={height}, width={width}'
)
def __init__(self, stack, parent=None):
super(init_GUI, self).__init__(parent)
self.font_h1 = QFont("Arial", 32)
self.font_p1 = QFont("Arial", 16)
self.queued_transformations = []
# create a dataManager object
self.sqlController = SqlController()
self.stack = stack
self.fileLocationManager = FileLocationManager(self.stack)
self.sqlController.get_animal_info(self.stack)
self.valid_sections = self.sqlController.get_valid_sections(stack)
self.valid_section_keys = sorted(list(self.valid_sections))
section_length = len(self.valid_section_keys)
self.curr_section_index = section_length // 2
self.prev_section_index = self.curr_section_index
self.next_section_index = self.curr_section_index
self.curr_section = self.valid_sections[self.valid_section_keys[
self.curr_section_index]]['destination']
self.prev_section = self.getPrevValidSection(self.curr_section_index)
self.next_section = self.getNextValidSection(self.curr_section_index)
self.initUI()
def adapt(stack):
"""
This method takes a tif file and runs it through opencv's adaptive equalization method.
Currently just working on the counter stain channel.
Args:
stack: the animal ID
Returns: nothing, but writes the new image to disk. To the normalized directory
"""
fileLocationManager = FileLocationManager(stack)
INPUT = fileLocationManager.cleaned
OUTPUT = fileLocationManager.normalized
image_name_list = sorted(os.listdir(INPUT))
tilesize = 16
for i, file in enumerate(tqdm(image_name_list)):
infile = os.path.join(INPUT, file)
cv2.imread(infile, cv2.)
try:
img = io.imread(infile)
except:
print('Could not open', infile)
return 0
img = get_last_2d(img)
clahe = cv2.createCLAHE(clipLimit=40.0, tileGridSize=(tilesize, tilesize))
img = clahe.apply(img)
outpath = os.path.join(OUTPUT, file)
cv2.imwrite(outpath, img.astype('uint16'))
def make_from_x_to_y_ini(stack, x, y, rostral_limit, caudal_limit,
dorsal_limit, ventral_limit):
'''
Creates operation configuration files that specify the cropping boxes for either the whole brain, or the brainstem.
'''
fileLocationManager = FileLocationManager(stack)
base_prep_id = ''
dest_prep_id = ''
if x == 'aligned':
base_prep_id = 'aligned'
elif x == 'padded':
base_prep_id = 'alignedPadded'
if y == 'wholeslice':
dest_prep_id = 'alignedWithMargin'
elif y == 'brainstem':
dest_prep_id = 'alignedBrainstemCrop'
fn = os.path.join(fileLocationManager.operation_configs,
'from_' + x + '_to_' + y + '.ini')
f = open(fn, "w")
f.write('[DEFAULT]\n')
f.write('type = crop\n\n')
f.write('base_prep_id = ' + base_prep_id + '\n')
f.write('dest_prep_id = ' + dest_prep_id + '\n\n')
f.write('rostral_limit = ' + str(rostral_limit) + '\n')
f.write('caudal_limit = ' + str(caudal_limit) + '\n')
f.write('dorsal_limit = ' + str(dorsal_limit) + '\n')
f.write('ventral_limit = ' + str(ventral_limit) + '\n')
f.write('resolution = thumbnail')
f.close()
def make_tif(animal, tif_id, file_id, testing=False):
fileLocationManager = FileLocationManager(animal)
sqlController = SqlController(animal)
INPUT = fileLocationManager.czi
OUTPUT = fileLocationManager.tif
start = time.time()
tif = sqlController.get_tif(tif_id)
slide = sqlController.get_slide(tif.slide_id)
czi_file = os.path.join(INPUT, slide.file_name)
section = sqlController.get_section(file_id)
tif_file = os.path.join(OUTPUT, section.file_name)
if not os.path.exists(czi_file) and not testing:
return 0
if os.path.exists(tif_file):
return 1
if testing:
command = ['touch', tif_file]
else:
command = ['/usr/local/share/bftools/bfconvert', '-bigtiff', '-separate',
'-series', str(tif.scene_index), '-channel', str(tif.channel-1), '-nooverwrite', czi_file, tif_file]
run(command)
end = time.time()
if os.path.exists(tif_file):
tif.file_size = os.path.getsize(tif_file)
tif.processing_duration = end - start
sqlController.update_row(tif)
return 1
def make_scenes(animal):
fileLocationManager = FileLocationManager(animal)
INPUT = fileLocationManager.tif
OUTPUT = os.path.join(fileLocationManager.thumbnail_web, 'scene')
os.makedirs(OUTPUT, exist_ok=True)
convert_commands = []
tifs = os.listdir(INPUT)
for tif in tifs:
tif_path = os.path.join(INPUT, tif)
if not tif.endswith('_C1.tif'):
continue
png = tif.replace('tif', 'png')
png_path = os.path.join(OUTPUT, png)
if os.path.exists(png_path):
continue
# convert tif to png
cmd = ['convert', tif_path, '-resize', '3.125%', '-normalize', png_path]
convert_commands.append(cmd)
with Pool(4) as p:
p.map(workernoshell, convert_commands)
def merge_channels(animal):
fileLocationManager = FileLocationManager(animal)
#CH1 = os.path.join(fileLocationManager.prep, 'CH1', '16_aligned')
CH3 = os.path.join(fileLocationManager.prep, 'CH3', '16_aligned')
OUTPUT = os.path.join(fileLocationManager.prep, 'CH3', '16_aligned_merged')
#ch1_files = sorted(os.listdir(CH1))
ch3_files = sorted(os.listdir(CH3))
os.makedirs(OUTPUT, exist_ok=True)
low_clahe = cv2.createCLAHE(clipLimit=8.0, tileGridSize=(18, 18))
high_clahe = cv2.createCLAHE(clipLimit=40.0, tileGridSize=(18, 18))
for ch3_file in tqdm(ch3_files):
outpath = os.path.join(OUTPUT, ch3_file)
if os.path.exists(outpath):
continue
#ch1_infile = os.path.join(CH1, ch1_file)
ch3_infile = os.path.join(CH3, ch3_file)
#ch1_img = cv2.imread(ch1_infile, cv2.IMREAD_UNCHANGED)
ch3_img = cv2.imread(ch3_infile, cv2.IMREAD_UNCHANGED)
#ch1_img = low_clahe.apply(ch1_img)
#ch1_img = ch1_img * 0.25
#ch1_img8 = (ch1_img / 256).astype('uint8')
ch3_img8 = (ch3_img / 256).astype('uint8')
ch3_img8 = high_clahe.apply(ch3_img8)
b = np.zeros(ch3_img8.shape).astype(np.uint8)
r = np.zeros(ch3_img8.shape).astype(np.uint8)
bgr_uint8 = np.dstack((b, ch3_img8, r)).astype(np.uint8)
cv2.imwrite(outpath, bgr_uint8)
def __init__(self, stack, parent=None):
super(init_GUI, self).__init__(parent)
self.font_h1 = QFont("Arial", 32)
self.font_p1 = QFont("Arial", 16)
self.stack = stack
self.fileLocationManager = FileLocationManager(self.stack)
self.sqlController = SqlController()
self.sqlController.get_animal_info(self.stack)
self.valid_sections = self.sqlController.get_valid_sections(stack)
self.valid_section_keys = sorted(list(self.valid_sections))
section_length = len(self.valid_section_keys)
self.curr_section_index = section_length // 2
self.prev_section_index = self.curr_section_index
self.next_section_index = self.curr_section_index
self.curr_section = self.valid_sections[self.valid_section_keys[
self.curr_section_index]]['destination']
self.prev_section = self.getPrevValidSection(self.curr_section_index)
self.next_section = self.getNextValidSection(self.curr_section_index)
self.curr_T = None
self.mode = 'view'
# self.mode = 'align'
self.transform_type = 'pairwise' # Can toggle to 'anchor'
# Increasing this number will brighten the images
self.curr_img_multiplier = 1
self.prev_img_multiplier = 1
self.initUI()
def get_transforms_filename(stack, anchor_fn=None):
if anchor_fn is None:
anchor_fn = sqlController.get_anchor_name(stack)
fileLocationManager = FileLocationManager(stack)
fp = os.path.join(fileLocationManager.brain_info,
'transforms_to_anchor.csv')
return fp
def create_prep2_section_limits(stack, lower_lim, upper_lim):
fileLocationManager = FileLocationManager(stack)
fn = os.path.join(fileLocationManager.brain_info,
'prep2_sectionLimits.ini')
f = open(fn, "w")
f.write('[DEFAULT]\n')
f.write('left_section_limit = ' + str(lower_lim) + '\n')
f.write('right_section_limit = ' + str(upper_lim) + '\n')
f.close()
def setup(stack):
"""
This script calls itself. The first time, the argument: op_id = from_none_to_padded
op_type initially equals warp
Args:
stack:
Returns:
"""
fileLocationManager = FileLocationManager(stack)
image_name_list = sorted(os.listdir(fileLocationManager.cleaned))
resol = 'thumbnail'
ops_in_prep_id = None
out_prep_id = 'alignedPadded'
op_id = 'from_none_to_padded'
op_seq = parse_operation_sequence(op_id, resol=resol, return_str=True, stack=stack)
ops_str_all_images = defaultdict(str)
for op_type, op_params_str_all_images, _, _ in op_seq:
for img_name, op_params_str in op_params_str_all_images.items():
# replace leading minus sign with ^ to satisfy argparse
if op_params_str.startswith('-'):
op_params_str = '^' + op_params_str[1:]
ops_str_all_images[img_name] += ' --op %s %s ' % (op_type, op_params_str)
# sequantial_dispatcher argument cannot be too long, so we must limit the number of images processed each time
batch_size = 100
infilepath = fileLocationManager.cleaned
outfilepath = fileLocationManager.aligned
for batch_id in range(0, len(image_name_list), batch_size):
script = os.path.join(os.getcwd(), 'alignment3a.py')
ops = ops_str_all_images[img_name]
input_fp = os.path.join(infilepath, img_name)
output_fp = os.path.join(outfilepath, img_name)
#cmd = 'python {} --input_fp {} --output_fp {} {}'.format(script, input_fp, output_fp)
#run_distributed(stack, cmd, argument_type='single', jobs_per_node=4, local_only=True, kwargs_list="")
"""
run_distributed(stack, 'python %(script)s --input_fp \"%%(input_fp)s\" --output_fp \"%%(output_fp)s\" %%(ops_str)s' % \
{'script': script},
kwargs_list=[{'ops_str': ops_str_all_images[img_name],
'input_fp': os.path.join(infilepath, img_name),
'output_fp': os.path.join(outfilepath, img_name)}
for img_name in image_name_list[batch_id:batch_id+batch_size]],
argument_type='single', jobs_per_node=4, local_only=True)
"""
run_distributed(stack, 'python %(script)s --input_fp \"%%(input_fp)s\" --output_fp \"%%(output_fp)s\" %%(ops_str)s' % \
{'script': script},
kwargs_list=[{'ops_str': ops_str_all_images[img_name],
'input_fp': os.path.join(infilepath, img_name),
'output_fp': os.path.join(outfilepath, img_name)}
for img_name in image_name_list[batch_id:batch_id+batch_size]],
argument_type='single', jobs_per_node=1, local_only=True)
def create_volume(animal):
output_resolution = '10.0um'
tb_resol = 'thumbnail'
fileLocationManager = FileLocationManager(animal)
INPUT = os.path.join(fileLocationManager.prep, 'CH1', 'thumbnail_aligned')
files = sorted(os.listdir(INPUT))
MASK = os.path.join(fileLocationManager.prep, 'thumbnail_masked_aligned')
# for sec in metadata_cache['valid_sections_all'][stack]:
section = 1
for infile in files:
img_rgb = os.path.join(INPUT, infile)
img = cv2.imread(img_rgb, cv2.IMREAD_GRAYSCALE)
#maskfile = os.path.join(MASK, infile)
#mask = io.imread(maskfile)
#img[~mask] = 0
images[section] = img
section += 1
# Specify isotropic resolution of the output volume.
voxel_size_um = convert_resolution_string_to_um(animal, output_resolution)
input_image_resolution_um = convert_resolution_string_to_um(
animal, tb_resol)
volume_images, origin_images = images_to_volume_v2(
images=images,
spacing_um=20.,
in_resol_um=input_image_resolution_um,
out_resol_um=voxel_size_um)
prep5_origin = load_cropbox_v2(stack=animal,
only_2d=True,
prep_id='alignedWithMargin')
loaded_cropbox_resol = 'thumbnail'
prep5_origin_wrt_prep1 = prep5_origin * convert_resolution_string_to_um(
stack=animal, resolution=loaded_cropbox_resol) / voxel_size_um
wholebrainWithMargin_origin = np.r_[
np.round(prep5_origin_wrt_prep1).astype(np.int)[[0, 2]], 0]
origin_brain = origin_images + wholebrainWithMargin_origin
OUTPUT = os.path.join(
'/net/birdstore/Active_Atlas_Data/data_root/CSHL_volumes', animal)
print('Shape of volume images', volume_images.shape)
outfile = os.path.join(OUTPUT, 'volume_images.npy')
print('Saving', OUTPUT)
np.save(outfile, np.ascontiguousarray(volume_images))
outfile = os.path.join(OUTPUT, 'origin_brain.npy')
print('Saving', outfile)
np.save(outfile, origin_brain)
def create_downsamples(animal, channel, suffix, downsample):
fileLocationManager = FileLocationManager(animal)
channel_outdir = f'C{channel}'
first_chunk = calculate_chunks(downsample, 0)
mips = [0, 1, 2, 3, 4, 5, 6, 7]
if downsample:
channel_outdir += 'T'
mips = [0, 1]
outpath = os.path.join(fileLocationManager.neuroglancer_data,
f'{channel_outdir}')
outpath = f'file://{outpath}'
if suffix is not None:
outpath += suffix
channel_outdir += "_rechunkme"
INPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data,
f'{channel_outdir}')
if not os.path.exists(INPUT_DIR):
print(f'DIR {INPUT_DIR} does not exist, exiting.')
sys.exit()
cloudpath = f"file://{INPUT_DIR}"
_, workers = get_cpus()
tq = LocalTaskQueue(parallel=workers)
tasks = tc.create_transfer_tasks(cloudpath,
dest_layer_path=outpath,
chunk_size=first_chunk,
mip=0,
skip_downsamples=True)
tq.insert(tasks)
tq.execute()
#mips = 7 shows good results in neuroglancer
for mip in mips:
cv = CloudVolume(outpath, mip)
chunks = calculate_chunks(downsample, mip)
factors = calculate_factors(downsample, mip)
tasks = tc.create_downsampling_tasks(cv.layer_cloudpath,
mip=mip,
num_mips=1,
factor=factors,
preserve_chunk_size=False,
compress=True,
chunk_size=chunks)
tq.insert(tasks)
tq.execute()
print("Done!")
def make_manual_anchor_points(stack, x_12N, y_12N, x_3N, y_3N, z_midline):
fileLocationManager = FileLocationManager(stack)
fn = os.path.join(fileLocationManager.brain_info,
'manual_anchor_points.ini')
f = open(fn, "w")
f.write('[DEFAULT]\n')
f.write('x_12N = ' + str(x_12N) + '\n')
f.write('y_12N = ' + str(y_12N) + '\n')
f.write('x_3N = ' + str(x_3N) + '\n')
f.write('y_3N = ' + str(y_3N) + '\n')
f.write('z_midline = ' + str(z_midline))
f.close()
def create_shell(animal):
start = timer()
sqlController = SqlController(animal)
fileLocationManager = FileLocationManager(animal)
INPUT = os.path.join(fileLocationManager.prep, 'CH1', 'thumbnail_aligned')
OUTPUT = os.path.join(fileLocationManager.neuroglancer_data, 'shell')
if os.path.exists(OUTPUT):
shutil.rmtree(OUTPUT)
os.makedirs(OUTPUT, exist_ok=True)
files = sorted(os.listdir(INPUT))
volume = []
for file in tqdm(files):
tif = io.imread(os.path.join(INPUT, file))
tif = mask_to_shell(tif)
volume.append(tif)
volume = np.array(volume).astype('uint8')
volume = np.swapaxes(volume, 0, 2)
"""
factor = (2, 2, 1)
volumes = tinybrain.downsample_segmentation(volume, factor=factor, num_mips=2, sparse=False)
volumes.insert(0, volume)
planar_resolution = sqlController.scan_run.resolution
resolution = int(planar_resolution * 1000 / 0.03125)
info = CloudVolume.create_new_info(
num_channels=1,
layer_type='segmentation',
data_type='uint8', # Channel images might be 'uint8'
encoding='raw', # raw, jpeg, compressed_segmentation, fpzip, kempressed
resolution=[resolution, resolution, 20000], # Voxel scaling, units are in nanometers
voxel_offset=[0, 0, 0], # x,y,z offset in voxels from the origin
chunk_size=[512, 512, 16], # units are voxels
volume_size=volume.shape, # e.g. a cubic millimeter dataset
)
path = 'file://' + OUTPUT
vol = CloudVolume(path, info=info, compress=False, progress=False)
for mip, volume in enumerate(volumes):
vol.add_scale(np.array(factor) ** mip)
vol.commit_info()
vol = CloudVolume(path, mip=mip, compress=False, progress=False)
vol[:, :, :] = volume[:, :, :]
vol.commit_info()
"""
end = timer()
print(f'Finito! Program took {end - start} seconds')
def run_elastix(animal, njobs):
"""
Sets up the arguments for running elastix in a sequence. Each file pair
creates a sub directory with the results. Uses a pool to spawn multiple processes
Args:
animal: the animal
limit: how many jobs you want to run.
Returns: nothing, just creates a lot of subdirs in the elastix directory.
"""
fileLocationManager = FileLocationManager(animal)
DIR = fileLocationManager.prep
INPUT = os.path.join(DIR, 'CH1', 'thumbnail_cleaned')
error = test_dir(animal, INPUT, downsample=True, same_size=True)
if len(error) > 0:
print(error)
sys.exit()
files = sorted(os.listdir(INPUT))
elastix_output_dir = fileLocationManager.elastix_dir
os.makedirs(elastix_output_dir, exist_ok=True)
param_file = os.path.join(os.getcwd(), 'utilities/alignment',
"Parameters_Rigid.txt")
commands = []
# previous file is the fixed image
# current file is the moving image
for i in range(1, len(files)):
prev_img_name = os.path.splitext(files[i - 1])[0]
curr_img_name = os.path.splitext(files[i])[0]
prev_fp = os.path.join(INPUT, files[i - 1])
curr_fp = os.path.join(INPUT, files[i])
new_dir = '{}_to_{}'.format(curr_img_name, prev_img_name)
output_subdir = os.path.join(elastix_output_dir, new_dir)
if os.path.exists(
output_subdir) and 'TransformParameters.0.txt' in os.listdir(
output_subdir):
continue
command = ['rm', '-rf', output_subdir]
subprocess.run(command)
os.makedirs(output_subdir, exist_ok=True)
cmd = [
ELASTIX_BIN, '-f', prev_fp, '-m', curr_fp, '-p', param_file,
'-out', output_subdir
]
commands.append(cmd)
with Pool(njobs) as p:
p.map(workernoshell, commands)
def __init__(self, parent=None):
super(init_GUI, self).__init__(parent)
self.font1 = QFont("Arial", 16)
self.font2 = QFont("Arial", 12)
self.stack = stack
self.fileLocationManager = FileLocationManager(self.stack)
self.sqlController = SqlController()
self.sqlController.get_animal_info(self.stack)
self.stain = self.sqlController.histology.counterstain
self.curr_step = self.sqlController.get_current_step_from_progress_ini(
self.stack)
self.initUI()
def create_points(animal, layer, url_id, create):
fileLocationManager = FileLocationManager(animal)
sqlController = SqlController(animal)
# Get src points data
df = sqlController.get_point_dataframe(url_id)
df = df[df['Layer'] == layer]
df.sort_values(by=['Section', 'X', 'Y'], inplace=True)
if create:
records = df[['X', 'Y', 'Section']].to_records(index=False)
coordinates = list(records)
width = sqlController.scan_run.width
height = sqlController.scan_run.height
sections = sqlController.get_section_count(animal)
layer = str(layer).replace(' ', '_')
OUTPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data, layer)
if os.path.exists(OUTPUT_DIR):
shutil.rmtree(OUTPUT_DIR)
os.makedirs(OUTPUT_DIR, exist_ok=True)
info = os.path.join(DATA_PATH, 'atlas_data', 'points', 'info')
outfile = os.path.join(OUTPUT_DIR, 'info')
with open(info, 'r+') as rf:
data = json.load(rf)
data['upper_bound'] = [width, height,
sections] # <--- add `id` value.
rf.seek(0) # <--- should reset file position to the beginning.
with open(outfile, 'w') as wf:
json.dump(data, wf, indent=4)
spatial_dir = os.path.join(OUTPUT_DIR, 'spatial0')
os.makedirs(spatial_dir)
total_count = len(
coordinates) # coordinates is a list of tuples (x,y,z)
with open(os.path.join(spatial_dir, '0_0_0.gz'), 'wb') as unzippedfile:
buf = struct.pack('<Q', total_count)
pt_buf = b''.join(
struct.pack('<3f', x, y, z) for (x, y, z) in coordinates)
buf += pt_buf
id_buf = struct.pack('<%sQ' % len(coordinates),
*range(len(coordinates)))
buf += id_buf
bufout = gzip.compress(buf)
unzippedfile.write(bufout)
else:
print(df['Layer'].unique())
print(df.head(25))
def convert_operation_to_arr(op, resol, inverse=False, return_str=False, stack=None):
if op['type'] == 'warp':
transforms_to_anchor = warp_image(op, resol, inverse, return_str, stack)
return transforms_to_anchor
elif op['type'] == 'crop':
cropboxes = crop_image(op, resol, inverse, return_str, stack)
return cropboxes
elif op['type'] == 'rotate':
fileLocationManager = FileLocationManager(stack)
image_name_list = sorted(os.listdir(fileLocationManager.masked))
return {img_name: op['how'] for img_name in image_name_list}
else:
raise Exception("Operation type specified by ini must be either warp, crop or rotate.")
def make_annotations(animal):
csvfile = os.path.join(CSV_PATH, f'{animal}_annotation.csv')
hand_annotations = pd.read_csv(csvfile)
hand_annotations['vertices'] = hand_annotations['vertices'] \
.apply(lambda x: x.replace(' ', ','))\
.apply(lambda x: x.replace('\n',','))\
.apply(lambda x: x.replace(',]',']'))\
.apply(lambda x: x.replace(',,', ','))\
.apply(lambda x: x.replace(',,', ','))\
.apply(lambda x: x.replace(',,', ',')).apply(lambda x: x.replace(',,', ','))
hand_annotations['vertices'] = hand_annotations['vertices'].apply(lambda x: ast.literal_eval(x))
structures = list(hand_annotations['name'].unique())
section_structure_vertices = defaultdict(dict)
for structure in tqdm(structures):
contour_annotations, first_sec, last_sec = get_contours_from_annotations(animal, structure, hand_annotations, densify=4)
for section in contour_annotations:
section_structure_vertices[section][structure] = contour_annotations[section][structure][1]
fileLocationManager = FileLocationManager(animal)
section_images = {}
INPUT = os.path.join(fileLocationManager.prep, 'CH1', 'thumbnail')
for file_name in tqdm(sorted(os.listdir(INPUT))):
filepath = os.path.join(INPUT, file_name)
img = io.imread(filepath)
section = int(file_name.split('.')[0])
for structure in section_structure_vertices[section]:
pts = section_structure_vertices[section][structure]
points = np.array(pts, dtype=np.float64)
points = (points / 32).astype(np.int32)
try:
cv2.polylines(img, [points], isClosed=True, color=(0, 0, 0), thickness=2)
except Exception as e:
print(f'Could not draw points with {structure} dtype {points.dtype} on {file_name}', e)
section_images[section] = img
OUTPUT = os.path.join(fileLocationManager.prep, 'CH1', 'annotations')
os.makedirs(OUTPUT, exist_ok=True)
for section in tqdm(section_images):
outpath = os.path.join(OUTPUT, str(section).zfill(3) + '.tif')
cv2.imwrite(outpath, section_images[section])
def parse_elastix(animal):
"""
After the elastix job is done, this goes into each subdirectory and parses the Transformation.0.txt file
Args:
animal: the animal
Returns: a dictionary of key=filename, value = coordinates
"""
fileLocationManager = FileLocationManager(animal)
DIR = fileLocationManager.prep
INPUT = os.path.join(DIR, 'CH1', 'thumbnail_cleaned')
error = test_dir(animal, INPUT, downsample=True, same_size=True)
if len(error) > 0:
print(error)
sys.exit()
files = sorted(os.listdir(INPUT))
midpoint_index = len(files) // 2
transformation_to_previous_section = {}
for i in range(1, len(files)):
fixed_index = os.path.splitext(files[i - 1])[0]
moving_index = os.path.splitext(files[i])[0]
transformation_to_previous_section[
i] = load_consecutive_section_transform(animal, moving_index,
fixed_index)
transformation_to_anchor_section = {}
# Converts every transformation
for moving_index in range(len(files)):
if moving_index == midpoint_index:
transformation_to_anchor_section[files[moving_index]] = np.eye(3)
elif moving_index < midpoint_index:
T_composed = np.eye(3)
for i in range(midpoint_index, moving_index, -1):
T_composed = np.dot(
np.linalg.inv(transformation_to_previous_section[i]),
T_composed)
transformation_to_anchor_section[files[moving_index]] = T_composed
else:
T_composed = np.eye(3)
for i in range(midpoint_index + 1, moving_index + 1):
T_composed = np.dot(transformation_to_previous_section[i],
T_composed)
transformation_to_anchor_section[files[moving_index]] = T_composed
return transformation_to_anchor_section
