def make_tifs(animal, channel, njobs):
"""
This method will:
1. Fetch the sections from the database
2. Yank the tif out of the czi file according to the index and channel with the bioformats tool.
3. Then updates the database with updated meta information
Args:
animal: the prep id of the animal
channel: the channel of the stack to process
njobs: number of jobs for parallel computing
compression: default is no compression so we can create jp2 files for CSHL. The files get
compressed using LZW when running create_preps.py
Returns:
nothing
"""
logger = get_logger(animal)
fileLocationManager = FileLocationManager(animal)
sqlController = SqlController(animal)
INPUT = fileLocationManager.czi
OUTPUT = fileLocationManager.tif
os.makedirs(OUTPUT, exist_ok=True)
sections = sqlController.get_distinct_section_filenames(animal, channel)
sqlController.set_task(animal, QC_IS_DONE_ON_SLIDES_IN_WEB_ADMIN)
sqlController.set_task(animal, CZI_FILES_ARE_CONVERTED_INTO_NUMBERED_TIFS_FOR_CHANNEL_1)
commands = []
for section in sections:
input_path = os.path.join(INPUT, section.czi_file)
output_path = os.path.join(OUTPUT, section.file_name)
cmd = ['/usr/local/share/bftools/bfconvert', '-bigtiff', '-separate', '-series', str(section.scene_index),
'-channel', str(section.channel_index), '-nooverwrite', input_path, output_path]
if not os.path.exists(input_path):
continue
if os.path.exists(output_path):
continue
commands.append(cmd)
with Pool(njobs) as p:
p.map(workernoshell, commands)
def make_low_resolution(animal, channel):
"""
Args:
takes the full resolution tifs and downsamples them.
animal: the prep id of the animal
channel: the channel of the stack to process
Returns:
list of commands
"""
sqlController = SqlController(animal)
if 'thion' in sqlController.histology.counterstain:
sqlController.set_task(animal, CREATE_CHANNEL_2_THUMBNAILS)
sqlController.set_task(animal, CREATE_CHANNEL_3_THUMBNAILS)
fileLocationManager = FileLocationManager(animal)
commands = []
INPUT = os.path.join(fileLocationManager.prep, f'CH{channel}', 'full')
##### Check if files in dir are valid
error = test_dir(animal, INPUT, downsample=False, same_size=False)
if len(error) > 0:
print(error)
sys.exit()
OUTPUT = os.path.join(fileLocationManager.prep, f'CH{channel}',
'thumbnail')
os.makedirs(OUTPUT, exist_ok=True)
tifs = sorted(os.listdir(INPUT))
for tif in tifs:
input_path = os.path.join(INPUT, tif)
output_path = os.path.join(OUTPUT, tif)
if os.path.exists(output_path):
continue
cmd = [
'convert', input_path, '-resize', '3.125%', '-compress', 'lzw',
output_path
]
commands.append(cmd)
with Pool(4) as p:
p.map(workernoshell, commands)
def make_full_resolution(animal, channel):
"""
Args:
animal: the prep id of the animal
channel: the channel of the stack to process
compress: Use the default LZW compression, otherwise just copy the file with the correct name
Returns:
list of commands
"""
fileLocationManager = FileLocationManager(animal)
sqlController = SqlController(animal)
if 'thion' in sqlController.histology.counterstain:
sqlController.set_task(animal, CREATE_CHANNEL_2_FULL_RES)
sqlController.set_task(animal, CREATE_CHANNEL_3_FULL_RES)
INPUT = os.path.join(fileLocationManager.tif)
##### Check if files in dir are valid
OUTPUT = os.path.join(fileLocationManager.prep, f'CH{channel}', 'full')
os.makedirs(OUTPUT, exist_ok=True)
sections = sqlController.get_sections(animal, channel)
for section_number, section in enumerate(tqdm(sections)):
input_path = os.path.join(INPUT, section.file_name)
output_path = os.path.join(OUTPUT,
str(section_number).zfill(3) + '.tif')
if not os.path.exists(input_path):
#print('Input tif does not exist', input_path)
continue
if os.path.exists(output_path):
continue
copyfile(input_path, output_path)
width, height = get_image_size(input_path)
sqlController.update_tif(section.id, width, height)
def masker(animal, channel, downsample, scale):
"""
Main method that starts the cleaning/rotating process.
:param animal: prep_id of the animal we are working on.
:param channel: channel {1,2,3}
:param flip: flip or flop or nothing
:param rotation: usually 1 for rotating 90 degrees
:param full: resolution, either full or thumbnail
:return: nothing, writes to disk the cleaned image
"""
sqlController = SqlController(animal)
fileLocationManager = FileLocationManager(animal)
channel_dir = 'CH{}'.format(channel)
CLEANED = os.path.join(fileLocationManager.prep, channel_dir,
'thumbnail_cleaned')
INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'thumbnail')
MASKS = os.path.join(fileLocationManager.prep, 'thumbnail_masked')
os.makedirs(CLEANED, exist_ok=True)
width = sqlController.scan_run.width
height = sqlController.scan_run.height
rotation = sqlController.scan_run.rotation
flip = sqlController.scan_run.flip
max_width = int(width * SCALING_FACTOR)
max_height = int(height * SCALING_FACTOR)
stain = sqlController.histology.counterstain
if channel == 1:
sqlController.set_task(animal, CLEAN_CHANNEL_1_THUMBNAIL_WITH_MASK)
if not downsample:
CLEANED = os.path.join(fileLocationManager.prep, channel_dir,
'full_cleaned')
os.makedirs(CLEANED, exist_ok=True)
INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'full')
MASKS = os.path.join(fileLocationManager.prep, 'full_masked')
max_width = width
max_height = height
error = test_dir(animal, INPUT, downsample, same_size=False)
if len(error) > 0:
print(error)
sys.exit()
files = sorted(os.listdir(INPUT))
progress_id = sqlController.get_progress_id(downsample, channel, 'CLEAN')
sqlController.set_task(animal, progress_id)
file_keys = []
for file in files:
infile = os.path.join(INPUT, file)
outpath = os.path.join(CLEANED, file)
if os.path.exists(outpath):
continue
maskfile = os.path.join(MASKS, file)
if 'thion' in stain.lower():
print('Not implemented.')
#fixed = fix_thion(infile, mask, maskfile, logger, rotation, flip, max_width, max_height)
else:
file_keys.append([
infile, outpath, maskfile, rotation, flip, max_width,
max_height, scale
])
start = timer()
workers = 4 # this is the upper limit. More than this and it crashes.
print(f'Working on {len(file_keys)} files with {workers} cpus')
with ProcessPoolExecutor(max_workers=workers) as executor:
executor.map(fix_ntb, sorted(file_keys))
end = timer()
print(f'Create cleaned files took {end - start} seconds total', end="\t")
print(f' { (end - start)/len(files)} per file')
def create_neuroglancer(animal, channel, downsample, debug=False):
fileLocationManager = FileLocationManager(animal)
sqlController = SqlController(animal)
channel_dir = f'CH{channel}'
channel_outdir = f'C{channel}T_rechunkme'
INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'thumbnail_aligned')
db_resolution = sqlController.scan_run.resolution
resolution = int(db_resolution * 1000 / SCALING_FACTOR)
workers, _ = get_cpus()
chunks = calculate_chunks(downsample, -1)
progress_id = sqlController.get_progress_id(downsample, channel, 'NEUROGLANCER')
sqlController.session.close()
if not downsample:
INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'full_aligned')
channel_outdir = f'C{channel}_rechunkme'
sqlController.set_task(animal, progress_id)
if 'thion' in sqlController.histology.counterstain:
sqlController.set_task(animal, RUN_PRECOMPUTE_NEUROGLANCER_CHANNEL_2_FULL_RES)
sqlController.set_task(animal, RUN_PRECOMPUTE_NEUROGLANCER_CHANNEL_3_FULL_RES)
resolution = int(db_resolution * 1000)
OUTPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data, f'{channel_outdir}')
error = test_dir(animal, INPUT, downsample, same_size=True)
if len(error) > 0 and not debug:
print(error)
sys.exit()
os.makedirs(OUTPUT_DIR, exist_ok=True)
files = sorted(os.listdir(INPUT))
midpoint = len(files) // 2
midfilepath = os.path.join(INPUT, files[midpoint])
midfile = io.imread(midfilepath, img_num=0)
height = midfile.shape[0]
width = midfile.shape[1]
num_channels = midfile.shape[2] if len(midfile.shape) > 2 else 1
file_keys = []
scales = (resolution, resolution, 20000)
volume_size = (width, height, len(files))
print('Volume shape:', volume_size)
ng = NumpyToNeuroglancer(animal, None, scales, 'image', midfile.dtype, num_channels=num_channels, chunk_size=chunks)
ng.init_precomputed(OUTPUT_DIR, volume_size, progress_id=progress_id)
for i, f in enumerate(files):
filepath = os.path.join(INPUT, f)
file_keys.append([i,filepath])
#ng.process_3channel([i, filepath])
#sys.exit()
start = timer()
print(f'Working on {len(file_keys)} files with {workers} cpus')
with ProcessPoolExecutor(max_workers=workers) as executor:
if num_channels == 1:
executor.map(ng.process_image, sorted(file_keys))
else:
executor.map(ng.process_3channel, sorted(file_keys))
end = timer()
print(f'Create volume method took {end - start} seconds')
ng.precomputed_vol.cache.flush()
tif.scene_index = series_index
channel_counter += 1
newtif = '{}_S{}_C{}.tif'.format(czi_file, scene_number,
channel_counter)
newtif = newtif.replace('.czi', '').replace('__', '_')
tif.file_name = newtif
tif.channel = channel_counter
tif.processing_duration = 0
tif.created = time.strftime('%Y-%m-%d %H:%M:%S')
session.add(tif)
section_number += 1
session.commit()
if __name__ == '__main__':
# Parsing argument
parser = argparse.ArgumentParser(description='Work on Animal')
parser.add_argument('--animal', help='Enter the animal', required=True)
parser.add_argument('--remove',
help='Enter true or false',
required=False,
default='false')
args = parser.parse_args()
animal = args.animal
remove = bool({'true': True, 'false': False}[str(args.remove).lower()])
make_meta(animal, remove)
sqlController = SqlController(animal)
sqlController.set_task(animal, SLIDES_ARE_SCANNED)
sqlController.set_task(animal, CZI_FILES_ARE_PLACED_ON_BIRDSTORE)
sqlController.set_task(animal, CZI_FILES_ARE_SCANNED_TO_GET_METADATA)
def run_offsets(animal, transforms, channel, downsample, masks, create_csv,
allen):
"""
This gets the dictionary from the above method, and uses the coordinates
to feed into the Imagemagick convert program. This method also uses a Pool to spawn multiple processes.
Args:
animal: the animal
transforms: the dictionary of file, coordinates
limit: number of jobs
Returns: nothing
"""
fileLocationManager = FileLocationManager(animal)
sqlController = SqlController(animal)
channel_dir = 'CH{}'.format(channel)
INPUT = os.path.join(fileLocationManager.prep, channel_dir,
'thumbnail_cleaned')
OUTPUT = os.path.join(fileLocationManager.prep, channel_dir,
'thumbnail_aligned')
if not downsample:
INPUT = os.path.join(fileLocationManager.prep, channel_dir,
'full_cleaned')
OUTPUT = os.path.join(fileLocationManager.prep, channel_dir,
'full_aligned')
error = test_dir(animal, INPUT, downsample=downsample, same_size=True)
if len(error) > 0 and not create_csv:
print(error)
sys.exit()
if masks:
INPUT = os.path.join(fileLocationManager.prep, 'rotated_masked')
error = test_dir(animal, INPUT, full=False, same_size=True)
if len(error) > 0:
print(error)
sys.exit()
OUTPUT = os.path.join(fileLocationManager.prep,
'rotated_aligned_masked')
os.makedirs(OUTPUT, exist_ok=True)
progress_id = sqlController.get_progress_id(downsample, channel, 'ALIGN')
sqlController.set_task(animal, progress_id)
warp_transforms = create_warp_transforms(animal, transforms, 'thumbnail',
downsample)
ordered_transforms = OrderedDict(sorted(warp_transforms.items()))
file_keys = []
r90 = np.array([[0, -1, 0], [1, 0, 0], [0, 0, 1]])
for i, (file, T) in enumerate(ordered_transforms.items()):
if allen:
ROT_DIR = os.path.join(fileLocationManager.root, animal,
'rotations')
rotfile = file.replace('tif', 'txt')
rotfile = os.path.join(ROT_DIR, rotfile)
R_cshl = np.loadtxt(rotfile)
R_cshl[0, 2] = R_cshl[0, 2] / 32
R_cshl[1, 2] = R_cshl[1, 2] / 32
R_cshl = R_cshl @ r90
R_cshl = np.linalg.inv(R_cshl)
R = T @ R_cshl
infile = os.path.join(INPUT, file)
outfile = os.path.join(OUTPUT, file)
if os.path.exists(outfile) and not create_csv:
continue
file_keys.append([i, infile, outfile, T])
if create_csv:
create_csv_data(animal, file_keys)
else:
start = timer()
workers, _ = get_cpus()
print(f'Working on {len(file_keys)} files with {workers} cpus')
with ProcessPoolExecutor(max_workers=workers) as executor:
executor.map(process_image, sorted(file_keys))
end = timer()
print(f'Create cleaned files took {end - start} seconds total',
end="\t")
print(f' { (end - start)/len(file_keys)} per file')
print('Finished')
def make_histogram(animal, channel):
"""
This method creates an individual histogram for each tif file by channel.
Args:
animal: the prep id of the animal
channel: the channel of the stack to process {1,2,3}
Returns:
nothing
"""
logger = get_logger(animal)
fileLocationManager = FileLocationManager(animal)
sqlController = SqlController(animal)
INPUT = os.path.join(fileLocationManager.prep, f'CH{channel}', 'thumbnail')
MASK_INPUT = fileLocationManager.thumbnail_masked
tifs = sqlController.get_sections(animal, channel)
error = test_dir(animal, INPUT, downsample=True, same_size=False)
if len(tifs) == 0:
error += " No sections in the database"
if len(error) > 0:
print(error)
sys.exit()
ch_dir = f'CH{channel}'
OUTPUT = os.path.join(fileLocationManager.histogram, ch_dir)
os.makedirs(OUTPUT, exist_ok=True)
progress_id = sqlController.get_progress_id(True, channel, 'HISTOGRAM')
sqlController.set_task(animal, progress_id)
for i, tif in enumerate(tqdm(tifs)):
filename = str(i).zfill(3) + '.tif'
input_path = os.path.join(INPUT, filename)
mask_path = os.path.join(MASK_INPUT, filename)
output_path = os.path.join(OUTPUT,
os.path.splitext(tif.file_name)[0] + '.png')
if not os.path.exists(input_path):
print('Input tif does not exist', input_path)
continue
if os.path.exists(output_path):
continue
try:
img = io.imread(input_path)
except:
logger.warning(f'Could not open {input_path}')
continue
try:
mask = io.imread(mask_path)
except:
logger.warning(f'Could not open {mask_path}')
continue
img = cv2.bitwise_and(img, img, mask=mask)
if img.shape[0] * img.shape[1] > 1000000000:
scale = 1 / float(2)
img = img[::int(1. / scale), ::int(1. / scale)]
try:
flat = img.flatten()
except:
logger.warning(f'Could not flat {input_path}')
continue
fig = plt.figure()
plt.rcParams['figure.figsize'] = [10, 6]
plt.hist(flat, flat.max(), [0, 10000], color=COLORS[channel])
plt.style.use('ggplot')
plt.yscale('log')
plt.grid(axis='y', alpha=0.75)
plt.xlabel('Value')
plt.ylabel('Frequency')
plt.title(f'{tif.file_name} @16bit')
plt.close()
fig.savefig(output_path, bbox_inches='tight')
cmd = [
'convert', input_path, '-resize', '3.125%', '-compress', 'lzw',
output_path
]
commands.append(cmd)
with Pool(4) as p:
p.map(workernoshell, commands)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Work on Animal')
parser.add_argument('--animal',
help='Enter the animal animal',
required=True)
parser.add_argument('--channel', help='Enter channel', required=True)
args = parser.parse_args()
animal = args.animal
channel = int(args.channel)
make_full_resolution(animal, channel)
make_low_resolution(animal, channel)
sqlController = SqlController(animal)
if channel == 1:
sqlController.update_scanrun(sqlController.scan_run.id)
progress_id = sqlController.get_progress_id(True, channel, 'TIF')
sqlController.set_task(animal, progress_id)
progress_id = sqlController.get_progress_id(False, channel, 'TIF')
sqlController.set_task(animal, progress_id)
