def masker(animal, channel, downsample, scale):
"""
Main method that starts the cleaning/rotating process.
:param animal: prep_id of the animal we are working on.
:param channel: channel {1,2,3}
:param flip: flip or flop or nothing
:param rotation: usually 1 for rotating 90 degrees
:param full: resolution, either full or thumbnail
:return: nothing, writes to disk the cleaned image
"""
sqlController = SqlController(animal)
fileLocationManager = FileLocationManager(animal)
channel_dir = 'CH{}'.format(channel)
CLEANED = os.path.join(fileLocationManager.prep, channel_dir,
'thumbnail_cleaned')
INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'thumbnail')
MASKS = os.path.join(fileLocationManager.prep, 'thumbnail_masked')
os.makedirs(CLEANED, exist_ok=True)
width = sqlController.scan_run.width
height = sqlController.scan_run.height
rotation = sqlController.scan_run.rotation
flip = sqlController.scan_run.flip
max_width = int(width * SCALING_FACTOR)
max_height = int(height * SCALING_FACTOR)
stain = sqlController.histology.counterstain
if channel == 1:
sqlController.set_task(animal, CLEAN_CHANNEL_1_THUMBNAIL_WITH_MASK)
if not downsample:
CLEANED = os.path.join(fileLocationManager.prep, channel_dir,
'full_cleaned')
os.makedirs(CLEANED, exist_ok=True)
INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'full')
MASKS = os.path.join(fileLocationManager.prep, 'full_masked')
max_width = width
max_height = height
error = test_dir(animal, INPUT, downsample, same_size=False)
if len(error) > 0:
print(error)
sys.exit()
files = sorted(os.listdir(INPUT))
progress_id = sqlController.get_progress_id(downsample, channel, 'CLEAN')
sqlController.set_task(animal, progress_id)
file_keys = []
for file in files:
infile = os.path.join(INPUT, file)
outpath = os.path.join(CLEANED, file)
if os.path.exists(outpath):
continue
maskfile = os.path.join(MASKS, file)
if 'thion' in stain.lower():
print('Not implemented.')
#fixed = fix_thion(infile, mask, maskfile, logger, rotation, flip, max_width, max_height)
else:
file_keys.append([
infile, outpath, maskfile, rotation, flip, max_width,
max_height, scale
])
start = timer()
workers = 4 # this is the upper limit. More than this and it crashes.
print(f'Working on {len(file_keys)} files with {workers} cpus')
with ProcessPoolExecutor(max_workers=workers) as executor:
executor.map(fix_ntb, sorted(file_keys))
end = timer()
print(f'Create cleaned files took {end - start} seconds total', end="\t")
print(f' { (end - start)/len(files)} per file')
def create_neuroglancer(animal, channel, downsample, debug=False):
fileLocationManager = FileLocationManager(animal)
sqlController = SqlController(animal)
channel_dir = f'CH{channel}'
channel_outdir = f'C{channel}T_rechunkme'
INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'thumbnail_aligned')
db_resolution = sqlController.scan_run.resolution
resolution = int(db_resolution * 1000 / SCALING_FACTOR)
workers, _ = get_cpus()
chunks = calculate_chunks(downsample, -1)
progress_id = sqlController.get_progress_id(downsample, channel, 'NEUROGLANCER')
sqlController.session.close()
if not downsample:
INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'full_aligned')
channel_outdir = f'C{channel}_rechunkme'
sqlController.set_task(animal, progress_id)
if 'thion' in sqlController.histology.counterstain:
sqlController.set_task(animal, RUN_PRECOMPUTE_NEUROGLANCER_CHANNEL_2_FULL_RES)
sqlController.set_task(animal, RUN_PRECOMPUTE_NEUROGLANCER_CHANNEL_3_FULL_RES)
resolution = int(db_resolution * 1000)
OUTPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data, f'{channel_outdir}')
error = test_dir(animal, INPUT, downsample, same_size=True)
if len(error) > 0 and not debug:
print(error)
sys.exit()
os.makedirs(OUTPUT_DIR, exist_ok=True)
files = sorted(os.listdir(INPUT))
midpoint = len(files) // 2
midfilepath = os.path.join(INPUT, files[midpoint])
midfile = io.imread(midfilepath, img_num=0)
height = midfile.shape[0]
width = midfile.shape[1]
num_channels = midfile.shape[2] if len(midfile.shape) > 2 else 1
file_keys = []
scales = (resolution, resolution, 20000)
volume_size = (width, height, len(files))
print('Volume shape:', volume_size)
ng = NumpyToNeuroglancer(animal, None, scales, 'image', midfile.dtype, num_channels=num_channels, chunk_size=chunks)
ng.init_precomputed(OUTPUT_DIR, volume_size, progress_id=progress_id)
for i, f in enumerate(files):
filepath = os.path.join(INPUT, f)
file_keys.append([i,filepath])
#ng.process_3channel([i, filepath])
#sys.exit()
start = timer()
print(f'Working on {len(file_keys)} files with {workers} cpus')
with ProcessPoolExecutor(max_workers=workers) as executor:
if num_channels == 1:
executor.map(ng.process_image, sorted(file_keys))
else:
executor.map(ng.process_3channel, sorted(file_keys))
end = timer()
print(f'Create volume method took {end - start} seconds')
ng.precomputed_vol.cache.flush()
def make_histogram(animal, channel):
"""
This method creates an individual histogram for each tif file by channel.
Args:
animal: the prep id of the animal
channel: the channel of the stack to process {1,2,3}
Returns:
nothing
"""
logger = get_logger(animal)
fileLocationManager = FileLocationManager(animal)
sqlController = SqlController(animal)
INPUT = os.path.join(fileLocationManager.prep, f'CH{channel}', 'thumbnail')
MASK_INPUT = fileLocationManager.thumbnail_masked
tifs = sqlController.get_sections(animal, channel)
error = test_dir(animal, INPUT, downsample=True, same_size=False)
if len(tifs) == 0:
error += " No sections in the database"
if len(error) > 0:
print(error)
sys.exit()
ch_dir = f'CH{channel}'
OUTPUT = os.path.join(fileLocationManager.histogram, ch_dir)
os.makedirs(OUTPUT, exist_ok=True)
progress_id = sqlController.get_progress_id(True, channel, 'HISTOGRAM')
sqlController.set_task(animal, progress_id)
for i, tif in enumerate(tqdm(tifs)):
filename = str(i).zfill(3) + '.tif'
input_path = os.path.join(INPUT, filename)
mask_path = os.path.join(MASK_INPUT, filename)
output_path = os.path.join(OUTPUT,
os.path.splitext(tif.file_name)[0] + '.png')
if not os.path.exists(input_path):
print('Input tif does not exist', input_path)
continue
if os.path.exists(output_path):
continue
try:
img = io.imread(input_path)
except:
logger.warning(f'Could not open {input_path}')
continue
try:
mask = io.imread(mask_path)
except:
logger.warning(f'Could not open {mask_path}')
continue
img = cv2.bitwise_and(img, img, mask=mask)
if img.shape[0] * img.shape[1] > 1000000000:
scale = 1 / float(2)
img = img[::int(1. / scale), ::int(1. / scale)]
try:
flat = img.flatten()
except:
logger.warning(f'Could not flat {input_path}')
continue
fig = plt.figure()
plt.rcParams['figure.figsize'] = [10, 6]
plt.hist(flat, flat.max(), [0, 10000], color=COLORS[channel])
plt.style.use('ggplot')
plt.yscale('log')
plt.grid(axis='y', alpha=0.75)
plt.xlabel('Value')
plt.ylabel('Frequency')
plt.title(f'{tif.file_name} @16bit')
plt.close()
fig.savefig(output_path, bbox_inches='tight')
def run_offsets(animal, transforms, channel, downsample, masks, create_csv,
allen):
"""
This gets the dictionary from the above method, and uses the coordinates
to feed into the Imagemagick convert program. This method also uses a Pool to spawn multiple processes.
Args:
animal: the animal
transforms: the dictionary of file, coordinates
limit: number of jobs
Returns: nothing
"""
fileLocationManager = FileLocationManager(animal)
sqlController = SqlController(animal)
channel_dir = 'CH{}'.format(channel)
INPUT = os.path.join(fileLocationManager.prep, channel_dir,
'thumbnail_cleaned')
OUTPUT = os.path.join(fileLocationManager.prep, channel_dir,
'thumbnail_aligned')
if not downsample:
INPUT = os.path.join(fileLocationManager.prep, channel_dir,
'full_cleaned')
OUTPUT = os.path.join(fileLocationManager.prep, channel_dir,
'full_aligned')
error = test_dir(animal, INPUT, downsample=downsample, same_size=True)
if len(error) > 0 and not create_csv:
print(error)
sys.exit()
if masks:
INPUT = os.path.join(fileLocationManager.prep, 'rotated_masked')
error = test_dir(animal, INPUT, full=False, same_size=True)
if len(error) > 0:
print(error)
sys.exit()
OUTPUT = os.path.join(fileLocationManager.prep,
'rotated_aligned_masked')
os.makedirs(OUTPUT, exist_ok=True)
progress_id = sqlController.get_progress_id(downsample, channel, 'ALIGN')
sqlController.set_task(animal, progress_id)
warp_transforms = create_warp_transforms(animal, transforms, 'thumbnail',
downsample)
ordered_transforms = OrderedDict(sorted(warp_transforms.items()))
file_keys = []
r90 = np.array([[0, -1, 0], [1, 0, 0], [0, 0, 1]])
for i, (file, T) in enumerate(ordered_transforms.items()):
if allen:
ROT_DIR = os.path.join(fileLocationManager.root, animal,
'rotations')
rotfile = file.replace('tif', 'txt')
rotfile = os.path.join(ROT_DIR, rotfile)
R_cshl = np.loadtxt(rotfile)
R_cshl[0, 2] = R_cshl[0, 2] / 32
R_cshl[1, 2] = R_cshl[1, 2] / 32
R_cshl = R_cshl @ r90
R_cshl = np.linalg.inv(R_cshl)
R = T @ R_cshl
infile = os.path.join(INPUT, file)
outfile = os.path.join(OUTPUT, file)
if os.path.exists(outfile) and not create_csv:
continue
file_keys.append([i, infile, outfile, T])
if create_csv:
create_csv_data(animal, file_keys)
else:
start = timer()
workers, _ = get_cpus()
print(f'Working on {len(file_keys)} files with {workers} cpus')
with ProcessPoolExecutor(max_workers=workers) as executor:
executor.map(process_image, sorted(file_keys))
end = timer()
print(f'Create cleaned files took {end - start} seconds total',
end="\t")
print(f' { (end - start)/len(file_keys)} per file')
print('Finished')
cmd = [
'convert', input_path, '-resize', '3.125%', '-compress', 'lzw',
output_path
]
commands.append(cmd)
with Pool(4) as p:
p.map(workernoshell, commands)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Work on Animal')
parser.add_argument('--animal',
help='Enter the animal animal',
required=True)
parser.add_argument('--channel', help='Enter channel', required=True)
args = parser.parse_args()
animal = args.animal
channel = int(args.channel)
make_full_resolution(animal, channel)
make_low_resolution(animal, channel)
sqlController = SqlController(animal)
if channel == 1:
sqlController.update_scanrun(sqlController.scan_run.id)
progress_id = sqlController.get_progress_id(True, channel, 'TIF')
sqlController.set_task(animal, progress_id)
progress_id = sqlController.get_progress_id(False, channel, 'TIF')
sqlController.set_task(animal, progress_id)
