def invert_elastix_transform_parameters(fixed: Path, tform_file: Path, param: Path, outdir: Path, threads: str):
"""
Invert the transform and get a new transform file
"""
if not common.test_installation('elastix'):
raise OSError('elastix not installed')
cmd = ['elastix',
'-t0', tform_file,
'-p', param,
'-f', fixed,
'-m', fixed,
'-out', outdir,
'-threads', threads # 11/09/18. This was set to 1. Can iversions take advantage of multithreading?
]
try:
subprocess.check_output(cmd)
except (Exception, subprocess.CalledProcessError) as e:
msg = f'Inverting transform file failed. cmd: {cmd}\n{str(e)}:'
logging.error(msg)
logging.exception(msg)
return False
return True
def _invert(self, volume, tform, outdir, threads=None):
"""
Using the iverted elastix transform paramter file, invert a volume with transformix
Parameters
----------
vol: str
path to volume to invert
tform: str
path to elastix transform parameter file
outdir: str
path to save transformix output
rename_output: str
rename the transformed volume to this
threads: str/None
number of threads for transformix to use. if None, use all available cpus
Returns
-------
str/bool
path to new img if succesful else False
"""
common.test_installation('transformix')
lm_basename = os.path.splitext(os.path.basename(volume))[0]
new_img_path = join(outdir, lm_basename + FILE_FORMAT)
cmd = [
'transformix',
'-in', volume,
'-tp', tform,
'-out', outdir
]
if threads:
cmd.extend(['-threads', str(threads)])
try:
subprocess.check_output(cmd)
except Exception as e:
logging.exception('transformix failed inverting volume: {} Is transformix installed?. Error: {}'.format(volume, e))
raise
try:
old_img = os.path.join(outdir, TRANSFORMIX_OUT)
os.rename(old_img, new_img_path)
except OSError:
return old_img
else:
return new_img_path
def __init__(self, config_path: Path, invertable, outdir, threads=None, noclobber=False):
"""
Inverts a series of volumes. A yaml config file specifies the order of inverted transform parameters
to use. This config file should be in the root of the directory containing these inverted tform dirs.
Also need to input a directory containing volumes/label maps etc to invert. These need to be in directories
named with the same name as the corresponding inverted tform file directories
Parameters
----------
config_path
path to yaml config containing the oder of the inverted directories to use. The directories containing
propagation tfransfrom files should be in the same diretory
threads: str/ None
number of threas to use. If None, use all available threads
invertable: str
path to object to invert (raw image, mask, label map etc)
outdir
where to store inverted volumes
invertable: str
dir or path. If dir, invert all objects within the subdirectories.
If path to object (eg. labelmap) invert that instead
noclobber: bool
if True do not overwrite already inverted labels
"""
self.noclobber = noclobber
common.test_installation('transformix')
self.config = cfg_load(config_path)
self.invertables = invertable
self.config_dir = config_path.parent # The dir containing the inverted elx param files
self.threads = threads
self.out_dir = outdir
common.mkdir_if_not_exists(self.out_dir)
self.elx_param_prefix = ELX_PARAM_PREFIX
self.PROPAGATION_TFORM_NAME = None # Set in subclasses
self.last_invert_dir = None # I thik this is used as a way to find volumes to do organ vol calculation on
def reverse_registration(config: Union[str, LamaConfig]):
common.test_installation('elastix')
if isinstance(config, (Path, str)):
config = LamaConfig(Path(config))
# threads = str(config['threads'])
inv_outdir = config.mkdir('inverted_transforms')
# Set the fixed volume to be the rigidly-aligned volume from the forward registration
paths = LamaSpecimenData(config.config_dir).setup()
fixed_vols_dir = paths.reg_dirs[0]
# Get the fixed vols
fixed_vol_paths = common.get_images_ignore_elx_itermediates(fixed_vols_dir)
# Get the fixed and moving images. They are flipped compared to the forward registration
moving_vol = config['fixed_volume']
# Do the forward registration for each image (usually just one image if using the jobrunner script)
for fixed_vol in fixed_vol_paths:
run_registration_schedule(config, fixed_vol, moving_vol, inv_outdir)
def batch_invert_transform_parameters(config: Union[Path, LamaConfig],
clobber=True, new_log:bool=False):
"""
Create new elastix TransformParameter files that can then be used by transformix to invert labelmaps, stats etc
Parameters
----------
config
path to original reg pipeline config file
clobber
if True overwrite inverted parameters present
new_log:
Whether to create a new log file. If called from another module, logging may happen there
"""
common.test_installation('elastix')
if isinstance(config, (Path, str)):
config = LamaConfig(config)
threads = str(config['threads'])
if new_log:
common.init_logging(config / 'invert_transforms.log')
reg_dirs = get_reg_dirs(config)
# Get the image basenames from the first stage registration folder (usually rigid)
# ignore images in non-relevent folder that may be present
volume_names = [x.stem for x in common.get_file_paths(reg_dirs[0], ignore_folders=[RESOLUTION_IMGS_DIR, IMG_PYRAMID_DIR])]
inv_outdir = config.mkdir('inverted_transforms')
stages_to_invert = defaultdict(list)
jobs: List[Dict] = []
reg_stage_dir: Path
for i, vol_id in enumerate(volume_names):
for reg_stage_dir in reg_dirs:
if not reg_stage_dir.is_dir():
logging.error('cannot find {}'.format(reg_stage_dir))
raise FileNotFoundError(f'Cannot find registration dir {reg_stage_dir}')
inv_stage_dir = inv_outdir / reg_stage_dir.name
specimen_stage_reg_dir = reg_stage_dir / vol_id
specimen_stage_inversion_dir = inv_stage_dir / vol_id
transform_file = common.getfile_startswith(specimen_stage_reg_dir, ELX_TRANSFORM_NAME)
parameter_file = common.getfile_startswith(reg_stage_dir, ELX_PARAM_PREFIX)
# Create the folder to put the specimen inversion parameter files in.
inv_stage_dir.mkdir(exist_ok=True)
# Add the stage to the inversion order config (in reverse order), if not already.
if reg_stage_dir.name not in stages_to_invert['label_propagation_order']:
stages_to_invert['label_propagation_order'].insert(0, reg_stage_dir.name)
if clobber:
common.mkdir_force(specimen_stage_inversion_dir) # Overwrite any inversion file that exist for a single specimen
# Each registration directory contains a metadata file, which contains the relative path to the fixed volume
reg_metadata = cfg_load(specimen_stage_reg_dir / common.INDV_REG_METADATA)
fixed_volume = (specimen_stage_reg_dir / reg_metadata['fixed_vol']).resolve()
# Invert the Transform parameters with options for normal image inversion
job = {
'specimen_stage_inversion_dir': specimen_stage_inversion_dir,
'parameter_file': abspath(parameter_file),
'transform_file': transform_file,
'fixed_volume': fixed_volume,
'param_file_output_name': 'inversion_parameters.txt',
'image_replacements': IMAGE_REPLACEMENTS,
'label_replacements': LABEL_REPLACEMENTS,
'image_transform_file': PROPAGATE_IMAGE_TRANSFORM,
'label_transform_file': PROPAGATE_LABEL_TRANFORM,
'clobber': clobber,
'threads': threads
}
jobs.append(job)
# By putting each inverison job (a single job per registration stage) we can speed things up a bit
# If we can get multithreded inversion in elastix we can remove this python multithreading
pool = Pool(8)
try:
pool.map(_invert_transform_parameters, jobs)
except KeyboardInterrupt:
print('terminating inversion')
pool.terminate()
pool.join()
# TODO: Should we replace the need for this invert.yaml?
reg_dir = Path(os.path.relpath(reg_stage_dir, inv_outdir))
stages_to_invert['registration_directory'] = str(reg_dir) # Doc why we need this
# Create a yaml config file so that inversions can be run seperatley
invert_config = config['inverted_transforms'] / PROPAGATE_CONFIG
with open(invert_config, 'w') as yf:
yf.write(yaml.dump(dict(stages_to_invert), default_flow_style=False))
def batch_invert_transform_parameters(config: Union[str, LamaConfig],
clobber=True,
new_log: bool = False):
"""
Create new elastix TransformParameter files that can then be used by transformix to invert labelmaps, stats etc
Parameters
----------
config
path to original reg pipeline config file
clobber
if True overwrite inverted parameters present
new_log:
Whether to create a new log file. If called from another module, logging may happen there
"""
common.test_installation('elastix')
if isinstance(config, Path):
config = LamaConfig(config)
threads = str(config['threads'])
if new_log:
common.init_logging(config / 'invert_transforms.log')
reg_dirs = get_reg_dirs(config)
# Get the image basenames from the first stage registration folder (usually rigid)
# ignore images in non-relevent folder that may be present
volume_names = [
x.stem for x in common.get_file_paths(reg_dirs[0],
ignore_folder=IGNORE_FOLDER)
]
inv_outdir = config.mkdir('inverted_transforms')
stages_to_invert = defaultdict(list)
jobs: List[Dict] = []
reg_stage_dir: Path
for i, vol_id in enumerate(volume_names):
label_replacements = {
'FinalBSplineInterpolationOrder': '0',
'FixedInternalImagePixelType': 'short',
'MovingInternalImagePixelType': 'short',
'ResultImagePixelType': 'unsigned char',
'WriteTransformParametersEachResolution': 'false',
'WriteResultImageAfterEachResolution': 'false'
}
image_replacements = {
'FinalBSplineInterpolationOrder': '3',
'FixedInternalImagePixelType': 'float',
'MovingInternalImagePixelType': 'float',
'ResultImagePixelType': 'float',
'WriteTransformParametersEachResolution': 'false',
'WriteResultImageAfterEachResolution': 'false'
}
for reg_stage_dir in reg_dirs:
if not reg_stage_dir.is_dir():
logging.error('cannot find {}'.format(reg_stage_dir))
raise FileNotFoundError(
f'Cannot find registration dir {reg_stage_dir}')
inv_stage_dir = inv_outdir / reg_stage_dir.name
specimen_stage_reg_dir = reg_stage_dir / vol_id
specimen_stage_inversion_dir = inv_stage_dir / vol_id
transform_file = common.getfile_startswith(specimen_stage_reg_dir,
ELX_TRANSFORM_PREFIX)
parameter_file = common.getfile_startswith(reg_stage_dir,
ELX_PARAM_PREFIX)
# Create the folder to put the specimen inversion parameter files in.
inv_stage_dir.mkdir(exist_ok=True)
# Add the stage to the inversion order config (in reverse order), if not already.
if reg_stage_dir.name not in stages_to_invert['inversion_order']:
stages_to_invert['inversion_order'].insert(
0, reg_stage_dir.name)
if clobber:
common.mkdir_force(
specimen_stage_inversion_dir
) # Overwrite any inversion file that exist for a single specimen
# Each registration directory contains a metadata file, which contains the relative path to the fixed volume
reg_metadata = cfg_load(specimen_stage_reg_dir /
common.INDV_REG_METADATA)
fixed_volume = (specimen_stage_reg_dir /
reg_metadata['fixed_vol']).resolve()
# Invert the Transform parameters with options for normal image inversion
job = {
'specimen_stage_inversion_dir': specimen_stage_inversion_dir,
'parameter_file': abspath(parameter_file),
'transform_file': transform_file,
'fixed_volume': fixed_volume,
'param_file_output_name': 'inversion_parameters.txt',
'image_replacements': image_replacements,
'label_replacements': label_replacements,
'image_transform_file': IMAGE_INVERTED_TRANSFORM,
'label_transform_file': LABEL_INVERTED_TRANFORM,
'clobber': clobber,
'threads': threads
}
jobs.append(job)
# Run the inversion jobs. Currently using only one thread as it seems that elastix now uses multiple threads on the
# Inversions
logging.info('inverting with {} threads: '.format(threads))
pool = Pool(
1
) # 17/09/18 If we can get multithreded inversion in elastix 4.9 we can remove the python multithreading
try:
pool.map(_invert_transform_parameters, jobs)
except KeyboardInterrupt:
print('terminating inversion')
pool.terminate()
pool.join()
# TODO: Should we replace the need for this invert.yaml?
reg_dir = Path(os.path.relpath(reg_stage_dir, inv_outdir))
stages_to_invert['registration_directory'] = str(
reg_dir) # Doc why we need this
# Create a yaml config file so that inversions can be run seperatley
invert_config = config['inverted_transforms'] / INVERT_CONFIG
with open(invert_config, 'w') as yf:
yf.write(yaml.dump(dict(stages_to_invert), default_flow_style=False))
def run(configfile: Path):
"""
This is the main function Lama script for generating data from registering volumes
It reads in the config file, creates directories, and initialises the registration process.
Looks for paths to inputs relative the directory containing the config file
Parameters
----------
param config
A toml config file
"""
try:
config = LamaConfig(configfile)
except OSError as e:
logging.error(f'Cannot open LAMA config file: {str(configfile)}\n{e}')
raise
except Exception as e:
raise (LamaConfigError(e))
config.mkdir('output_dir')
qc_dir = config.mkdir('qc_dir')
config.mkdir('average_folder')
config.mkdir('root_reg_dir')
# TODO find the histogram batch code
# if not config['no_qc']:
# input_histogram_dir = config.mkdir('input_image_histograms')
# make_histograms(config['inputs'], input_histogram_dir)
logpath = config.config_path.parent / LOG_FILE # Make log in same directory as config file
common.init_logging(logpath)
if not common.test_installation('elastix'):
raise OSError('Make sure elastix is installed')
# Catch ctr-c signals so we can write that to logs
# signal.signal(signal.SIGTERM, common.service_shutdown)
signal.signal(signal.SIGINT, common.service_shutdown)
mem_monitor = MonitorMemory(Path(config['output_dir']).absolute())
# Disable QC output?
no_qc: bool = config['no_qc']
logging.info(common.git_log()
) # If running from a git repo, log the branch and commit #
logging.info("Registration started")
final_registration_dir = run_registration_schedule(config)
make_deformations_at_different_scales(config)
create_glcms(config, final_registration_dir)
if config['skip_transform_inversion']:
logging.info('Skipping inversion of transforms')
else:
logging.info('inverting transforms')
batch_invert_transform_parameters(config)
logging.info('inverting volumes')
invert_volumes(config)
if config['label_map']:
generate_organ_volumes(config)
if not generate_staging_data(config):
logging.warning('No staging data generated')
# Write out the names of the registration dirs in the order they were run
with open(config['root_reg_dir'] / REG_DIR_ORDER, 'w') as fh:
for reg_stage in config['registration_stage_params']:
fh.write(f'{reg_stage["stage_id"]}\n')
if not no_qc:
if config['skip_transform_inversion']:
inverted_label_overlay_dir = None
else:
inverted_label_overlay_dir = config.mkdir(
'inverted_label_overlay_dir')
# registered_midslice_dir = config.mkdir('registered_midslice_dir')
make_qc_images(config.config_dir, config['fixed_volume'], qc_dir)
mem_monitor.stop()
return True
def run(configfile: Path):
"""
This is the main function Lama script for generating data from registering volumes
It reads in the config file, creates directories, and initialises the registration process.
Looks for paths to inputs relative the directory containing the config file
Parameters
----------
param config
A toml config file
"""
try:
config = LamaConfig(configfile)
except OSError as e:
logging.error(f'Cannot open LAMA config file: {str(configfile)}\n{e}')
raise
except Exception as e:
raise (LamaConfigError(e))
config.mkdir('output_dir')
qc_dir = config.mkdir('qc_dir')
config.mkdir('average_folder')
config.mkdir('root_reg_dir')
# TODO find the histogram batch code
# if not config['no_qc']:
# input_histogram_dir = config.mkdir('input_image_histograms')
# make_histograms(config['inputs'], input_histogram_dir)
logpath = config.config_path.parent / LOG_FILE # Make log in same directory as config file
common.init_logging(logpath)
if not common.test_installation('elastix'):
raise OSError('Make sure elastix is installed')
# Catch ctr-c signals so we can write that to logs
# signal.signal(signal.SIGTERM, common.service_shutdown)
signal.signal(signal.SIGINT, common.service_shutdown)
mem_monitor = MonitorMemory(Path(config['output_dir']).absolute())
# Disable QC output?
no_qc: bool = config['no_qc']
logging.info(common.git_log()
) # If running from a git repo, log the branch and commit #
logging.info("Registration started")
first_stage_only = config['skip_forward_registration']
# If we only want the reverse label propagation we just need the initial rigid registration to act as the
# Fixed image for the moving populaiton average
final_registration_dir = run_registration_schedule(
config, first_stage_only=first_stage_only)
if not first_stage_only:
neg_jac = make_deformations_at_different_scales(config)
folding_report(neg_jac,
config['output_dir'],
config['label_info'],
outdir=config['output_dir'])
create_glcms(config, final_registration_dir)
# Write out the names of the registration dirs in the order they were run
with open(config['root_reg_dir'] / REG_DIR_ORDER_CFG, 'w') as fh:
for reg_stage in config['registration_stage_params']:
fh.write(f'{reg_stage["stage_id"]}\n')
if first_stage_only:
break
if config['skip_transform_inversion']:
logging.info('Skipping inversion of transforms')
else:
logging.info('inverting transforms')
if config['label_propagation'] == 'reverse_registration':
reverse_registration(config)
else: # invert_transform method is the default
batch_invert_transform_parameters(config)
logging.info('propagating volumes')
invert_volumes(config)
# Now that labels have been inverted, should we delete the transorm files?
if config['delete_inverted_transforms']:
shutil.rmtree(config['output_dir'] / 'inverted_transforms')
if config['label_map']:
generate_organ_volumes(config)
if config['seg_plugin_dir']:
plugin_interface.secondary_segmentation(config)
if not generate_staging_data(config):
logging.warning('No staging data generated')
if not no_qc:
rev_reg = True if config[
'label_propagation'] == 'reverse_registration' else False
make_qc_images(config.config_dir,
config['fixed_volume'],
qc_dir,
mask=None,
reverse_reg_propagation=rev_reg)
mem_monitor.stop()
return True