def estimate_acceleration_regression(self,
template_specifications,
dataset_specifications,
model_options={},
estimator_options={},
write_output=True):
# Check and completes the input parameters.
template_specifications, model_options, estimator_options = self.further_initialization(
'Regression', template_specifications, model_options,
dataset_specifications, estimator_options)
# Instantiate dataset
dataset = create_dataset(template_specifications,
dimension=model_options['dimension'],
**dataset_specifications)
assert (
dataset.is_time_series()
), "Cannot estimate an acceleration controlled regression from a non-time-series dataset."
# Instantiate model
statistical_model = AccelerationRegression(template_specifications,
**model_options)
# Instantiate estimator
estimator = self.__instantiate_estimator(statistical_model,
dataset,
self.output_dir,
estimator_options,
default=ScipyOptimize)
# Launch
self.__launch_estimator(estimator, write_output)
return statistical_model
def estimate_acceleration_gompertz_regression(self,
template_specifications,
dataset_specifications,
model_options={},
estimator_options={},
write_output=True):
# Check and completes the input parameters
template_specifications, model_options, estimator_options = self.further_initialization(
'Regression', template_specifications, model_options,
dataset_specifications, estimator_options)
# Instantiate dataset
dataset = create_dataset(template_specifications,
dimension=model_options['dimension'],
**dataset_specifications)
assert (
dataset.is_time_series()
), "Cannot estimate an acceleration controlled regression from a non-time-series dataset."
print("CALLING V2")
# Instantiate model
statistical_model = AccelerationGompertzRegressionV2(
template_specifications, **model_options)
#target_times = dataset.times[0]
#target_objects = dataset.deformable_objects[0]
#last_object = target_objects[len(target_objects) - 1]
#the_image = last_object.get_data()
#statistical_model.set_A(the_image['image_intensities'])
# Instantiate estimator.
estimator = self.__instantiate_estimator(statistical_model,
dataset,
self.output_dir,
estimator_options,
default=ScipyOptimize)
# Launch.
self.__launch_estimator(estimator, write_output)
return statistical_model
def estimate_geodesic_regression(xml_parameters):
print('')
print('[ estimate_geodesic_regression function ]')
print('')
"""
Create the dataset object.
"""
dataset = create_dataset(xml_parameters.dataset_filenames,
xml_parameters.visit_ages,
xml_parameters.subject_ids,
xml_parameters.template_specifications)
assert dataset.is_time_series(
), "Cannot run a geodesic regression on a non-time_series dataset."
"""
Create the model object.
"""
model = instantiate_geodesic_regression_model(xml_parameters, dataset)
"""
Create the estimator object.
"""
if xml_parameters.optimization_method_type == 'GradientAscent'.lower():
estimator = GradientAscent()
estimator.initial_step_size = xml_parameters.initial_step_size
estimator.scale_initial_step_size = xml_parameters.scale_initial_step_size
estimator.line_search_shrink = xml_parameters.line_search_shrink
estimator.line_search_expand = xml_parameters.line_search_expand
elif xml_parameters.optimization_method_type == 'ScipyLBFGS'.lower():
estimator = ScipyOptimize()
estimator.memory_length = xml_parameters.memory_length
if not model.freeze_template and model.use_sobolev_gradient and estimator.memory_length > 1:
print(
'>> Using a Sobolev gradient for the template data with the ScipyLBFGS estimator memory length '
'being larger than 1. Beware: that can be tricky.')
# estimator.memory_length = 1
# msg = 'Impossible to use a Sobolev gradient for the template data with the ScipyLBFGS estimator memory ' \
# 'length being larger than 1. Overriding the "memory_length" option, now set to "1".'
# warnings.warn(msg)
else:
estimator = GradientAscent()
estimator.initial_step_size = xml_parameters.initial_step_size
estimator.scale_initial_step_size = xml_parameters.scale_initial_step_size
estimator.line_search_shrink = xml_parameters.line_search_shrink
estimator.line_search_expand = xml_parameters.line_search_expand
msg = 'Unknown optimization-method-type: \"' + xml_parameters.optimization_method_type \
+ '\". Defaulting to GradientAscent.'
warnings.warn(msg)
estimator.max_iterations = xml_parameters.max_iterations
estimator.max_line_search_iterations = xml_parameters.max_line_search_iterations
estimator.convergence_tolerance = xml_parameters.convergence_tolerance
estimator.print_every_n_iters = xml_parameters.print_every_n_iters
estimator.save_every_n_iters = xml_parameters.save_every_n_iters
estimator.dataset = dataset
estimator.statistical_model = model
"""
Launch.
"""
if not os.path.exists(Settings().output_dir):
os.makedirs(Settings().output_dir)
model.name = 'GeodesicRegression'
print('')
print('[ update method of the ' + estimator.name + ' optimizer ]')
start_time = time.time()
estimator.update()
estimator.write()
end_time = time.time()
print('>> Estimation took: ' +
str(time.strftime("%H:%M:%S", time.gmtime(end_time - start_time))))
return model
def estimate_longitudinal_atlas(xml_parameters):
print('')
print('[ estimate_longitudinal_atlas function ]')
print('')
"""
Create the dataset object.
"""
dataset = create_dataset(xml_parameters.dataset_filenames,
xml_parameters.visit_ages,
xml_parameters.subject_ids,
xml_parameters.template_specifications)
"""
Create the model object.
"""
model, individual_RER = instantiate_longitudinal_atlas_model(
xml_parameters, dataset)
"""
Create the estimator object.
"""
if xml_parameters.optimization_method_type == 'GradientAscent'.lower():
estimator = GradientAscent()
estimator.initial_step_size = xml_parameters.initial_step_size
estimator.scale_initial_step_size = xml_parameters.scale_initial_step_size
estimator.max_line_search_iterations = xml_parameters.max_line_search_iterations
estimator.line_search_shrink = xml_parameters.line_search_shrink
estimator.line_search_expand = xml_parameters.line_search_expand
elif xml_parameters.optimization_method_type == 'ScipyLBFGS'.lower():
estimator = ScipyOptimize()
estimator.max_line_search_iterations = xml_parameters.max_line_search_iterations
estimator.memory_length = xml_parameters.memory_length
if not model.is_frozen[
'template_data'] and model.use_sobolev_gradient and estimator.memory_length > 1:
print(
'>> Using a Sobolev gradient for the template data with the ScipyLBFGS estimator memory length '
'being larger than 1. Beware: that can be tricky.')
# estimator.memory_length = 1
# msg = 'Impossible to use a Sobolev gradient for the template data with the ScipyLBFGS estimator memory ' \
# 'length being larger than 1. Overriding the "memory_length" option, now set to "1".'
# warnings.warn(msg)
elif xml_parameters.optimization_method_type == 'McmcSaem'.lower():
sampler = SrwMhwgSampler()
# Onset age proposal distribution.
onset_age_proposal_distribution = MultiScalarNormalDistribution()
onset_age_proposal_distribution.set_variance_sqrt(
xml_parameters.onset_age_proposal_std)
sampler.individual_proposal_distributions[
'onset_age'] = onset_age_proposal_distribution
# Log-acceleration proposal distribution.
log_acceleration_proposal_distribution = MultiScalarNormalDistribution(
)
log_acceleration_proposal_distribution.set_variance_sqrt(
xml_parameters.log_acceleration_proposal_std)
sampler.individual_proposal_distributions[
'log_acceleration'] = log_acceleration_proposal_distribution
# Sources proposal distribution.
sources_proposal_distribution = MultiScalarNormalDistribution()
sources_proposal_distribution.set_variance_sqrt(
xml_parameters.sources_proposal_std)
sampler.individual_proposal_distributions[
'sources'] = sources_proposal_distribution
estimator = McmcSaem()
estimator.sampler = sampler
estimator.sample_every_n_mcmc_iters = xml_parameters.sample_every_n_mcmc_iters
estimator.print_every_n_iters = xml_parameters.print_every_n_iters
# Gradient-based estimator.
# estimator.gradient_based_estimator = ScipyOptimize()
# estimator.gradient_based_estimator.memory_length = 5
estimator.gradient_based_estimator = GradientAscent()
estimator.gradient_based_estimator.initial_step_size = xml_parameters.initial_step_size
estimator.gradient_based_estimator.scale_initial_step_size = False
estimator.gradient_based_estimator.line_search_shrink = xml_parameters.line_search_shrink
estimator.gradient_based_estimator.line_search_expand = xml_parameters.line_search_expand
estimator.gradient_based_estimator.statistical_model = model
estimator.gradient_based_estimator.dataset = dataset
estimator.gradient_based_estimator.optimized_log_likelihood = 'class2'
estimator.gradient_based_estimator.max_iterations = 5
estimator.gradient_based_estimator.max_line_search_iterations = xml_parameters.max_line_search_iterations
estimator.gradient_based_estimator.convergence_tolerance = xml_parameters.convergence_tolerance
estimator.gradient_based_estimator.print_every_n_iters = 1
estimator.gradient_based_estimator.save_every_n_iters = 100000
else:
estimator = GradientAscent()
estimator.initial_step_size = xml_parameters.initial_step_size
estimator.scale_initial_step_size = xml_parameters.scale_initial_step_size
estimator.max_line_search_iterations = xml_parameters.max_line_search_iterations
estimator.line_search_shrink = xml_parameters.line_search_shrink
estimator.line_search_expand = xml_parameters.line_search_expand
msg = 'Unknown optimization-method-type: \"' + xml_parameters.optimization_method_type \
+ '\". Defaulting to GradientAscent.'
warnings.warn(msg)
estimator.optimized_log_likelihood = xml_parameters.optimized_log_likelihood
estimator.max_iterations = xml_parameters.max_iterations
estimator.convergence_tolerance = xml_parameters.convergence_tolerance
estimator.print_every_n_iters = xml_parameters.print_every_n_iters
estimator.save_every_n_iters = xml_parameters.save_every_n_iters
estimator.dataset = dataset
estimator.statistical_model = model
# Initial random effects realizations
estimator.individual_RER = individual_RER
"""
Launch.
"""
if not os.path.exists(Settings().output_dir):
os.makedirs(Settings().output_dir)
model.name = 'LongitudinalAtlas'
print('')
print('[ update method of the ' + estimator.name + ' optimizer ]')
print('')
start_time = time.time()
estimator.update()
estimator.write()
end_time = time.time()
print('>> Estimation took: ' +
str(time.strftime("%H:%M:%S", time.gmtime(end_time - start_time))))
return model
filename_xml.text = 'sample_%d/SimulatedData__Reconstruction__%s__subject_s%d__tp_%d__age_%.2f%s' \
% (sample_index, obj_name, i, j, age, obj_extension)
filename_xml.set('object_id', obj_name)
dataset_xml_path = 'data_set__sample_' + str(sample_index) + '.xml'
doc = parseString((et.tostring(dataset_xml).decode('utf-8').replace(
'\n', '').replace('\t', ''))).toprettyxml()
np.savetxt(dataset_xml_path, [doc], fmt='%s')
"""
Create a dataset object from the xml, and compute the residuals.
"""
xml_parameters._read_dataset_xml(dataset_xml_path)
dataset = create_dataset(
xml_parameters.template_specifications,
visit_ages=xml_parameters.visit_ages,
dataset_filenames=xml_parameters.dataset_filenames,
subject_ids=xml_parameters.subject_ids,
dimension=global_dimension)
if global_add_noise:
(template_data, template_points, control_points, momenta,
modulation_matrix) = model._fixed_effects_to_torch_tensors(False)
sources, onset_ages, accelerations = model._individual_RER_to_torch_tensors(
individual_RER, False)
absolute_times, tmin, tmax = model._compute_absolute_times(
dataset.times, onset_ages, accelerations)
model._update_spatiotemporal_reference_frame(
template_points, control_points, momenta, modulation_matrix,
tmin, tmax)
residuals = model._compute_residuals(dataset, template_data,
absolute_times, sources)
def estimate_longitudinal_registration(xml_parameters, overwrite=True):
print('')
print('[ estimate_longitudinal_registration function ]')
"""
Prepare the loop over each subject.
"""
registration_output_path = Settings().output_dir
full_dataset_filenames = xml_parameters.dataset_filenames
full_visit_ages = xml_parameters.visit_ages
full_subject_ids = xml_parameters.subject_ids
number_of_subjects = len(full_dataset_filenames)
xml_parameters.save_every_n_iters = 100000 # Don't waste time saving intermediate results.
# Global parameter.
initial_modulation_matrix_shape = read_2D_array(xml_parameters.initial_modulation_matrix).shape
if len(initial_modulation_matrix_shape) > 1:
global_number_of_sources = initial_modulation_matrix_shape[1]
else:
global_number_of_sources = 1
"""
Launch the individual longitudinal registrations.
"""
# Multi-threaded version.
if False and Settings().number_of_threads > 1: # It is already multi-threaded at the level below.
pool = Pool(processes=Settings().number_of_threads)
args = [(i, Settings().serialize(), xml_parameters, registration_output_path,
full_subject_ids, full_dataset_filenames, full_visit_ages)
for i in range(number_of_subjects)]
pool.map(estimate_longitudinal_registration_for_subject, args, overwrite)
pool.close()
pool.join()
# Single thread version.
else:
for i in range(number_of_subjects):
estimate_longitudinal_registration_for_subject((
i, Settings().serialize(), xml_parameters, registration_output_path,
full_subject_ids, full_dataset_filenames, full_visit_ages), overwrite)
"""
Gather all the individual registration results.
"""
print('')
print('[ save the aggregated registration parameters of all subjects ]')
print('')
# Gather the individual random effect realizations.
onset_ages = np.zeros((number_of_subjects,))
log_accelerations = np.zeros((number_of_subjects,))
sources = np.zeros((number_of_subjects, global_number_of_sources))
for i in range(number_of_subjects):
subject_registration_output_path = os.path.join(
registration_output_path, 'LongitudinalRegistration__subject_' + full_subject_ids[i])
onset_ages[i] = np.loadtxt(os.path.join(
subject_registration_output_path, 'LongitudinalRegistration__EstimatedParameters__OnsetAges.txt'))
log_accelerations[i] = np.loadtxt(os.path.join(
subject_registration_output_path, 'LongitudinalRegistration__EstimatedParameters__LogAccelerations.txt'))
sources[i] = np.loadtxt(os.path.join(
subject_registration_output_path, 'LongitudinalRegistration__EstimatedParameters__Sources.txt'))
individual_RER = {}
individual_RER['sources'] = sources
individual_RER['onset_age'] = onset_ages
individual_RER['log_acceleration'] = log_accelerations
# Write temporarily those files.
temporary_output_path = os.path.join(registration_output_path, 'tmp')
if os.path.isdir(temporary_output_path):
shutil.rmtree(temporary_output_path)
os.mkdir(temporary_output_path)
path_to_onset_ages = os.path.join(temporary_output_path, 'onset_ages.txt')
path_to_log_accelerations = os.path.join(temporary_output_path, 'log_acceleration.txt')
path_to_sources = os.path.join(temporary_output_path, 'sources.txt')
np.savetxt(path_to_onset_ages, onset_ages)
np.savetxt(path_to_log_accelerations, log_accelerations)
np.savetxt(path_to_sources, sources)
# Construct the aggregated longitudinal atlas model, and save it.
xml_parameters.dataset_filenames = full_dataset_filenames
xml_parameters.visit_ages = full_visit_ages
xml_parameters.subject_ids = full_subject_ids
xml_parameters.initial_onset_ages = path_to_onset_ages
xml_parameters.initial_log_accelerations = path_to_log_accelerations
xml_parameters.initial_sources = path_to_sources
Settings().output_dir = registration_output_path
if not os.path.isdir(Settings().output_dir):
os.mkdir(Settings().output_dir)
dataset = create_dataset(xml_parameters.dataset_filenames, xml_parameters.visit_ages,
xml_parameters.subject_ids, xml_parameters.template_specifications)
model, _ = instantiate_longitudinal_atlas_model(xml_parameters, dataset)
model.name = 'LongitudinalRegistration'
model.write(dataset, None, individual_RER)
def estimate_longitudinal_registration_for_subject(args, overwrite=True):
i, general_settings, xml_parameters, registration_output_path, \
full_subject_ids, full_dataset_filenames, full_visit_ages = args
Settings().initialize(general_settings)
"""
Create the dataset object.
"""
xml_parameters.dataset_filenames = [full_dataset_filenames[i]]
xml_parameters.visit_ages = [full_visit_ages[i]]
xml_parameters.subject_ids = [full_subject_ids[i]]
dataset = create_dataset([full_dataset_filenames[i]], [full_visit_ages[i]], [full_subject_ids[i]],
xml_parameters.template_specifications)
"""
Create a dedicated output folder for the current subject, adapt the global settings.
"""
subject_registration_output_path = os.path.join(
registration_output_path, 'LongitudinalRegistration__subject_' + full_subject_ids[i])
if not overwrite and os.path.isdir(subject_registration_output_path):
return None
print('')
print('[ longitudinal registration of subject ' + full_subject_ids[i] + ' ]')
print('')
if os.path.isdir(subject_registration_output_path):
shutil.rmtree(subject_registration_output_path)
os.mkdir(subject_registration_output_path)
Settings().output_dir = subject_registration_output_path
Settings().state_file = os.path.join(Settings().output_dir, 'pydef_state.p')
"""
Create the model object.
"""
model, individual_RER = instantiate_longitudinal_atlas_model(xml_parameters, dataset)
# In case of given initial random effect realizations, select only the relevant ones.
for (xml_parameter, random_effect_name) \
in zip([xml_parameters.initial_onset_ages,
xml_parameters.initial_log_accelerations,
xml_parameters.initial_sources],
['onset_age', 'log_acceleration', 'sources']):
if xml_parameter is not None and individual_RER[random_effect_name].shape[0] > 1:
individual_RER[random_effect_name] = np.array([individual_RER[random_effect_name][i]])
"""
Create the estimator object.
"""
if xml_parameters.optimization_method_type == 'GradientAscent'.lower():
estimator = GradientAscent()
estimator.initial_step_size = xml_parameters.initial_step_size
estimator.max_line_search_iterations = xml_parameters.max_line_search_iterations
estimator.line_search_shrink = xml_parameters.line_search_shrink
estimator.line_search_expand = xml_parameters.line_search_expand
elif xml_parameters.optimization_method_type == 'ScipyLBFGS'.lower():
estimator = ScipyOptimize()
estimator.max_line_search_iterations = xml_parameters.max_line_search_iterations
estimator.memory_length = xml_parameters.memory_length
if not model.is_frozen['template_data'] and model.use_sobolev_gradient and estimator.memory_length > 1:
print('>> Using a Sobolev gradient for the template data with the ScipyLBFGS estimator memory length '
'being larger than 1. Beware: that can be tricky.')
elif xml_parameters.optimization_method_type == 'ScipyPowell'.lower():
estimator = ScipyOptimize()
estimator.method = 'Powell'
elif xml_parameters.optimization_method_type == 'McmcSaem'.lower():
sampler = SrwMhwgSampler()
momenta_proposal_distribution = MultiScalarNormalDistribution()
# initial_control_points = model.get_control_points()
# momenta_proposal_distribution.set_mean(np.zeros(initial_control_points.size,))
momenta_proposal_distribution.set_variance_sqrt(xml_parameters.momenta_proposal_std)
sampler.individual_proposal_distributions['momenta'] = momenta_proposal_distribution
estimator = McmcSaem()
estimator.sampler = sampler
estimator.sample_every_n_mcmc_iters = xml_parameters.sample_every_n_mcmc_iters
else:
estimator = GradientAscent()
estimator.initial_step_size = xml_parameters.initial_step_size
estimator.max_line_search_iterations = xml_parameters.max_line_search_iterations
estimator.line_search_shrink = xml_parameters.line_search_shrink
estimator.line_search_expand = xml_parameters.line_search_expand
msg = 'Unknown optimization-method-type: \"' + xml_parameters.optimization_method_type \
+ '\". Defaulting to GradientAscent.'
warnings.warn(msg)
estimator.max_iterations = xml_parameters.max_iterations
estimator.convergence_tolerance = xml_parameters.convergence_tolerance
estimator.print_every_n_iters = xml_parameters.print_every_n_iters
estimator.save_every_n_iters = xml_parameters.save_every_n_iters
estimator.dataset = dataset
estimator.statistical_model = model
# Initial random effects realizations
estimator.individual_RER = individual_RER
"""
Launch.
"""
if not os.path.exists(Settings().output_dir): os.makedirs(Settings().output_dir)
model.name = 'LongitudinalRegistration'
print('')
print('[ update method of the ' + estimator.name + ' optimizer ]')
try:
start_time = time.time()
estimator.update()
model._write_model_parameters(estimator.individual_RER)
end_time = time.time()
except RuntimeError as error:
print('>> Failure of the longitudinal registration procedure for subject %s: %s' % (full_subject_ids[i], error))
if not (estimator.name.lower() == 'scipyoptimize' and estimator.method.lower() == 'scipypowell'):
print('>> Second try with the ScipyPowell optimiser.')
estimator = ScipyOptimize()
estimator.method = 'Powell'
estimator.max_iterations = xml_parameters.max_iterations
estimator.convergence_tolerance = xml_parameters.convergence_tolerance
estimator.print_every_n_iters = xml_parameters.print_every_n_iters
estimator.save_every_n_iters = xml_parameters.save_every_n_iters
estimator.dataset = dataset
estimator.statistical_model = model
estimator.individual_RER = individual_RER
start_time = time.time()
estimator.update()
model._write_model_parameters(estimator.individual_RER)
end_time = time.time()
print('')
print('>> Estimation took: ' + str(time.strftime("%H:%M:%S", time.gmtime(end_time - start_time))))
return model