def _validate_dimension(self, X, sample_ols_confounder):
if X.shape[0] != sample_ols_confounder.shape[0]:
err_msg = 'Number of samples (N=' + str(
X.shape[0]) + ') is not the same as number of cases (N=' + str(
sample_ols_confounder.shape[0]) + ') in confounder!'
logger.error(err_msg)
raise ValueError(err_msg)
def inverse_transform(self, X, y=None, **kwargs):
if not self.extract_mode == 'vec':
msg = "BrainMask extract_mode={} is not supported with inverse_transform".format(self.extract_mode)
logger.error(msg)
raise NotImplementedError(msg)
return self.masker.inverse_transform(X)
def save_backmapping(self, filename: str, backmapping):
try:
if isinstance(backmapping, list):
backmapping = np.asarray(backmapping)
try:
from nibabel.nifti1 import Nifti1Image
if isinstance(backmapping, Nifti1Image):
backmapping.to_filename(
os.path.join(self.output_settings.results_folder,
filename + '.nii.gz'))
except ImportError:
pass
finally:
if isinstance(backmapping, np.ndarray):
if backmapping.size > 1000:
np.savez(
os.path.join(self.output_settings.results_folder,
filename + '.npz'), backmapping)
else:
np.savetxt(os.path.join(
self.output_settings.results_folder,
filename + '.csv'),
backmapping,
delimiter=',')
else:
with open(
os.path.join(self.output_settings.results_folder,
filename + '.p'), 'wb') as f:
pickle.dump(backmapping, f)
except Exception as e:
logger.error("Could not save backmapped feature importances")
logger.error(e)
def get_format_info_from_first_image(X):
img = None
if isinstance(X, str):
img = image.load_img(X)
elif isinstance(X, list) or isinstance(X, np.ndarray):
if isinstance(X[0], str):
img = image.load_img(X[0])
elif isinstance(X[0], nib.Nifti1Image):
img = X[0]
elif isinstance(X, nib.Nifti1Image):
img = X
else:
error_msg = "Can only process strings as file paths to nifti images or nifti image object"
logger.error(error_msg)
raise ValueError(error_msg)
if img is not None:
if len(img.shape) > 3:
img_shape = img.shape[:3]
else:
img_shape = img.shape
return img.affine, img_shape
else:
raise ValueError(
"Could not load image for affine and shape definition.")
def _add_mask_to_library(self, mask_name: str = '', target_affine=None, target_shape=None, mask_threshold=0.5):
# Todo: find solution for multiprocessing spaming
if mask_name in self.photon_masks.keys():
original_mask_object = self.photon_masks[mask_name]
else:
logger.debug("Checking custom mask")
original_mask_object = self._check_custom_mask(mask_name)
mask_object = MaskObject(name=mask_name, mask_file=original_mask_object.mask_file)
#mask_object.mask = image.threshold_img(mask_object.mask_file, threshold=mask_threshold)
mask_object.mask = image.math_img('img > {}'.format(mask_threshold), img=mask_object.mask_file)
if target_affine is not None and target_shape is not None:
mask_object.mask = self._resample(mask_object.mask, target_affine=target_affine, target_shape=target_shape)
# check if roi is empty
if np.sum(mask_object.mask.dataobj != 0) == 0:
mask_object.is_empty = True
msg = 'No voxels in mask after resampling (' + mask_object.name + ').'
logger.error(msg)
raise ValueError(msg)
AtlasLibrary.LIBRARY[(mask_object.name, str(target_affine), str(target_shape), str(mask_threshold))] = mask_object
logger.debug("BrainMask: Done adding mask to library!")
def create(cls, metric: str) -> Optional[Callable]:
"""
Searches for the metric by name and instantiates the according calculation function
:param metric: the name of the metric as encoded in the ELEMENT_DICTIONARY
:type metric: str
:return: a callable instance of the metric calculation
"""
if metric in Scorer.ELEMENT_DICTIONARY:
try:
desired_class_info = Scorer.ELEMENT_DICTIONARY[metric]
desired_class_home = desired_class_info[0]
desired_class_name = desired_class_info[1]
imported_module = __import__(desired_class_home, globals(),
locals(), desired_class_name, 0)
desired_class = getattr(imported_module, desired_class_name)
scoring_method = desired_class
return scoring_method
except AttributeError as ae:
logger.error('ValueError: Could not find according class: ' +
Scorer.ELEMENT_DICTIONARY[metric])
elif metric in Scorer.CUSTOM_ELEMENT_DICTIONARY:
return Scorer.CUSTOM_ELEMENT_DICTIONARY[metric]
else:
logger.error('NameError: Metric not supported right now:' + metric)
# raise Warning('Metric not supported right now:', metric)
return None
def greater_is_better_distinction(metric: str) -> bool:
if metric in Scorer.ELEMENT_DICTIONARY:
# for now do a simple hack and set greater_is_better
# by looking at error/score in metric name
specifier = Scorer.ELEMENT_DICTIONARY[metric][2]
if specifier == 'score':
return True
elif specifier == 'error':
return False
else:
# Todo: better error checking?
error_msg = "Metric not suitable for optimizer."
logger.error(error_msg)
raise NameError(error_msg)
elif metric in Scorer.CUSTOM_ELEMENT_DICTIONARY:
# Check if it's a greater_is_better metric by calling it with example values
metric_callable = Scorer.create(metric)
y_true = [1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0]
y_pred = [0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0]
true_true = metric_callable(y_true, y_true)
true_pred = metric_callable(y_true, y_pred)
return true_true > true_pred
else:
logger.error('Specify valid metric to choose best config.')
raise NameError('Specify valid metric to choose best config.')
def optimize(self):
"""
Optimizes the marginal log likelihood and returns the best found
hyperparameter configuration theta.
:return: theta: Hyperparameter vector that maximizes the marginal log likelihood
:rtype: np.ndarray(H)
"""
# Start optimization from the previous hyperparameter configuration
p0 = self.gp.kernel.vector
p0 = np.append(p0, np.log(self.noise))
if self.use_gradients:
bounds = [(-10, 10)] * (len(self.kernel) + 1)
theta, _, _ = optimize.fmin_l_bfgs_b(self.nll,
p0,
fprime=self.grad_nll,
bounds=bounds)
else:
try:
results = optimize.minimize(self.nll, p0)
theta = results.x
except ValueError:
logger.error(
"Fabolas.GaussianProcess: Could not find a valid hyperparameter configuration! Use initial configuration"
)
theta = p0
return theta
def transform(self, X, y=None, **kwargs):
if self.base_element.cache_folder is not None:
# make sure we cache individually
self.base_element.single_subject_caching = True
self.base_element.caching = True
if self.nr_of_processes > 1:
if self.base_element.cache_folder is not None:
# at first apply the transformation on several cores, everything gets written to the cache,
# so the next step only has to reload the data ...
self.apply_transform_parallelized(X)
else:
logger.error(
"Cannot use parallelization without a cache folder specified in the hyperpipe."
"Using single core instead")
logger.debug("NeuroBranch " + self.name +
" is collecting data from the different cores...")
X_new, _, _ = self.base_element.transform(X)
# check if we have a list of niftis, should avoid this, except when output_image = True
if not self.output_img:
if ((isinstance(X_new, list) and len(X_new) > 0) or
(isinstance(X_new, np.ndarray)
and len(X_new.shape) == 1)) and isinstance(
X_new[0], Nifti1Image):
X_new = np.asarray([i.dataobj for i in X_new])
return X_new, y, kwargs
def transform(cls, X):
n_subjects = 1
load_data = None
msg = None
# load all niftis to memory
if isinstance(X, list) or isinstance(X, np.ndarray):
n_subjects = len(X)
if all([isinstance(x, str) for x in X]):
load_data = image.load_img(X)
elif all([isinstance(x, np.ndarray) for x in X]):
n_subjects = X.shape[0]
load_data = image.load_img(X)
elif all([isinstance(x, Nifti1Image) for x in X]):
load_data = image.load_img(X)
else:
msg = "Cannot interpret the types in the given input_data list."
elif isinstance(X, str):
load_data = image.load_img(X)
elif isinstance(X, Nifti1Image):
load_data = X
else:
msg = "Can only process strings as file paths to nifti images or nifti image object"
if msg:
logger.error(msg)
raise ValueError(msg)
return load_data, n_subjects
def __iadd__(self, pipe_element):
"""
Add an element to the neuro branch. Only neuro pipeline elements are allowed.
Returns self
Parameters
----------
* `pipe_element` [PipelineElement]:
The transformer object to add. Should be registered in the Neuro module.
"""
if pipe_element.name in NeuroBranch.NEURO_ELEMENTS:
# as the neuro branch is parallelized and processes several images subsequently on
# different cores, we need to stop the children to process on several cores as well
pipe_element.base_element.output_img = True
self.elements.append(pipe_element)
self._prepare_pipeline()
elif isinstance(pipe_element, CallbackElement):
self.elements.append(pipe_element)
self._prepare_pipeline()
else:
logger.error(
"PipelineElement {} is not part of the Neuro module:".format(
pipe_element.name))
return self
def get_random_value(self, definite_list: bool = True):
if definite_list:
return random.choice(self.values)
else:
msg = "The PhotonHyperparam has no own random function."
logger.error(msg)
raise ValueError(msg)
def save_backmapping(self, filename: str, backmapping):
try:
if isinstance(backmapping, np.ndarray):
if len(backmapping) > 1000:
np.savez(
os.path.join(self.output_settings.results_folder,
filename + ".npz"),
backmapping,
)
else:
np.savetxt(
os.path.join(self.output_settings.results_folder,
filename + ".csv"),
backmapping,
delimiter=",",
)
elif isinstance(backmapping, Nifti1Image):
backmapping.to_filename(
os.path.join(self.output_settings.results_folder,
filename + ".nii.gz"))
else:
with open(
os.path.join(self.output_settings.results_folder,
filename + ".p"),
"wb",
) as f:
pickle.dump(backmapping, f)
except Exception as e:
logger.error("Could not save backmapped feature importances")
logger.error(e)
def draw_patches(patch_x, patch_size):
if isinstance(patch_x, str):
patch_x = np.ascontiguousarray(load_img(patch_x).get_data())
elif isinstance(patch_x, Nifti1Image):
patch_x = np.ascontiguousarray(patch_x.dataobj)
elif isinstance(patch_x, np.ndarray):
if len(patch_x.shape) == 1:
patch_x = list(patch_x)
elif isinstance(patch_x, list):
pass
else:
msg = "Could not read input data."
logger.error(msg)
raise ValueError(msg)
if isinstance(patch_x, list):
if all([isinstance(px, str) for px in patch_x]):
patch_x = [np.ascontiguousarray(load_img(px).get_data()) for px in patch_x]
return_list = []
for p in patch_x:
return_list.append(PatchImages.draw_patch_from_mri(p, patch_size))
return return_list
return PatchImages.draw_patch_from_mri(patch_x, patch_size)
def voxel_size(self, voxel_size):
if isinstance(voxel_size, int):
self._voxel_size = [voxel_size, voxel_size, voxel_size]
elif isinstance(voxel_size, list) and len(voxel_size) == 3 and all(isinstance(x, int) for x in voxel_size):
self._voxel_size = voxel_size
else:
msg = "ResampleImages expected voxel_size as int or a list of three ints like [3, 3, 3]."
logger.error(msg)
raise ValueError(msg)
def get_random_value(self, definite_list: bool = False):
if definite_list:
if not self.values:
msg = "No values were set. Please use transform method."
logger.error(msg)
raise ValueError(msg)
return random.choice(self.values)
else:
return random.uniform(self.start, self.stop)
def register(self, photon_name: str, class_str: str, element_type: str):
"""
Save element information to the JSON file
Parameters:
-----------
* 'photon_name' [str]:
The string literal with which you want to access the class
* 'class_str' [str]:
The namespace of the class, like in the import statement
* 'element_type' [str]:
Can be 'Estimator' or 'Transformer'
* 'custom_folder' [str]:
All registrations are saved to this folder
"""
# check if folder exists
if not PhotonRegistry.CUSTOM_ELEMENTS_FOLDER:
raise ValueError(
"To register an element, specify a custom elements folder when instantiating the registry "
"module. Example: registry = PhotonRegistry('/MY/CUSTOM/ELEMENTS/FOLDER)"
)
if not element_type == "Estimator" and not element_type == "Transformer":
raise ValueError(
"Variable element_type must be 'Estimator' or 'Transformer'")
duplicate = self._check_duplicate(
photon_name=photon_name,
class_str=class_str,
content=PhotonRegistry.CUSTOM_ELEMENTS)
if not duplicate:
python_file = os.path.join(PhotonRegistry.CUSTOM_ELEMENTS_FOLDER,
class_str.split('.')[0] + '.py')
if not os.path.isfile(python_file):
raise FileNotFoundError(
"Couldn't find python file {} in your custom elements folder. "
"Please copy your file into this folder first!".format(
python_file))
# add new element
PhotonRegistry.CUSTOM_ELEMENTS[
photon_name] = class_str, element_type
# write back to file
self._write_to_json(PhotonRegistry.CUSTOM_ELEMENTS)
logger.info('Adding PipelineElement ' + class_str +
' to CustomElements.json as "' + photon_name + '".')
# activate custom elements
self.activate()
# check custom element
logger.info("Running tests on custom element...")
return self._run_tests(photon_name, element_type)
else:
logger.error('Could not register element!')
def _fit_dummy(self):
if self.dummy_estimator is not None:
logger.info("Running Dummy Estimator...")
try:
if isinstance(self._validation_X, np.ndarray):
if len(self._validation_X.shape) > 2:
logger.info(
"Skipping dummy estimator because of too many dimensions"
)
self.result_object.dummy_results = None
return
dummy_y = np.reshape(self._validation_y, (-1, 1))
self.dummy_estimator.fit(dummy_y, self._validation_y)
train_scores = InnerFoldManager.score(
self.dummy_estimator,
self._validation_X,
self._validation_y,
metrics=self.optimization_info.metrics,
)
# fill result tree with fold information
inner_fold = MDBInnerFold()
inner_fold.training = train_scores
if self.cross_validaton_info.eval_final_performance:
test_scores = InnerFoldManager.score(
self.dummy_estimator,
self._test_X,
self._test_y,
metrics=self.optimization_info.metrics,
)
print_metrics("DUMMY", test_scores.metrics)
inner_fold.validation = test_scores
self.result_object.dummy_results = inner_fold
# performaceConstraints: DummyEstimator
if self.constraint_objects is not None:
dummy_constraint_objs = [
opt
for opt in self.constraint_objects
if isinstance(opt, DummyPerformance)
]
if dummy_constraint_objs:
for dummy_constraint_obj in dummy_constraint_objs:
dummy_constraint_obj.set_dummy_performance(
self.result_object.dummy_results
)
return inner_fold
except Exception as e:
logger.error(e)
logger.info("Skipping dummy because of error..")
return None
else:
logger.info("Skipping dummy ..")
def write_result_tree_to_file(self):
try:
local_file = os.path.join(self.output_settings.results_folder,
"photon_result_file.p")
file_opened = open(local_file, "wb")
pickle.dump(self.results.to_son(), file_opened)
file_opened.close()
except OSError as e:
logger.error("Could not write results to local file")
logger.error(str(e))
def _check_custom_atlas(atlas_file):
if not path.isfile(atlas_file):
msg = "Cannot find custom atlas {}".format(atlas_file)
logger.error(msg)
raise FileNotFoundError(msg)
labels_file = path.split(atlas_file)[0] + '_labels.txt'
if not path.isfile(labels_file):
msg = "Didn't find .txt file with ROI labels. Using indices as labels."
logger.warning(msg)
warnings.warn(msg)
return AtlasObject(name=atlas_file, path=atlas_file, labels_file=labels_file)
def __init__(self, voxel_size: Union[int, List] = 3, interpolation: str = 'nearest', output_img: bool = False):
super(ResampleImages, self).__init__(output_img=output_img)
self._voxel_size = None
self.voxel_size = voxel_size
if interpolation in ['continuous', 'linear', 'nearest']:
self.interpolation = interpolation
else:
msg = "Got unexpected interpolation. Please use one of ['continuous', 'linear' 'nearest']"
logger.error(msg)
raise NameError(msg)
def _resample(mask, target_affine, target_shape):
if target_affine is not None and target_shape is not None:
mask = image.resample_img(mask, target_affine=target_affine, target_shape=target_shape, interpolation='nearest')
# check orientations
orient_data = ''.join(nib.aff2axcodes(target_affine))
orient_roi = ''.join(nib.aff2axcodes(mask.affine))
if not orient_roi == orient_data:
msg = 'Orientation of mask and data are not the same: ' + \
orient_roi + ' (mask) vs. ' + orient_data + ' (data)'
logger.error(msg)
raise ValueError(msg)
return mask
def write_result_tree_to_file(self):
try:
local_file = os.path.join(self.output_settings.results_folder,
'photon_result_file.json')
# file_opened = open(local_file, 'wb')
with open(local_file, 'w') as outfile:
json.dump(self.results.to_son(),
outfile,
default=self.convert_to_json_serializable)
except OSError as e:
logger.error("Could not write results to local file")
logger.error(str(e))
def _check_custom_atlas(atlas_file):
if not path.isfile(atlas_file):
raise FileNotFoundError(
"Cannot find custom atlas {}".format(atlas_file))
labels_file = path.split(atlas_file)[0] + "_labels.txt"
if not path.isfile(labels_file):
logger.error(
"Didn't find .txt file with ROI labels. Using indices as labels."
)
return AtlasObject(name=atlas_file,
path=atlas_file,
labels_file=labels_file)
def save(self):
if self.output_settings.mongodb_connect_url:
connect(self.output_settings.mongodb_connect_url,
alias='photon_core')
logger.info('Write results to mongodb...')
try:
self.results.save()
except DocumentTooLarge as e:
logger.error(
'Could not save document into MongoDB: Document too large')
if self.output_settings.save_output:
logger.info("Writing results to project folder...")
self.write_result_tree_to_file()
def get_random_value(self, definite_list: bool = True):
"""
Method for random search to get random parameter based on its domain.
Parameters
----------
* `definite_list` [bool: True]:
Choice of transform param to discret list or not.
"""
if definite_list:
return random.choice(self.values)
else:
msg = "The PHOTONAI hyperparam has no own random function."
logger.error(msg)
raise NotImplementedError(msg)
def predict(self, X, **kwargs):
if len(self.hyperpipes_to_fit) == 0:
msg = "No hyperpipes to predict. Did you remember to fit or load the Atlas Mapper?"
logger.error(msg)
raise Exception(msg)
X_extracted, _, _ = self.neuro_element.transform(X)
X_extracted = AtlasMapper._reshape_roi_data(X_extracted)
predictions = dict()
for roi, infos in self.hyperpipe_infos.items():
roi_index = infos['roi_index']
predictions[roi] = self.hyperpipes_to_fit[roi].predict(
X_extracted[roi_index], **kwargs)
return predictions
def raise_PhotonaiNotImplementedError(msg: str):
"""
Photonai Raise a NotImplemented error, and loogging.error of
the same msg.
Parameters
----------
msg: The formatted or unformatted sting sent to error and log.
Returns
-------
Traps into an error that is probably not recoverable.
"""
logger.error(msg)
raise PhotonaiNotImplementedError(msg)
def get_format_info_from_first_image(X):
img, n_subjects = NiftiConverter.transform(X)
if n_subjects > 1:
img = img.slicer[:, :, :, 0]
if img is not None:
if len(img.shape) > 3:
img_shape = img.shape[:3]
else:
img_shape = img.shape
return img.affine, img_shape
else:
msg = "Could not load image for affine and shape definition."
logger.error(msg)
raise ValueError(msg)
def _resample(mask, target_affine, target_shape):
if target_affine is not None and target_shape is not None:
mask = image.resample_img(
mask,
target_affine=target_affine,
target_shape=target_shape,
interpolation="nearest",
)
# check orientations
orient_data = "".join(nib.aff2axcodes(target_affine))
orient_roi = "".join(nib.aff2axcodes(mask.affine))
if not orient_roi == orient_data:
logger.error(
"Orientation of mask and data are not the same: " +
orient_roi + " (mask) vs. " + orient_data + " (data)")
return mask
