def _validate_dimension(self, X, sample_ols_confounder): if X.shape[0] != sample_ols_confounder.shape[0]: err_msg = 'Number of samples (N=' + str( X.shape[0]) + ') is not the same as number of cases (N=' + str( sample_ols_confounder.shape[0]) + ') in confounder!' logger.error(err_msg) raise ValueError(err_msg)
def inverse_transform(self, X, y=None, **kwargs): if not self.extract_mode == 'vec': msg = "BrainMask extract_mode={} is not supported with inverse_transform".format(self.extract_mode) logger.error(msg) raise NotImplementedError(msg) return self.masker.inverse_transform(X)
def save_backmapping(self, filename: str, backmapping): try: if isinstance(backmapping, list): backmapping = np.asarray(backmapping) try: from nibabel.nifti1 import Nifti1Image if isinstance(backmapping, Nifti1Image): backmapping.to_filename( os.path.join(self.output_settings.results_folder, filename + '.nii.gz')) except ImportError: pass finally: if isinstance(backmapping, np.ndarray): if backmapping.size > 1000: np.savez( os.path.join(self.output_settings.results_folder, filename + '.npz'), backmapping) else: np.savetxt(os.path.join( self.output_settings.results_folder, filename + '.csv'), backmapping, delimiter=',') else: with open( os.path.join(self.output_settings.results_folder, filename + '.p'), 'wb') as f: pickle.dump(backmapping, f) except Exception as e: logger.error("Could not save backmapped feature importances") logger.error(e)
def get_format_info_from_first_image(X): img = None if isinstance(X, str): img = image.load_img(X) elif isinstance(X, list) or isinstance(X, np.ndarray): if isinstance(X[0], str): img = image.load_img(X[0]) elif isinstance(X[0], nib.Nifti1Image): img = X[0] elif isinstance(X, nib.Nifti1Image): img = X else: error_msg = "Can only process strings as file paths to nifti images or nifti image object" logger.error(error_msg) raise ValueError(error_msg) if img is not None: if len(img.shape) > 3: img_shape = img.shape[:3] else: img_shape = img.shape return img.affine, img_shape else: raise ValueError( "Could not load image for affine and shape definition.")
def _add_mask_to_library(self, mask_name: str = '', target_affine=None, target_shape=None, mask_threshold=0.5): # Todo: find solution for multiprocessing spaming if mask_name in self.photon_masks.keys(): original_mask_object = self.photon_masks[mask_name] else: logger.debug("Checking custom mask") original_mask_object = self._check_custom_mask(mask_name) mask_object = MaskObject(name=mask_name, mask_file=original_mask_object.mask_file) #mask_object.mask = image.threshold_img(mask_object.mask_file, threshold=mask_threshold) mask_object.mask = image.math_img('img > {}'.format(mask_threshold), img=mask_object.mask_file) if target_affine is not None and target_shape is not None: mask_object.mask = self._resample(mask_object.mask, target_affine=target_affine, target_shape=target_shape) # check if roi is empty if np.sum(mask_object.mask.dataobj != 0) == 0: mask_object.is_empty = True msg = 'No voxels in mask after resampling (' + mask_object.name + ').' logger.error(msg) raise ValueError(msg) AtlasLibrary.LIBRARY[(mask_object.name, str(target_affine), str(target_shape), str(mask_threshold))] = mask_object logger.debug("BrainMask: Done adding mask to library!")
def create(cls, metric: str) -> Optional[Callable]: """ Searches for the metric by name and instantiates the according calculation function :param metric: the name of the metric as encoded in the ELEMENT_DICTIONARY :type metric: str :return: a callable instance of the metric calculation """ if metric in Scorer.ELEMENT_DICTIONARY: try: desired_class_info = Scorer.ELEMENT_DICTIONARY[metric] desired_class_home = desired_class_info[0] desired_class_name = desired_class_info[1] imported_module = __import__(desired_class_home, globals(), locals(), desired_class_name, 0) desired_class = getattr(imported_module, desired_class_name) scoring_method = desired_class return scoring_method except AttributeError as ae: logger.error('ValueError: Could not find according class: ' + Scorer.ELEMENT_DICTIONARY[metric]) elif metric in Scorer.CUSTOM_ELEMENT_DICTIONARY: return Scorer.CUSTOM_ELEMENT_DICTIONARY[metric] else: logger.error('NameError: Metric not supported right now:' + metric) # raise Warning('Metric not supported right now:', metric) return None
def greater_is_better_distinction(metric: str) -> bool: if metric in Scorer.ELEMENT_DICTIONARY: # for now do a simple hack and set greater_is_better # by looking at error/score in metric name specifier = Scorer.ELEMENT_DICTIONARY[metric][2] if specifier == 'score': return True elif specifier == 'error': return False else: # Todo: better error checking? error_msg = "Metric not suitable for optimizer." logger.error(error_msg) raise NameError(error_msg) elif metric in Scorer.CUSTOM_ELEMENT_DICTIONARY: # Check if it's a greater_is_better metric by calling it with example values metric_callable = Scorer.create(metric) y_true = [1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0] y_pred = [0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0] true_true = metric_callable(y_true, y_true) true_pred = metric_callable(y_true, y_pred) return true_true > true_pred else: logger.error('Specify valid metric to choose best config.') raise NameError('Specify valid metric to choose best config.')
def optimize(self): """ Optimizes the marginal log likelihood and returns the best found hyperparameter configuration theta. :return: theta: Hyperparameter vector that maximizes the marginal log likelihood :rtype: np.ndarray(H) """ # Start optimization from the previous hyperparameter configuration p0 = self.gp.kernel.vector p0 = np.append(p0, np.log(self.noise)) if self.use_gradients: bounds = [(-10, 10)] * (len(self.kernel) + 1) theta, _, _ = optimize.fmin_l_bfgs_b(self.nll, p0, fprime=self.grad_nll, bounds=bounds) else: try: results = optimize.minimize(self.nll, p0) theta = results.x except ValueError: logger.error( "Fabolas.GaussianProcess: Could not find a valid hyperparameter configuration! Use initial configuration" ) theta = p0 return theta
def transform(self, X, y=None, **kwargs): if self.base_element.cache_folder is not None: # make sure we cache individually self.base_element.single_subject_caching = True self.base_element.caching = True if self.nr_of_processes > 1: if self.base_element.cache_folder is not None: # at first apply the transformation on several cores, everything gets written to the cache, # so the next step only has to reload the data ... self.apply_transform_parallelized(X) else: logger.error( "Cannot use parallelization without a cache folder specified in the hyperpipe." "Using single core instead") logger.debug("NeuroBranch " + self.name + " is collecting data from the different cores...") X_new, _, _ = self.base_element.transform(X) # check if we have a list of niftis, should avoid this, except when output_image = True if not self.output_img: if ((isinstance(X_new, list) and len(X_new) > 0) or (isinstance(X_new, np.ndarray) and len(X_new.shape) == 1)) and isinstance( X_new[0], Nifti1Image): X_new = np.asarray([i.dataobj for i in X_new]) return X_new, y, kwargs
def transform(cls, X): n_subjects = 1 load_data = None msg = None # load all niftis to memory if isinstance(X, list) or isinstance(X, np.ndarray): n_subjects = len(X) if all([isinstance(x, str) for x in X]): load_data = image.load_img(X) elif all([isinstance(x, np.ndarray) for x in X]): n_subjects = X.shape[0] load_data = image.load_img(X) elif all([isinstance(x, Nifti1Image) for x in X]): load_data = image.load_img(X) else: msg = "Cannot interpret the types in the given input_data list." elif isinstance(X, str): load_data = image.load_img(X) elif isinstance(X, Nifti1Image): load_data = X else: msg = "Can only process strings as file paths to nifti images or nifti image object" if msg: logger.error(msg) raise ValueError(msg) return load_data, n_subjects
def __iadd__(self, pipe_element): """ Add an element to the neuro branch. Only neuro pipeline elements are allowed. Returns self Parameters ---------- * `pipe_element` [PipelineElement]: The transformer object to add. Should be registered in the Neuro module. """ if pipe_element.name in NeuroBranch.NEURO_ELEMENTS: # as the neuro branch is parallelized and processes several images subsequently on # different cores, we need to stop the children to process on several cores as well pipe_element.base_element.output_img = True self.elements.append(pipe_element) self._prepare_pipeline() elif isinstance(pipe_element, CallbackElement): self.elements.append(pipe_element) self._prepare_pipeline() else: logger.error( "PipelineElement {} is not part of the Neuro module:".format( pipe_element.name)) return self
def get_random_value(self, definite_list: bool = True): if definite_list: return random.choice(self.values) else: msg = "The PhotonHyperparam has no own random function." logger.error(msg) raise ValueError(msg)
def save_backmapping(self, filename: str, backmapping): try: if isinstance(backmapping, np.ndarray): if len(backmapping) > 1000: np.savez( os.path.join(self.output_settings.results_folder, filename + ".npz"), backmapping, ) else: np.savetxt( os.path.join(self.output_settings.results_folder, filename + ".csv"), backmapping, delimiter=",", ) elif isinstance(backmapping, Nifti1Image): backmapping.to_filename( os.path.join(self.output_settings.results_folder, filename + ".nii.gz")) else: with open( os.path.join(self.output_settings.results_folder, filename + ".p"), "wb", ) as f: pickle.dump(backmapping, f) except Exception as e: logger.error("Could not save backmapped feature importances") logger.error(e)
def draw_patches(patch_x, patch_size): if isinstance(patch_x, str): patch_x = np.ascontiguousarray(load_img(patch_x).get_data()) elif isinstance(patch_x, Nifti1Image): patch_x = np.ascontiguousarray(patch_x.dataobj) elif isinstance(patch_x, np.ndarray): if len(patch_x.shape) == 1: patch_x = list(patch_x) elif isinstance(patch_x, list): pass else: msg = "Could not read input data." logger.error(msg) raise ValueError(msg) if isinstance(patch_x, list): if all([isinstance(px, str) for px in patch_x]): patch_x = [np.ascontiguousarray(load_img(px).get_data()) for px in patch_x] return_list = [] for p in patch_x: return_list.append(PatchImages.draw_patch_from_mri(p, patch_size)) return return_list return PatchImages.draw_patch_from_mri(patch_x, patch_size)
def voxel_size(self, voxel_size): if isinstance(voxel_size, int): self._voxel_size = [voxel_size, voxel_size, voxel_size] elif isinstance(voxel_size, list) and len(voxel_size) == 3 and all(isinstance(x, int) for x in voxel_size): self._voxel_size = voxel_size else: msg = "ResampleImages expected voxel_size as int or a list of three ints like [3, 3, 3]." logger.error(msg) raise ValueError(msg)
def get_random_value(self, definite_list: bool = False): if definite_list: if not self.values: msg = "No values were set. Please use transform method." logger.error(msg) raise ValueError(msg) return random.choice(self.values) else: return random.uniform(self.start, self.stop)
def register(self, photon_name: str, class_str: str, element_type: str): """ Save element information to the JSON file Parameters: ----------- * 'photon_name' [str]: The string literal with which you want to access the class * 'class_str' [str]: The namespace of the class, like in the import statement * 'element_type' [str]: Can be 'Estimator' or 'Transformer' * 'custom_folder' [str]: All registrations are saved to this folder """ # check if folder exists if not PhotonRegistry.CUSTOM_ELEMENTS_FOLDER: raise ValueError( "To register an element, specify a custom elements folder when instantiating the registry " "module. Example: registry = PhotonRegistry('/MY/CUSTOM/ELEMENTS/FOLDER)" ) if not element_type == "Estimator" and not element_type == "Transformer": raise ValueError( "Variable element_type must be 'Estimator' or 'Transformer'") duplicate = self._check_duplicate( photon_name=photon_name, class_str=class_str, content=PhotonRegistry.CUSTOM_ELEMENTS) if not duplicate: python_file = os.path.join(PhotonRegistry.CUSTOM_ELEMENTS_FOLDER, class_str.split('.')[0] + '.py') if not os.path.isfile(python_file): raise FileNotFoundError( "Couldn't find python file {} in your custom elements folder. " "Please copy your file into this folder first!".format( python_file)) # add new element PhotonRegistry.CUSTOM_ELEMENTS[ photon_name] = class_str, element_type # write back to file self._write_to_json(PhotonRegistry.CUSTOM_ELEMENTS) logger.info('Adding PipelineElement ' + class_str + ' to CustomElements.json as "' + photon_name + '".') # activate custom elements self.activate() # check custom element logger.info("Running tests on custom element...") return self._run_tests(photon_name, element_type) else: logger.error('Could not register element!')
def _fit_dummy(self): if self.dummy_estimator is not None: logger.info("Running Dummy Estimator...") try: if isinstance(self._validation_X, np.ndarray): if len(self._validation_X.shape) > 2: logger.info( "Skipping dummy estimator because of too many dimensions" ) self.result_object.dummy_results = None return dummy_y = np.reshape(self._validation_y, (-1, 1)) self.dummy_estimator.fit(dummy_y, self._validation_y) train_scores = InnerFoldManager.score( self.dummy_estimator, self._validation_X, self._validation_y, metrics=self.optimization_info.metrics, ) # fill result tree with fold information inner_fold = MDBInnerFold() inner_fold.training = train_scores if self.cross_validaton_info.eval_final_performance: test_scores = InnerFoldManager.score( self.dummy_estimator, self._test_X, self._test_y, metrics=self.optimization_info.metrics, ) print_metrics("DUMMY", test_scores.metrics) inner_fold.validation = test_scores self.result_object.dummy_results = inner_fold # performaceConstraints: DummyEstimator if self.constraint_objects is not None: dummy_constraint_objs = [ opt for opt in self.constraint_objects if isinstance(opt, DummyPerformance) ] if dummy_constraint_objs: for dummy_constraint_obj in dummy_constraint_objs: dummy_constraint_obj.set_dummy_performance( self.result_object.dummy_results ) return inner_fold except Exception as e: logger.error(e) logger.info("Skipping dummy because of error..") return None else: logger.info("Skipping dummy ..")
def write_result_tree_to_file(self): try: local_file = os.path.join(self.output_settings.results_folder, "photon_result_file.p") file_opened = open(local_file, "wb") pickle.dump(self.results.to_son(), file_opened) file_opened.close() except OSError as e: logger.error("Could not write results to local file") logger.error(str(e))
def _check_custom_atlas(atlas_file): if not path.isfile(atlas_file): msg = "Cannot find custom atlas {}".format(atlas_file) logger.error(msg) raise FileNotFoundError(msg) labels_file = path.split(atlas_file)[0] + '_labels.txt' if not path.isfile(labels_file): msg = "Didn't find .txt file with ROI labels. Using indices as labels." logger.warning(msg) warnings.warn(msg) return AtlasObject(name=atlas_file, path=atlas_file, labels_file=labels_file)
def __init__(self, voxel_size: Union[int, List] = 3, interpolation: str = 'nearest', output_img: bool = False): super(ResampleImages, self).__init__(output_img=output_img) self._voxel_size = None self.voxel_size = voxel_size if interpolation in ['continuous', 'linear', 'nearest']: self.interpolation = interpolation else: msg = "Got unexpected interpolation. Please use one of ['continuous', 'linear' 'nearest']" logger.error(msg) raise NameError(msg)
def _resample(mask, target_affine, target_shape): if target_affine is not None and target_shape is not None: mask = image.resample_img(mask, target_affine=target_affine, target_shape=target_shape, interpolation='nearest') # check orientations orient_data = ''.join(nib.aff2axcodes(target_affine)) orient_roi = ''.join(nib.aff2axcodes(mask.affine)) if not orient_roi == orient_data: msg = 'Orientation of mask and data are not the same: ' + \ orient_roi + ' (mask) vs. ' + orient_data + ' (data)' logger.error(msg) raise ValueError(msg) return mask
def write_result_tree_to_file(self): try: local_file = os.path.join(self.output_settings.results_folder, 'photon_result_file.json') # file_opened = open(local_file, 'wb') with open(local_file, 'w') as outfile: json.dump(self.results.to_son(), outfile, default=self.convert_to_json_serializable) except OSError as e: logger.error("Could not write results to local file") logger.error(str(e))
def _check_custom_atlas(atlas_file): if not path.isfile(atlas_file): raise FileNotFoundError( "Cannot find custom atlas {}".format(atlas_file)) labels_file = path.split(atlas_file)[0] + "_labels.txt" if not path.isfile(labels_file): logger.error( "Didn't find .txt file with ROI labels. Using indices as labels." ) return AtlasObject(name=atlas_file, path=atlas_file, labels_file=labels_file)
def save(self): if self.output_settings.mongodb_connect_url: connect(self.output_settings.mongodb_connect_url, alias='photon_core') logger.info('Write results to mongodb...') try: self.results.save() except DocumentTooLarge as e: logger.error( 'Could not save document into MongoDB: Document too large') if self.output_settings.save_output: logger.info("Writing results to project folder...") self.write_result_tree_to_file()
def get_random_value(self, definite_list: bool = True): """ Method for random search to get random parameter based on its domain. Parameters ---------- * `definite_list` [bool: True]: Choice of transform param to discret list or not. """ if definite_list: return random.choice(self.values) else: msg = "The PHOTONAI hyperparam has no own random function." logger.error(msg) raise NotImplementedError(msg)
def predict(self, X, **kwargs): if len(self.hyperpipes_to_fit) == 0: msg = "No hyperpipes to predict. Did you remember to fit or load the Atlas Mapper?" logger.error(msg) raise Exception(msg) X_extracted, _, _ = self.neuro_element.transform(X) X_extracted = AtlasMapper._reshape_roi_data(X_extracted) predictions = dict() for roi, infos in self.hyperpipe_infos.items(): roi_index = infos['roi_index'] predictions[roi] = self.hyperpipes_to_fit[roi].predict( X_extracted[roi_index], **kwargs) return predictions
def raise_PhotonaiNotImplementedError(msg: str): """ Photonai Raise a NotImplemented error, and loogging.error of the same msg. Parameters ---------- msg: The formatted or unformatted sting sent to error and log. Returns ------- Traps into an error that is probably not recoverable. """ logger.error(msg) raise PhotonaiNotImplementedError(msg)
def get_format_info_from_first_image(X): img, n_subjects = NiftiConverter.transform(X) if n_subjects > 1: img = img.slicer[:, :, :, 0] if img is not None: if len(img.shape) > 3: img_shape = img.shape[:3] else: img_shape = img.shape return img.affine, img_shape else: msg = "Could not load image for affine and shape definition." logger.error(msg) raise ValueError(msg)
def _resample(mask, target_affine, target_shape): if target_affine is not None and target_shape is not None: mask = image.resample_img( mask, target_affine=target_affine, target_shape=target_shape, interpolation="nearest", ) # check orientations orient_data = "".join(nib.aff2axcodes(target_affine)) orient_roi = "".join(nib.aff2axcodes(mask.affine)) if not orient_roi == orient_data: logger.error( "Orientation of mask and data are not the same: " + orient_roi + " (mask) vs. " + orient_data + " (data)") return mask