def activations(self, value):
"""
Setter for act_func. Checks if strategy is supported.
:param value:
:return:
"""
if not type(value) in [list, str]:
raise ValueError("act_func type is not supported. Please use one of [list, float]")
else:
if not self._hidden_layer_sizes:
self._activations = value
else:
if type(value) == str:
if value in __supported_activations__.keys():
self._activations = [value]*len(self.hidden_layer_sizes)
logger.warning("activations with type str converted to type list.")
else:
raise ValueError(
"activations not supported. Please use one of: " + str(__supported_activations__.keys()))
elif len(value) != len(self.hidden_layer_sizes):
raise ValueError("activations length missmatched layer length.")
elif any(act not in __supported_activations__.keys() for act in value):
raise ValueError("activations not supported. Please use one of: "+str(__supported_activations__.keys()))
else:
self._activations = value
def fit(self, X, y, reload_weights: bool = False):
# set weights to initial weights to achieve a weight reset
self.model.set_weights(self.init_weights)
# allow labels to be encoded before being passed to the model
# by default self.encode_targets returns identity of y
y = self.encode_targets(y)
# use callbacks only when size of training set is above 100
if X.shape[0] > 100:
# get pseudo validation set for keras callbacks
# fit the model
self.model.fit(X,
y,
batch_size=self.nn_batch_size,
validation_split=0.1,
epochs=self.epochs,
callbacks=self.callbacks,
verbose=self.verbosity)
else:
# fit the model
logger.warning(
'Cannot use Keras Callbacks because of small sample size.')
self.model.fit(X,
y,
batch_size=self.nn_batch_size,
epochs=self.epochs,
callbacks=self.callbacks,
verbose=self.verbosity)
return self
def shall_continue(self, config_item):
"""
Function to evaluate if the constraint is reached.
If it returns True, the testing of the configuration is continued.
If it returns False, further testing of the configuration is skipped
to increase speed of the hyperparameter search.
Parameters
----------
* 'config_item' [MDBConfig]:
All performance metrics and other scoring information for all configuration's performance.
Can be used to evaluate if the configuration has any potential to serve the model's learning task.
"""
if self.metric == "unknown":
logger.warning(
"The metric is not known. Please check the metric: " +
self.metric + ". " +
"Performance constraints are constantly True.")
return True
if self.metric not in config_item.inner_folds[0].validation.metrics:
logger.warning("The metric is not calculated. Please insert " +
self.metric + " to Hyperpipe.metrics. " +
"Performance constraints are constantly False.")
return False
if self._greater_is_better:
if self.strategy.name == 'first':
if config_item.inner_folds[0].validation.metrics[
self.metric] < self.threshold:
return False
elif self.strategy.name == 'all':
if any(item < self.threshold for item in [
x.validation.metrics[self.metric]
for x in config_item.inner_folds
]):
return False
elif self.strategy.name == 'mean':
if np.mean([
x.validation.metrics[self.metric]
for x in config_item.inner_folds
]) < self.threshold:
return False
return True
else:
if self.strategy.name == 'first':
if config_item.inner_folds[0].validation.metrics[
self.metric] > self.threshold:
return False
elif self.strategy.name == 'all':
if any(item > self.threshold for item in [
x.validation.metrics[self.metric]
for x in config_item.inner_folds
]):
return False
elif self.strategy.name == 'mean':
if np.mean([
x.validation.metrics[self.metric]
for x in config_item.inner_folds
]) > self.threshold:
return False
return True
def register_custom_metric(
cls, metric: Union[Metric_Type,
Tuple[str, Metric_Type]]) -> Optional[str]:
if cls.dynamic_keras_import is None:
cls.try_import_keras()
# if metric is already a string, don't do anything
if isinstance(metric, str):
return metric
# derive name from metric class unless it is explicitly given with a tuple
if metric is not None:
if isinstance(metric, Tuple):
metric_name = metric[0]
metric = metric[1]
elif cls.dynamic_keras_import is not None and isinstance(
metric, cls.dynamic_keras_import.metrics.Metric):
metric_name = "custom_" + str(metric.__module__) + '.' + str(
type(metric).__name__)
else:
metric_name = "custom_" + str(metric.__module__) + '.' + str(
metric.__name__)
metric_name = metric_name.lower()
else:
raise ValueError("Metric is None")
# Check if metric_name is already registered
if metric_name in Scorer.CUSTOM_ELEMENT_DICTIONARY:
warn_text = 'Custom metric name ' + metric_name + ' is ambiguous. Please specify metric as tuple with ' + \
'cooresponding name (e.g. instead of metrics=[keras.metrics.Accuracy] use ' \
'metrics=[(\'MetricName1\', keras.metrics.Accuracy)]. Only the first occurance of this ' \
'metric will be used!'
logger.warning(warn_text)
raise Warning(warn_text)
return None
# derive a metric function from the given object
if cls.dynamic_keras_import is not None and (
(isinstance(metric, type) and issubclass(
metric, cls.dynamic_keras_import.metrics.Metric)) or
isinstance(metric, cls.dynamic_keras_import.metrics.Metric)):
if isinstance(metric, type) and issubclass(
metric, cls.dynamic_keras_import.metrics.Metric):
metric_obj = metric()
else:
metric_obj = metric
def metric_func(y_true, y_pred):
metric_obj.reset_states()
metric_obj.update_state(y_true=y_true, y_pred=y_pred)
return float(
cls.dynamic_keras_import.backend.eval(metric_obj.result()))
Scorer.CUSTOM_ELEMENT_DICTIONARY[metric_name] = metric_func
elif callable(metric):
Scorer.CUSTOM_ELEMENT_DICTIONARY[metric_name] = metric
return metric_name
def __init__(self, patch_size=25, nr_of_processes=1):
super(PatchImages, self).__init__(output_img=True)
# Todo: give cache folder to mother class
self.nr_of_processes = nr_of_processes
self.patch_size = patch_size
msg = "Use PatchImages wisely: not tested in content."
logger.warning(msg)
warnings.warn(msg)
def _check_custom_atlas(atlas_file):
if not path.isfile(atlas_file):
msg = "Cannot find custom atlas {}".format(atlas_file)
logger.error(msg)
raise FileNotFoundError(msg)
labels_file = path.split(atlas_file)[0] + '_labels.txt'
if not path.isfile(labels_file):
msg = "Didn't find .txt file with ROI labels. Using indices as labels."
logger.warning(msg)
warnings.warn(msg)
return AtlasObject(name=atlas_file, path=atlas_file, labels_file=labels_file)
def list_rois(self, atlas: str):
"""
ROI listing of specific atlas
:param atlas: str, atlas name
:return: roi_names: list, list of ROIs
"""
if atlas not in self.ATLAS_DICTIONARY.keys():
msg = 'Atlas {} is not supported.'.format(atlas)
logger.warning(msg)
warnings.warn(msg)
roi_names = []
else:
atlas = self.get_atlas(atlas)
roi_names = [roi.label for roi in atlas.roi_list]
return roi_names
def metric(self, value):
"""
Setter for attribute metric.
:param value: metric value
:return:
"""
try:
self._metric = value
self._greater_is_better = Scorer.greater_is_better_distinction(
self._metric)
except NameError:
self._metric = "unknown"
logger.warning(
"Your metric is not supported. Performance constraints are constantly False."
)
def load_from_mongodb(self, mongodb_connect_url: str, pipe_name: str):
connect(mongodb_connect_url, alias="photon_core")
results = list(MDBHyperpipe.objects.raw({'name': pipe_name}))
if len(results) == 1:
self.results = results[0]
elif len(results) > 1:
self.results = MDBHyperpipe.objects.order_by([
("computation_start_time", DESCENDING)
]).raw({
'name': pipe_name
}).first()
warn_text = 'Found multiple hyperpipes with that name. Returning most recent one.'
logger.warning(warn_text)
raise Warning(warn_text)
else:
raise FileNotFoundError('Could not load hyperpipe from MongoDB.')
def fwhm(self, fwhm):
if isinstance(fwhm, int): # allowing int to improve optimization
self._fwhm = [fwhm, fwhm, fwhm]
elif isinstance(fwhm, list) and len(fwhm) == 3 and all(isinstance(x, int) for x in fwhm):
self._fwhm = fwhm
elif fwhm == 'fast':
self._fwhm = fwhm
elif fwhm is None:
self._fwhm = None
warn_msg = "the fwhm in SmoothImages is None, no filtering is performed (useful when just " \
"removal of non-finite values is needed). "
logger.warning(warn_msg)
warnings.warn(warn_msg)
else:
msg = "SmoothImages expected fwhm as int, as str=='fast' or a list of three ints like [3, 3, 3]."
logger.error(msg)
raise ValueError(msg)
def dropout_rate(self, value):
"""
Setter for dropout_rate. Checks if strategy is supported.
:param value:
:return:
"""
if not type(value) in [list, float]:
raise ValueError("Dropout type is not supported. Please use one of [list, float]")
else:
if not self._hidden_layer_sizes:
self._dropout_rate = value
else:
if type(value) == float:
self._dropout_rate = [value]*len(self.hidden_layer_sizes)
logger.warning("Dropout with type float converted to type list.")
elif len(value) != len(self.hidden_layer_sizes):
raise ValueError("Dropout length missmatched layer length.")
else:
self._dropout_rate = value
def prepare(self, pipeline_elements: list, maximize_metric: bool):
self.hyperparameter_list = []
self.maximize_metric = maximize_metric
# build space
space = []
for pipe_element in pipeline_elements:
if pipe_element.__class__.__name__ == 'Switch':
error_msg = 'Scikit-Optimize cannot operate in the specified hyperparameter space with a Switch ' \
'element. We recommend the use of SMAC.'
logger.error(error_msg)
raise ValueError(error_msg)
if hasattr(pipe_element, 'hyperparameters'):
for name, value in pipe_element.hyperparameters.items():
# if we only have one value we do not need to optimize
if isinstance(value, list) and len(value) < 2:
self.constant_dictionary[name] = value[0]
continue
if isinstance(value,
PhotonCategorical) and len(value.values) < 2:
self.constant_dictionary[name] = value.values[0]
continue
skopt_param = self._convert_PHOTON_to_skopt_space(
value, name)
if skopt_param is not None:
space.append(skopt_param)
if len(space) == 0:
logger.warning(
"Did not find any hyperparameters to convert into skopt space")
self.optimizer = None
else:
self.optimizer = Optimizer(space,
"ET",
acq_func=self.acq_func,
acq_func_kwargs=self.acq_func_kwargs)
self.ask = self.ask_generator()
def plot_optimizer_history(self,
metric,
title: str = 'Optimizer History',
type: str = 'plot',
reduce_scatter_by: Union[int, str] = 'auto',
file: str = None):
"""
:param metric: specify metric that has been stored within the PHOTON results tree
:param type: 'plot' or 'scatter'
:param reduce_scatter_by: integer or string ('auto'), reduce the number of points plotted by scatter
:param file: specify a filename if you want to save the plot
:return:
"""
if metric not in self.results.hyperpipe_info.metrics:
raise ValueError(
'Metric "{}" not stored in results tree'.format(metric))
config_evaluations = self.get_config_evaluations()
minimum_config_evaluations = self.get_minimum_config_evaluations()
# handle different lengths
min_corresponding = len(min(config_evaluations[metric], key=len))
config_evaluations_corres = [
configs[:min_corresponding]
for configs in config_evaluations[metric]
]
minimum_config_evaluations_corres = [
configs[:min_corresponding]
for configs in minimum_config_evaluations[metric]
]
mean = np.nanmean(np.asarray(config_evaluations_corres), axis=0)
mean_min = np.nanmean(np.asarray(minimum_config_evaluations_corres),
axis=0)
greater_is_better = Scorer.greater_is_better_distinction(metric)
if greater_is_better:
caption = 'Maximum'
else:
caption = 'Minimum'
plt.figure()
if type == 'plot':
plt.plot(np.arange(0, len(mean)),
mean,
'-',
color='gray',
label='Mean Performance')
elif type == 'scatter':
# now do smoothing
if isinstance(reduce_scatter_by, str):
if reduce_scatter_by != 'auto':
logger.warning(
'{} is not a valid smoothing_kernel specifier. Falling back to "auto".'
.format(reduce_scatter_by))
# if auto, then calculate size of reduce_scatter_by so that 75 points on x remain
# smallest reduce_scatter_by should be 1
reduce_scatter_by = max(
[np.floor(min_corresponding / 75).astype(int), 1])
if reduce_scatter_by > 1:
plt.plot([], [],
' ',
label="scatter reduced by factor {}".format(
reduce_scatter_by))
for i, fold in enumerate(config_evaluations[metric]):
# add a few None so that list can be divided by smoothing_kernel
remaining = len(fold) % reduce_scatter_by
if remaining:
fold.extend([np.nan] * (reduce_scatter_by - remaining))
# calculate mean over every n named_steps so that plot is less cluttered
reduced_fold = np.nanmean(np.asarray(fold).reshape(
-1, reduce_scatter_by),
axis=1)
reduced_xfit = np.arange(reduce_scatter_by / 2,
len(fold),
step=reduce_scatter_by)
if i == len(config_evaluations[metric]) - 1:
plt.scatter(reduced_xfit,
np.asarray(reduced_fold),
color='gray',
alpha=0.5,
label='Performance',
marker='.')
else:
plt.scatter(reduced_xfit,
np.asarray(reduced_fold),
color='gray',
alpha=0.5,
marker='.')
else:
raise ValueError('Please specify either "plot" or "scatter".')
plt.plot(np.arange(0, len(mean_min)),
mean_min,
'-',
color='black',
label='Mean {} Performance'.format(caption))
for i, fold in enumerate(minimum_config_evaluations[metric]):
xfit = np.arange(0, len(fold))
plt.plot(xfit, fold, '-', color='black', alpha=0.5)
plt.ylabel(metric.replace('_', ' '))
plt.xlabel('No of Evaluations')
plt.legend()
plt.title(title)
if file:
plt.savefig(file)
else:
if self.output_settings:
file = os.path.join(self.output_settings.results_folder,
"optimizer_history.png")
plt.savefig(file)
plt.close()
def transform(self):
if self.range_type == "geomspace" and self.start == 0:
error_message = "Geometric sequence cannot include zero"
logger.error(error_message)
raise ValueError(error_message)
if self.range_type == "range" and self.start > self.stop:
warn_message = "NumberRange or one of its subclasses is empty cause np.arange " + \
"does not deal with start greater than stop."
logger.warning(warn_message)
values = []
if self.range_type == "range":
if not self.step:
values = np.arange(self.start,
self.stop,
dtype=self.num_type,
**self.range_params)
else:
values = np.arange(self.start,
self.stop,
self.step,
dtype=self.num_type,
**self.range_params)
elif self.range_type == "linspace":
if self.num:
values = np.linspace(self.start,
self.stop,
num=self.num,
dtype=self.num_type,
**self.range_params)
else:
values = np.linspace(self.start,
self.stop,
dtype=self.num_type,
**self.range_params)
elif self.range_type == "logspace":
if self.num_type == np.int32:
raise ValueError(
"Cannot use logspace for integer, use geomspace instead.")
if self.num:
values = np.logspace(self.start,
self.stop,
num=self.num,
dtype=self.num_type,
**self.range_params)
else:
values = np.logspace(self.start,
self.stop,
dtype=self.num_type,
**self.range_params)
elif self.range_type == "geomspace":
if self.num:
values = np.geomspace(self.start,
self.stop,
num=self.num,
dtype=self.num_type,
**self.range_params)
else:
values = np.geomspace(self.start,
self.stop,
dtype=self.num_type,
**self.range_params)
# convert to python datatype because mongodb needs it
try:
self.values = [values[i].item() for i in range(len(values))]
except:
msg = "PHOTON can not guarantee full mongodb support since you chose a non [np.integer, np.floating] " \
"subtype in NumberType.dtype."
logger.warn(msg)
raise Warning(msg)
self.values = values