def wait_next(self, async_evaluator):
future = async_evaluator.wait_next()
if future.result is not None:
evaluation = future.result
if isinstance(evaluation, Sequence):
# This signature is currently only for an ASHA evaluation result
evaluation, loss, rung, full_evaluation = evaluation
if not full_evaluation:
# We don't process low-fidelity evaluations here (for now?).
return future
individual = evaluation.individual
log_event(
log,
TOKENS.EVALUATION_RESULT,
individual.fitness.start_time,
individual.fitness.wallclock_time,
individual.fitness.process_time,
individual.fitness.values,
individual._id,
individual.pipeline_str(),
)
if self._evaluate_callback is not None:
self._evaluate_callback(evaluation)
elif future.exception is not None:
log.warning(
f"Error raised during evaluation: {str(future.exception)}.")
return future
def wait_next(self, async_evaluator):
future = async_evaluator.wait_next()
if future.result is not None:
evaluation = future.result
if isinstance(evaluation, Sequence):
# Only evaluation with this signature is currently an ASHA evaluation result
evaluation, loss, rung, full_evaluation = evaluation # not re-assignment of evaluation
if not full_evaluation:
# We don't process low-fidelity evaluations here (for now?).
return future
individual = evaluation.individual
log_event(log, TOKENS.EVALUATION_RESULT,
individual.fitness.start_time,
individual.fitness.wallclock_time,
individual.fitness.process_time,
individual.fitness.values, individual._id,
individual.pipeline_str())
if self._evaluate_callback is not None:
self._evaluate_callback(evaluation)
elif future.exception is not None:
log.warning(
f'Encountered exception while evaluating individual: {str(future.exception)}.'
)
return future
def start_activity(self,
activity: str,
time_limit: Optional[int] = None,
activity_meta: Optional[List[Any]] = None) -> Iterator[Stopwatch]:
""" Mark the start of a new activity and automatically time its duration.
TimeManager does not currently support nested activities.
Parameters
----------
activity: str
Name of the activity for reference in current activity or later look-ups.
time_limit: int, optional (default=None)
Intended time limit of the activity in seconds. Used to calculate time remaining.
activity_meta: List[Any], optional (default=None)
Any additional information about the activity to be logged.
Returns
-------
ContextManager
A context manager which when exited notes the end of the started activity.
"""
if activity_meta is None:
activity_meta = []
log_event(log, TOKENS.PHASE_START, activity, *activity_meta)
with Stopwatch() as sw:
self.current_activity = Activity(activity, sw, time_limit)
self.activities.append(self.current_activity)
yield sw
self.current_activity = None
log_event(log, TOKENS.PHASE_END, activity, *activity_meta)
log.info("{} took {:.4f}s.".format(activity, sw.elapsed_time))
def mutate(self, ind: Individual, *args, **kwargs):
def mutate_with_log():
new_individual = ind.copy_as_new()
mutator = self._mutate(new_individual, *args, **kwargs)
log_args = [
TOKENS.MUTATION, new_individual._id, ind._id, mutator.__name__
]
return new_individual, log_args
ind, log_args = self.try_until_new(mutate_with_log)
log_event(log, *log_args)
return ind
def mate(self, ind1: Individual, ind2: Individual, *args, **kwargs):
def mate_with_log():
new_individual1, new_individual2 = ind1.copy_as_new(
), ind2.copy_as_new()
self._mate(new_individual1, new_individual2, *args, **kwargs)
log_args = [
TOKENS.CROSSOVER, new_individual1._id, ind1._id, ind2._id
]
return new_individual1, log_args
individual, log_args = self.try_until_new(mate_with_log)
log_event(log, *log_args)
return individual
def evaluate_pipeline(
pipeline,
x,
y_train,
timeout: float,
metrics="accuracy",
cv=5,
logger=None,
subsample=None,
) -> Tuple:
""" Score `pipeline` with k-fold CV according to `metrics` on (a subsample of) X, y
Returns
-------
Tuple:
prediction: np.ndarray if successful, None if not
scores: tuple with one float per metric, each value is -inf on fail.
estimators: list of fitted pipelines if successful, None if not
error: None if successful, otherwise an Exception
"""
if not logger:
logger = log
if not object_is_valid_pipeline(pipeline):
raise TypeError(
f"Pipeline must not be None and requires fit, predict, steps.")
start_datetime = datetime.now()
prediction, estimators = None, None
# default score for e.g. timeout or failure
scores = tuple([float("-inf")] * len(metrics))
with stopit.ThreadingTimeout(timeout) as c_mgr:
try:
if isinstance(subsample, int) and subsample < len(y_train):
sampler = ShuffleSplit(n_splits=1,
train_size=subsample,
random_state=0)
idx, _ = next(sampler.split(x))
x, y_train = x.iloc[idx, :], y_train[idx]
prediction, scores, estimators = cross_val_predict_score(
pipeline, x, y_train, cv=cv, metrics=metrics)
except stopit.TimeoutException:
# This exception is handled by the ThreadingTimeout context manager.
raise
except KeyboardInterrupt:
raise
except Exception as e:
if isinstance(logger, MultiprocessingLogger):
logger.debug(f"{type(e)} raised during evaluation.")
else:
logger.debug(f"{type(e)} raised during evaluation.",
exc_info=True)
single_line_pipeline = str(pipeline).replace("\n", "")
log_event(
logger,
TOKENS.EVALUATION_ERROR,
start_datetime,
single_line_pipeline,
type(e),
e,
)
return prediction, scores, estimators, e
if c_mgr.state == c_mgr.INTERRUPTED:
# A TimeoutException was raised, but not by the context manager.
# This indicates that the outer context manager (the ea) timed out.
logger.info("Outer-timeout during evaluation of {}".format(pipeline))
raise stopit.utils.TimeoutException()
if not c_mgr:
# For now we treat an eval timeout the same way as
# e.g. NaN exceptions and use the default score.
single_line_pipeline = str(pipeline).replace("\n", "")
log_event(logger, TOKENS.EVALUATION_TIMEOUT, start_datetime,
single_line_pipeline)
logger.debug(f"Timeout after {timeout}s: {pipeline}")
return prediction, scores, estimators, stopit.TimeoutException()
return prediction, tuple(scores), estimators, None
def evaluate_pipeline(pipeline,
X,
y_train,
timeout: float,
metrics='accuracy',
cv=5,
logger=None,
subsample=None) -> Tuple:
""" Score `pipeline` with k-fold CV according to `metrics` on (a subsample of) X, y
Returns
-------
Tuple:
prediction: np.ndarray if successful, None if not
scores: tuple with one float per metric, each value is -inf on fail.
estimators: list of fitted pipelines if successful, None if not
error: None if successful, otherwise an Exception
"""
if not logger:
logger = log
if not object_is_valid_pipeline(pipeline):
raise ValueError(
'Pipeline is not valid. Must not be None and have `fit`, `predict` and `steps`.'
)
start_datetime = datetime.now()
prediction, estimators = None, None
scores = tuple([float('-inf')] *
len(metrics)) # default score for e.g. timeout or failure
with stopit.ThreadingTimeout(timeout) as c_mgr:
try:
if isinstance(subsample, int) and subsample < len(y_train):
idx, _ = next(
ShuffleSplit(n_splits=1,
train_size=subsample,
random_state=0).split(X))
X, y_train = X.iloc[idx, :], y_train[idx]
prediction, scores, estimators = cross_val_predict_score(
pipeline, X, y_train, cv=cv, metrics=metrics)
except stopit.TimeoutException:
# This exception is handled by the ThreadingTimeout context manager.
raise
except KeyboardInterrupt:
raise
except Exception as e:
if isinstance(logger, MultiprocessingLogger):
logger.debug(
'{} encountered while evaluating pipeline.'.format(
type(e)))
else:
logger.debug(
'{} encountered while evaluating pipeline.'.format(
type(e)),
exc_info=True)
single_line_pipeline = str(pipeline).replace('\n', '')
log_event(logger, TOKENS.EVALUATION_ERROR, start_datetime,
single_line_pipeline, type(e), e)
return prediction, scores, None, e
if c_mgr.state == c_mgr.INTERRUPTED:
# A TimeoutException was raised, but not by the context manager.
# This indicates that the outer context manager (the ea) timed out.
logger.info("Outer-timeout during evaluation of {}".format(pipeline))
raise stopit.utils.TimeoutException()
if not c_mgr:
# For now we treat an eval timeout the same way as e.g. NaN exceptions and use the default score.
single_line_pipeline = str(pipeline).replace('\n', '')
log_event(logger, TOKENS.EVALUATION_TIMEOUT, start_datetime,
single_line_pipeline)
logger.debug("Timeout after {}s: {}".format(timeout, pipeline))
return prediction, scores, estimators, None
for _ in range(8):
start_new_job()
while ((maximum_max_rung_evaluations is None)
or (len(individuals_by_rung[max_rung]) <
maximum_max_rung_evaluations)):
future = operations.wait_next(async_)
if future.result is not None:
evaluation, loss, rung, _ = future.result
individual = evaluation.individual
individuals_by_rung[rung].append((loss, individual))
# Due to `evaluate` returning additional information (like the rung),
# evaluations are not automatically logged, so we do it here.
log_event(log, ASHA_LOG_TOKEN, rung,
individual.fitness.wallclock_time,
individual.fitness.values, individual._id,
individual.pipeline_str())
start_new_job()
highest_rung_reached = max(rungs)
except stopit.TimeoutException:
log.info('ASHA ended due to timeout.')
highest_rung_reached = max(
rung for rung, individuals in individuals_by_rung.items()
if individuals != [])
if highest_rung_reached != max(rungs):
raise RuntimeWarning("Highest rung not reached.")
finally:
for rung, individuals in individuals_by_rung.items():
log.info('[{}] {}'.format(rung, len(individuals)))
def __init__(
self,
scoring: Union[str, Metric, Iterable[str],
Iterable[Metric]] = "filled_in_by_child_class",
regularize_length: bool = True,
max_pipeline_length: Optional[int] = None,
config: Dict = None,
random_state: Optional[int] = None,
max_total_time: int = 3600,
max_eval_time: Optional[int] = None,
n_jobs: Optional[int] = None,
verbosity: int = logging.WARNING,
keep_analysis_log: Optional[str] = "gama.log",
search_method: BaseSearch = AsyncEA(),
post_processing_method: BasePostProcessing = BestFitPostProcessing(),
cache: Optional[str] = None,
):
"""
Parameters
----------
scoring: str, Metric or Tuple
Specifies the/all metric(s) to optimize towards.
A string will be converted to Metric.
A tuple must specify each metric with the same type (e.g. all str).
See :ref:`Metrics` for built-in metrics.
regularize_length: bool (default=True)
If True, add pipeline length as an optimization metric.
Short pipelines should then be preferred over long ones.
max_pipeline_length: int, optional (default=None)
If set, limit the maximum number of steps in any evaluated pipeline.
Encoding and imputation are excluded.
config: Dict
Specifies available components and their valid hyperparameter settings.
For more information, see :ref:`search_space_configuration`.
random_state: int, optional (default=None)
Seed for the random number generators used in the process.
However, with `n_jobs > 1`,
there will be randomization introduced by multi-processing.
For reproducible results, set this and use `n_jobs=1`.
max_total_time: positive int (default=3600)
Time in seconds that can be used for the `fit` call.
max_eval_time: positive int, optional (default=None)
Time in seconds that can be used to evaluate any one single individual.
If None, set to 0.1 * max_total_time.
n_jobs: int, optional (default=None)
The amount of parallel processes that may be created to speed up `fit`.
Accepted values are positive integers, -1 or None.
If -1 is specified, multiprocessing.cpu_count() processes are created.
If None is specified, multiprocessing.cpu_count() / 2 processes are created.
verbosity: int (default=logging.WARNING)
Sets the level of log messages to be automatically output to terminal.
keep_analysis_log: str, optional (default='gama.log')
If non-empty str, specify filepath where the log should be stored.
If `None`, no log is stored.
search_method: BaseSearch (default=AsyncEA())
Search method to use to find good pipelines. Should be instantiated.
post_processing_method: BasePostProcessing (default=BestFitPostProcessing())
Post-processing method to create a model after the search phase.
Should be an instantiated subclass of BasePostProcessing.
cache: str, optional (default=None)
Directory to use to save intermediate results during search.
If set to None, generate a unique cache name.
"""
register_stream_log(verbosity)
if keep_analysis_log is not None:
register_file_log(keep_analysis_log)
if keep_analysis_log is not None and not os.path.isabs(
keep_analysis_log):
keep_analysis_log = os.path.abspath(keep_analysis_log)
arguments = ",".join([
f"{k}={v}" for (k, v) in locals().items()
if k not in ["self", "config"]
])
log.info(f"Using GAMA version {__version__}.")
log.info(f"{self.__class__.__name__}({arguments})")
log_event(log, TOKENS.INIT, arguments)
if n_jobs is None:
n_jobs = multiprocessing.cpu_count() // 2
log.debug("n_jobs defaulted to %d", n_jobs)
if max_total_time is None or max_total_time <= 0:
raise ValueError(
f"Expect positive int for max_total_time, got {max_total_time}."
)
if max_eval_time is not None and max_eval_time <= 0:
raise ValueError(
f"Expect None or positive int for max_eval_time, got {max_eval_time}."
)
if n_jobs < -1 or n_jobs == 0:
raise ValueError(
f"n_jobs should be -1 or positive int but is {n_jobs}.")
AsyncEvaluator.n_jobs = n_jobs
if max_eval_time is None:
max_eval_time = round(0.1 * max_total_time)
if max_eval_time > max_total_time:
log.warning(
f"max_eval_time ({max_eval_time}) > max_total_time ({max_total_time}) "
f"is not allowed. max_eval_time set to {max_total_time}.")
max_eval_time = max_total_time
self._max_eval_time = max_eval_time
self._time_manager = TimeKeeper(max_total_time)
self._metrics: Tuple[Metric, ...] = scoring_to_metric(scoring)
self._regularize_length = regularize_length
self._search_method: BaseSearch = search_method
self._post_processing = post_processing_method
if random_state is not None:
random.seed(random_state)
np.random.seed(random_state)
self._x: Optional[pd.DataFrame] = None
self._y: Optional[pd.DataFrame] = None
self._basic_encoding_pipeline: Optional[Pipeline] = None
self._fixed_pipeline_extension: List[Tuple[str, TransformerMixin]] = []
self._inferred_dtypes: List[Type] = []
self.model: object = None
self._final_pop: List[Individual] = []
self._subscribers: Dict[str, List[Callable]] = defaultdict(list)
if not cache:
cache = f"cache_{str(uuid.uuid4())}"
if isinstance(post_processing_method, EnsemblePostProcessing):
self._evaluation_library = EvaluationLibrary(
m=post_processing_method.hyperparameters["max_models"],
n=post_processing_method.hyperparameters["hillclimb_size"],
cache_directory=cache,
)
else:
# Don't keep memory-heavy evaluation meta-data (predictions, estimators)
self._evaluation_library = EvaluationLibrary(m=0,
cache_directory=cache)
self.evaluation_completed(self._evaluation_library.save_evaluation)
self._pset, parameter_checks = pset_from_config(config)
max_start_length = 3 if max_pipeline_length is None else max_pipeline_length
self._operator_set = OperatorSet(
mutate=partial(
random_valid_mutation_in_place,
primitive_set=self._pset,
max_length=max_pipeline_length,
),
mate=partial(random_crossover, max_length=max_pipeline_length),
create_from_population=partial(create_from_population,
cxpb=0.2,
mutpb=0.8),
create_new=partial(
create_random_expression,
primitive_set=self._pset,
max_length=max_start_length,
),
compile_=compile_individual,
eliminate=eliminate_from_pareto,
evaluate_callback=self._on_evaluation_completed,
completed_evaluations=self._evaluation_library.lookup,
)
def async_ea(
ops: OperatorSet,
output: List[Individual],
start_candidates: List[Individual],
restart_callback: Optional[Callable[[], bool]] = None,
max_n_evaluations: Optional[int] = None,
population_size: int = 50,
) -> List[Individual]:
""" Perform asynchronous evolutionary optimization with given operators.
Parameters
----------
ops: OperatorSet
Operator set with `evaluate`, `create`, `individual` and `eliminate` functions.
output: List[Individual]
A list which contains the set of best found individuals during search.
start_candidates: List[Individual]
A list with candidate individuals which should be used to start search from.
restart_callback: Callable[[], bool], optional (default=None)
Function which takes no arguments and returns True if search restart.
max_n_evaluations: int, optional (default=None)
If specified, only a maximum of `max_n_evaluations` individuals are evaluated.
If None, the algorithm will be run indefinitely.
population_size: int (default=50)
Maximum number of individuals in the population at any time.
Returns
-------
List[Individual]
The individuals currently in the population.
"""
if max_n_evaluations is not None and max_n_evaluations <= 0:
raise ValueError(
f"n_evaluations must be non-negative or None, is {max_n_evaluations}."
)
max_pop_size = population_size
logger = MultiprocessingLogger()
evaluate_log = partial(ops.evaluate, logger=logger)
current_population = output
n_evaluated_individuals = 0
with AsyncEvaluator() as async_:
should_restart = True
while should_restart:
should_restart = False
current_population[:] = []
log.info("Starting EA with new population.")
for individual in start_candidates:
async_.submit(evaluate_log, individual)
while (max_n_evaluations is None) or (n_evaluated_individuals <
max_n_evaluations):
future = ops.wait_next(async_)
logger.flush_to_log(log)
if future.exception is None:
individual = future.result.individual
current_population.append(individual)
if len(current_population) > max_pop_size:
to_remove = ops.eliminate(current_population, 1)
log_event(log, TOKENS.EA_REMOVE_IND, to_remove[0])
current_population.remove(to_remove[0])
if len(current_population) > 2:
new_individual = ops.create(current_population, 1)[0]
async_.submit(evaluate_log, new_individual)
should_restart = restart_callback is not None and restart_callback(
)
n_evaluated_individuals += 1
if should_restart:
log.info(
"Restart criterion met. Creating new random population."
)
log_event(log, TOKENS.EA_RESTART, n_evaluated_individuals)
start_candidates = [
ops.individual() for _ in range(max_pop_size)
]
break
return current_population
def __init__(
self,
scoring: Union[str, Metric, Tuple[Union[str, Metric],
...]] = 'filled_in_by_child_class',
regularize_length: bool = True,
max_pipeline_length: Optional[int] = None,
config: Dict = None,
random_state: Optional[int] = None,
max_total_time: int = 3600,
max_eval_time: Optional[int] = None,
n_jobs: Optional[int] = None,
verbosity: int = logging.WARNING,
keep_analysis_log: Optional[str] = 'gama.log',
search_method: BaseSearch = AsyncEA(),
post_processing_method: BasePostProcessing = BestFitPostProcessing()):
"""
Parameters
----------
scoring: str, Metric or Tuple
Specifies the/all metric(s) to optimize towards. A string will be converted to Metric. A tuple must
specify each metric with the same type (i.e. all str or all Metric). See :ref:`Metrics` for built-in
metrics.
regularize_length: bool
If True, add pipeline length as an optimization metric (preferring short over long).
max_pipeline_length: int, optional (default=None)
If set, limit the maximum number of steps in any evaluated pipeline. Encoding and imputation are excluded.
config: a dictionary which specifies available components and their valid hyperparameter settings
For more information, see :ref:`search_space_configuration`.
random_state: int, optional (default=None)
If an integer is passed, this will be the seed for the random number generators used in the process.
However, with `n_jobs > 1`, there will be randomization introduced by multi-processing.
For reproducible results, set this and use `n_jobs=1`.
max_total_time: positive int (default=3600)
Time in seconds that can be used for the `fit` call.
max_eval_time: positive int, optional (default=None)
Time in seconds that can be used to evaluate any one single individual.
If None, set to 0.1 * max_total_time.
n_jobs: int, optional (default=None)
The amount of parallel processes that may be created to speed up `fit`.
Accepted values are positive integers, -1 or None.
If -1 is specified, multiprocessing.cpu_count() processes are created.
If None is specified, multiprocessing.cpu_count() / 2 processes are created.
verbosity: int (default=logging.WARNING)
Sets the level of log messages to be automatically output to terminal.
keep_analysis_log: str, optional (default='gama.log')
If non-empty str, specifies the path (and name) where the log should be stored, e.g. /output/gama.log.
If `None`, no log is stored.
search_method: BaseSearch (default=AsyncEA())
Search method to use to find good pipelines. Should be instantiated.
post_processing_method: BasePostProcessing (default=BestFitPostProcessing())
Post-processing method to create a model after the search phase. Should be instantiated.
"""
register_stream_log(verbosity)
if keep_analysis_log is not None:
register_file_log(keep_analysis_log)
if keep_analysis_log is not None and not os.path.isabs(
keep_analysis_log):
keep_analysis_log = os.path.abspath(keep_analysis_log)
arguments = ','.join([
'{}={}'.format(k, v) for (k, v) in locals().items() if k not in [
'self', 'config', 'gamalog', 'file_handler',
'stdout_streamhandler'
]
])
log.info('Using GAMA version {}.'.format(__version__))
log.info('{}({})'.format(self.__class__.__name__, arguments))
log_event(log, TOKENS.INIT, arguments)
if n_jobs is None:
n_jobs = multiprocessing.cpu_count() // 2
log.debug('n_jobs defaulted to %d', n_jobs)
if max_total_time is None or max_total_time <= 0:
raise ValueError(
f"max_total_time should be integer greater than zero but is {max_total_time}."
)
if max_eval_time is not None and max_eval_time <= 0:
raise ValueError(
f"max_eval_time should be None or integer greater than zero but is {max_eval_time}."
)
if n_jobs < -1 or n_jobs == 0:
raise ValueError(
f"n_jobs should be -1 or positive integer but is {n_jobs}.")
elif n_jobs != -1:
# AsyncExecutor defaults to using multiprocessing.cpu_count(), i.e. n_jobs=-1
AsyncEvaluator.n_jobs = n_jobs
if max_eval_time is None:
max_eval_time = 0.1 * max_total_time
if max_eval_time > max_total_time:
log.warning(
f"max_eval_time ({max_eval_time}) > max_total_time ({max_total_time}) is not allowed. "
f"max_eval_time set to {max_total_time}.")
max_eval_time = max_total_time
self._max_eval_time = max_eval_time
self._time_manager = TimeKeeper(max_total_time)
self._metrics: Tuple[Metric] = scoring_to_metric(scoring)
self._regularize_length = regularize_length
self._search_method: BaseSearch = search_method
self._post_processing = post_processing_method
if random_state is not None:
random.seed(random_state)
np.random.seed(random_state)
self._X: Optional[pd.DataFrame] = None
self._y: Optional[pd.DataFrame] = None
self._basic_encoding_pipeline: Optional[Pipeline] = None
self._inferred_dtypes: List[Type] = []
self.model: object = None
self._final_pop = None
self._subscribers = defaultdict(list)
if isinstance(post_processing_method, EnsemblePostProcessing):
self._evaluation_library = EvaluationLibrary(
m=post_processing_method.hyperparameters['max_models'],
n=post_processing_method.hyperparameters['hillclimb_size'],
)
else:
# Don't keep memory-heavy evaluation meta-data (predictions, estimators)
self._evaluation_library = EvaluationLibrary(m=0)
self.evaluation_completed(self._evaluation_library.save_evaluation)
self._pset, parameter_checks = pset_from_config(config)
max_start_length = 3 if max_pipeline_length is None else max_pipeline_length
self._operator_set = OperatorSet(
mutate=partial(random_valid_mutation_in_place,
primitive_set=self._pset,
max_length=max_pipeline_length),
mate=partial(random_crossover, max_length=max_pipeline_length),
create_from_population=partial(create_from_population,
cxpb=0.2,
mutpb=0.8),
create_new=partial(create_random_expression,
primitive_set=self._pset,
max_length=max_start_length),
compile_=compile_individual,
eliminate=eliminate_from_pareto,
evaluate_callback=self._on_evaluation_completed)