def execute(
self,
config,
dataset=None,
training_set=None,
validation_set=None,
test_set=None,
training_set_metadata=None,
data_format=None,
experiment_name="hyperopt",
model_name="run",
# model_load_path=None,
# model_resume_path=None,
skip_save_training_description=False,
skip_save_training_statistics=False,
skip_save_model=False,
skip_save_progress=False,
skip_save_log=False,
skip_save_processed_input=True,
skip_save_unprocessed_output=False,
skip_save_predictions=False,
skip_save_eval_stats=False,
output_directory="results",
gpus=None,
gpu_memory_limit=None,
allow_parallel_threads=True,
backend=None,
random_seed=default_random_seed,
debug=False,
**kwargs) -> HyperoptResults:
ctx = multiprocessing.get_context('spawn')
if gpus is None:
gpus = get_available_gpus_cuda_string()
if gpus is not None:
num_available_cpus = ctx.cpu_count()
if self.num_workers > num_available_cpus:
logger.warning(
"WARNING: num_workers={}, num_available_cpus={}. "
"To avoid bottlenecks setting num workers to be less "
"or equal to number of available cpus is suggested".format(
self.num_workers, num_available_cpus))
if isinstance(gpus, int):
gpus = str(gpus)
gpus = gpus.strip()
gpu_ids = gpus.split(",")
num_gpus = len(gpu_ids)
available_gpu_memory_list = get_available_gpu_memory()
gpu_ids_meta = {}
if num_gpus < self.num_workers:
fraction = (num_gpus / self.num_workers) - self.epsilon
for gpu_id in gpu_ids:
available_gpu_memory = available_gpu_memory_list[int(
gpu_id)]
required_gpu_memory = fraction * available_gpu_memory
if gpu_memory_limit is None:
logger.warning(
'WARNING: Setting gpu_memory_limit to {} '
'as there available gpus are {} '
'and the num of workers is {} '
'and the available gpu memory for gpu_id '
'{} is {}'.format(required_gpu_memory, num_gpus,
self.num_workers, gpu_id,
available_gpu_memory))
new_gpu_memory_limit = required_gpu_memory - \
(
self.TF_REQUIRED_MEMORY_PER_WORKER * self.num_workers)
else:
new_gpu_memory_limit = gpu_memory_limit
if new_gpu_memory_limit > available_gpu_memory:
logger.warning(
'WARNING: Setting gpu_memory_limit to available gpu '
'memory {} minus an epsilon as the value specified is greater than '
'available gpu memory.'.format(
available_gpu_memory))
new_gpu_memory_limit = available_gpu_memory - self.epsilon_memory
if required_gpu_memory < new_gpu_memory_limit:
if required_gpu_memory > 0.5 * available_gpu_memory:
if available_gpu_memory != new_gpu_memory_limit:
logger.warning(
'WARNING: Setting gpu_memory_limit to available gpu '
'memory {} minus an epsilon as the gpus would be underutilized for '
'the parallel processes otherwise'.
format(available_gpu_memory))
new_gpu_memory_limit = available_gpu_memory - self.epsilon_memory
else:
logger.warning(
'WARNING: Setting gpu_memory_limit to {} '
'as the available gpus are {} and the num of workers '
'are {} and the available gpu memory for gpu_id '
'{} is {}'.format(required_gpu_memory,
num_gpus,
self.num_workers, gpu_id,
available_gpu_memory))
new_gpu_memory_limit = required_gpu_memory
else:
logger.warning(
'WARNING: gpu_memory_limit could be increased to {} '
'as the available gpus are {} and the num of workers '
'are {} and the available gpu memory for gpu_id '
'{} is {}'.format(required_gpu_memory,
num_gpus, self.num_workers,
gpu_id,
available_gpu_memory))
process_per_gpu = int(available_gpu_memory /
new_gpu_memory_limit)
gpu_ids_meta[gpu_id] = {
"gpu_memory_limit": new_gpu_memory_limit,
"process_per_gpu": process_per_gpu
}
else:
for gpu_id in gpu_ids:
gpu_ids_meta[gpu_id] = {
"gpu_memory_limit": gpu_memory_limit,
"process_per_gpu": 1
}
manager = ctx.Manager()
self.queue = manager.Queue()
for gpu_id in gpu_ids:
process_per_gpu = gpu_ids_meta[gpu_id]["process_per_gpu"]
gpu_memory_limit = gpu_ids_meta[gpu_id]["gpu_memory_limit"]
for _ in range(process_per_gpu):
gpu_id_meta = {
"gpu_id": gpu_id,
"gpu_memory_limit": gpu_memory_limit
}
self.queue.put(gpu_id_meta)
pool = ctx.Pool(self.num_workers, ParallelExecutor.init_worker)
try:
trial_results = []
trials = 0
while not self.hyperopt_sampler.finished():
sampled_parameters = self.hyperopt_sampler.sample_batch()
hyperopt_parameters = []
for i, parameters in enumerate(sampled_parameters):
modified_config = substitute_parameters(
copy.deepcopy(config), parameters)
trial_id = trials + i
hyperopt_parameters.append(
dict(
parameters=parameters,
config=modified_config,
eval_split=self.split,
dataset=dataset,
training_set=training_set,
validation_set=validation_set,
test_set=test_set,
training_set_metadata=training_set_metadata,
data_format=data_format,
experiment_name=f'{experiment_name}_{trial_id}',
model_name=model_name,
# model_load_path=model_load_path,
# model_resume_path=model_resume_path,
skip_save_training_description=
skip_save_training_description,
skip_save_training_statistics=
skip_save_training_statistics,
skip_save_model=skip_save_model,
skip_save_progress=skip_save_progress,
skip_save_log=skip_save_log,
# needed because of concurrent HDF5 writes
skip_save_processed_input=True,
skip_save_unprocessed_output=
skip_save_unprocessed_output,
skip_save_predictions=skip_save_predictions,
skip_save_eval_stats=skip_save_eval_stats,
output_directory=output_directory,
gpus=gpus,
gpu_memory_limit=gpu_memory_limit,
allow_parallel_threads=allow_parallel_threads,
backend=backend,
random_seed=random_seed,
debug=debug,
))
trials += len(sampled_parameters)
if gpus is not None:
batch_results = pool.map(self._run_experiment_gpu,
hyperopt_parameters)
else:
batch_results = pool.map(self._run_experiment,
hyperopt_parameters)
self.hyperopt_sampler.update_batch(
(result.parameters, result.metric_score)
for result in batch_results)
trial_results.extend(batch_results)
finally:
pool.close()
pool.join()
ordered_trials = self.sort_hyperopt_results(trial_results)
return HyperoptResults(ordered_trials=ordered_trials)
def execute(
self,
model_definition,
data_df=None,
data_train_df=None,
data_validation_df=None,
data_test_df=None,
data_csv=None,
data_train_csv=None,
data_validation_csv=None,
data_test_csv=None,
data_hdf5=None,
data_train_hdf5=None,
data_validation_hdf5=None,
data_test_hdf5=None,
train_set_metadata_json=None,
experiment_name="hyperopt",
model_name="run",
# model_load_path=None,
# model_resume_path=None,
skip_save_training_description=False,
skip_save_training_statistics=False,
skip_save_model=False,
skip_save_progress=False,
skip_save_log=False,
skip_save_processed_input=False,
skip_save_unprocessed_output=False,
skip_save_test_predictions=False,
skip_save_test_statistics=False,
output_directory="results",
gpus=None,
gpu_memory_limit=None,
allow_parallel_threads=True,
use_horovod=False,
random_seed=default_random_seed,
debug=False,
**kwargs
):
ctx = multiprocessing.get_context('spawn')
if gpus is None:
gpus = get_available_gpus_cuda_string()
if gpus is not None:
num_available_cpus = ctx.cpu_count()
if self.num_workers > num_available_cpus:
logger.warning(
"WARNING: num_workers={}, num_available_cpus={}. "
"To avoid bottlenecks setting num workers to be less "
"or equal to number of available cpus is suggested".format(
self.num_workers, num_available_cpus
)
)
if isinstance(gpus, int):
gpus = str(gpus)
gpus = gpus.strip()
gpu_ids = gpus.split(",")
num_gpus = len(gpu_ids)
available_gpu_memory_list = get_available_gpu_memory()
gpu_ids_meta = {}
if num_gpus < self.num_workers:
fraction = (num_gpus / self.num_workers) - self.epsilon
for gpu_id in gpu_ids:
available_gpu_memory = available_gpu_memory_list[
int(gpu_id)]
required_gpu_memory = fraction * available_gpu_memory
if gpu_memory_limit is None:
logger.warning(
'WARNING: Setting gpu_memory_limit to {} '
'as there available gpus are {} '
'and the num of workers is {} '
'and the available gpu memory for gpu_id '
'{} is {}'.format(
required_gpu_memory, num_gpus,
self.num_workers,
gpu_id, available_gpu_memory)
)
new_gpu_memory_limit = required_gpu_memory - \
(
self.TF_REQUIRED_MEMORY_PER_WORKER * self.num_workers)
else:
new_gpu_memory_limit = gpu_memory_limit
if new_gpu_memory_limit > available_gpu_memory:
logger.warning(
'WARNING: Setting gpu_memory_limit to available gpu '
'memory {} minus an epsilon as the value specified is greater than '
'available gpu memory.'.format(
available_gpu_memory)
)
new_gpu_memory_limit = available_gpu_memory - self.epsilon_memory
if required_gpu_memory < new_gpu_memory_limit:
if required_gpu_memory > 0.5 * available_gpu_memory:
if available_gpu_memory != new_gpu_memory_limit:
logger.warning(
'WARNING: Setting gpu_memory_limit to available gpu '
'memory {} minus an epsilon as the gpus would be underutilized for '
'the parallel processes otherwise'.format(
available_gpu_memory)
)
new_gpu_memory_limit = available_gpu_memory - self.epsilon_memory
else:
logger.warning(
'WARNING: Setting gpu_memory_limit to {} '
'as the available gpus are {} and the num of workers '
'are {} and the available gpu memory for gpu_id '
'{} is {}'.format(
required_gpu_memory, num_gpus,
self.num_workers,
gpu_id, available_gpu_memory)
)
new_gpu_memory_limit = required_gpu_memory
else:
logger.warning(
'WARNING: gpu_memory_limit could be increased to {} '
'as the available gpus are {} and the num of workers '
'are {} and the available gpu memory for gpu_id '
'{} is {}'.format(
required_gpu_memory, num_gpus,
self.num_workers,
gpu_id, available_gpu_memory)
)
process_per_gpu = int(
available_gpu_memory / new_gpu_memory_limit)
gpu_ids_meta[gpu_id] = {
"gpu_memory_limit": new_gpu_memory_limit,
"process_per_gpu": process_per_gpu}
else:
for gpu_id in gpu_ids:
gpu_ids_meta[gpu_id] = {
"gpu_memory_limit": gpu_memory_limit,
"process_per_gpu": 1}
manager = ctx.Manager()
self.queue = manager.Queue()
for gpu_id in gpu_ids:
process_per_gpu = gpu_ids_meta[gpu_id]["process_per_gpu"]
gpu_memory_limit = gpu_ids_meta[gpu_id]["gpu_memory_limit"]
for _ in range(process_per_gpu):
gpu_id_meta = {"gpu_id": gpu_id,
"gpu_memory_limit": gpu_memory_limit}
self.queue.put(gpu_id_meta)
pool = ctx.Pool(self.num_workers,
ParallelExecutor.init_worker)
try:
hyperopt_results = []
trials = 0
while not self.hyperopt_sampler.finished():
sampled_parameters = self.hyperopt_sampler.sample_batch()
hyperopt_parameters = []
for i, parameters in enumerate(sampled_parameters):
modified_model_definition = substitute_parameters(
copy.deepcopy(model_definition), parameters)
trial_id = trials + i
hyperopt_parameters.append(
{
"parameters": parameters,
"model_definition": modified_model_definition,
"eval_split": self.split,
"data_df": data_df,
"data_train_df": data_train_df,
"data_validation_df": data_validation_df,
"data_test_df": data_test_df,
"data_csv": data_csv,
"data_train_csv": data_train_csv,
"data_validation_csv": data_validation_csv,
"data_test_csv": data_test_csv,
"data_hdf5": data_hdf5,
"data_train_hdf5": data_train_hdf5,
"data_validation_hdf5": data_validation_hdf5,
"data_test_hdf5": data_test_hdf5,
"train_set_metadata_json": train_set_metadata_json,
"experiment_name": f'{experiment_name}_{trial_id}',
"model_name": model_name,
# model_load_path:model_load_path,
# model_resume_path:model_resume_path,
'skip_save_training_description': skip_save_training_description,
'skip_save_training_statistics': skip_save_training_statistics,
'skip_save_model': skip_save_model,
'skip_save_progress': skip_save_progress,
'skip_save_log': skip_save_log,
'skip_save_processed_input': skip_save_processed_input,
'skip_save_unprocessed_output': skip_save_unprocessed_output,
'skip_save_test_predictions': skip_save_test_predictions,
'skip_save_test_statistics': skip_save_test_statistics,
'output_directory': output_directory,
'gpus': gpus,
'gpu_memory_limit': gpu_memory_limit,
'allow_parallel_threads': allow_parallel_threads,
'use_horovod': use_horovod,
'random_seed': random_seed,
'debug': debug,
}
)
trials += len(sampled_parameters)
if gpus is not None:
batch_results = pool.map(self._train_and_eval_model_gpu,
hyperopt_parameters)
else:
batch_results = pool.map(self._train_and_eval_model,
hyperopt_parameters)
self.hyperopt_sampler.update_batch(
(result["parameters"], result["metric_score"]) for result
in
batch_results
)
hyperopt_results.extend(batch_results)
finally:
pool.close()
pool.join()
hyperopt_results = self.sort_hyperopt_results(hyperopt_results)
return hyperopt_results