def execute( self, config, dataset=None, training_set=None, validation_set=None, test_set=None, training_set_metadata=None, data_format=None, experiment_name="hyperopt", model_name="run", # model_load_path=None, # model_resume_path=None, skip_save_training_description=False, skip_save_training_statistics=False, skip_save_model=False, skip_save_progress=False, skip_save_log=False, skip_save_processed_input=True, skip_save_unprocessed_output=False, skip_save_predictions=False, skip_save_eval_stats=False, output_directory="results", gpus=None, gpu_memory_limit=None, allow_parallel_threads=True, backend=None, random_seed=default_random_seed, debug=False, **kwargs) -> HyperoptResults: ctx = multiprocessing.get_context('spawn') if gpus is None: gpus = get_available_gpus_cuda_string() if gpus is not None: num_available_cpus = ctx.cpu_count() if self.num_workers > num_available_cpus: logger.warning( "WARNING: num_workers={}, num_available_cpus={}. " "To avoid bottlenecks setting num workers to be less " "or equal to number of available cpus is suggested".format( self.num_workers, num_available_cpus)) if isinstance(gpus, int): gpus = str(gpus) gpus = gpus.strip() gpu_ids = gpus.split(",") num_gpus = len(gpu_ids) available_gpu_memory_list = get_available_gpu_memory() gpu_ids_meta = {} if num_gpus < self.num_workers: fraction = (num_gpus / self.num_workers) - self.epsilon for gpu_id in gpu_ids: available_gpu_memory = available_gpu_memory_list[int( gpu_id)] required_gpu_memory = fraction * available_gpu_memory if gpu_memory_limit is None: logger.warning( 'WARNING: Setting gpu_memory_limit to {} ' 'as there available gpus are {} ' 'and the num of workers is {} ' 'and the available gpu memory for gpu_id ' '{} is {}'.format(required_gpu_memory, num_gpus, self.num_workers, gpu_id, available_gpu_memory)) new_gpu_memory_limit = required_gpu_memory - \ ( self.TF_REQUIRED_MEMORY_PER_WORKER * self.num_workers) else: new_gpu_memory_limit = gpu_memory_limit if new_gpu_memory_limit > available_gpu_memory: logger.warning( 'WARNING: Setting gpu_memory_limit to available gpu ' 'memory {} minus an epsilon as the value specified is greater than ' 'available gpu memory.'.format( available_gpu_memory)) new_gpu_memory_limit = available_gpu_memory - self.epsilon_memory if required_gpu_memory < new_gpu_memory_limit: if required_gpu_memory > 0.5 * available_gpu_memory: if available_gpu_memory != new_gpu_memory_limit: logger.warning( 'WARNING: Setting gpu_memory_limit to available gpu ' 'memory {} minus an epsilon as the gpus would be underutilized for ' 'the parallel processes otherwise'. format(available_gpu_memory)) new_gpu_memory_limit = available_gpu_memory - self.epsilon_memory else: logger.warning( 'WARNING: Setting gpu_memory_limit to {} ' 'as the available gpus are {} and the num of workers ' 'are {} and the available gpu memory for gpu_id ' '{} is {}'.format(required_gpu_memory, num_gpus, self.num_workers, gpu_id, available_gpu_memory)) new_gpu_memory_limit = required_gpu_memory else: logger.warning( 'WARNING: gpu_memory_limit could be increased to {} ' 'as the available gpus are {} and the num of workers ' 'are {} and the available gpu memory for gpu_id ' '{} is {}'.format(required_gpu_memory, num_gpus, self.num_workers, gpu_id, available_gpu_memory)) process_per_gpu = int(available_gpu_memory / new_gpu_memory_limit) gpu_ids_meta[gpu_id] = { "gpu_memory_limit": new_gpu_memory_limit, "process_per_gpu": process_per_gpu } else: for gpu_id in gpu_ids: gpu_ids_meta[gpu_id] = { "gpu_memory_limit": gpu_memory_limit, "process_per_gpu": 1 } manager = ctx.Manager() self.queue = manager.Queue() for gpu_id in gpu_ids: process_per_gpu = gpu_ids_meta[gpu_id]["process_per_gpu"] gpu_memory_limit = gpu_ids_meta[gpu_id]["gpu_memory_limit"] for _ in range(process_per_gpu): gpu_id_meta = { "gpu_id": gpu_id, "gpu_memory_limit": gpu_memory_limit } self.queue.put(gpu_id_meta) pool = ctx.Pool(self.num_workers, ParallelExecutor.init_worker) try: trial_results = [] trials = 0 while not self.hyperopt_sampler.finished(): sampled_parameters = self.hyperopt_sampler.sample_batch() hyperopt_parameters = [] for i, parameters in enumerate(sampled_parameters): modified_config = substitute_parameters( copy.deepcopy(config), parameters) trial_id = trials + i hyperopt_parameters.append( dict( parameters=parameters, config=modified_config, eval_split=self.split, dataset=dataset, training_set=training_set, validation_set=validation_set, test_set=test_set, training_set_metadata=training_set_metadata, data_format=data_format, experiment_name=f'{experiment_name}_{trial_id}', model_name=model_name, # model_load_path=model_load_path, # model_resume_path=model_resume_path, skip_save_training_description= skip_save_training_description, skip_save_training_statistics= skip_save_training_statistics, skip_save_model=skip_save_model, skip_save_progress=skip_save_progress, skip_save_log=skip_save_log, # needed because of concurrent HDF5 writes skip_save_processed_input=True, skip_save_unprocessed_output= skip_save_unprocessed_output, skip_save_predictions=skip_save_predictions, skip_save_eval_stats=skip_save_eval_stats, output_directory=output_directory, gpus=gpus, gpu_memory_limit=gpu_memory_limit, allow_parallel_threads=allow_parallel_threads, backend=backend, random_seed=random_seed, debug=debug, )) trials += len(sampled_parameters) if gpus is not None: batch_results = pool.map(self._run_experiment_gpu, hyperopt_parameters) else: batch_results = pool.map(self._run_experiment, hyperopt_parameters) self.hyperopt_sampler.update_batch( (result.parameters, result.metric_score) for result in batch_results) trial_results.extend(batch_results) finally: pool.close() pool.join() ordered_trials = self.sort_hyperopt_results(trial_results) return HyperoptResults(ordered_trials=ordered_trials)
def execute( self, model_definition, data_df=None, data_train_df=None, data_validation_df=None, data_test_df=None, data_csv=None, data_train_csv=None, data_validation_csv=None, data_test_csv=None, data_hdf5=None, data_train_hdf5=None, data_validation_hdf5=None, data_test_hdf5=None, train_set_metadata_json=None, experiment_name="hyperopt", model_name="run", # model_load_path=None, # model_resume_path=None, skip_save_training_description=False, skip_save_training_statistics=False, skip_save_model=False, skip_save_progress=False, skip_save_log=False, skip_save_processed_input=False, skip_save_unprocessed_output=False, skip_save_test_predictions=False, skip_save_test_statistics=False, output_directory="results", gpus=None, gpu_memory_limit=None, allow_parallel_threads=True, use_horovod=False, random_seed=default_random_seed, debug=False, **kwargs ): ctx = multiprocessing.get_context('spawn') if gpus is None: gpus = get_available_gpus_cuda_string() if gpus is not None: num_available_cpus = ctx.cpu_count() if self.num_workers > num_available_cpus: logger.warning( "WARNING: num_workers={}, num_available_cpus={}. " "To avoid bottlenecks setting num workers to be less " "or equal to number of available cpus is suggested".format( self.num_workers, num_available_cpus ) ) if isinstance(gpus, int): gpus = str(gpus) gpus = gpus.strip() gpu_ids = gpus.split(",") num_gpus = len(gpu_ids) available_gpu_memory_list = get_available_gpu_memory() gpu_ids_meta = {} if num_gpus < self.num_workers: fraction = (num_gpus / self.num_workers) - self.epsilon for gpu_id in gpu_ids: available_gpu_memory = available_gpu_memory_list[ int(gpu_id)] required_gpu_memory = fraction * available_gpu_memory if gpu_memory_limit is None: logger.warning( 'WARNING: Setting gpu_memory_limit to {} ' 'as there available gpus are {} ' 'and the num of workers is {} ' 'and the available gpu memory for gpu_id ' '{} is {}'.format( required_gpu_memory, num_gpus, self.num_workers, gpu_id, available_gpu_memory) ) new_gpu_memory_limit = required_gpu_memory - \ ( self.TF_REQUIRED_MEMORY_PER_WORKER * self.num_workers) else: new_gpu_memory_limit = gpu_memory_limit if new_gpu_memory_limit > available_gpu_memory: logger.warning( 'WARNING: Setting gpu_memory_limit to available gpu ' 'memory {} minus an epsilon as the value specified is greater than ' 'available gpu memory.'.format( available_gpu_memory) ) new_gpu_memory_limit = available_gpu_memory - self.epsilon_memory if required_gpu_memory < new_gpu_memory_limit: if required_gpu_memory > 0.5 * available_gpu_memory: if available_gpu_memory != new_gpu_memory_limit: logger.warning( 'WARNING: Setting gpu_memory_limit to available gpu ' 'memory {} minus an epsilon as the gpus would be underutilized for ' 'the parallel processes otherwise'.format( available_gpu_memory) ) new_gpu_memory_limit = available_gpu_memory - self.epsilon_memory else: logger.warning( 'WARNING: Setting gpu_memory_limit to {} ' 'as the available gpus are {} and the num of workers ' 'are {} and the available gpu memory for gpu_id ' '{} is {}'.format( required_gpu_memory, num_gpus, self.num_workers, gpu_id, available_gpu_memory) ) new_gpu_memory_limit = required_gpu_memory else: logger.warning( 'WARNING: gpu_memory_limit could be increased to {} ' 'as the available gpus are {} and the num of workers ' 'are {} and the available gpu memory for gpu_id ' '{} is {}'.format( required_gpu_memory, num_gpus, self.num_workers, gpu_id, available_gpu_memory) ) process_per_gpu = int( available_gpu_memory / new_gpu_memory_limit) gpu_ids_meta[gpu_id] = { "gpu_memory_limit": new_gpu_memory_limit, "process_per_gpu": process_per_gpu} else: for gpu_id in gpu_ids: gpu_ids_meta[gpu_id] = { "gpu_memory_limit": gpu_memory_limit, "process_per_gpu": 1} manager = ctx.Manager() self.queue = manager.Queue() for gpu_id in gpu_ids: process_per_gpu = gpu_ids_meta[gpu_id]["process_per_gpu"] gpu_memory_limit = gpu_ids_meta[gpu_id]["gpu_memory_limit"] for _ in range(process_per_gpu): gpu_id_meta = {"gpu_id": gpu_id, "gpu_memory_limit": gpu_memory_limit} self.queue.put(gpu_id_meta) pool = ctx.Pool(self.num_workers, ParallelExecutor.init_worker) try: hyperopt_results = [] trials = 0 while not self.hyperopt_sampler.finished(): sampled_parameters = self.hyperopt_sampler.sample_batch() hyperopt_parameters = [] for i, parameters in enumerate(sampled_parameters): modified_model_definition = substitute_parameters( copy.deepcopy(model_definition), parameters) trial_id = trials + i hyperopt_parameters.append( { "parameters": parameters, "model_definition": modified_model_definition, "eval_split": self.split, "data_df": data_df, "data_train_df": data_train_df, "data_validation_df": data_validation_df, "data_test_df": data_test_df, "data_csv": data_csv, "data_train_csv": data_train_csv, "data_validation_csv": data_validation_csv, "data_test_csv": data_test_csv, "data_hdf5": data_hdf5, "data_train_hdf5": data_train_hdf5, "data_validation_hdf5": data_validation_hdf5, "data_test_hdf5": data_test_hdf5, "train_set_metadata_json": train_set_metadata_json, "experiment_name": f'{experiment_name}_{trial_id}', "model_name": model_name, # model_load_path:model_load_path, # model_resume_path:model_resume_path, 'skip_save_training_description': skip_save_training_description, 'skip_save_training_statistics': skip_save_training_statistics, 'skip_save_model': skip_save_model, 'skip_save_progress': skip_save_progress, 'skip_save_log': skip_save_log, 'skip_save_processed_input': skip_save_processed_input, 'skip_save_unprocessed_output': skip_save_unprocessed_output, 'skip_save_test_predictions': skip_save_test_predictions, 'skip_save_test_statistics': skip_save_test_statistics, 'output_directory': output_directory, 'gpus': gpus, 'gpu_memory_limit': gpu_memory_limit, 'allow_parallel_threads': allow_parallel_threads, 'use_horovod': use_horovod, 'random_seed': random_seed, 'debug': debug, } ) trials += len(sampled_parameters) if gpus is not None: batch_results = pool.map(self._train_and_eval_model_gpu, hyperopt_parameters) else: batch_results = pool.map(self._train_and_eval_model, hyperopt_parameters) self.hyperopt_sampler.update_batch( (result["parameters"], result["metric_score"]) for result in batch_results ) hyperopt_results.extend(batch_results) finally: pool.close() pool.join() hyperopt_results = self.sort_hyperopt_results(hyperopt_results) return hyperopt_results