def _test_resource_allocation_and_time_limit(num_jobs, num_folds_parallel, time_limit): num_cpus = get_cpu_count() num_gpus = get_gpu_count_all() time_start = time.time() fold_fitting_strategy = _construct_dummy_fold_strategy( time_limit=time_limit, num_folds_parallel=num_folds_parallel) for i in range(num_jobs): fold_fitting_strategy.schedule_fold_model_fit(dict()) resources, batches, num_parallel_jobs = fold_fitting_strategy._get_resource_suggestions( len(fold_fitting_strategy.jobs)) time_elapsed = time.time() - time_start time_remaining = time_limit - time_elapsed time_limit_fold = fold_fitting_strategy._get_fold_time_limit( len(fold_fitting_strategy.jobs)) num_cpus_per_job = resources.get('num_cpus', 0) num_gpus_per_job = resources.get('num_gpus', 0) assert batches >= 1 if batches > 1: assert num_jobs <= num_parallel_jobs * batches <= (num_jobs + num_parallel_jobs) assert num_cpus_per_job * num_parallel_jobs <= num_cpus if num_gpus != 0: assert num_gpus_per_job * num_parallel_jobs <= num_gpus else: assert num_gpus_per_job == 0 assert math.isclose(time_limit_fold, (time_remaining / batches), abs_tol=0.5)
def load(cls, path, verbosity=2): """Load previously saved predictor. Parameters ---------- path : str The file name for saved pickle file. If `path` is a directory, will try to load the file `image_predictor.ag` in this directory. verbosity : int, default = 2 Verbosity levels range from 0 to 4 and control how much information is printed. Higher levels correspond to more detailed print statements (you can set verbosity = 0 to suppress warnings). If using logging, you can alternatively control amount of information printed via logger.setLevel(L), where L ranges from 0 to 50 (Note: higher values of L correspond to fewer print statements, opposite of verbosity levels) """ if os.path.isdir(path): path = os.path.join(path, 'image_predictor.ag') with open(path, 'rb') as fid: gpu_count = get_gpu_count_all() if gpu_count > 0: obj = pickle.load(fid) else: obj = CPU_Unpickler(fid).load() obj._verbosity = verbosity return obj
def __init__(self, config=None, logger=None): super(ObjectDetection, self).__init__() self._fit_summary = {} self._logger = logger if logger is not None else logging.getLogger(__name__) self._logger.setLevel(logging.INFO) self._fit_summary = {} self._results = {} # cpu and gpu setting cpu_count = get_cpu_count() gpu_count = get_gpu_count_all() # default settings if not config: if gpu_count < 1: self._logger.info('No GPU detected/allowed, using most conservative search space.') config = LiteConfig() else: config = DefaultConfig() config = config.asdict() else: if not config.get('dist_ip_addrs', None): ngpus_per_trial = config.get('ngpus_per_trial', gpu_count) ngpus_per_trial = min(ngpus_per_trial, gpu_count) if ngpus_per_trial < 1: self._logger.info('No GPU detected/allowed, using most conservative search space.') default_config = LiteConfig() else: default_config = DefaultConfig() config = default_config.merge(config, allow_new_key=True).asdict() # adjust cpu/gpu resources if not config.get('dist_ip_addrs', None): nthreads_per_trial = config.get('nthreads_per_trial', cpu_count) nthreads_per_trial = min(nthreads_per_trial, cpu_count) ngpus_per_trial = config.get('ngpus_per_trial', gpu_count) if ngpus_per_trial > gpu_count: ngpus_per_trial = gpu_count self._logger.warning( "The number of requested GPUs is greater than the number of available GPUs." "Reduce the number to %d", ngpus_per_trial) else: raise ValueError('Please specify `nthreads_per_trial` and `ngpus_per_trial` ' 'given that dist workers are available') # fix estimator-transfer relationship estimator = config.get('estimator', None) transfer = config.get('transfer', None) if estimator is not None and transfer is not None: if isinstance(estimator, ag.Space): estimator = estimator.data elif isinstance(estimator, str): estimator = [estimator] if isinstance(transfer, ag.Space): transfer = transfer.data elif isinstance(transfer, str): transfer = [transfer] valid_transfer = [] for e in estimator: for t in transfer: if e in t: valid_transfer.append(t) if not valid_transfer: raise ValueError(f'No matching `transfer` model for {estimator}') if len(valid_transfer) == 1: config['transfer'] = valid_transfer[0] else: config['transfer'] = ag.Categorical(*valid_transfer) # additional configs config['num_workers'] = nthreads_per_trial config['gpus'] = [int(i) for i in range(ngpus_per_trial)] config['seed'] = config.get('seed', np.random.randint(32,767)) config['final_fit'] = False self._cleanup_disk = config.get('cleanup_disk', True) self._config = config # scheduler options self.search_strategy = config.get('search_strategy', 'random') self.search_options = config.get('search_options', {}) self.scheduler_options = { 'resource': {'num_cpus': nthreads_per_trial, 'num_gpus': ngpus_per_trial}, 'checkpoint': config.get('checkpoint', 'checkpoint/exp1.ag'), 'num_trials': config.get('num_trials', 2), 'time_out': config.get('time_limits', 60 * 60), 'resume': (len(config.get('resume', '')) > 0), 'visualizer': config.get('visualizer', 'none'), 'time_attr': 'epoch', 'reward_attr': 'map_reward', 'dist_ip_addrs': config.get('dist_ip_addrs', None), 'searcher': self.search_strategy, 'search_options': self.search_options, 'max_reward': config.get('max_reward', 0.9)}
def _get_num_gpus_available(): return get_gpu_count_all()
def __init__(self, config=None, logger=None, problem_type=None): super(ImageClassification, self).__init__() if problem_type is None: problem_type = MULTICLASS self._problem_type = problem_type self._fit_summary = {} self._logger = logger if logger is not None else logging.getLogger(__name__) self._logger.setLevel(logging.INFO) self._fit_summary = {} self._results = {} # cpu and gpu setting cpu_count = get_cpu_count() gpu_count = get_gpu_count_all() # default settings if not config: if gpu_count < 1: self._logger.info('No GPU detected/allowed, using most conservative search space.') config = LiteConfig() else: config = DefaultConfig() config = config.asdict() else: if not config.get('dist_ip_addrs', None): ngpus_per_trial = config.get('ngpus_per_trial', gpu_count) ngpus_per_trial = min(ngpus_per_trial, gpu_count) if ngpus_per_trial < 1: self._logger.info('No GPU detected/allowed, using most conservative search space.') default_config = LiteConfig() else: default_config = DefaultConfig() config = default_config.merge(config, allow_new_key=True).asdict() # adjust cpu/gpu resources if not config.get('dist_ip_addrs', None): nthreads_per_trial = config.get('nthreads_per_trial', cpu_count) nthreads_per_trial = min(nthreads_per_trial, cpu_count) ngpus_per_trial = config.get('ngpus_per_trial', gpu_count) if ngpus_per_trial > gpu_count: ngpus_per_trial = gpu_count self._logger.warning( "The number of requested GPUs is greater than the number of available GPUs." "Reduce the number to %d", ngpus_per_trial) else: raise ValueError('Please specify `nthreads_per_trial` and `ngpus_per_trial` ' 'given that dist workers are available') # additional configs config['num_workers'] = nthreads_per_trial config['gpus'] = [int(i) for i in range(ngpus_per_trial)] config['seed'] = config.get('seed', np.random.randint(32,767)) config['final_fit'] = False self._cleanup_disk = config.get('cleanup_disk', True) self._config = config # scheduler options self.scheduler = config.get('scheduler', 'local') self.search_strategy = config.get('search_strategy', 'random') self.search_options = config.get('search_options', {}) self.scheduler_options = { 'resource': {'num_cpus': nthreads_per_trial, 'num_gpus': ngpus_per_trial}, 'checkpoint': config.get('checkpoint', 'checkpoint/exp1.ag'), 'num_trials': config.get('num_trials', 2), 'time_out': config.get('time_limits', 60 * 60), 'resume': (len(config.get('resume', '')) > 0), 'visualizer': config.get('visualizer', 'none'), 'time_attr': 'epoch', 'reward_attr': 'acc_reward', 'dist_ip_addrs': config.get('dist_ip_addrs', None), 'searcher': self.search_strategy, 'search_options': self.search_options, 'max_reward': config.get('max_reward', 0.95)}