def build_config_space(self): ''' Build ray config space from flattened spec.search for ray spec passed to run_experiments() Specify a config space in spec using `"{key}__{space_type}": {v}`. Where {space_type} is 'grid_search' of ray.tune, or any function name of np.random. - grid_search: str, int, float. v = list of choices - choice: str, int, float. v = list of choices - randint: int. v = [low, high) - uniform: float. v = [low, high) - normal: float. v = [mean, stdev) E.g. `"lr__uniform": [0.001, 0.1]`, and it will sample `lr` using np.random.uniform(0.001, 0.1) If any key uses 'grid_search', it will be combined exhaustively in combination with other random sampling. ''' config_space = {} for k, v in util.flatten_dict(self.experiment.spec['search']).items(): if '__' in k: key, space_type = k.split('__') else: key, space_type = k, 'grid_search' if space_type == 'grid_search': config_space[key] = grid_search(v) elif space_type == 'choice': config_space[key] = lambda spec, v=v: random.choice(v) else: np_fn = getattr(np.random, space_type) config_space[key] = lambda spec, v=v: np_fn(*v) # RaySearch.lab_trial on ray.remote is not thread-safe, we have to carry the trial_index from config, created at the top level on the same thread, ticking once a trial (one samping). config_space[ 'trial_index'] = lambda spec: self.experiment.info_space.tick( 'trial')['trial'] return config_space
def build_config_space(experiment): ''' Build ray config space from flattened spec.search Specify a config space in spec using `"{key}__{space_type}": {v}`. Where `{space_type}` is `grid_search` of `ray.tune`, or any function name of `np.random`: - `grid_search`: str/int/float. v = list of choices - `choice`: str/int/float. v = list of choices - `randint`: int. v = [low, high) - `uniform`: float. v = [low, high) - `normal`: float. v = [mean, stdev) For example: - `"explore_anneal_epi__randint": [10, 60],` will sample integers uniformly from 10 to 60 for `explore_anneal_epi`, - `"lr__uniform": [0.001, 0.1]`, and it will sample `lr` using `np.random.uniform(0.001, 0.1)` If any key uses `grid_search`, it will be combined exhaustively in combination with other random sampling. ''' config_space = {} for k, v in util.flatten_dict(experiment.spec['search']).items(): if '__' in k: key, space_type = k.split('__') else: key, space_type = k, 'grid_search' if space_type == 'grid_search': config_space[key] = grid_search(v) elif space_type == 'choice': config_space[key] = lambda spec, v=v: random.choice(v) else: np_fn = getattr(np.random, space_type) config_space[key] = lambda spec, v=v: np_fn(*v) return config_space
def build_config_space(experiment): ''' Build ray config space from flattened spec.search Specify a config space in spec using `"{key}__{space_type}": {v}`. Where `{space_type}` is `grid_search` of `ray.tune`, or any function name of `np.random`: - `grid_search`: str/int/float. v = list of choices - `choice`: str/int/float. v = list of choices - `randint`: int. v = [low, high) - `uniform`: float. v = [low, high) - `normal`: float. v = [mean, stdev) For example: - `"explore_anneal_epi__randint": [10, 60],` will sample integers uniformly from 10 to 60 for `explore_anneal_epi`, - `"lr__uniform": [0.001, 0.1]`, and it will sample `lr` using `np.random.uniform(0.001, 0.1)` If any key uses `grid_search`, it will be combined exhaustively in combination with other random sampling. ''' config_space = {} for k, v in util.flatten_dict(experiment.spec['search']).items(): if '__' in k: key, space_type = k.split('__') else: key, space_type = k, 'grid_search' if space_type == 'grid_search': config_space[key] = grid_search(v) elif space_type == 'choice': config_space[key] = lambda spec, v=v: random.choice(v) else: np_fn = getattr(np.random, space_type) config_space[key] = lambda spec, v=v: np_fn(*v) return config_space
def calc_population_size(experiment): '''Calculate the population size for RandomSearch or EvolutionarySearch''' pop_size = 2 # x2 for more search coverage for k, v in util.flatten_dict(experiment.spec['search']).items(): if '__' in k: key, space_type = k.split('__') else: key, space_type = k, 'grid_search' if space_type in ('grid_search', 'choice'): pop_size *= len(v) else: pop_size *= 3 return pop_size
def calc_population_size(experiment): '''Calculate the population size for RandomSearch or EvolutionarySearch''' pop_size = 2 # x2 for more search coverage for k, v in util.flatten_dict(experiment.spec['search']).items(): if '__' in k: key, space_type = k.split('__') else: key, space_type = k, 'grid_search' if space_type in ('grid_search', 'choice'): pop_size *= len(v) else: pop_size *= 3 return pop_size
def calc_population_size(experiment): '''Calculate the population size for RandomSearch or EvolutionarySearch''' pop_size = 2 # start with x2 for better sampling coverage for k, v in util.flatten_dict(experiment.spec['search']).items(): if '__' in k: key, space_type = k.split('__') else: key, space_type = k, 'grid_search' if space_type in ('grid_search', 'choice'): pop_size *= len(v) elif space_type == 'randint': pop_size *= (v[1] - v[0]) else: pop_size *= 5 return pop_size
def test_flatten_dict(d, flat_d): assert util.flatten_dict(d) == flat_d
def test_flatten_dict(d, flat_d): assert util.flatten_dict(d) == flat_d
def test_flatten_dict(test_dict): assert util.flatten_dict(test_dict) == test_dict assert util.flatten_dict({'a': {'b': 1}}) == {'a.b': 1} assert util.flatten_dict({'a': {'b': 1}}) == {'a.b': 1} assert util.flatten_dict({'a': {'b': 1}}, sep='_') == {'a_b': 1}